add simple #lang to output files

info.rkt

@@ -10,8 +10,8 @@
   "typed-racket-doc"
   "typed-racket-lib" ;; why?
 ))
-(define pkg-desc "For measuring")
-(define version "0.4")
+(define pkg-desc "Benchmark harness")
+(define version "1.0")
 (define pkg-authors '(ben))
 (define scribblings '(("scribblings/gtp-measure.scrbl" ())))
 (define raco-commands '(("gtp-measure" (submod gtp-measure/private/raco main) "Collect performance data" #f)))

output/README.md

@@ -0,0 +1,35 @@
+output
+===
+
+Output formats for `gtp-measure`.
+
+
+### `#lang gtp-measure/output/file`
+
+Output for a single file, contains a sequence of lines:
+```
+DATA
+...
+```
+
+Where `DATA` is one of the following:
++ `(TIME-STR ...)`
+  each `TIME-STR` reports CPU time, real time, and gc time
++ `timeout N`
+  where `N` is a natural number; see `key:time-limit` in docs
++ `ERR-STR`
+  a Racket runtime error message
+
+
+### `#lang gtp-measure/output/typed-untyped`
+
+Output for a mixed-typed program, contains a sequence of lines:
+
+```
+(CONFIG-STR FILE-DATA)
+...
+```
+
+where `CONFIG-STR` is the name of a typed/untyped configuration
+and `FILE-DATA` is a result for a single file (see above)
+

output/file.rkt

@@ -0,0 +1,26 @@
+#lang racket/base
+
+;; Defines a #lang for single-file output
+
+;; TODO parse and validate the data
+
+(provide
+  (rename-out [-#%module-begin #%module-begin]))
+
+(require
+  (for-syntax
+    racket/base
+    syntax/parse))
+
+;; -----------------------------------------------------------------------------
+
+(define-syntax (-#%module-begin stx)
+  (syntax-parse stx
+   [(_ . data*)
+    #`(#%module-begin '#,(format "~e" (syntax->datum #'data*)))]))
+
+(module* reader syntax/module-reader
+  gtp-measure/output/file
+  #:read s:read
+  #:read-syntax s:read-syntax
+  (require (prefix-in s: scribble/reader)))

output/test/file-0.rktd

@@ -0,0 +1,2 @@
+#lang gtp-measure/output/file
+timeout 44

output/test/typed-untyped-0.rkt

@@ -0,0 +1,33 @@
+#lang gtp-measure/output/typed-untyped
+("00000" ("cpu time: 566 real time: 567 gc time: 62" "cpu time: 577 real time: 578 gc time: 62" "cpu time: 562 real time: 564 gc time: 60" "cpu time: 570 real time: 571 gc time: 61" "cpu time: 564 real time: 566 gc time: 64" "cpu time: 562 real time: 563 gc time: 60" "cpu time: 575 real time: 576 gc time: 61" "cpu time: 561 real time: 562 gc time: 64"))
+("00001" ("cpu time: 820 real time: 822 gc time: 46" "cpu time: 793 real time: 795 gc time: 44" "cpu time: 818 real time: 820 gc time: 48" "cpu time: 797 real time: 799 gc time: 45" "cpu time: 805 real time: 807 gc time: 45" "cpu time: 801 real time: 803 gc time: 48" "cpu time: 805 real time: 807 gc time: 46" "cpu time: 823 real time: 825 gc time: 45"))
+("00010" ("cpu time: 561 real time: 562 gc time: 46" "cpu time: 565 real time: 566 gc time: 44" "cpu time: 552 real time: 553 gc time: 47" "cpu time: 555 real time: 556 gc time: 45" "cpu time: 545 real time: 546 gc time: 43" "cpu time: 550 real time: 551 gc time: 46" "cpu time: 560 real time: 561 gc time: 44" "cpu time: 553 real time: 555 gc time: 46"))
+("00011" ("cpu time: 805 real time: 807 gc time: 47" "cpu time: 813 real time: 815 gc time: 45" "cpu time: 806 real time: 808 gc time: 45" "cpu time: 808 real time: 810 gc time: 47" "cpu time: 814 real time: 816 gc time: 51" "cpu time: 817 real time: 819 gc time: 47" "cpu time: 815 real time: 817 gc time: 45" "cpu time: 813 real time: 815 gc time: 52"))
+("00100" ("cpu time: 552 real time: 553 gc time: 46" "cpu time: 552 real time: 553 gc time: 53" "cpu time: 552 real time: 553 gc time: 48" "cpu time: 549 real time: 551 gc time: 46" "cpu time: 545 real time: 547 gc time: 47" "cpu time: 552 real time: 554 gc time: 49" "cpu time: 542 real time: 543 gc time: 46" "cpu time: 555 real time: 557 gc time: 46"))
+("00101" ("cpu time: 812 real time: 814 gc time: 52" "cpu time: 810 real time: 812 gc time: 46" "cpu time: 820 real time: 822 gc time: 46" "cpu time: 818 real time: 820 gc time: 48" "cpu time: 807 real time: 808 gc time: 46" "cpu time: 820 real time: 822 gc time: 49" "cpu time: 809 real time: 811 gc time: 46" "cpu time: 825 real time: 827 gc time: 48"))
+("00110" ("cpu time: 551 real time: 553 gc time: 47" "cpu time: 554 real time: 556 gc time: 53" "cpu time: 558 real time: 559 gc time: 45" "cpu time: 558 real time: 559 gc time: 45" "cpu time: 561 real time: 562 gc time: 46" "cpu time: 565 real time: 567 gc time: 48" "cpu time: 551 real time: 553 gc time: 48" "cpu time: 552 real time: 553 gc time: 46"))
+("00111" ("cpu time: 840 real time: 842 gc time: 47" "cpu time: 840 real time: 842 gc time: 57" "cpu time: 809 real time: 811 gc time: 49" "cpu time: 812 real time: 814 gc time: 47" "cpu time: 813 real time: 815 gc time: 49" "cpu time: 835 real time: 837 gc time: 47" "cpu time: 828 real time: 830 gc time: 47" "cpu time: 819 real time: 821 gc time: 50"))
+("01000" ("cpu time: 1359 real time: 1363 gc time: 51" "cpu time: 1452 real time: 1456 gc time: 52" "cpu time: 1416 real time: 1419 gc time: 51" "cpu time: 1398 real time: 1401 gc time: 52" "cpu time: 1441 real time: 1444 gc time: 51" "cpu time: 1401 real time: 1405 gc time: 53" "cpu time: 1394 real time: 1397 gc time: 51" "cpu time: 1394 real time: 1397 gc time: 51"))
+("01001" ("cpu time: 1550 real time: 1554 gc time: 53" "cpu time: 1510 real time: 1513 gc time: 50" "cpu time: 1529 real time: 1533 gc time: 52" "cpu time: 1629 real time: 1633 gc time: 51" "cpu time: 1498 real time: 1502 gc time: 53" "cpu time: 1553 real time: 1557 gc time: 53" "cpu time: 1572 real time: 1576 gc time: 51" "cpu time: 1475 real time: 1478 gc time: 53"))
+("01010" ("cpu time: 675 real time: 677 gc time: 48" "cpu time: 675 real time: 677 gc time: 46" "cpu time: 687 real time: 688 gc time: 47" "cpu time: 671 real time: 673 gc time: 46" "cpu time: 680 real time: 682 gc time: 49" "cpu time: 683 real time: 685 gc time: 49" "cpu time: 677 real time: 678 gc time: 46" "cpu time: 676 real time: 678 gc time: 47"))
+("01011" ("cpu time: 765 real time: 767 gc time: 46" "cpu time: 776 real time: 777 gc time: 50" "cpu time: 781 real time: 783 gc time: 46" "cpu time: 777 real time: 779 gc time: 47" "cpu time: 782 real time: 784 gc time: 48" "cpu time: 766 real time: 768 gc time: 46" "cpu time: 768 real time: 770 gc time: 45" "cpu time: 779 real time: 781 gc time: 46"))
+("01100" ("cpu time: 1411 real time: 1415 gc time: 60" "cpu time: 1406 real time: 1409 gc time: 53" "cpu time: 1389 real time: 1392 gc time: 52" "cpu time: 1400 real time: 1403 gc time: 53" "cpu time: 1406 real time: 1409 gc time: 51" "cpu time: 1378 real time: 1381 gc time: 51" "cpu time: 1508 real time: 1511 gc time: 53" "cpu time: 1424 real time: 1428 gc time: 53"))
+("01101" ("cpu time: 1614 real time: 1618 gc time: 51" "cpu time: 1510 real time: 1514 gc time: 50" "cpu time: 1608 real time: 1612 gc time: 50" "cpu time: 1553 real time: 1557 gc time: 52" "cpu time: 1543 real time: 1547 gc time: 50" "cpu time: 1479 real time: 1483 gc time: 60" "cpu time: 1508 real time: 1512 gc time: 50" "cpu time: 1551 real time: 1556 gc time: 53"))
+("01110" ("cpu time: 684 real time: 685 gc time: 46" "cpu time: 680 real time: 682 gc time: 46" "cpu time: 678 real time: 679 gc time: 48" "cpu time: 699 real time: 701 gc time: 47" "cpu time: 671 real time: 673 gc time: 46" "cpu time: 677 real time: 679 gc time: 48" "cpu time: 684 real time: 686 gc time: 49" "cpu time: 676 real time: 677 gc time: 46"))
+("01111" ("cpu time: 773 real time: 774 gc time: 46" "cpu time: 830 real time: 832 gc time: 52" "cpu time: 782 real time: 784 gc time: 47" "cpu time: 765 real time: 767 gc time: 45" "cpu time: 774 real time: 776 gc time: 49" "cpu time: 771 real time: 772 gc time: 45" "cpu time: 765 real time: 767 gc time: 45" "cpu time: 774 real time: 775 gc time: 51"))
+("10000" ("cpu time: 553 real time: 554 gc time: 49" "cpu time: 546 real time: 547 gc time: 47" "cpu time: 542 real time: 543 gc time: 47" "cpu time: 551 real time: 552 gc time: 47" "cpu time: 548 real time: 549 gc time: 48" "cpu time: 554 real time: 555 gc time: 47" "cpu time: 564 real time: 565 gc time: 47" "cpu time: 552 real time: 553 gc time: 47"))
+("10001" ("cpu time: 810 real time: 812 gc time: 48" "cpu time: 815 real time: 817 gc time: 50" "cpu time: 808 real time: 810 gc time: 48" "cpu time: 808 real time: 810 gc time: 48" "cpu time: 814 real time: 816 gc time: 49" "cpu time: 810 real time: 812 gc time: 50" "cpu time: 808 real time: 810 gc time: 48" "cpu time: 824 real time: 826 gc time: 52"))
+("10010" ("cpu time: 561 real time: 562 gc time: 47" "cpu time: 559 real time: 561 gc time: 49" "cpu time: 568 real time: 569 gc time: 50" "cpu time: 557 real time: 558 gc time: 49" "cpu time: 552 real time: 553 gc time: 47" "cpu time: 555 real time: 556 gc time: 47" "cpu time: 562 real time: 563 gc time: 47" "cpu time: 552 real time: 553 gc time: 47"))
+("10011" ("cpu time: 815 real time: 817 gc time: 49" "cpu time: 824 real time: 826 gc time: 50" "cpu time: 808 real time: 810 gc time: 50" "cpu time: 826 real time: 828 gc time: 49" "cpu time: 823 real time: 825 gc time: 51" "cpu time: 827 real time: 829 gc time: 54" "cpu time: 820 real time: 822 gc time: 48" "cpu time: 821 real time: 823 gc time: 56"))
+("10100" ("cpu time: 550 real time: 551 gc time: 48" "cpu time: 556 real time: 557 gc time: 51" "cpu time: 543 real time: 544 gc time: 48" "cpu time: 556 real time: 557 gc time: 49" "cpu time: 564 real time: 565 gc time: 48" "cpu time: 562 real time: 563 gc time: 51" "cpu time: 556 real time: 557 gc time: 52" "cpu time: 562 real time: 563 gc time: 49"))
+("10101" ("cpu time: 834 real time: 836 gc time: 51" "cpu time: 813 real time: 815 gc time: 51" "cpu time: 831 real time: 832 gc time: 56" "cpu time: 809 real time: 810 gc time: 49" "cpu time: 832 real time: 834 gc time: 52" "cpu time: 823 real time: 825 gc time: 52" "cpu time: 843 real time: 845 gc time: 51" "cpu time: 814 real time: 816 gc time: 49"))
+("10110" ("cpu time: 560 real time: 561 gc time: 49" "cpu time: 553 real time: 554 gc time: 51" "cpu time: 559 real time: 560 gc time: 48" "cpu time: 560 real time: 561 gc time: 49" "cpu time: 558 real time: 559 gc time: 49" "cpu time: 556 real time: 557 gc time: 48" "cpu time: 553 real time: 554 gc time: 49" "cpu time: 561 real time: 563 gc time: 49"))
+("10111" ("cpu time: 853 real time: 855 gc time: 49" "cpu time: 819 real time: 821 gc time: 53" "cpu time: 845 real time: 847 gc time: 53" "cpu time: 818 real time: 820 gc time: 52" "cpu time: 824 real time: 826 gc time: 50" "cpu time: 839 real time: 841 gc time: 52" "cpu time: 824 real time: 826 gc time: 50" "cpu time: 834 real time: 836 gc time: 51"))
+("11000" ("cpu time: 1394 real time: 1398 gc time: 55" "cpu time: 1383 real time: 1386 gc time: 57" "cpu time: 1853 real time: 1858 gc time: 57" "cpu time: 1423 real time: 1427 gc time: 60" "cpu time: 1431 real time: 1435 gc time: 55" "cpu time: 1441 real time: 1445 gc time: 57" "cpu time: 1386 real time: 1389 gc time: 56" "cpu time: 1456 real time: 1459 gc time: 56"))
+("11001" ("cpu time: 1513 real time: 1517 gc time: 56" "cpu time: 1505 real time: 1509 gc time: 55" "cpu time: 1580 real time: 1584 gc time: 54" "cpu time: 1511 real time: 1515 gc time: 55" "cpu time: 1506 real time: 1510 gc time: 55" "cpu time: 1570 real time: 1574 gc time: 57" "cpu time: 1563 real time: 1567 gc time: 55" "cpu time: 1532 real time: 1536 gc time: 54"))
+("11010" ("cpu time: 686 real time: 687 gc time: 50" "cpu time: 697 real time: 699 gc time: 61" "cpu time: 683 real time: 685 gc time: 52" "cpu time: 685 real time: 687 gc time: 50" "cpu time: 678 real time: 680 gc time: 50" "cpu time: 685 real time: 687 gc time: 50" "cpu time: 683 real time: 685 gc time: 50" "cpu time: 678 real time: 680 gc time: 50"))
+("11011" ("cpu time: 764 real time: 765 gc time: 48" "cpu time: 778 real time: 780 gc time: 50" "cpu time: 792 real time: 794 gc time: 54" "cpu time: 776 real time: 778 gc time: 50" "cpu time: 787 real time: 789 gc time: 51" "cpu time: 769 real time: 770 gc time: 48" "cpu time: 780 real time: 782 gc time: 48" "cpu time: 778 real time: 780 gc time: 52"))
+("11100" ("cpu time: 1386 real time: 1389 gc time: 55" "cpu time: 1424 real time: 1427 gc time: 55" "cpu time: 1398 real time: 1401 gc time: 56" "cpu time: 1387 real time: 1391 gc time: 56" "cpu time: 1399 real time: 1403 gc time: 55" "cpu time: 1399 real time: 1402 gc time: 56" "cpu time: 1402 real time: 1406 gc time: 55" "cpu time: 1481 real time: 1485 gc time: 55"))
+("11101" ("cpu time: 1539 real time: 1543 gc time: 62" "cpu time: 1545 real time: 1549 gc time: 57" "cpu time: 1508 real time: 1511 gc time: 64" "cpu time: 1507 real time: 1511 gc time: 57" "cpu time: 1545 real time: 1550 gc time: 58" "cpu time: 1534 real time: 1538 gc time: 56" "cpu time: 1485 real time: 1489 gc time: 54" "cpu time: 1532 real time: 1536 gc time: 55"))
+("11110" ("cpu time: 675 real time: 677 gc time: 49" "cpu time: 690 real time: 692 gc time: 51" "cpu time: 696 real time: 698 gc time: 51" "cpu time: 696 real time: 697 gc time: 51" "cpu time: 689 real time: 690 gc time: 49" "cpu time: 677 real time: 678 gc time: 49" "cpu time: 671 real time: 673 gc time: 49" "cpu time: 681 real time: 682 gc time: 51"))
+("11111" ("cpu time: 782 real time: 784 gc time: 47" "cpu time: 814 real time: 816 gc time: 49" "cpu time: 782 real time: 784 gc time: 51" "cpu time: 778 real time: 780 gc time: 48" "cpu time: 774 real time: 775 gc time: 47" "cpu time: 773 real time: 775 gc time: 48" "cpu time: 804 real time: 806 gc time: 49" "cpu time: 778 real time: 780 gc time: 47"))

output/typed-untyped.rkt

@@ -0,0 +1,26 @@
+#lang racket/base
+
+;; Defines a #lang for typed/untyped output
+
+;; TODO parse and validate the data
+
+(provide
+  (rename-out [-#%module-begin #%module-begin]))
+
+(require
+  (for-syntax
+    racket/base
+    syntax/parse))
+
+;; -----------------------------------------------------------------------------
+
+(define-syntax (-#%module-begin stx)
+  (syntax-parse stx
+   [(_ . data*)
+    #`(#%module-begin '#,(format "~e" (syntax->datum #'data*)))]))
+
+(module* reader syntax/module-reader
+  gtp-measure/output/file
+  #:read s:read
+  #:read-syntax s:read-syntax
+  (require (prefix-in s: scribble/reader)))

private/parse.rkt

@@ -247,7 +247,7 @@
   (test-case "racket-filenames"
     (let ((v (racket-filenames TEST)))
       (check-equal? (set-count v) 7)
-      (check set=? v (set (string->path "infinite-loop-file-target.rkt") (string->path "sample-file-target.rkt") (string->path "sample-manifest-target.rkt") (string->path "sample-manifest-target-config.rkt") (string->path "manifest1.rkt") (string->path "manifest2.rkt")))))
+      (check set=? v (set (string->path "infinite-loop-file-target.rkt") (string->path "sample-file-target.rkt") (string->path "sample-manifest-target.rkt") (string->path "sample-manifest-target-bin.rkt") (string->path "sample-manifest-target-config.rkt") (string->path "manifest1.rkt") (string->path "manifest2.rkt")))))
 
   (test-case "manifest->targets"
     (check-equal? (manifest->targets M-TGT)

private/task.rkt

@@ -101,6 +101,7 @@
 
 (define INPUT-EXTENSION #".in")
 (define OUTPUT-EXTENSION #".out")
+(define LANG-PREFIX "gtp-measure/output/")
 
 (define (task-print t port write?)
   (if write?
@@ -451,14 +452,18 @@
 
 (define (file->subtask* in-file out-file config)
   (define thunk
-    (make-file-timer in-file config))
+    (let ([main (make-file-timer in-file config)])
+      (lambda ([out-port (current-output-port)])
+        (write-lang! out-port "file")
+        (main))))
   (list (make-gtp-measure-subtask out-file thunk config)))
 
 (define (typed-untyped->subtask* tu-dir configuration-dir in-file* out-file* config)
   (define entry-file (path->string (build-path configuration-dir (config-ref config key:entry-point))))
   (for/list ([in-file (in-list in-file*)]
              [out-file (in-list out-file*)])
     (define (thunk [out-port (current-output-port)])
+      (write-lang! out-port "typed-untyped")
       (define total-configs (count-configurations in-file))
       (define fmt (make-progress-counter total-configs "configuration"))
       (with-input-from-file in-file
@@ -589,6 +594,9 @@
   (lambda (i)
     (format "[~a~a/~a]" units (fmt i) total)))
 
+(define (write-lang! out-port suffix)
+  (fprintf out-port "#lang ~a~a~n" LANG-PREFIX suffix))
+
 ;; =============================================================================
 
 (module+ test
@@ -729,8 +737,9 @@
         (begin0
           (file->lines test-file)
           (delete-file test-file))))
-    (check-equal? (length out-str*) (config-ref config key:iterations))
-    (check-true (andmap time-line? out-str*)))
+    (check-equal? (length out-str*) (+ 1 (config-ref config key:iterations)))
+    (check-equal? (car out-str*) "#lang gtp-measure/output/file")
+    (check-true (andmap time-line? (cdr out-str*))))
 
   (filesystem-test-case "copy-configuration!"
     (define configuration-dir (build-path TEST-DIR "sample-typed-untyped-configuration"))
@@ -771,15 +780,16 @@
                   (delete-directory/files configuration-dir)
                   (delete-file in-file)
                   (delete-file out-file)))))
-    (check-equal? (length out-str*) 4)
+    (check-equal? (length out-str*) 5)
     (check-equal?
       (for/sum ([msg (in-list (hash-ref log-hash 'debug))]
                 #:when (regexp-match? #rx"deleting zo folder" msg))
         1)
-      (length out-str*)
+      (- (length out-str*) 1)
       "missing evidence that 'zo' files deleted between configurations")
+    (check-equal? (car out-str*) "#lang gtp-measure/output/typed-untyped")
     (for ((c (in-list configuration*))
-          (str (in-list out-str*)))
+          (str (in-list (cdr out-str*))))
       (define v
         (with-input-from-string str read))
       (check-pred list? v)