text/scanner: add examples

Added examples for use of Mode, Whitespace, and IsIdentRune properties.

Fixes #23768

Change-Id: I2528e14fde63a4476f3c25510bf0c5b73f38ba5d
Reviewed-on: https://go-review.googlesource.com/93199
Run-TryBot: Robert Griesemer <[email protected]>
Reviewed-by: Robert Griesemer <[email protected]>

Author: awoodbeck

src/text/scanner/example_test.go

@@ -1,4 +1,4 @@
-// Copyright 2015 The Go Authors. All rights reserved.
+// Copyright 2018 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"strings"
 	"text/scanner"
+	"unicode"
 )
 
 func Example() {
@@ -16,6 +17,7 @@ func Example() {
 if a > 10 {
 	someParsable = text
 }`
+
 	var s scanner.Scanner
 	s.Init(strings.NewReader(src))
 	s.Filename = "example"
@@ -34,3 +36,105 @@ if a > 10 {
 	// example:4:17: text
 	// example:5:1: }
 }
+
+func Example_isIdentRune() {
+	const src = "%var1 var2%"
+
+	var s scanner.Scanner
+	s.Init(strings.NewReader(src))
+	s.Filename = "default"
+
+	for tok := s.Scan(); tok != scanner.EOF; tok = s.Scan() {
+		fmt.Printf("%s: %s\n", s.Position, s.TokenText())
+	}
+
+	fmt.Println()
+	s.Init(strings.NewReader(src))
+	s.Filename = "percent"
+
+	// treat leading '%' as part of an identifier
+	s.IsIdentRune = func(ch rune, i int) bool {
+		return ch == '%' && i == 0 || unicode.IsLetter(ch) || unicode.IsDigit(ch) && i > 0
+	}
+
+	for tok := s.Scan(); tok != scanner.EOF; tok = s.Scan() {
+		fmt.Printf("%s: %s\n", s.Position, s.TokenText())
+	}
+
+	// Output:
+	// default:1:1: %
+	// default:1:2: var1
+	// default:1:7: var2
+	// default:1:11: %
+	//
+	// percent:1:1: %var1
+	// percent:1:7: var2
+	// percent:1:11: %
+}
+
+func Example_mode() {
+	const src = `
+    // Comment begins at column 5.
+
+This line should not be included in the output.
+
+/*
+This multiline comment
+should be extracted in
+its entirety.
+*/
+`
+
+	var s scanner.Scanner
+	s.Init(strings.NewReader(src))
+	s.Filename = "comments"
+	s.Mode ^= scanner.SkipComments // don't skip comments
+
+	for tok := s.Scan(); tok != scanner.EOF; tok = s.Scan() {
+		txt := s.TokenText()
+		if strings.HasPrefix(txt, "//") || strings.HasPrefix(txt, "/*") {
+			fmt.Printf("%s: %s\n", s.Position, txt)
+		}
+	}
+
+	// Output:
+	// comments:2:5: // Comment begins at column 5.
+	// comments:6:1: /*
+	// This multiline comment
+	// should be extracted in
+	// its entirety.
+	// */
+}
+
+func Example_whitespace() {
+	// tab-separated values
+	const src = `aa	ab	ac	ad
+ba	bb	bc	bd
+ca	cb	cc	cd
+da	db	dc	dd`
+
+	var (
+		col, row int
+		s        scanner.Scanner
+		tsv      [4][4]string // large enough for example above
+	)
+	s.Init(strings.NewReader(src))
+	s.Whitespace ^= 1<<'\t' | 1<<'\n' // don't skip tabs and new lines
+
+	for tok := s.Scan(); tok != scanner.EOF; tok = s.Scan() {
+		switch tok {
+		case '\n':
+			row++
+			col = 0
+		case '\t':
+			col++
+		default:
+			tsv[row][col] = s.TokenText()
+		}
+	}
+
+	fmt.Print(tsv)
+
+	// Output:
+	// [[aa ab ac ad] [ba bb bc bd] [ca cb cc cd] [da db dc dd]]
+}