cmd/names-lens: distill db of names from names tiles

cmd/names-lens/names-lens.go

@@ -0,0 +1,125 @@
+package main
+
+import (
+	"bytes"
+	"compress/gzip"
+	"encoding/json"
+	"io"
+	"log"
+	"os"
+	"strconv"
+	"strings"
+
+	"crawshaw.io/sqlite"
+	"filippo.io/sunlight"
+	"github.com/schollz/progressbar/v3"
+	"golang.org/x/mod/sumdb/tlog"
+)
+
+func main() {
+	conn, err := sqlite.OpenConn(os.Args[1], 0)
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer conn.Close()
+
+	if err := ExecScript(conn, `
+		CREATE TABLE IF NOT EXISTS names (name TEXT PRIMARY KEY) WITHOUT ROWID;
+		PRAGMA journal_mode = WAL;
+		PRAGMA synchronous = OFF;
+		PRAGMA cache_size = 100000;
+		PRAGMA temp_store = MEMORY;
+	`); err != nil {
+		log.Fatal(err)
+	}
+
+	insStmt, err := conn.Prepare("INSERT OR IGNORE INTO names (name) VALUES ($1);")
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	root, err := os.OpenRoot(os.Args[2])
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	checkpoint, err := root.ReadFile("checkpoint")
+	if err != nil {
+		log.Fatal(err)
+	}
+	_, rest, _ := strings.Cut(string(checkpoint), "\n")
+	size, _, _ := strings.Cut(rest, "\n")
+	end, err := strconv.ParseInt(size, 10, 64)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	bar := progressbar.Default(end)
+	for n := range end / 256 {
+		tile := sunlight.TilePath(tlog.Tile{H: 8, L: -2, N: n, W: 256})
+		data, err := root.ReadFile(tile)
+		if os.IsNotExist(err) {
+			continue // Skip missing tiles.
+		}
+		if err != nil {
+			log.Fatal(err)
+		}
+		r, err := gzip.NewReader(bytes.NewReader(data))
+		if err != nil {
+			log.Fatal(err)
+		}
+		d := json.NewDecoder(r)
+		for {
+			var entry struct {
+				DNS []string
+			}
+			if err := d.Decode(&entry); err != nil {
+				if err == io.EOF {
+					break
+				}
+				log.Fatal(err)
+			}
+			for _, name := range entry.DNS {
+				if err := insStmt.Reset(); err != nil {
+					log.Fatal(err)
+				}
+				insStmt.SetText("$1", name)
+				if _, err := insStmt.Step(); err != nil {
+					log.Fatal(err)
+				}
+			}
+			bar.Add(1)
+		}
+	}
+
+	if err := ExecScript(conn, `
+		PRAGMA sycnhronous = NORMAL;
+		PRAGMA journal_mode = MEMORY;
+		VACUUM;
+	`); err != nil {
+		log.Fatal(err)
+	}
+}
+
+func ExecScript(conn *sqlite.Conn, queries string) (err error) {
+	for {
+		queries = strings.TrimSpace(queries)
+		if queries == "" {
+			break
+		}
+		var stmt *sqlite.Stmt
+		var trailingBytes int
+		stmt, trailingBytes, err = conn.PrepareTransient(queries)
+		if err != nil {
+			return err
+		}
+		usedBytes := len(queries) - trailingBytes
+		queries = queries[usedBytes:]
+		_, err := stmt.Step()
+		stmt.Finalize()
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}

go.mod

@@ -1,6 +1,6 @@
 module filippo.io/sunlight
 
-go 1.24.4
+go 1.25.0
 
 require (
 	crawshaw.io/sqlite v0.3.3-0.20220618202545-d1964889ea3c
@@ -12,6 +12,7 @@ require (
 	github.com/aws/smithy-go v1.20.3
 	github.com/google/certificate-transparency-go v1.3.2
 	github.com/prometheus/client_golang v1.22.0
+	github.com/schollz/progressbar/v3 v3.18.0
 	golang.org/x/crypto v0.39.0
 	golang.org/x/mod v0.25.0
 	golang.org/x/net v0.40.0
@@ -53,11 +54,14 @@ require (
 	github.com/jackc/puddle/v2 v2.2.2 // indirect
 	github.com/jmespath/go-jmespath v0.4.1-0.20220621161143-b0104c826a24 // indirect
 	github.com/kr/text v0.2.0 // indirect
+	github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/prometheus/client_model v0.6.1 // indirect
 	github.com/prometheus/common v0.62.0 // indirect
 	github.com/prometheus/procfs v0.15.1 // indirect
+	github.com/rivo/uniseg v0.4.7 // indirect
 	golang.org/x/sys v0.33.0 // indirect
+	golang.org/x/term v0.32.0 // indirect
 	golang.org/x/text v0.26.0 // indirect
 	golang.org/x/time v0.11.0 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20250303144028-a0af3efb3deb // indirect

go.sum

@@ -58,6 +58,8 @@ github.com/canonical/go-sp800.90a-drbg v0.0.0-20210314144037-6eeb1040d6c3 h1:oe6
 github.com/canonical/go-sp800.90a-drbg v0.0.0-20210314144037-6eeb1040d6c3/go.mod h1:qdP0gaj0QtgX2RUZhnlVrceJ+Qln8aSlDyJwelLLFeM=
 github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
 github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/chengxilo/virtualterm v1.0.4 h1:Z6IpERbRVlfB8WkOmtbHiDbBANU7cimRIof7mk9/PwM=
+github.com/chengxilo/virtualterm v1.0.4/go.mod h1:DyxxBZz/x1iqJjFxTFcr6/x+jSpqN0iwWCOK1q10rlY=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
@@ -102,6 +104,10 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
+github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc=
+github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
+github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ=
+github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
@@ -115,8 +121,12 @@ github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ
 github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I=
 github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc=
 github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
+github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
+github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
 github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M=
 github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA=
+github.com/schollz/progressbar/v3 v3.18.0 h1:uXdoHABRFmNIjUfte/Ex7WtuyVslrw2wVPQmCN62HpA=
+github.com/schollz/progressbar/v3 v3.18.0/go.mod h1:IsO3lpbaGuzh8zIMzgY3+J8l4C8GjO0Y9S69eFvNsec=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
@@ -144,6 +154,8 @@ golang.org/x/sync v0.15.0 h1:KWH3jNZsfyT6xfAfKiz6MRNmd46ByHDYaZ7KSkCtdW8=
 golang.org/x/sync v0.15.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
 golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw=
 golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
+golang.org/x/term v0.32.0 h1:DR4lr0TjUs3epypdhTOkMmuF5CDFJ/8pOnbzMZPQ7bg=
+golang.org/x/term v0.32.0/go.mod h1:uZG1FhGx848Sqfsq4/DlJr3xGGsYMu/L5GW4abiaEPQ=
 golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M=
 golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA=
 golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0=