diff --git a/cmd/names-lens/names-lens.go b/cmd/names-lens/names-lens.go new file mode 100644 index 0000000..805a45b --- /dev/null +++ b/cmd/names-lens/names-lens.go @@ -0,0 +1,125 @@ +package main + +import ( + "bytes" + "compress/gzip" + "encoding/json" + "io" + "log" + "os" + "strconv" + "strings" + + "crawshaw.io/sqlite" + "filippo.io/sunlight" + "github.com/schollz/progressbar/v3" + "golang.org/x/mod/sumdb/tlog" +) + +func main() { + conn, err := sqlite.OpenConn(os.Args[1], 0) + if err != nil { + log.Fatal(err) + } + defer conn.Close() + + if err := ExecScript(conn, ` + CREATE TABLE IF NOT EXISTS names (name TEXT PRIMARY KEY) WITHOUT ROWID; + PRAGMA journal_mode = WAL; + PRAGMA synchronous = OFF; + PRAGMA cache_size = 100000; + PRAGMA temp_store = MEMORY; + `); err != nil { + log.Fatal(err) + } + + insStmt, err := conn.Prepare("INSERT OR IGNORE INTO names (name) VALUES ($1);") + if err != nil { + log.Fatal(err) + } + + root, err := os.OpenRoot(os.Args[2]) + if err != nil { + log.Fatal(err) + } + + checkpoint, err := root.ReadFile("checkpoint") + if err != nil { + log.Fatal(err) + } + _, rest, _ := strings.Cut(string(checkpoint), "\n") + size, _, _ := strings.Cut(rest, "\n") + end, err := strconv.ParseInt(size, 10, 64) + if err != nil { + log.Fatal(err) + } + + bar := progressbar.Default(end) + for n := range end / 256 { + tile := sunlight.TilePath(tlog.Tile{H: 8, L: -2, N: n, W: 256}) + data, err := root.ReadFile(tile) + if os.IsNotExist(err) { + continue // Skip missing tiles. + } + if err != nil { + log.Fatal(err) + } + r, err := gzip.NewReader(bytes.NewReader(data)) + if err != nil { + log.Fatal(err) + } + d := json.NewDecoder(r) + for { + var entry struct { + DNS []string + } + if err := d.Decode(&entry); err != nil { + if err == io.EOF { + break + } + log.Fatal(err) + } + for _, name := range entry.DNS { + if err := insStmt.Reset(); err != nil { + log.Fatal(err) + } + insStmt.SetText("$1", name) + if _, err := insStmt.Step(); err != nil { + log.Fatal(err) + } + } + bar.Add(1) + } + } + + if err := ExecScript(conn, ` + PRAGMA sycnhronous = NORMAL; + PRAGMA journal_mode = MEMORY; + VACUUM; + `); err != nil { + log.Fatal(err) + } +} + +func ExecScript(conn *sqlite.Conn, queries string) (err error) { + for { + queries = strings.TrimSpace(queries) + if queries == "" { + break + } + var stmt *sqlite.Stmt + var trailingBytes int + stmt, trailingBytes, err = conn.PrepareTransient(queries) + if err != nil { + return err + } + usedBytes := len(queries) - trailingBytes + queries = queries[usedBytes:] + _, err := stmt.Step() + stmt.Finalize() + if err != nil { + return err + } + } + return nil +} diff --git a/go.mod b/go.mod index 73187f3..a6364b0 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module filippo.io/sunlight -go 1.24.4 +go 1.25.0 require ( crawshaw.io/sqlite v0.3.3-0.20220618202545-d1964889ea3c @@ -12,6 +12,7 @@ require ( github.com/aws/smithy-go v1.20.3 github.com/google/certificate-transparency-go v1.3.2 github.com/prometheus/client_golang v1.22.0 + github.com/schollz/progressbar/v3 v3.18.0 golang.org/x/crypto v0.39.0 golang.org/x/mod v0.25.0 golang.org/x/net v0.40.0 @@ -53,11 +54,14 @@ require ( github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/jmespath/go-jmespath v0.4.1-0.20220621161143-b0104c826a24 // indirect github.com/kr/text v0.2.0 // indirect + github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/common v0.62.0 // indirect github.com/prometheus/procfs v0.15.1 // indirect + github.com/rivo/uniseg v0.4.7 // indirect golang.org/x/sys v0.33.0 // indirect + golang.org/x/term v0.32.0 // indirect golang.org/x/text v0.26.0 // indirect golang.org/x/time v0.11.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20250303144028-a0af3efb3deb // indirect diff --git a/go.sum b/go.sum index e9295b0..d9dc713 100644 --- a/go.sum +++ b/go.sum @@ -58,6 +58,8 @@ github.com/canonical/go-sp800.90a-drbg v0.0.0-20210314144037-6eeb1040d6c3 h1:oe6 github.com/canonical/go-sp800.90a-drbg v0.0.0-20210314144037-6eeb1040d6c3/go.mod h1:qdP0gaj0QtgX2RUZhnlVrceJ+Qln8aSlDyJwelLLFeM= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chengxilo/virtualterm v1.0.4 h1:Z6IpERbRVlfB8WkOmtbHiDbBANU7cimRIof7mk9/PwM= +github.com/chengxilo/virtualterm v1.0.4/go.mod h1:DyxxBZz/x1iqJjFxTFcr6/x+jSpqN0iwWCOK1q10rlY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= @@ -102,6 +104,10 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= +github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ= +github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -115,8 +121,12 @@ github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I= github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= +github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= +github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= +github.com/schollz/progressbar/v3 v3.18.0 h1:uXdoHABRFmNIjUfte/Ex7WtuyVslrw2wVPQmCN62HpA= +github.com/schollz/progressbar/v3 v3.18.0/go.mod h1:IsO3lpbaGuzh8zIMzgY3+J8l4C8GjO0Y9S69eFvNsec= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -144,6 +154,8 @@ golang.org/x/sync v0.15.0 h1:KWH3jNZsfyT6xfAfKiz6MRNmd46ByHDYaZ7KSkCtdW8= golang.org/x/sync v0.15.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/term v0.32.0 h1:DR4lr0TjUs3epypdhTOkMmuF5CDFJ/8pOnbzMZPQ7bg= +golang.org/x/term v0.32.0/go.mod h1:uZG1FhGx848Sqfsq4/DlJr3xGGsYMu/L5GW4abiaEPQ= golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M= golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA= golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0=