Skip to content

Split detection engine into pkg/quickdup library, keep cmd/quickdup as CLI #10

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
rogeralsing merged 1 commit into from
Jul 15, 2026
Merged

Split detection engine into pkg/quickdup library, keep cmd/quickdup as CLI #10

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 7 additions & 5 deletions cmd/quickdup/compare.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@ import (
"path/filepath"
"sort"
"strings"

"github.com/asynkron/Asynkron.QuickDup/pkg/quickdup"
)

// runCompare compares duplicate patterns between two git commits
Expand Down Expand Up @@ -106,7 +108,7 @@ func runCompare(baseRef, headRef, subdir, ext, exclude string, minOccur, minRank
}

headOccur := make(map[string]int)
headPatterns := make(map[string]JSONPattern)
headPatterns := make(map[string]quickdup.JSONPattern)
for _, p := range headResults.Patterns {
headOccur[p.Hash] = p.Occurrences
headPatterns[p.Hash] = p
Expand All @@ -122,7 +124,7 @@ func runCompare(baseRef, headRef, subdir, ext, exclude string, minOccur, minRank
baseCount int
headCount int
removed int
pattern JSONPattern
pattern quickdup.JSONPattern
}
var lingeringPatterns []lingering

Expand Down Expand Up @@ -187,12 +189,12 @@ func runCompare(baseRef, headRef, subdir, ext, exclude string, minOccur, minRank
}
}

func loadJSONResults(path string) JSONOutput {
func loadJSONResults(path string) quickdup.JSONOutput {
data, err := os.ReadFile(path)
if err != nil {
return JSONOutput{}
return quickdup.JSONOutput{}
}
var output JSONOutput
var output quickdup.JSONOutput
json.Unmarshal(data, &output)
return output
}
Expand Down
49 changes: 23 additions & 26 deletions cmd/quickdup/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,9 @@ import (
"runtime"
"strings"
"time"
)

// Active strategy (set from --strategy flag)
var activeStrategy Strategy
var debugEnabled bool
"github.com/asynkron/Asynkron.QuickDup/pkg/quickdup"
)

// Default comment prefixes by file extension
var commentPrefixes = map[string]string{
Expand Down Expand Up @@ -93,8 +91,6 @@ var commentPrefixes = map[string]string{
".vbs": "'",
}

var commentPrefix string

func main() {
path := flag.String("path", ".", "Path to scan")
filePath := flag.String("file", "", "Scan a single file (overrides --path)")
Expand All @@ -118,7 +114,8 @@ func main() {
debug := flag.Bool("debug", false, "Print verbose progress for long-running phases")
timeoutSeconds := flag.Int("timeout", 20, "Hard timeout in seconds (0 disables)")
flag.Parse()
debugEnabled = *debug
quickdup.Debug = *debug
quickdup.ProgressOutput = os.Stdout
if *timeoutSeconds > 0 {
timeout := time.Duration(*timeoutSeconds) * time.Second
go func() {
Expand All @@ -133,14 +130,14 @@ func main() {
}

// Select strategy
strategies := map[string]Strategy{
"word-indent": &WordIndentStrategy{},
"normalized-indent": &NormalizedIndentStrategy{},
"word-only": &WordOnlyStrategy{},
"inlineable": &InlineableStrategy{},
strategies := map[string]quickdup.Strategy{
"word-indent": &quickdup.WordIndentStrategy{},
"normalized-indent": &quickdup.NormalizedIndentStrategy{},
"word-only": &quickdup.WordOnlyStrategy{},
"inlineable": &quickdup.InlineableStrategy{},
}
if s, ok := strategies[*strategyName]; ok {
activeStrategy = s
quickdup.ActiveStrategy = s
} else {
fmt.Fprintf(os.Stderr, "Unknown strategy: %s\n", *strategyName)
os.Exit(1)
Expand Down Expand Up @@ -218,15 +215,15 @@ func main() {

// Auto-detect comment prefix from extension, allow override
if *comment != "" {
commentPrefix = *comment
quickdup.CommentPrefix = *comment
} else if prefix, ok := commentPrefixes[extension]; ok {
commentPrefix = prefix
quickdup.CommentPrefix = prefix
} else {
commentPrefix = "//" // fallback default
quickdup.CommentPrefix = "//" // fallback default
}

// Load user-ignored hashes from ignore.json
userIgnored := LoadIgnoredHashes(folder, *strategyName)
userIgnored := quickdup.LoadIgnoredHashes(folder, *strategyName)
PrintIgnoredPatterns(len(userIgnored))

// First pass: count files
Expand Down Expand Up @@ -277,16 +274,16 @@ func main() {
PrintScanStart(totalFiles, runtime.NumCPU())

parseStart := time.Now()
var cache *FileCache
var cache *quickdup.FileCache
if !*noCache {
cache = loadCache(folder, *strategyName)
cache = quickdup.LoadCache(folder, *strategyName)
}

fileData, cacheHits, cacheMisses := parseFilesWithCache(files, cache)
fileData, cacheHits, cacheMisses := quickdup.ParseFilesWithCache(files, cache)
Comment on lines +279 to +282

// Save updated cache
if !*noCache && cacheMisses > 0 {
saveCache(folder, *strategyName, files, fileData)
quickdup.SaveCache(folder, *strategyName, files, fileData)
}
parseTime := time.Since(parseStart)

Expand All @@ -301,13 +298,13 @@ func main() {
// Phase 2: Pattern detection with growth
detectStart := time.Now()
PrintDetectStart()
patterns := detectPatterns(fileData, len(fileData), *minOccur, *minSize, *maxSize, *keepOverlaps)
patterns := quickdup.DetectPatterns(fileData, len(fileData), *minOccur, *minSize, *maxSize, *keepOverlaps)
detectTime := time.Since(detectStart)
PrintDetectComplete(detectTime)

// Filter and score matches
filterStart := time.Now()
matches, filterStats := FilterPatterns(patterns, FilterConfig{
matches, filterStats := quickdup.FilterPatterns(patterns, quickdup.FilterConfig{
MinOccur: *minOccur,
MinRank: *minRank,
MinSimilarity: *minSimilarity,
Expand All @@ -318,7 +315,7 @@ func main() {
// Report results
PrintFilterComplete(filterTime, filterStats.SkippedBlocked, filterStats.SkippedLowRank, filterStats.SkippedLowSimilarity, *minRank, *minSimilarity)

top := TopN(matches, *topN)
top := quickdup.TopN(matches, *topN)

if *githubAnnotations {
PrintGitHubAnnotations(top, len(top), *githubLevel, *gitDiff, changedFiles)
Expand Down Expand Up @@ -379,7 +376,7 @@ func parseSelectRange(s string) (skip, limit int, err error) {
}

// selectMatches returns a slice of matches starting at skip with at most limit items
func selectMatches(matches []PatternMatch, skip, limit int) []PatternMatch {
func selectMatches(matches []quickdup.PatternMatch, skip, limit int) []quickdup.PatternMatch {
if skip >= len(matches) {
return nil
}
Expand All @@ -391,7 +388,7 @@ func selectMatches(matches []PatternMatch, skip, limit int) []PatternMatch {
}

// selectJSONPatterns returns a slice of JSON patterns starting at skip with at most limit items
func selectJSONPatterns(patterns []JSONPattern, skip, limit int) []JSONPattern {
func selectJSONPatterns(patterns []quickdup.JSONPattern, skip, limit int) []quickdup.JSONPattern {
if skip >= len(patterns) {
return nil
}
Expand Down
29 changes: 15 additions & 14 deletions cmd/quickdup/output.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"strings"
"time"

"github.com/asynkron/Asynkron.QuickDup/pkg/quickdup"
"github.com/charmbracelet/glamour"
"github.com/charmbracelet/lipgloss"
)
Expand Down Expand Up @@ -123,7 +124,7 @@ func PrintIgnoredPatterns(count int) {
}

// PrintGitHubAnnotations outputs GitHub Actions annotations for matches
func PrintGitHubAnnotations(matches []PatternMatch, top int, githubLevel string, gitDiff string, changedFiles map[string]bool) {
func PrintGitHubAnnotations(matches []quickdup.PatternMatch, top int, githubLevel string, gitDiff string, changedFiles map[string]bool) {
annotationCount := 0
for _, m := range matches[:top] {
loc := m.Locations[0]
Expand Down Expand Up @@ -153,7 +154,7 @@ func PrintMatchSummary(matchCount, minOccur, top int) {
}

// PrintMatches prints the top matches with their locations
func PrintMatches(matches []PatternMatch, top int) {
func PrintMatches(matches []quickdup.PatternMatch, top int) {
for i, m := range matches[:top] {
fmt.Printf("\n%s %s %s %s %s %s\n",
theme.Summary.Render(fmt.Sprintf("Pattern %d", i+1)),
Expand All @@ -172,7 +173,7 @@ func PrintMatches(matches []PatternMatch, top int) {
}

// PrintHotspots prints the duplication hotspots
func PrintHotspots(matches []PatternMatch) {
func PrintHotspots(matches []quickdup.PatternMatch) {
// Count duplicated lines per file
fileDupLines := make(map[string]int)
for _, m := range matches {
Expand Down Expand Up @@ -279,7 +280,7 @@ var langFromExt = map[string]string{
}

// normalizeIndent removes common leading whitespace from lines
func normalizeIndent(entries []Entry) []string {
func normalizeIndent(entries []quickdup.Entry) []string {
if len(entries) == 0 {
return nil
}
Expand Down Expand Up @@ -331,7 +332,7 @@ func normalizeIndent(entries []Entry) []string {
}

// PrintDetailedMatches prints detailed pattern matches with source code using glow
func PrintDetailedMatches(matches []PatternMatch, ext string) {
func PrintDetailedMatches(matches []quickdup.PatternMatch, ext string) {
lang := langFromExt[ext]
if lang == "" {
lang = strings.TrimPrefix(ext, ".")
Expand Down Expand Up @@ -496,13 +497,13 @@ func renderWithGlow(markdown string) {
}

// ReadJSONResults reads results from a JSON file
func ReadJSONResults(path string) ([]JSONPattern, error) {
func ReadJSONResults(path string) ([]quickdup.JSONPattern, error) {
data, err := os.ReadFile(path)
if err != nil {
return nil, err
}

var output JSONOutput
var output quickdup.JSONOutput
if err := json.Unmarshal(data, &output); err != nil {
return nil, err
}
Expand All @@ -511,7 +512,7 @@ func ReadJSONResults(path string) ([]JSONPattern, error) {
}

// PrintDetailedMatchesFromJSON prints detailed pattern matches from JSON results
func PrintDetailedMatchesFromJSON(patterns []JSONPattern, ext string) {
func PrintDetailedMatchesFromJSON(patterns []quickdup.JSONPattern, ext string) {
lang := langFromExt[ext]
if lang == "" {
lang = strings.TrimPrefix(ext, ".")
Expand Down Expand Up @@ -631,22 +632,22 @@ func readSourceLines(filename string, startLine, count int) []string {
}

// WriteJSONResults writes the results to a JSON file
func WriteJSONResults(matches []PatternMatch, outputPath string) error {
jsonOutput := JSONOutput{
func WriteJSONResults(matches []quickdup.PatternMatch, outputPath string) error {
jsonOutput := quickdup.JSONOutput{
TotalPatterns: len(matches),
Patterns: make([]JSONPattern, 0, len(matches)),
Patterns: make([]quickdup.JSONPattern, 0, len(matches)),
}

for _, m := range matches {
locs := make([]JSONLocation, len(m.Locations))
locs := make([]quickdup.JSONLocation, len(m.Locations))
for i, loc := range m.Locations {
locs[i] = JSONLocation{
locs[i] = quickdup.JSONLocation{
Filename: loc.Filename,
LineStart: loc.LineStart,
}
}

jsonOutput.Patterns = append(jsonOutput.Patterns, JSONPattern{
jsonOutput.Patterns = append(jsonOutput.Patterns, quickdup.JSONPattern{
Hash: fmt.Sprintf("%016x", m.Hash),
Score: m.Score,
Complexity: m.Complexity,
Expand Down
Loading
Loading