Skip to content

Commit 79caf03

Browse files
ChadiiiChadiii
authored
Merge pull request #13 from flagship-io/fix/custom_regex
append custom regex & add custom regex to extract info
2 parents 3a4935c + 2a24071 commit 79caf03

File tree

5 files changed

+74
-109
lines changed

5 files changed

+74
-109
lines changed

‎internal/files/search.go‎

Lines changed: 16 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -4,13 +4,13 @@ import (
44
"encoding/json"
55
"fmt"
66
"io/ioutil"
7-
"log"
87
"path/filepath"
98
"regexp"
109
"strings"
1110

1211
"github.com/flagship-io/codebase-analyzer/internal/model"
1312
"github.com/flagship-io/codebase-analyzer/pkg/config"
13+
"github.com/sirupsen/logrus"
1414
"github.com/thoas/go-funk"
1515
)
1616

@@ -67,14 +67,14 @@ func SearchFiles(cfg *config.Config, path string, resultChannel chan model.FileS
6767

6868
// Get file extension to choose matching regex
6969
ext := filepath.Ext(path)
70-
var flagRegexes []model.FlagRegex
70+
var regexes []string
7171
for _, extRegex := range model.LanguageRegexes {
72-
regxp := regexp.MustCompile(extRegex.ExtensionRegex)
72+
regxp := regexp.MustCompile(extRegex.FileExtension)
7373
if regxp.Match([]byte(ext)) {
74-
flagRegexes = extRegex.FlagRegexes
74+
regexes = append(regexes, extRegex.Regexes...)
7575
}
7676
}
77-
if len(flagRegexes) == 0 {
77+
if len(regexes) == 0 {
7878
resultChannel <- model.FileSearchResult{
7979
File: path,
8080
Results: nil,
@@ -84,15 +84,15 @@ func SearchFiles(cfg *config.Config, path string, resultChannel chan model.FileS
8484
}
8585

8686
// Add default regex for flags in commentaries
87-
flagRegexes = append(flagRegexes, model.FlagRegex{
88-
FieldRegex: `fs:flag:(.+)`,
89-
})
87+
regexes = append(regexes,
88+
`fs:flag:(.+)`,
89+
)
9090

9191
results := []model.SearchResult{}
9292

9393
flagIndexes := [][]int{}
94-
for _, flagRegex := range flagRegexes {
95-
regxp := regexp.MustCompile(flagRegex.FieldRegex)
94+
for _, regex := range regexes {
95+
regxp := regexp.MustCompile(regex)
9696
flagLineIndexes := regxp.FindAllStringIndex(fileContentStr, -1)
9797

9898
for _, flagLineIndex := range flagLineIndexes {
@@ -102,12 +102,16 @@ func SearchFiles(cfg *config.Config, path string, resultChannel chan model.FileS
102102

103103
for _, submatchIndex := range submatchIndexes {
104104
if len(submatchIndex) < 3 {
105-
log.Printf("Did not find the flag key in file %s. Code : %s", path, submatch)
105+
logrus.WithFields(logrus.Fields{
106+
"reason": "Did not find the flag key in file " + path + ". Code: " + submatch,
107+
}).Error("Key not found")
106108
continue
107109
}
108110

109111
if len(submatchIndex) < 6 {
110-
log.Printf("Did not find the flag default value in file %s. Code : %s", path, submatch)
112+
logrus.WithFields(logrus.Fields{
113+
"reason": "Did not find the flag default value in file " + path + " . Code: " + submatch,
114+
}).Warn("Type unknown")
111115
flagIndexes = append(flagIndexes, []int{
112116
flagLineIndex[0] + submatchIndex[2],
113117
flagLineIndex[0] + submatchIndex[3],

‎internal/model/language_regexes.go‎

Lines changed: 46 additions & 86 deletions
Original file line numberDiff line numberDiff line change
@@ -6,132 +6,92 @@ import (
66
)
77

88
type LanguageRegex struct {
9-
ExtensionRegex string `json:"extension_regex"`
10-
FlagRegexes []FlagRegex `json:"flag_regexes"`
11-
}
12-
13-
type FlagRegex struct {
14-
FieldRegex string `json:"field_regex"`
9+
FileExtension string `json:"file_extension"`
10+
Regexes []string `json:"regexes"`
1511
}
1612

1713
var LanguageRegexes = []LanguageRegex{
1814
{
19-
ExtensionRegex: `\.[jt]sx?$`,
20-
FlagRegexes: []FlagRegex{
21-
{
22-
FieldRegex: `useFsFlag[(](?:\s*['"](.*)['"]\s*,\s*(["'].*\s*[^"]*["']|[^)]*))\s*[)]`, // SDK React V3
23-
},
24-
{
25-
FieldRegex: `['"]?key['"]?\s*\:\s*['"](.+?)['"](?:.*\s*)['"]?defaultValue['"]?\s*\:\s*(['"].*['"]|[^\r\n\t\f\v,}]+).*[},]?`, // SDK JS V2 && SDK React V2
26-
},
27-
{
28-
FieldRegex: `getFlag[(](?:\s*["'](.*)["']\s*,\s*(["'].*\s*[^"]*["']|[^)]*))\s*[)]`, // SDK JS V3
29-
},
15+
FileExtension: `\.[jt]sx?$`,
16+
Regexes: []string{
17+
`useFsFlag[(](?:\s*['"](.*)['"]\s*,\s*(["'].*\s*[^"]*["']|[^)]*))\s*[)]`, // SDK React V3
18+
`['"]?key['"]?\s*\:\s*['"](.+?)['"](?:.*\s*)['"]?defaultValue['"]?\s*\:\s*(['"].*['"]|[^\r\n\t\f\v,}]+).*[},]?`, // SDK JS V2 && SDK React V2
19+
`getFlag[(](?:\s*["'](.*)["']\s*,\s*(["'].*\s*[^"]*["']|[^)]*))\s*[)]`, // SDK JS V3
3020
},
3121
},
22+
3223
{
33-
ExtensionRegex: `\.go$`,
34-
FlagRegexes: []FlagRegex{
35-
{
36-
FieldRegex: `\.GetModification(?:String|Number|Bool|Object|Array)\(\s*["']([\w\-]+)['"]\s*,\s*(["'][^"]*['"]|[+-]?(?:\d*[.])?\d+|true|false|False|True)(?:\s*,\s*(?:true|false|\d+|"[^"]*"))?\s*\)`, // SDK GO V2
37-
},
24+
FileExtension: `\.go$`,
25+
Regexes: []string{
26+
`\.GetModification(?:String|Number|Bool|Object|Array)\(\s*["']([\w\-]+)['"]\s*,\s*(["'][^"]*['"]|[+-]?(?:\d*[.])?\d+|true|false|False|True)(?:\s*,\s*(?:true|false|\d+|"[^"]*"))?\s*\)`, // SDK GO V2
3827
},
3928
},
4029
{
41-
ExtensionRegex: `\.py$`,
42-
FlagRegexes: []FlagRegex{
43-
{
44-
FieldRegex: `\.get_modification\(\s*["']([\w\-]+)['"]\s*,\s*(["'][^"]*['"]|[+-]?(?:\d*[.])?\d+|true|false|False|True)(?:\s*,\s*(?:true|false|True|False|\d+|"[^"]*"))?\s*\)`, // SDK PYTHON V2
45-
},
30+
FileExtension: `\.py$`,
31+
Regexes: []string{
32+
`\.get_modification\(\s*["']([\w\-]+)['"]\s*,\s*(["'][^"]*['"]|[+-]?(?:\d*[.])?\d+|true|false|False|True)(?:\s*,\s*(?:true|false|True|False|\d+|"[^"]*"))?\s*\)`, // SDK PYTHON V2
4633
},
4734
},
4835
{
49-
ExtensionRegex: `\.java$`,
50-
FlagRegexes: []FlagRegex{
51-
{
52-
FieldRegex: `\.getModification\(\s*["']([\w\-]+)['"]\s*,\s*(["'][^"]*['"]|[+-]?(?:\d*[.])?\d+|true|false|False|True)(?:\s*,\s*(?:true|false|\d+|"[^"]*"))?\s*\)`, // SDK JAVA V2
53-
},
54-
{
55-
FieldRegex: `\.getFlag[(](?:\s*["'](.*)["']\s*,\s*(["'].*\s*[^"]*["']|[^)]*))\s*[)]`, // SDK JAVA V3
56-
},
36+
FileExtension: `\.java$`,
37+
Regexes: []string{
38+
`\.getModification\(\s*["']([\w\-]+)['"]\s*,\s*(["'][^"]*['"]|[+-]?(?:\d*[.])?\d+|true|false|False|True)(?:\s*,\s*(?:true|false|\d+|"[^"]*"))?\s*\)`, // SDK JAVA V2
39+
`\.getFlag[(](?:\s*["'](.*)["']\s*,\s*(["'].*\s*[^"]*["']|[^)]*))\s*[)]`, // SDK JAVA V3
5740
},
5841
},
5942
{
60-
ExtensionRegex: `\.php$`,
61-
FlagRegexes: []FlagRegex{
62-
{
63-
FieldRegex: `\-\>getModification\(\s*["']([\w\-]+)['"]\s*,\s*(["'][^"]*['"]|[+-]?(?:\d*[.])?\d+|true|false|False|True)(?:\s*,\s*(?:true|false|\d+|"[^"]*"))?\s*\)`, // SDK PHP V1 && SDK PHP V2
64-
},
65-
{
66-
FieldRegex: `\-\>getFlag[(](?:\s*["'](.*)["']\s*,\s*(["'].*\s*[^"]*["']|[^)]*))\s*[)]`, // SDK PHP V3
67-
},
43+
FileExtension: `\.php$`,
44+
Regexes: []string{
45+
`\-\>getModification\(\s*["']([\w\-]+)['"]\s*,\s*(["'][^"]*['"]|[+-]?(?:\d*[.])?\d+|true|false|False|True)(?:\s*,\s*(?:true|false|\d+|"[^"]*"))?\s*\)`, // SDK PHP V1 && SDK PHP V2
46+
`\-\>getFlag[(](?:\s*["'](.*)["']\s*,\s*(["'].*\s*[^"]*["']|[^)]*))\s*[)]`, // SDK PHP V3
6847
},
6948
},
7049
{
71-
ExtensionRegex: `\.kt$`,
72-
FlagRegexes: []FlagRegex{
73-
{
74-
FieldRegex: `\.getModification\(\s*["']([\w\-]+)['"]\s*,\s*(["'][^"]*['"]|[+-]?(?:\d*[.])?\d+|true|false|False|True)(?:\s*,\s*(?:true|false|\d+|"[^"]*"))?\s*\)`, // SDK ANDROID V2
75-
},
76-
{
77-
FieldRegex: `\.getFlag[(](?:\s*["'](.*)["']\s*,\s*(["'].*\s*[^"]*["']|[^)]*))\s*[)]`, // SDK ANDROID V3
78-
},
50+
FileExtension: `\.kt$`,
51+
Regexes: []string{
52+
`\.getModification\(\s*["']([\w\-]+)['"]\s*,\s*(["'][^"]*['"]|[+-]?(?:\d*[.])?\d+|true|false|False|True)(?:\s*,\s*(?:true|false|\d+|"[^"]*"))?\s*\)`, // SDK ANDROID V2
53+
`\.getFlag[(](?:\s*["'](.*)["']\s*,\s*(["'].*\s*[^"]*["']|[^)]*))\s*[)]`, // SDK ANDROID V3
54+
7955
},
8056
},
8157
{
82-
ExtensionRegex: `\.swift$`,
83-
FlagRegexes: []FlagRegex{
84-
{
85-
FieldRegex: `\.getModification\(\s*["'](\w+)['"]\s*,\s*default(?:String|Double|Float|Int|Bool|Json|Array)\s*:\s*(["'][^"]*['"]|[+-]?(?:\d*[.])?\d+|true|false|False|True)\s*(?:,\s*activate\s*:\s*(?:true|false|\d+|"[^"]*"))?\s*\)`, // SDK iOS V2
86-
},
87-
{
88-
FieldRegex: `\.getFlag[(]\s*key\s*:\s*(?:\s*["'](.*)["']\s*,\s*defaultValue\s*:\s*(["'].*\s*[^"]*["']|[^)]*))\s*[)]`, // SDK iOS V3
89-
},
58+
FileExtension: `\.swift$`,
59+
Regexes: []string{
60+
`\.getModification\(\s*["'](\w+)['"]\s*,\s*default(?:String|Double|Float|Int|Bool|Json|Array)\s*:\s*(["'][^"]*['"]|[+-]?(?:\d*[.])?\d+|true|false|False|True)\s*(?:,\s*activate\s*:\s*(?:true|false|\d+|"[^"]*"))?\s*\)`, // SDK iOS V2
61+
`\.getFlag[(]\s*key\s*:\s*(?:\s*["'](.*)["']\s*,\s*defaultValue\s*:\s*(["'].*\s*[^"]*["']|[^)]*))\s*[)]`, // SDK iOS V3
9062
},
9163
},
9264
{
93-
ExtensionRegex: `\.m$`,
94-
FlagRegexes: []FlagRegex{
95-
{
96-
FieldRegex: `getModification\s*:\s*@\s*['"](.+?)['"](?:\s*)default(?:String|Double|Bool|Float|Int|Json|Array):\@?\s*(['"].+?['"]|YES|NO|TRUE|FALSE|true|false|[+-]?(?:\d*[.])?\d+)?`, // SDK iOS V2
97-
},
98-
{
99-
FieldRegex: `getFlagWithKey\s*:\s*\@['"](.+?)['"](?:\s*)['"]?defaultValue['"]?\s*\:\s*\@?\s*(.+?)\s*[\]]`, // SDK iOS V3
100-
},
65+
FileExtension: `\.m$`,
66+
Regexes: []string{
67+
`getModification\s*:\s*@\s*['"](.+?)['"](?:\s*)default(?:String|Double|Bool|Float|Int|Json|Array):\@?\s*(['"].+?['"]|YES|NO|TRUE|FALSE|true|false|[+-]?(?:\d*[.])?\d+)?`, // SDK iOS V2
68+
`getFlagWithKey\s*:\s*\@['"](.+?)['"](?:\s*)['"]?defaultValue['"]?\s*\:\s*\@?\s*(.+?)\s*[\]]`, // SDK iOS V3
10169
},
10270
},
10371
{
104-
ExtensionRegex: `\.[fc]s$`,
105-
FlagRegexes: []FlagRegex{
106-
{
107-
FieldRegex: `\.GetModification\(\s*["']([\w\-]+)['"]\s*,\s*(["'][^"]*['"]|[+-]?(?:\d*[.])?\d+|true|false|False|True)(?:\s*,\s*(?:true|false|\d+|"[^"]*"))?\s*\)`, // SDK .NET V1
108-
},
109-
{
110-
FieldRegex: `\.GetFlag\(\s*["']([\w\-]+)['"]\s*,\s*(["'][^"]*['"]|[+-]?(?:\d*[.])?\d+|true|false|False|True)(?:\s*,\s*(?:true|false|\d+|"[^"]*"))?\s*\)`, // SDK .NET V3
111-
},
72+
FileExtension: `\.[fc]s$`,
73+
Regexes: []string{
74+
`\.GetModification\(\s*["']([\w\-]+)['"]\s*,\s*(["'][^"]*['"]|[+-]?(?:\d*[.])?\d+|true|false|False|True)(?:\s*,\s*(?:true|false|\d+|"[^"]*"))?\s*\)`, // SDK .NET V1
75+
`\.GetFlag\(\s*["']([\w\-]+)['"]\s*,\s*(["'][^"]*['"]|[+-]?(?:\d*[.])?\d+|true|false|False|True)(?:\s*,\s*(?:true|false|\d+|"[^"]*"))?\s*\)`, // SDK .NET V3
11276
},
11377
},
11478
{
115-
ExtensionRegex: `\.vb$`,
116-
FlagRegexes: []FlagRegex{
117-
{
118-
FieldRegex: `\.GetModification\(\s*["']([\w\-]+)['"]\s*,\s*(["'][^"]*['"]|[+-]?(?:\d*[.])?\d+|true|false|False|True)(?:\s*,\s*(?:true|True|false|False|\d+|"[^"]*"))?\s*\)`, // SDK .NET V1
119-
},
120-
{
121-
FieldRegex: `\.GetFlag\(\s*["']([\w\-]+)['"]\s*,\s*(["'][^"]*['"]|[+-]?(?:\d*[.])?\d+|true|false|False|True)(?:\s*,\s*(?:true|false|\d+|"[^"]*"))?\s*\)`, // SDK .NET V3
122-
},
79+
FileExtension: `\.vb$`,
80+
Regexes: []string{
81+
`\.GetModification\(\s*["']([\w\-]+)['"]\s*,\s*(["'][^"]*['"]|[+-]?(?:\d*[.])?\d+|true|false|False|True)(?:\s*,\s*(?:true|True|false|False|\d+|"[^"]*"))?\s*\)`, // SDK .NET V1
82+
`\.GetFlag\(\s*["']([\w\-]+)['"]\s*,\s*(["'][^"]*['"]|[+-]?(?:\d*[.])?\d+|true|false|False|True)(?:\s*,\s*(?:true|false|\d+|"[^"]*"))?\s*\)`, // SDK .NET V3
12383
},
12484
},
12585
}
12686

12787
func AddCustomRegexes(customRegexJSON string) {
128-
customRegexes := []LanguageRegex{}
129-
err := json.Unmarshal([]byte(customRegexJSON), &customRegexes)
88+
regexes := []LanguageRegex{}
89+
err := json.Unmarshal([]byte(customRegexJSON), &regexes)
13090

13191
if err != nil {
13292
log.Printf("Error when parsing custom regexes : %v", err)
13393
return
13494
}
13595

136-
LanguageRegexes = append(LanguageRegexes, customRegexes...)
96+
LanguageRegexes = append(LanguageRegexes, regexes...)
13797
}

‎internal/model/language_regexes_test.go‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -8,10 +8,10 @@ import (
88
)
99

1010
func TestCustomRegex(t *testing.T) {
11-
AddCustomRegexes(`[{"extension_regex":".tsx?","flag_regexes":[{"field_regex":"\\s*['\"](.+?)['\"]"}]}]`)
11+
AddCustomRegexes(`[{"file_extension":".tsx?","regexes":["\\s*['\"](.+?)['\"]"]}]`)
1212

1313
found := funk.Find(LanguageRegexes, func(languageRegex LanguageRegex) bool {
14-
return languageRegex.ExtensionRegex == ".tsx?"
14+
return languageRegex.FileExtension == ".tsx?"
1515
})
1616

1717
assert.NotNil(t, found)
@@ -20,7 +20,7 @@ func TestCustomRegex(t *testing.T) {
2020
assert.True(t, ok)
2121
assert.NotNil(t, foundLanguageRegex)
2222

23-
assert.Equal(t, ".tsx?", foundLanguageRegex.ExtensionRegex)
24-
assert.Equal(t, 1, len(foundLanguageRegex.FlagRegexes))
25-
assert.Equal(t, "\\s*['\"](.+?)['\"]", foundLanguageRegex.FlagRegexes[0].FieldRegex)
23+
assert.Equal(t, ".tsx?", foundLanguageRegex.FileExtension)
24+
assert.Equal(t, 1, len(foundLanguageRegex.Regexes))
25+
assert.Equal(t, "\\s*['\"](.+?)['\"]", foundLanguageRegex.Regexes[0])
2626
}

‎pkg/handler/extract.go‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,13 @@ import (
1010

1111
// ExtractFlagsInfo extract all flag usage information for code
1212
func ExtractFlagsInfo(cfg *config.Config) ([]model.FileSearchResult, error) {
13+
14+
// Environment variables for the project
15+
16+
if cfg.SearchCustomRegex != "" {
17+
model.AddCustomRegexes(cfg.SearchCustomRegex)
18+
}
19+
1320
// List all files within the current directory
1421
filePaths, err := files.ListFiles(cfg.Directory, cfg.FilesToExcludes)
1522

‎pkg/handler/handler.go‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,17 +4,11 @@ import (
44
log "github.com/sirupsen/logrus"
55

66
"github.com/flagship-io/codebase-analyzer/internal/api"
7-
"github.com/flagship-io/codebase-analyzer/internal/model"
87
"github.com/flagship-io/codebase-analyzer/pkg/config"
98
)
109

1110
// AnalyzeCode loads and checks environment variables, extract flags from code and send flag infos to Flagship API
1211
func AnalyzeCode(cfg *config.Config) error {
13-
// Environment variables for the project
14-
15-
if cfg.SearchCustomRegex != "" {
16-
model.AddCustomRegexes(cfg.SearchCustomRegex)
17-
}
1812

1913
results, err := ExtractFlagsInfo(cfg)
2014

0 commit comments

Comments
 (0)