From f8aa1fa3b537dd4262d6523622a45771af556a15 Mon Sep 17 00:00:00 2001 From: Hammad Afzal Date: Sat, 8 Jun 2024 21:53:49 +0500 Subject: [PATCH 1/2] feat(endpoints): Added new endpoints --- go.mod | 3 + go.sum | 6 + handlers/archives.go | 171 ++++++++++++++++++++++++ handlers/mail-config.go | 158 ++++++++++++++++++++++ handlers/robots-txt.go | 90 +++++++++++++ handlers/screenshot.go | 71 ++++++++++ handlers/security-txt.go | 115 ++++++++++++++++ handlers/sitemap.go | 141 ++++++++++++++++++++ handlers/ssl.go | 50 +++++++ handlers/status.go | 113 ++++++++++++++++ handlers/tech-stack.go | 62 +++++++++ handlers/threats.go | 278 +++++++++++++++++++++++++++++++++++++++ handlers/txt-records.go | 60 +++++++++ handlers/whois.go | 155 ++++++++++++++++++++++ server/server.go | 12 ++ 15 files changed, 1485 insertions(+) create mode 100644 handlers/archives.go create mode 100644 handlers/mail-config.go create mode 100644 handlers/robots-txt.go create mode 100644 handlers/screenshot.go create mode 100644 handlers/security-txt.go create mode 100644 handlers/sitemap.go create mode 100644 handlers/ssl.go create mode 100644 handlers/status.go create mode 100644 handlers/tech-stack.go create mode 100644 handlers/threats.go create mode 100644 handlers/txt-records.go create mode 100644 handlers/whois.go diff --git a/go.mod b/go.mod index 1581613..2099cb4 100644 --- a/go.mod +++ b/go.mod @@ -19,6 +19,8 @@ require ( github.com/kr/pretty v0.3.1 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + golang.org/x/mod v0.16.0 // indirect + golang.org/x/tools v0.19.0 // indirect gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect ) @@ -26,6 +28,7 @@ require ( github.com/PuerkitoBio/goquery v1.9.2 github.com/aeden/traceroute v0.0.0-20210211061815-03f5f7cb7908 github.com/jarcoal/httpmock v1.3.1 + github.com/miekg/dns v1.1.59 github.com/stretchr/testify v1.9.0 golang.org/x/net v0.25.0 golang.org/x/sys v0.20.0 // indirect diff --git a/go.sum b/go.sum index ec4cd93..2246a02 100644 --- a/go.sum +++ b/go.sum @@ -37,6 +37,8 @@ github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0 github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/maxatome/go-testdeep v1.12.0 h1:Ql7Go8Tg0C1D/uMMX59LAoYK7LffeJQ6X2T04nTH68g= github.com/maxatome/go-testdeep v1.12.0/go.mod h1:lPZc/HAcJMP92l7yI6TRz1aZN5URwUBUAfUNvrclaNM= +github.com/miekg/dns v1.1.59 h1:C9EXc/UToRwKLhK5wKU/I4QVsBUc8kE6MkHBkeypWZs= +github.com/miekg/dns v1.1.59/go.mod h1:nZpewl5p6IvctfgrckopVx2OlSEHPRO/U4SYkRklrEk= github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32 h1:W6apQkHrMkS0Muv8G/TipAy/FJl/rCYT0+EuS8+Z0z4= github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms= github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde h1:x0TT0RDC7UhAVbbWWBzr41ElhJx5tXPWkIHA2HWPRuw= @@ -53,6 +55,8 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.16.0 h1:QX4fJ0Rr5cPQCF7O9lh9Se4pmwfwskqZfq5moyldzic= +golang.org/x/mod v0.16.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= @@ -87,6 +91,8 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.19.0 h1:tfGCXNR1OsFG+sVdLAitlpjAvD/I6dHDKnYrpEZUHkw= +golang.org/x/tools v0.19.0/go.mod h1:qoJWxmGSIBmAeriMx19ogtrEPrGtDbPK634QFIcLAhc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= diff --git a/handlers/archives.go b/handlers/archives.go new file mode 100644 index 0000000..711af5e --- /dev/null +++ b/handlers/archives.go @@ -0,0 +1,171 @@ +package handlers + +import ( + "encoding/json" + "fmt" + "math" + "net/http" + "strconv" + "strings" + "time" +) + +const archiveAPIURL = "https://web.archive.org/cdx/search/cdx" + +func convertTimestampToDate(timestamp string) (time.Time, error) { + year, err := strconv.Atoi(timestamp[0:4]) + if err != nil { + return time.Time{}, err + } + month, err := strconv.Atoi(timestamp[4:6]) + if err != nil { + return time.Time{}, err + } + day, err := strconv.Atoi(timestamp[6:8]) + if err != nil { + return time.Time{}, err + } + hour, err := strconv.Atoi(timestamp[8:10]) + if err != nil { + return time.Time{}, err + } + minute, err := strconv.Atoi(timestamp[10:12]) + if err != nil { + return time.Time{}, err + } + second, err := strconv.Atoi(timestamp[12:14]) + if err != nil { + return time.Time{}, err + } + return time.Date(year, time.Month(month), day, hour, minute, second, 0, time.UTC), nil +} + +func countPageChanges(results [][]string) int { + prevDigest := "" + changeCount := -1 + for _, curr := range results { + if curr[2] != prevDigest { + prevDigest = curr[2] + changeCount++ + } + } + return changeCount +} + +func getAveragePageSize(scans [][]string) int { + totalSize := 0 + for _, scan := range scans { + size, err := strconv.Atoi(scan[3]) + if err != nil { + continue + } + totalSize += size + } + return totalSize / len(scans) +} + +func getScanFrequency(firstScan, lastScan time.Time, totalScans, changeCount int) map[string]float64 { + formatToTwoDecimal := func(num float64) float64 { + return math.Round(num*100) / 100 + } + + dayFactor := lastScan.Sub(firstScan).Hours() / 24 + daysBetweenScans := formatToTwoDecimal(dayFactor / float64(totalScans)) + daysBetweenChanges := formatToTwoDecimal(dayFactor / float64(changeCount)) + scansPerDay := formatToTwoDecimal(float64(totalScans-1) / dayFactor) + changesPerDay := formatToTwoDecimal(float64(changeCount) / dayFactor) + + // Handle NaN values + if math.IsNaN(daysBetweenScans) { + daysBetweenScans = 0 + } + if math.IsNaN(daysBetweenChanges) { + daysBetweenChanges = 0 + } + if math.IsNaN(scansPerDay) { + scansPerDay = 0 + } + if math.IsNaN(changesPerDay) { + changesPerDay = 0 + } + + return map[string]float64{ + "daysBetweenScans": daysBetweenScans, + "daysBetweenChanges": daysBetweenChanges, + "scansPerDay": scansPerDay, + "changesPerDay": changesPerDay, + } +} + +func getWaybackData(url string) (map[string]interface{}, error) { + cdxUrl := fmt.Sprintf("%s?url=%s&output=json&fl=timestamp,statuscode,digest,length,offset", archiveAPIURL, url) + + resp, err := http.Get(cdxUrl) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + var data [][]string + err = json.NewDecoder(resp.Body).Decode(&data) + if err != nil { + return nil, err + } + + if len(data) <= 1 { + return map[string]interface{}{ + "skipped": "Site has never before been archived via the Wayback Machine", + }, nil + } + + // Remove the header row + data = data[1:] + + firstScan, err := convertTimestampToDate(data[0][0]) + if err != nil { + return nil, err + } + lastScan, err := convertTimestampToDate(data[len(data)-1][0]) + if err != nil { + return nil, err + } + totalScans := len(data) + changeCount := countPageChanges(data) + + return map[string]interface{}{ + "firstScan": firstScan.Format(time.RFC3339), + "lastScan": lastScan.Format(time.RFC3339), + "totalScans": totalScans, + "changeCount": changeCount, + "averagePageSize": getAveragePageSize(data), + "scanFrequency": getScanFrequency(firstScan, lastScan, totalScans, changeCount), + "scans": data, + "scanUrl": url, + }, nil +} + +func HandleArchives() http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + urlParam := r.URL.Query().Get("url") + if urlParam == "" { + http.Error(w, "missing 'url' parameter", http.StatusBadRequest) + return + } + + if !strings.HasPrefix(urlParam, "http://") && !strings.HasPrefix(urlParam, "https://") { + urlParam = "http://" + urlParam + } + + data, err := getWaybackData(urlParam) + if err != nil { + http.Error(w, fmt.Sprintf("Error fetching Wayback data: %v", err), http.StatusInternalServerError) + return + } + + w.Header().Set("Content-Type", "application/json") + err = json.NewEncoder(w).Encode(data) + if err != nil { + http.Error(w, fmt.Sprintf("Error encoding response: %v", err), http.StatusInternalServerError) + } + }) +} diff --git a/handlers/mail-config.go b/handlers/mail-config.go new file mode 100644 index 0000000..2c8b044 --- /dev/null +++ b/handlers/mail-config.go @@ -0,0 +1,158 @@ +package handlers + +import ( + "errors" + "net/http" + "net/url" + "strings" + + "github.com/miekg/dns" +) + +func ResolveMx(domain string) ([]*dns.MX, int, error) { + c := new(dns.Client) + m := new(dns.Msg) + m.SetQuestion(dns.Fqdn(domain), dns.TypeMX) + r, _, err := c.Exchange(m, "8.8.8.8:53") + if err != nil { + return nil, dns.RcodeServerFailure, err + } + if r.Rcode != dns.RcodeSuccess { + return nil, r.Rcode, &dns.Error{} + } + var mxRecords []*dns.MX + for _, ans := range r.Answer { + if mx, ok := ans.(*dns.MX); ok { + mxRecords = append(mxRecords, mx) + } + } + if len(mxRecords) == 0 { + return nil, dns.RcodeNameError, nil + } + return mxRecords, dns.RcodeSuccess, nil +} + +func ResolveTxt(domain string) ([]string, int, error) { + c := new(dns.Client) + m := new(dns.Msg) + m.SetQuestion(dns.Fqdn(domain), dns.TypeTXT) + r, _, err := c.Exchange(m, "8.8.8.8:53") + if err != nil { + return nil, dns.RcodeServerFailure, err + } + if r.Rcode != dns.RcodeSuccess { + return nil, r.Rcode, &dns.Error{} + } + var txtRecords []string + for _, ans := range r.Answer { + if txt, ok := ans.(*dns.TXT); ok { + txtRecords = append(txtRecords, txt.Txt...) + } + } + if len(txtRecords) == 0 { + return nil, dns.RcodeNameError, nil + } + return txtRecords, dns.RcodeSuccess, nil +} + +func HandleMailConfig() http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + urlParam := r.URL.Query().Get("url") + if urlParam == "" { + JSONError(w, errors.New("URL parameter is required"), http.StatusBadRequest) + return + } + + if !strings.HasPrefix(urlParam, "http://") && !strings.HasPrefix(urlParam, "https://") { + urlParam = "http://" + urlParam + } + + parsedURL, err := url.Parse(urlParam) + if err != nil { + JSONError(w, errors.New("Invalid URL"), http.StatusBadRequest) + return + } + domain := parsedURL.Hostname() + if domain == "" { + domain = parsedURL.Path + } + + mxRecords, rcode, err := ResolveMx(domain) + if err != nil { + JSONError(w, err, http.StatusInternalServerError) + return + } + + if rcode == dns.RcodeNameError || rcode == dns.RcodeServerFailure { + JSON(w, map[string]string{"skipped": "No mail server in use on this domain"}, http.StatusOK) + return + } + + txtRecords, rcode, err := ResolveTxt(domain) + if err != nil { + JSONError(w, err, http.StatusInternalServerError) + return + } + + if rcode == dns.RcodeNameError || rcode == dns.RcodeServerFailure { + JSON(w, map[string]string{"skipped": "No mail server in use on this domain"}, http.StatusOK) + return + } + + emailTxtRecords := filterEmailTxtRecords(txtRecords) + mailServices := identifyMailServices(emailTxtRecords, mxRecords) + + JSON(w, map[string]interface{}{ + "mxRecords": mxRecords, + "txtRecords": emailTxtRecords, + "mailServices": mailServices, + }, http.StatusOK) + }) +} + +func filterEmailTxtRecords(records []string) []string { + var emailTxtRecords []string + for _, record := range records { + if strings.HasPrefix(record, "v=spf1") || + strings.HasPrefix(record, "v=DKIM1") || + strings.HasPrefix(record, "v=DMARC1") || + strings.HasPrefix(record, "protonmail-verification=") || + strings.HasPrefix(record, "google-site-verification=") || + strings.HasPrefix(record, "MS=") || + strings.HasPrefix(record, "zoho-verification=") || + strings.HasPrefix(record, "titan-verification=") || + strings.Contains(record, "bluehost.com") { + emailTxtRecords = append(emailTxtRecords, record) + } + } + return emailTxtRecords +} + +func identifyMailServices(emailTxtRecords []string, mxRecords []*dns.MX) []map[string]string { + var mailServices []map[string]string + for _, record := range emailTxtRecords { + if strings.HasPrefix(record, "protonmail-verification=") { + mailServices = append(mailServices, map[string]string{"provider": "ProtonMail", "value": strings.Split(record, "=")[1]}) + } else if strings.HasPrefix(record, "google-site-verification=") { + mailServices = append(mailServices, map[string]string{"provider": "Google Workspace", "value": strings.Split(record, "=")[1]}) + } else if strings.HasPrefix(record, "MS=") { + mailServices = append(mailServices, map[string]string{"provider": "Microsoft 365", "value": strings.Split(record, "=")[1]}) + } else if strings.HasPrefix(record, "zoho-verification=") { + mailServices = append(mailServices, map[string]string{"provider": "Zoho", "value": strings.Split(record, "=")[1]}) + } else if strings.HasPrefix(record, "titan-verification=") { + mailServices = append(mailServices, map[string]string{"provider": "Titan", "value": strings.Split(record, "=")[1]}) + } else if strings.Contains(record, "bluehost.com") { + mailServices = append(mailServices, map[string]string{"provider": "BlueHost", "value": record}) + } + } + + for _, mx := range mxRecords { + if strings.Contains(mx.Mx, "yahoodns.net") { + mailServices = append(mailServices, map[string]string{"provider": "Yahoo", "value": mx.Mx}) + } else if strings.Contains(mx.Mx, "mimecast.com") { + mailServices = append(mailServices, map[string]string{"provider": "Mimecast", "value": mx.Mx}) + } + } + + return mailServices +} diff --git a/handlers/robots-txt.go b/handlers/robots-txt.go new file mode 100644 index 0000000..6456cb2 --- /dev/null +++ b/handlers/robots-txt.go @@ -0,0 +1,90 @@ +package handlers + +import ( + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strings" +) + +func ParseRobotsTxt(content string) map[string][]map[string]string { + lines := strings.Split(content, "\n") + rules := []map[string]string{} + + for _, line := range lines { + line = strings.TrimSpace(line) + match := "" + + if strings.HasPrefix(strings.ToLower(line), "allow:") { + match = "Allow" + } else if strings.HasPrefix(strings.ToLower(line), "disallow:") { + match = "Disallow" + } else if strings.HasPrefix(strings.ToLower(line), "user-agent:") { + match = "User-agent" + } + + if match != "" { + val := strings.TrimSpace(strings.SplitN(line, ":", 2)[1]) + rule := map[string]string{ + "lbl": match, + "val": val, + } + rules = append(rules, rule) + } + } + + return map[string][]map[string]string{"robots": rules} +} + +func HandleRobotsTxt() http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + urlParam := r.URL.Query().Get("url") + if urlParam == "" { + JSONError(w, errors.New("url query parameter is required"), http.StatusBadRequest) + return + } + + if !strings.HasPrefix(urlParam, "http://") && !strings.HasPrefix(urlParam, "https://") { + urlParam = "http://" + urlParam + } + + parsedURL, err := url.Parse(urlParam) + if err != nil { + JSONError(w, errors.New("Invalid url query parameter"), http.StatusBadRequest) + return + } + + robotsURL := fmt.Sprintf("%s://%s/robots.txt", parsedURL.Scheme, parsedURL.Host) + + resp, err := http.Get(robotsURL) + if err != nil { + JSONError(w, fmt.Errorf("Error fetching robots.txt: %s", err.Error()), http.StatusInternalServerError) + return + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + JSON(w, map[string]interface{}{ + "error": "Failed to fetch robots.txt", + "statusCode": resp.StatusCode, + }, resp.StatusCode) + return + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + JSONError(w, fmt.Errorf("Error reading robots.txt: %s", err.Error()), http.StatusInternalServerError) + return + } + + parsedData := ParseRobotsTxt(string(body)) + if robots, ok := parsedData["robots"]; !ok || len(robots) == 0 { + JSON(w, map[string]string{"skipped": "No robots.txt file present, unable to continue"}, http.StatusOK) + return + } + + JSON(w, parsedData, http.StatusOK) + }) +} diff --git a/handlers/screenshot.go b/handlers/screenshot.go new file mode 100644 index 0000000..2c1eff0 --- /dev/null +++ b/handlers/screenshot.go @@ -0,0 +1,71 @@ +package handlers + +import ( + "context" + "encoding/base64" + "encoding/json" + "errors" + "net/http" + "net/url" + "strings" + + "github.com/chromedp/chromedp" +) + +type screenshotResponse struct { + Image string `json:"image"` +} + +func takeScreenshot(targetURL string) (*screenshotResponse, error) { + if targetURL == "" { + return nil, errors.New("URL is missing from queryStringParameters") + } + + if !strings.HasPrefix(targetURL, "http://") && !strings.HasPrefix(targetURL, "https://") { + targetURL = "http://" + targetURL + } + + parsedURL, err := url.ParseRequestURI(targetURL) + if err != nil { + return nil, errors.New("URL provided is invalid") + } + + ctx, cancel := chromedp.NewContext(context.Background()) + defer cancel() + + var buf []byte + if err := chromedp.Run(ctx, + chromedp.EmulateViewport(800, 600), + chromedp.Navigate(parsedURL.String()), + chromedp.WaitReady("body", chromedp.ByQuery), + chromedp.FullScreenshot(&buf, 90), + ); err != nil { + return nil, err + } + + base64Screenshot := base64.StdEncoding.EncodeToString(buf) + return &screenshotResponse{ + Image: base64Screenshot, + }, nil +} + +func HandleScreenshot() http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + targetURL := r.URL.Query().Get("url") + if targetURL == "" { + http.Error(w, "missing 'url' parameter", http.StatusBadRequest) + return + } + + data, err := takeScreenshot(targetURL) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(data); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + } + }) +} diff --git a/handlers/security-txt.go b/handlers/security-txt.go new file mode 100644 index 0000000..879ca23 --- /dev/null +++ b/handlers/security-txt.go @@ -0,0 +1,115 @@ +package handlers + +import ( + "errors" + "fmt" + "io/ioutil" + "net/http" + "net/url" + "strings" +) + +var SECURITY_TXT_PATHS = []string{ + "/security.txt", + "/.well-known/security.txt", +} + +func parseResult(result string) map[string]string { + output := make(map[string]string) + counts := make(map[string]int) + lines := strings.Split(result, "\n") + regex := ":\\s*" + + for _, line := range lines { + if !strings.HasPrefix(line, "#") && !strings.HasPrefix(line, "-----") && strings.TrimSpace(line) != "" { + parts := strings.SplitN(line, regex, 2) + if len(parts) == 2 { + key := strings.TrimSpace(parts[0]) + value := strings.TrimSpace(parts[1]) + if _, exists := output[key]; exists { + counts[key]++ + key = fmt.Sprintf("%s%d", key, counts[key]) + } + output[key] = value + } + } + } + + return output +} + +func isPgpSigned(result string) bool { + return strings.Contains(result, "-----BEGIN PGP SIGNED MESSAGE-----") +} + +func HandleSecurityTxt() http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + urlParam := r.URL.Query().Get("url") + if urlParam == "" { + JSONError(w, errors.New("url query parameter is required"), http.StatusBadRequest) + return + } + + if !strings.HasPrefix(urlParam, "http://") && !strings.HasPrefix(urlParam, "https://") { + urlParam = "https://" + urlParam + } + + parsedURL, err := url.Parse(urlParam) + if err != nil { + JSONError(w, errors.New("Invalid url query parameter"), http.StatusBadRequest) + return + } + parsedURL.Path = "" + + for _, path := range SECURITY_TXT_PATHS { + result, err := fetchSecurityTxt(parsedURL, path) + if err != nil { + JSONError(w, err, http.StatusInternalServerError) + return + } + + if result != "" && strings.Contains(result, " 0 { + return sitemap, nil + } + + var urlset URLSet + if err := xml.Unmarshal(body, &urlset); err == nil && len(urlset.URLs) > 0 { + return urlset, nil + } + + return nil, errors.New("invalid sitemap format") +} + +func getSitemapURLFromRobotsTxt(baseURL string) (string, error) { + client := http.Client{ + Timeout: hardTimeout, + } + + resp, err := client.Get(fmt.Sprintf("%s/robots.txt", baseURL)) + if err != nil { + return "", err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return "", errors.New("failed to fetch robots.txt") + } + + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return "", err + } + + lines := strings.Split(string(body), "\n") + for _, line := range lines { + if strings.HasPrefix(strings.ToLower(strings.TrimSpace(line)), "sitemap:") { + parts := strings.SplitN(line, " ", 2) + if len(parts) == 2 { + return strings.TrimSpace(parts[1]), nil + } + } + } + + return "", errors.New("no sitemap found in robots.txt") +} diff --git a/handlers/ssl.go b/handlers/ssl.go new file mode 100644 index 0000000..a93f5d2 --- /dev/null +++ b/handlers/ssl.go @@ -0,0 +1,50 @@ +package handlers + +import ( + "crypto/tls" + "errors" + "fmt" + "net/http" + "net/url" +) + +func HandleSSL() http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + urlParam := r.URL.Query().Get("url") + if urlParam == "" { + JSONError(w, errors.New("url query parameter is required"), http.StatusBadRequest) + return + } + + parsedURL, err := url.Parse(urlParam) + if err != nil { + JSONError(w, errors.New("invalid URL format"), http.StatusBadRequest) + return + } + + options := &tls.Config{ + ServerName: parsedURL.Hostname(), + InsecureSkipVerify: true, // Skip certificate validation + } + + conn, err := tls.Dial("tcp", parsedURL.Host+":443", options) + if err != nil { + JSONError(w, fmt.Errorf("error establishing TLS connection: %s", err.Error()), http.StatusInternalServerError) + return + } + defer conn.Close() + + state := conn.ConnectionState() + if len(state.PeerCertificates) == 0 { + JSONError(w, errors.New("no certificate presented by the server"), http.StatusInternalServerError) + return + } + + cert := state.PeerCertificates[0] + + // Remove the raw field from the certificate + cert.Raw = nil + + JSON(w, cert, http.StatusOK) + }) +} diff --git a/handlers/status.go b/handlers/status.go new file mode 100644 index 0000000..b840271 --- /dev/null +++ b/handlers/status.go @@ -0,0 +1,113 @@ +package handlers + +import ( + "encoding/json" + "errors" + "net/http" + "net/http/httptrace" + "strings" + "time" +) + +type responseData struct { + IsUp bool `json:"isUp"` + DNSLookupTime float64 `json:"dnsLookupTime"` + ResponseTime float64 `json:"responseTime"` + ResponseCode int `json:"responseCode"` +} + +func fetchURL(url string) (*responseData, error) { + if url == "" { + return nil, errors.New("you must provide a URL query parameter!") + } + + var dnsStart, dnsEnd, startTime time.Time + var responseCode int + + trace := &httptrace.ClientTrace{ + DNSStart: func(_ httptrace.DNSStartInfo) { + dnsStart = time.Now() + }, + DNSDone: func(_ httptrace.DNSDoneInfo) { + dnsEnd = time.Now() + }, + } + + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return nil, err + } + req = req.WithContext(httptrace.WithClientTrace(req.Context(), trace)) + + startTime = time.Now() + client := &http.Client{ + CheckRedirect: func(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse + }, + } + resp, err := client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + // Follow redirects + for resp.StatusCode >= 300 && resp.StatusCode < 400 { + loc, err := resp.Location() + if err != nil { + return nil, err + } + + req, err := http.NewRequest("GET", loc.String(), nil) + if err != nil { + return nil, err + } + req = req.WithContext(httptrace.WithClientTrace(req.Context(), trace)) + + resp, err = client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + } + + responseCode = resp.StatusCode + if responseCode != 200 { + return nil, errors.New("received non-success response code: " + http.StatusText(responseCode)) + } + + dnsLookupTime := dnsEnd.Sub(dnsStart).Seconds() * 1000 // Convert to milliseconds + responseTime := time.Since(startTime).Seconds() * 1000 // Convert to milliseconds + + return &responseData{ + IsUp: true, + DNSLookupTime: dnsLookupTime, + ResponseTime: responseTime, + ResponseCode: responseCode, + }, nil +} + +func HandleStatus() http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + urlParam := r.URL.Query().Get("url") + if urlParam == "" { + http.Error(w, "missing 'url' parameter", http.StatusBadRequest) + return + } + + if !strings.HasPrefix(urlParam, "http://") && !strings.HasPrefix(urlParam, "https://") { + urlParam = "http://" + urlParam + } + + data, err := fetchURL(urlParam) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(data); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + } + }) +} diff --git a/handlers/tech-stack.go b/handlers/tech-stack.go new file mode 100644 index 0000000..f5b6112 --- /dev/null +++ b/handlers/tech-stack.go @@ -0,0 +1,62 @@ +package handlers + +import ( + "errors" + "net/http" + "strings" + + "github.com/PuerkitoBio/goquery" +) + +func extractTechnologies(url string) ([]string, error) { + resp, err := http.Get(url) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + doc, err := goquery.NewDocumentFromReader(resp.Body) + if err != nil { + return nil, err + } + + technologies := make([]string, 0) + + doc.Find("script[src], link[href]").Each(func(i int, s *goquery.Selection) { + if src, exists := s.Attr("src"); exists { + technologies = append(technologies, src) + } + if href, exists := s.Attr("href"); exists { + technologies = append(technologies, href) + } + }) + + return technologies, nil +} + +func HandleTechStack() http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + urlParam := r.URL.Query().Get("url") + if urlParam == "" { + JSONError(w, errors.New("Missing 'url' parameter"), http.StatusBadRequest) + return + } + + if !strings.HasPrefix(urlParam, "http://") && !strings.HasPrefix(urlParam, "https://") { + urlParam = "http://" + urlParam + } + + technologies, err := extractTechnologies(urlParam) + if err != nil { + JSONError(w, err, http.StatusInternalServerError) + return + } + + if len(technologies) == 0 { + JSONError(w, errors.New("Unable to find any technologies for site"), http.StatusInternalServerError) + return + } + + JSON(w, technologies, http.StatusOK) + }) +} diff --git a/handlers/threats.go b/handlers/threats.go new file mode 100644 index 0000000..edb1cf0 --- /dev/null +++ b/handlers/threats.go @@ -0,0 +1,278 @@ +package handlers + +import ( + "bytes" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strings" + + "golang.org/x/net/html" +) + +const ( + googleSafeBrowsingAPI = "https://safebrowsing.googleapis.com/v4/threatMatches:find" + urlHausAPI = "https://urlhaus-api.abuse.ch/v1/host/" + phishTankAPI = "https://checkurl.phishtank.com/checkurl/" + cloudmersiveAPI = "https://api.cloudmersive.com/virus/scan/website" +) + +type GoogleSafeBrowsingResponse struct { + Matches []Match `json:"matches,omitempty"` +} + +type Match struct { + ThreatType string `json:"threatType,omitempty"` + PlatformType string `json:"platformType,omitempty"` + ThreatEntryType string `json:"threatEntryType,omitempty"` + Threat Threat `json:"threat,omitempty"` + CacheDuration string `json:"cacheDuration,omitempty"` + HashPrefix string `json:"hashPrefix,omitempty"` + ThreatEntryMetadata Metadata `json:"threatEntryMetadata,omitempty"` +} + +type Threat struct { + URL string `json:"url,omitempty"` +} + +type Metadata struct { + Entries []Entry `json:"entries,omitempty"` +} + +type Entry struct { + Key string `json:"key,omitempty"` + Value string `json:"value,omitempty"` +} + +type UrlHausResponse struct { + URLs []string `json:"urls,omitempty"` +} + +type PhishTankResponse struct { + Title string `json:"title"` + Body string `json:"body"` +} + +type CloudmersiveResponse struct { + Error string `json:"error,omitempty"` + IsSuccess bool `json:"isSuccess,omitempty"` + Message string `json:"message,omitempty"` +} + +func getGoogleSafeBrowsingResult(urlParam string) (*GoogleSafeBrowsingResponse, error) { + apiKey := "" // Set your Google Safe Browsing API key here + if apiKey == "" { + return nil, errors.New("GOOGLE_CLOUD_API_KEY is required for the Google Safe Browsing check") + } + + requestBody := map[string]interface{}{ + "threatInfo": map[string]interface{}{ + "threatTypes": []string{"MALWARE", "SOCIAL_ENGINEERING", "UNWANTED_SOFTWARE", "POTENTIALLY_HARMFUL_APPLICATION", "API_ABUSE"}, + "platformTypes": []string{"ANY_PLATFORM"}, + "threatEntryTypes": []string{"URL"}, + "threatEntries": []map[string]string{ + {"url": urlParam}, + }, + }, + } + + requestJSON, err := json.Marshal(requestBody) + if err != nil { + return nil, err + } + + response, err := http.Post(fmt.Sprintf("%s?key=%s", googleSafeBrowsingAPI, apiKey), "application/json", bytes.NewBuffer(requestJSON)) + if err != nil { + return nil, err + } + defer response.Body.Close() + + var googleSafeBrowsingResponse GoogleSafeBrowsingResponse + if err := json.NewDecoder(response.Body).Decode(&googleSafeBrowsingResponse); err != nil { + return nil, err + } + + return &googleSafeBrowsingResponse, nil +} + +func getUrlHausResult(urlParam string) (*UrlHausResponse, error) { + parsedURL, err := url.Parse(urlParam) + if err != nil { + return nil, err + } + + domain := parsedURL.Hostname() + if domain == "" { + return nil, errors.New("invalid URL format") + } + + response, err := http.PostForm(urlHausAPI, url.Values{"host": {domain}}) + if err != nil { + return nil, err + } + defer response.Body.Close() + + body, err := io.ReadAll(response.Body) + if err != nil { + return nil, err + } + + var urlHausResponse UrlHausResponse + if err := json.Unmarshal(body, &urlHausResponse); err != nil { + return nil, err + } + + return &urlHausResponse, nil +} + +func getPhishTankResult(urlParam string) (*PhishTankResponse, error) { + encodedURL := base64.StdEncoding.EncodeToString([]byte(urlParam)) + endpoint := fmt.Sprintf("%s?url=%s", phishTankAPI, encodedURL) + + response, err := http.Get(endpoint) + if err != nil { + return nil, err + } + defer response.Body.Close() + + body, err := io.ReadAll(response.Body) + if err != nil { + return nil, err + } + + // Parse HTML + doc, err := html.Parse(strings.NewReader(string(body))) + if err != nil { + return nil, err + } + + // Extract title and body + title := extractTitle(doc) + bodyContent := extractBody(doc) + + // Create PhishTankResponse + phishTankResponse := &PhishTankResponse{ + Title: title, + Body: bodyContent, + } + + return phishTankResponse, nil +} + +func extractTitle(doc *html.Node) string { + var title string + var f func(*html.Node) + f = func(n *html.Node) { + if n.Type == html.ElementNode && n.Data == "title" { + title = strings.TrimSpace(n.FirstChild.Data) + return + } + for c := n.FirstChild; c != nil; c = c.NextSibling { + f(c) + } + } + f(doc) + return title +} + +func extractBody(doc *html.Node) string { + var bodyContent string + var f func(*html.Node) + f = func(n *html.Node) { + if n.Type == html.ElementNode && n.Data == "body" { + for c := n.FirstChild; c != nil; c = c.NextSibling { + if c.Type == html.TextNode { + bodyContent += strings.TrimSpace(c.Data) + "\n" + } + } + return + } + for c := n.FirstChild; c != nil; c = c.NextSibling { + f(c) + } + } + f(doc) + return bodyContent +} + +func getCloudmersiveResult(urlParam string) (*CloudmersiveResponse, error) { + apiKey := "" // Set your Cloudmersive API key here + if apiKey == "" { + return nil, errors.New("CLOUDMERSIVE_API_KEY is required for the Cloudmersive check") + } + + data := url.Values{} + data.Set("Url", urlParam) + + client := &http.Client{} + request, err := http.NewRequest("POST", cloudmersiveAPI, strings.NewReader(data.Encode())) + if err != nil { + return nil, err + } + request.Header.Set("Content-Type", "application/x-www-form-urlencoded") + request.Header.Set("Apikey", apiKey) + + response, err := client.Do(request) + if err != nil { + return nil, err + } + defer response.Body.Close() + + var cloudmersiveResponse CloudmersiveResponse + if err := json.NewDecoder(response.Body).Decode(&cloudmersiveResponse); err != nil { + return nil, err + } + + return &cloudmersiveResponse, nil +} + +func HandleThreats() http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + urlParam := r.URL.Query().Get("url") + if urlParam == "" { + JSONError(w, errors.New("url query parameter is required"), http.StatusBadRequest) + return + } + + if !strings.HasPrefix(urlParam, "http://") && !strings.HasPrefix(urlParam, "https://") { + urlParam = "https://" + urlParam + } + + urlHausResult, err := getUrlHausResult(urlParam) + if err != nil { + JSONError(w, err, http.StatusInternalServerError) + return + } + + phishTankResult, err := getPhishTankResult(urlParam) + if err != nil { + JSONError(w, err, http.StatusInternalServerError) + return + } + + cloudmersiveResult, err := getCloudmersiveResult(urlParam) + if err != nil { + JSONError(w, err, http.StatusInternalServerError) + return + } + + googleSafeBrowsingResult, err := getGoogleSafeBrowsingResult(urlParam) + if err != nil { + JSONError(w, err, http.StatusInternalServerError) + return + } + + response := map[string]interface{}{ + "urlHaus": urlHausResult, + "phishTank": phishTankResult, + "cloudmersive": cloudmersiveResult, + "googleSafeBrowsing": googleSafeBrowsingResult, + } + + JSON(w, response, http.StatusOK) + }) +} diff --git a/handlers/txt-records.go b/handlers/txt-records.go new file mode 100644 index 0000000..e00e9c9 --- /dev/null +++ b/handlers/txt-records.go @@ -0,0 +1,60 @@ +package handlers + +import ( + "errors" + "net" + "net/http" + "net/url" + "strings" +) + +func HandleTXTRecords() http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + urlParam := r.URL.Query().Get("url") + if urlParam == "" { + JSONError(w, errors.New("url query parameter is required"), http.StatusBadRequest) + return + } + + if !strings.HasPrefix(urlParam, "http://") && !strings.HasPrefix(urlParam, "https://") { + urlParam = "https://" + urlParam + } + + parsedURL, err := url.Parse(urlParam) + if err != nil { + JSONError(w, errors.New("invalid URL format"), http.StatusBadRequest) + return + } + + txtRecords, err := resolveTXTRecords(parsedURL.Hostname()) + if err != nil { + JSONError(w, err, http.StatusInternalServerError) + return + } + + readableTxtRecords := parseTXTRecords(txtRecords) + + JSON(w, readableTxtRecords, http.StatusOK) + }) +} + +func resolveTXTRecords(hostname string) ([]string, error) { + txtRecords, err := net.LookupTXT(hostname) + if err != nil { + return nil, err + } + return txtRecords, nil +} + +func parseTXTRecords(txtRecords []string) map[string]string { + readableTxtRecords := make(map[string]string) + for _, recordString := range txtRecords { + splitRecord := strings.SplitN(recordString, "=", 2) + if len(splitRecord) == 2 { + key := splitRecord[0] + value := splitRecord[1] + readableTxtRecords[key] = value + } + } + return readableTxtRecords +} diff --git a/handlers/whois.go b/handlers/whois.go new file mode 100644 index 0000000..dba2e26 --- /dev/null +++ b/handlers/whois.go @@ -0,0 +1,155 @@ +package handlers + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "net/http" + "regexp" + "strings" +) + +const internicHostname = "whois.internic.net" +const myAPIURL = "https://whois-api-zeta.vercel.app/" + +func getBaseDomain(url string) (string, error) { + protocol := "" + if strings.HasPrefix(url, "http://") { + protocol = "http://" + } else if strings.HasPrefix(url, "https://") { + protocol = "https://" + } + noProtocolURL := strings.Replace(url, protocol, "", 1) + parsed, err := parseDomain(noProtocolURL) + if err != nil { + return "", err + } + return protocol + parsed, nil +} +func parseDomain(url string) (string, error) { + parts := strings.Split(url, "/") + if len(parts) > 0 { + domainParts := strings.Split(parts[0], ".") + if len(domainParts) < 2 { + return "", errors.New("invalid URL") + } + return domainParts[len(domainParts)-2] + "." + domainParts[len(domainParts)-1], nil + } + return "", errors.New("invalid URL") +} + +func parseWhoisData(data string) map[string]string { + result := make(map[string]string) + lines := strings.Split(data, "\r\n") + + var lastKey string + + for _, line := range lines { + index := strings.Index(line, ":") + if index == -1 { + if lastKey != "" { + result[lastKey] += " " + strings.TrimSpace(line) + } + continue + } + key := strings.TrimSpace(line[:index]) + value := strings.TrimSpace(line[index+1:]) + if len(value) == 0 { + continue + } + key = regexp.MustCompile(`\W+`).ReplaceAllString(key, "_") + lastKey = key + + result[key] = value + } + + return result +} + +func fetchFromInternic(hostname string) (map[string]string, error) { + conn, err := net.Dial("tcp", internicHostname+":43") + if err != nil { + return nil, err + } + defer conn.Close() + + _, err = conn.Write([]byte(hostname + "\r\n")) + if err != nil { + return nil, err + } + + var buf bytes.Buffer + _, err = buf.ReadFrom(conn) + if err != nil { + return nil, err + } + + parsedData := parseWhoisData(buf.String()) + + if _, ok := parsedData["No_match_for"]; ok { + return nil, errors.New("No matches found for domain in internic database") + } + + return parsedData, nil +} + +func fetchFromMyAPI(hostname string) (map[string]interface{}, error) { + resp, err := http.Post(myAPIURL, "application/json", strings.NewReader(fmt.Sprintf(`{"domain": "%s"}`, hostname))) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + // Log the response body + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + var result map[string]interface{} + err = json.Unmarshal(body, &result) + if err != nil { + return nil, err + } + + return result, nil +} + +func HandleWhois() http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + url := r.URL.Query().Get("url") + if url == "" { + http.Error(w, "missing 'url' parameter", http.StatusBadRequest) + return + } + + hostname, err := getBaseDomain(url) + if err != nil { + http.Error(w, fmt.Sprintf("Unable to parse URL: %v", err), http.StatusInternalServerError) + return + } + + internicData, err := fetchFromInternic(hostname) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + myAPIData, err := fetchFromMyAPI(hostname) + if err != nil { + http.Error(w, fmt.Sprintf("Error fetching data from your API: %v", err), http.StatusInternalServerError) + return + } + + response := map[string]interface{}{ + "internicData": internicData, + "myAPIData": myAPIData, + } + + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(response) + }) +} diff --git a/server/server.go b/server/server.go index 9629d34..ff34d01 100644 --- a/server/server.go +++ b/server/server.go @@ -44,6 +44,18 @@ func (s *Server) routes() { s.mux.Handle("GET /api/social-tags", handlers.HandleGetSocialTags()) s.mux.Handle("GET /api/tls", handlers.HandleTLS()) s.mux.Handle("GET /api/trace-route", handlers.HandleTraceRoute()) + s.mux.Handle("/api/mail-config", handlers.HandleMailConfig()) + s.mux.Handle("/api/robots-txt", handlers.HandleRobotsTxt()) + s.mux.Handle("/api/security-txt", handlers.HandleSecurityTxt()) + s.mux.Handle("/api/sitemap", handlers.HandleSitemap()) + s.mux.Handle("/api/ssl", handlers.HandleSSL()) + s.mux.Handle("/api/threats", handlers.HandleThreats()) + s.mux.Handle("/api/txt-records", handlers.HandleTXTRecords()) + s.mux.Handle("/api/whois", handlers.HandleWhois()) + s.mux.Handle("/api/archives", handlers.HandleArchives()) + s.mux.Handle("/api/status", handlers.HandleStatus()) + s.mux.Handle("/api/screenshot", handlers.HandleScreenshot()) + s.mux.Handle("/api/tech-stack", handlers.HandleTechStack()) } func (s *Server) Run() error { From a234574a1e275389fa407fb3311b7bf92e6bce83 Mon Sep 17 00:00:00 2001 From: Hammad Afzal Date: Fri, 14 Jun 2024 02:45:10 +0500 Subject: [PATCH 2/2] RF: Code Refactoring and cleanup --- go.mod | 6 ++- go.sum | 8 ++++ handlers/archives.go | 83 ++++++++++++++++------------------------ handlers/mail-config.go | 24 ++---------- handlers/robots-txt.go | 18 ++------- handlers/screenshot.go | 17 ++------ handlers/security-txt.go | 23 +++-------- handlers/sitemap.go | 17 ++++---- handlers/ssl.go | 15 ++------ handlers/status.go | 13 ++----- handlers/tech-stack.go | 13 ++----- handlers/threats.go | 18 ++++----- handlers/txt-records.go | 12 ++---- handlers/whois.go | 9 +++-- 14 files changed, 98 insertions(+), 178 deletions(-) diff --git a/go.mod b/go.mod index 08ecc69..b014e20 100644 --- a/go.mod +++ b/go.mod @@ -17,9 +17,11 @@ require ( github.com/josharian/intern v1.0.0 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/mailru/easyjson v0.7.7 // indirect + github.com/miekg/dns v1.1.61 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - golang.org/x/mod v0.16.0 // indirect - golang.org/x/tools v0.19.0 // indirect + golang.org/x/mod v0.18.0 // indirect + golang.org/x/sync v0.7.0 // indirect + golang.org/x/tools v0.22.0 // indirect gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect ) diff --git a/go.sum b/go.sum index a297f7e..d7417e9 100644 --- a/go.sum +++ b/go.sum @@ -31,6 +31,8 @@ github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80 h1:6Yzfa6GP0rIo/kUL github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/miekg/dns v1.1.61 h1:nLxbwF3XxhwVSm8g9Dghm9MHPaUZuqhPiGL+675ZmEs= +github.com/miekg/dns v1.1.61/go.mod h1:mnAarhS3nWaW+NVP2wTkYVIZyHNJ098SJZUki3eykwQ= github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde h1:x0TT0RDC7UhAVbbWWBzr41ElhJx5tXPWkIHA2HWPRuw= github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= @@ -47,6 +49,8 @@ golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.16.0 h1:QX4fJ0Rr5cPQCF7O9lh9Se4pmwfwskqZfq5moyldzic= golang.org/x/mod v0.16.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.18.0 h1:5+9lSbEzPSdWkH32vYPBwEpX8KwDbM52Ud9xBUvNlb0= +golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= @@ -57,6 +61,8 @@ golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -83,6 +89,8 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.19.0 h1:tfGCXNR1OsFG+sVdLAitlpjAvD/I6dHDKnYrpEZUHkw= golang.org/x/tools v0.19.0/go.mod h1:qoJWxmGSIBmAeriMx19ogtrEPrGtDbPK634QFIcLAhc= +golang.org/x/tools v0.22.0 h1:gqSGLZqv+AI9lIQzniJ0nZDRG5GBPsSi+DRNHWNz6yA= +golang.org/x/tools v0.22.0/go.mod h1:aCwcsjqvq7Yqt6TNyX7QMU2enbQ/Gt0bo6krSeEri+c= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= diff --git a/handlers/archives.go b/handlers/archives.go index 711af5e..b403d1e 100644 --- a/handlers/archives.go +++ b/handlers/archives.go @@ -5,39 +5,16 @@ import ( "fmt" "math" "net/http" + "net/url" "strconv" - "strings" "time" ) const archiveAPIURL = "https://web.archive.org/cdx/search/cdx" func convertTimestampToDate(timestamp string) (time.Time, error) { - year, err := strconv.Atoi(timestamp[0:4]) - if err != nil { - return time.Time{}, err - } - month, err := strconv.Atoi(timestamp[4:6]) - if err != nil { - return time.Time{}, err - } - day, err := strconv.Atoi(timestamp[6:8]) - if err != nil { - return time.Time{}, err - } - hour, err := strconv.Atoi(timestamp[8:10]) - if err != nil { - return time.Time{}, err - } - minute, err := strconv.Atoi(timestamp[10:12]) - if err != nil { - return time.Time{}, err - } - second, err := strconv.Atoi(timestamp[12:14]) - if err != nil { - return time.Time{}, err - } - return time.Date(year, time.Month(month), day, hour, minute, second, 0, time.UTC), nil + mask := "20060102150405" + return time.Parse(mask, timestamp) } func countPageChanges(results [][]string) int { @@ -64,9 +41,9 @@ func getAveragePageSize(scans [][]string) int { return totalSize / len(scans) } -func getScanFrequency(firstScan, lastScan time.Time, totalScans, changeCount int) map[string]float64 { - formatToTwoDecimal := func(num float64) float64 { - return math.Round(num*100) / 100 +func getScanFrequency(firstScan, lastScan time.Time, totalScans, changeCount int) map[string]string { + formatToTwoDecimal := func(num float64) string { + return fmt.Sprintf("%.2f", num) } dayFactor := lastScan.Sub(firstScan).Hours() / 24 @@ -75,21 +52,20 @@ func getScanFrequency(firstScan, lastScan time.Time, totalScans, changeCount int scansPerDay := formatToTwoDecimal(float64(totalScans-1) / dayFactor) changesPerDay := formatToTwoDecimal(float64(changeCount) / dayFactor) - // Handle NaN values - if math.IsNaN(daysBetweenScans) { - daysBetweenScans = 0 + if math.IsNaN(dayFactor / float64(totalScans)) { + daysBetweenScans = "0.00" } - if math.IsNaN(daysBetweenChanges) { - daysBetweenChanges = 0 + if math.IsNaN(dayFactor / float64(changeCount)) { + daysBetweenChanges = "0.00" } - if math.IsNaN(scansPerDay) { - scansPerDay = 0 + if math.IsNaN(float64(totalScans-1) / dayFactor) { + scansPerDay = "0.00" } - if math.IsNaN(changesPerDay) { - changesPerDay = 0 + if math.IsNaN(float64(changeCount) / dayFactor) { + changesPerDay = "0.00" } - return map[string]float64{ + return map[string]string{ "daysBetweenScans": daysBetweenScans, "daysBetweenChanges": daysBetweenChanges, "scansPerDay": scansPerDay, @@ -97,10 +73,14 @@ func getScanFrequency(firstScan, lastScan time.Time, totalScans, changeCount int } } -func getWaybackData(url string) (map[string]interface{}, error) { +func getWaybackData(url *url.URL) (map[string]interface{}, error) { cdxUrl := fmt.Sprintf("%s?url=%s&output=json&fl=timestamp,statuscode,digest,length,offset", archiveAPIURL, url) - resp, err := http.Get(cdxUrl) + client := http.Client{ + Timeout: 60 * time.Second, + } + + resp, err := client.Get(cdxUrl) if err != nil { return nil, err } @@ -118,9 +98,18 @@ func getWaybackData(url string) (map[string]interface{}, error) { }, nil } + if len(data) < 1 { + return nil, fmt.Errorf("data slice is empty") + } + // Remove the header row data = data[1:] + if len(data) < 1 { + return nil, fmt.Errorf("data slice became empty after removing the first element") + } + + // Access the first element of the remaining data firstScan, err := convertTimestampToDate(data[0][0]) if err != nil { return nil, err @@ -146,17 +135,13 @@ func getWaybackData(url string) (map[string]interface{}, error) { func HandleArchives() http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - urlParam := r.URL.Query().Get("url") - if urlParam == "" { - http.Error(w, "missing 'url' parameter", http.StatusBadRequest) + rawURL, err := extractURL(r) + if err != nil { + JSONError(w, ErrMissingURLParameter, http.StatusBadRequest) return } - if !strings.HasPrefix(urlParam, "http://") && !strings.HasPrefix(urlParam, "https://") { - urlParam = "http://" + urlParam - } - - data, err := getWaybackData(urlParam) + data, err := getWaybackData(rawURL) if err != nil { http.Error(w, fmt.Sprintf("Error fetching Wayback data: %v", err), http.StatusInternalServerError) return diff --git a/handlers/mail-config.go b/handlers/mail-config.go index 2c8b044..35dc44f 100644 --- a/handlers/mail-config.go +++ b/handlers/mail-config.go @@ -1,9 +1,7 @@ package handlers import ( - "errors" "net/http" - "net/url" "strings" "github.com/miekg/dns" @@ -57,27 +55,13 @@ func ResolveTxt(domain string) ([]string, int, error) { func HandleMailConfig() http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - urlParam := r.URL.Query().Get("url") - if urlParam == "" { - JSONError(w, errors.New("URL parameter is required"), http.StatusBadRequest) - return - } - - if !strings.HasPrefix(urlParam, "http://") && !strings.HasPrefix(urlParam, "https://") { - urlParam = "http://" + urlParam - } - - parsedURL, err := url.Parse(urlParam) + rawURL, err := extractURL(r) if err != nil { - JSONError(w, errors.New("Invalid URL"), http.StatusBadRequest) + JSONError(w, ErrMissingURLParameter, http.StatusBadRequest) return } - domain := parsedURL.Hostname() - if domain == "" { - domain = parsedURL.Path - } - mxRecords, rcode, err := ResolveMx(domain) + mxRecords, rcode, err := ResolveMx(rawURL.Hostname()) if err != nil { JSONError(w, err, http.StatusInternalServerError) return @@ -88,7 +72,7 @@ func HandleMailConfig() http.Handler { return } - txtRecords, rcode, err := ResolveTxt(domain) + txtRecords, rcode, err := ResolveTxt(rawURL.Hostname()) if err != nil { JSONError(w, err, http.StatusInternalServerError) return diff --git a/handlers/robots-txt.go b/handlers/robots-txt.go index 6456cb2..11f8bcf 100644 --- a/handlers/robots-txt.go +++ b/handlers/robots-txt.go @@ -1,11 +1,9 @@ package handlers import ( - "errors" "fmt" "io" "net/http" - "net/url" "strings" ) @@ -40,23 +38,13 @@ func ParseRobotsTxt(content string) map[string][]map[string]string { func HandleRobotsTxt() http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - urlParam := r.URL.Query().Get("url") - if urlParam == "" { - JSONError(w, errors.New("url query parameter is required"), http.StatusBadRequest) - return - } - - if !strings.HasPrefix(urlParam, "http://") && !strings.HasPrefix(urlParam, "https://") { - urlParam = "http://" + urlParam - } - - parsedURL, err := url.Parse(urlParam) + rawURL, err := extractURL(r) if err != nil { - JSONError(w, errors.New("Invalid url query parameter"), http.StatusBadRequest) + JSONError(w, ErrMissingURLParameter, http.StatusBadRequest) return } - robotsURL := fmt.Sprintf("%s://%s/robots.txt", parsedURL.Scheme, parsedURL.Host) + robotsURL := fmt.Sprintf("%s://%s/robots.txt", rawURL.Scheme, rawURL.Host) resp, err := http.Get(robotsURL) if err != nil { diff --git a/handlers/screenshot.go b/handlers/screenshot.go index 2c1eff0..f9f60c2 100644 --- a/handlers/screenshot.go +++ b/handlers/screenshot.go @@ -7,7 +7,6 @@ import ( "errors" "net/http" "net/url" - "strings" "github.com/chromedp/chromedp" ) @@ -17,14 +16,6 @@ type screenshotResponse struct { } func takeScreenshot(targetURL string) (*screenshotResponse, error) { - if targetURL == "" { - return nil, errors.New("URL is missing from queryStringParameters") - } - - if !strings.HasPrefix(targetURL, "http://") && !strings.HasPrefix(targetURL, "https://") { - targetURL = "http://" + targetURL - } - parsedURL, err := url.ParseRequestURI(targetURL) if err != nil { return nil, errors.New("URL provided is invalid") @@ -51,13 +42,13 @@ func takeScreenshot(targetURL string) (*screenshotResponse, error) { func HandleScreenshot() http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - targetURL := r.URL.Query().Get("url") - if targetURL == "" { - http.Error(w, "missing 'url' parameter", http.StatusBadRequest) + rawURL, err := extractURL(r) + if err != nil { + JSONError(w, ErrMissingURLParameter, http.StatusBadRequest) return } - data, err := takeScreenshot(targetURL) + data, err := takeScreenshot(rawURL.String()) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return diff --git a/handlers/security-txt.go b/handlers/security-txt.go index 879ca23..f69a290 100644 --- a/handlers/security-txt.go +++ b/handlers/security-txt.go @@ -1,9 +1,8 @@ package handlers import ( - "errors" "fmt" - "io/ioutil" + "io" "net/http" "net/url" "strings" @@ -44,25 +43,15 @@ func isPgpSigned(result string) bool { func HandleSecurityTxt() http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - urlParam := r.URL.Query().Get("url") - if urlParam == "" { - JSONError(w, errors.New("url query parameter is required"), http.StatusBadRequest) - return - } - - if !strings.HasPrefix(urlParam, "http://") && !strings.HasPrefix(urlParam, "https://") { - urlParam = "https://" + urlParam - } - - parsedURL, err := url.Parse(urlParam) + rawURL, err := extractURL(r) if err != nil { - JSONError(w, errors.New("Invalid url query parameter"), http.StatusBadRequest) + JSONError(w, ErrMissingURLParameter, http.StatusBadRequest) return } - parsedURL.Path = "" + rawURL.Path = "" for _, path := range SECURITY_TXT_PATHS { - result, err := fetchSecurityTxt(parsedURL, path) + result, err := fetchSecurityTxt(rawURL, path) if err != nil { JSONError(w, err, http.StatusInternalServerError) return @@ -106,7 +95,7 @@ func fetchSecurityTxt(baseURL *url.URL, path string) (string, error) { return "", nil } - body, err := ioutil.ReadAll(resp.Body) + body, err := io.ReadAll(resp.Body) if err != nil { return "", err } diff --git a/handlers/sitemap.go b/handlers/sitemap.go index 4058e9c..313a07f 100644 --- a/handlers/sitemap.go +++ b/handlers/sitemap.go @@ -4,6 +4,7 @@ import ( "encoding/xml" "errors" "fmt" + "io" "io/ioutil" "net/http" "strings" @@ -30,17 +31,13 @@ type URL struct { func HandleSitemap() http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - urlParam := r.URL.Query().Get("url") - if urlParam == "" { - JSONError(w, errors.New("url query parameter is required"), http.StatusBadRequest) + rawURL, err := extractURL(r) + if err != nil { + JSONError(w, ErrMissingURLParameter, http.StatusBadRequest) return } - if !strings.HasPrefix(urlParam, "http://") && !strings.HasPrefix(urlParam, "https://") { - urlParam = "https://" + urlParam - } - - sitemapURL := fmt.Sprintf("%s/sitemap.xml", urlParam) + sitemapURL := fmt.Sprintf("%s/sitemap.xml", rawURL.String()) sitemap, err := fetchSitemap(sitemapURL) if err != nil { @@ -51,7 +48,7 @@ func HandleSitemap() http.Handler { // If sitemap not found, try to fetch it from robots.txt if err.Error() == "404" { - sitemapURL, err = getSitemapURLFromRobotsTxt(urlParam) + sitemapURL, err = getSitemapURLFromRobotsTxt(rawURL.Hostname()) if err != nil { JSON(w, map[string]string{"skipped": "No sitemap found"}, http.StatusOK) return @@ -122,7 +119,7 @@ func getSitemapURLFromRobotsTxt(baseURL string) (string, error) { return "", errors.New("failed to fetch robots.txt") } - body, err := ioutil.ReadAll(resp.Body) + body, err := io.ReadAll(resp.Body) if err != nil { return "", err } diff --git a/handlers/ssl.go b/handlers/ssl.go index a93f5d2..f075025 100644 --- a/handlers/ssl.go +++ b/handlers/ssl.go @@ -5,29 +5,22 @@ import ( "errors" "fmt" "net/http" - "net/url" ) func HandleSSL() http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - urlParam := r.URL.Query().Get("url") - if urlParam == "" { - JSONError(w, errors.New("url query parameter is required"), http.StatusBadRequest) - return - } - - parsedURL, err := url.Parse(urlParam) + rawURL, err := extractURL(r) if err != nil { - JSONError(w, errors.New("invalid URL format"), http.StatusBadRequest) + JSONError(w, ErrMissingURLParameter, http.StatusBadRequest) return } options := &tls.Config{ - ServerName: parsedURL.Hostname(), + ServerName: rawURL.Hostname(), InsecureSkipVerify: true, // Skip certificate validation } - conn, err := tls.Dial("tcp", parsedURL.Host+":443", options) + conn, err := tls.Dial("tcp", rawURL.Host+":443", options) if err != nil { JSONError(w, fmt.Errorf("error establishing TLS connection: %s", err.Error()), http.StatusInternalServerError) return diff --git a/handlers/status.go b/handlers/status.go index b840271..51f03c5 100644 --- a/handlers/status.go +++ b/handlers/status.go @@ -5,7 +5,6 @@ import ( "errors" "net/http" "net/http/httptrace" - "strings" "time" ) @@ -89,17 +88,13 @@ func fetchURL(url string) (*responseData, error) { func HandleStatus() http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - urlParam := r.URL.Query().Get("url") - if urlParam == "" { - http.Error(w, "missing 'url' parameter", http.StatusBadRequest) + rawURL, err := extractURL(r) + if err != nil { + JSONError(w, ErrMissingURLParameter, http.StatusBadRequest) return } - if !strings.HasPrefix(urlParam, "http://") && !strings.HasPrefix(urlParam, "https://") { - urlParam = "http://" + urlParam - } - - data, err := fetchURL(urlParam) + data, err := fetchURL(rawURL.String()) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return diff --git a/handlers/tech-stack.go b/handlers/tech-stack.go index f5b6112..6ac9435 100644 --- a/handlers/tech-stack.go +++ b/handlers/tech-stack.go @@ -3,7 +3,6 @@ package handlers import ( "errors" "net/http" - "strings" "github.com/PuerkitoBio/goquery" ) @@ -36,17 +35,13 @@ func extractTechnologies(url string) ([]string, error) { func HandleTechStack() http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - urlParam := r.URL.Query().Get("url") - if urlParam == "" { - JSONError(w, errors.New("Missing 'url' parameter"), http.StatusBadRequest) + rawURL, err := extractURL(r) + if err != nil { + JSONError(w, ErrMissingURLParameter, http.StatusBadRequest) return } - if !strings.HasPrefix(urlParam, "http://") && !strings.HasPrefix(urlParam, "https://") { - urlParam = "http://" + urlParam - } - - technologies, err := extractTechnologies(urlParam) + technologies, err := extractTechnologies(rawURL.String()) if err != nil { JSONError(w, err, http.StatusInternalServerError) return diff --git a/handlers/threats.go b/handlers/threats.go index edb1cf0..28f2e1a 100644 --- a/handlers/threats.go +++ b/handlers/threats.go @@ -232,35 +232,31 @@ func getCloudmersiveResult(urlParam string) (*CloudmersiveResponse, error) { func HandleThreats() http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - urlParam := r.URL.Query().Get("url") - if urlParam == "" { - JSONError(w, errors.New("url query parameter is required"), http.StatusBadRequest) + rawURL, err := extractURL(r) + if err != nil { + JSONError(w, ErrMissingURLParameter, http.StatusBadRequest) return } - if !strings.HasPrefix(urlParam, "http://") && !strings.HasPrefix(urlParam, "https://") { - urlParam = "https://" + urlParam - } - - urlHausResult, err := getUrlHausResult(urlParam) + urlHausResult, err := getUrlHausResult(rawURL.String()) if err != nil { JSONError(w, err, http.StatusInternalServerError) return } - phishTankResult, err := getPhishTankResult(urlParam) + phishTankResult, err := getPhishTankResult(rawURL.String()) if err != nil { JSONError(w, err, http.StatusInternalServerError) return } - cloudmersiveResult, err := getCloudmersiveResult(urlParam) + cloudmersiveResult, err := getCloudmersiveResult(rawURL.String()) if err != nil { JSONError(w, err, http.StatusInternalServerError) return } - googleSafeBrowsingResult, err := getGoogleSafeBrowsingResult(urlParam) + googleSafeBrowsingResult, err := getGoogleSafeBrowsingResult(rawURL.String()) if err != nil { JSONError(w, err, http.StatusInternalServerError) return diff --git a/handlers/txt-records.go b/handlers/txt-records.go index e00e9c9..5ae4fce 100644 --- a/handlers/txt-records.go +++ b/handlers/txt-records.go @@ -10,17 +10,13 @@ import ( func HandleTXTRecords() http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - urlParam := r.URL.Query().Get("url") - if urlParam == "" { - JSONError(w, errors.New("url query parameter is required"), http.StatusBadRequest) + rawURL, err := extractURL(r) + if err != nil { + JSONError(w, ErrMissingURLParameter, http.StatusBadRequest) return } - if !strings.HasPrefix(urlParam, "http://") && !strings.HasPrefix(urlParam, "https://") { - urlParam = "https://" + urlParam - } - - parsedURL, err := url.Parse(urlParam) + parsedURL, err := url.Parse(rawURL.String()) if err != nil { JSONError(w, errors.New("invalid URL format"), http.StatusBadRequest) return diff --git a/handlers/whois.go b/handlers/whois.go index dba2e26..92d3e19 100644 --- a/handlers/whois.go +++ b/handlers/whois.go @@ -120,13 +120,14 @@ func fetchFromMyAPI(hostname string) (map[string]interface{}, error) { func HandleWhois() http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - url := r.URL.Query().Get("url") - if url == "" { - http.Error(w, "missing 'url' parameter", http.StatusBadRequest) + rawURL, err := extractURL(r) + if err != nil { + JSONError(w, ErrMissingURLParameter, http.StatusBadRequest) return } - hostname, err := getBaseDomain(url) + hostname, err := getBaseDomain(rawURL.Hostname()) + if err != nil { http.Error(w, fmt.Sprintf("Unable to parse URL: %v", err), http.StatusInternalServerError) return