Merge pull request #21 from HuntDownProject/release/v1.0.3

Release/v1.0.3

Author: neriberto

.github/workflows/auto-pr.yml

@@ -4,11 +4,11 @@ on:
   workflow_dispatch:
   push:
     branches:
-      - dev
+      - develop
   pull_request:
     branches:
       - main
-      - dev
+      - develop
       - feature*
     paths:
       - '**.go'
@@ -38,12 +38,6 @@ jobs:
         run: go test ./...
         working-directory: .
 
-      #- name: Integration Tests
-      #  env:
-      #    GH_ACTION: true
-      #  run: bash run.sh
-      #  working-directory: integration_tests/
-
       - name: Install
         run: go install
         working-directory: cmd/hednsextractor/

.github/workflows/build-test.yml

@@ -6,9 +6,7 @@ import (
 	"strconv"
 
 	"github.com/HuntDownProject/hednsextractor/utils"
-	"github.com/projectdiscovery/goflags"
 	"github.com/projectdiscovery/gologger"
-	"github.com/projectdiscovery/gologger/levels"
 )
 
 var (
@@ -17,72 +15,35 @@ var (
 
 func main() {
 
+	// Parse the stdin
 	utils.ParseStdin()
 
-	flagSet := goflags.NewFlagSet()
-	flagSet.Marshal = true
-	flagSet.SetDescription("HEDnsExtractor - A suite for hunting suspicious targets, expose domains and phishing discovery!")
-	flagSet.BoolVar(&utils.OptionCmd.Onlydomains, "only-domains", false, "show only domains")
-	flagSet.BoolVar(&utils.OptionCmd.Onlynetworks, "only-networks", false, "show only networks")
-	flagSet.StringVar(&utils.OptionCmd.Workflow, "workflow", "", "Workflow config")
-	flagSet.StringVar(&utils.OptionCmd.Target, "target", "", "IP Address or Network to query")
-	flagSet.BoolVar(&utils.OptionCmd.Silent, "silent", false, "show silent output")
-	flagSet.BoolVar(&utils.OptionCmd.Verbose, "verbose", false, "show verbose output")
-
-	flagSet.CreateGroup("configuration", "Configuration",
-		flagSet.StringVar(&utils.OptionCmd.Config, "config", utils.DefaultConfigLocation, "flag config file"),
-	)
-
-	flagSet.CreateGroup("config", "Virustotal",
-		flagSet.BoolVar(&utils.OptionCmd.Vtscore, "vt", false, "show Virustotal score"),
-		flagSet.StringVar(&utils.OptionCmd.VtApiKey, "vt-api-key", "", "Virustotal API Key"),
-		flagSet.StringVar(&utils.OptionCmd.VtscoreValue, "vt-score", "0", "Minimum Virustotal score to show"),
-	)
-
-	if err := flagSet.Parse(); err != nil {
-		gologger.Fatal().Msgf("Could not parse flags: %s\n", err)
-	}
-
-	if utils.OptionCmd.Verbose {
-		gologger.DefaultLogger.SetMaxLevel(levels.LevelVerbose)
-	} else if utils.OptionCmd.Silent {
-		gologger.DefaultLogger.SetMaxLevel(levels.LevelSilent)
-	} else {
-		gologger.DefaultLogger.SetMaxLevel(levels.LevelInfo)
-	}
-
-	if utils.OptionCmd.Vtscore && utils.OptionCmd.VtApiKey == "" {
-		gologger.Fatal().Msgf("A Virustotal API Key is needed in config file: %s\n", utils.DefaultConfigLocation)
-	}
+	// Load parameters from command line and configuration file
+	utils.LoadParameters()
 
 	// Show Banner
 	utils.ShowBanner()
 
-	// read the targets from yaml
-	var c utils.Conf
+	// read the Workflow from yaml
+	var workflow utils.Workflow
 	if utils.OptionCmd.Workflow != "" {
-		c.GetConf(utils.OptionCmd.Workflow)
+		workflow.GetConf(utils.OptionCmd.Workflow)
 
-		for i := range c.Domains {
-			utils.IdentifyTarget(c.Domains[i])
+		for i := range workflow.Domains {
+			utils.IdentifyTarget(workflow.Domains[i])
 		}
 
-		for i := range c.Ipaddrs {
-			utils.IdentifyTarget(c.Ipaddrs[i])
+		for i := range workflow.Ipaddrs {
+			utils.IdentifyTarget(workflow.Ipaddrs[i])
 		}
 
-		for i := range c.Networks {
-			utils.IdentifyTarget(c.Networks[i])
+		for i := range workflow.Networks {
+			utils.IdentifyTarget(workflow.Networks[i])
 		}
 	}
 
-	// Look into target parameter to grab the IPv4s and Networks
-	if utils.OptionCmd.Target != "" {
-		gologger.Verbose().Msgf("Identifying networks for %s", utils.OptionCmd.Target)
-		utils.IdentifyTarget(utils.OptionCmd.Target)
-	}
-
-	utils.RunCrawler()
+	hurricane := utils.Hurricane{}
+	hurricane.RunCrawler()
 
 	if utils.OptionCmd.Vtscore && !utils.OptionCmd.Silent {
 		gologger.Info().Msgf("Filtering with Virustotal with a mininum score %s", utils.OptionCmd.VtscoreValue)
@@ -92,8 +53,8 @@ func main() {
 		var bMatchedPTR = false
 		var bMatchedDomain = false
 
-		if c.Regex != "" {
-			var re = regexp.MustCompile(c.Regex)
+		if workflow.Regex != "" {
+			var re = regexp.MustCompile(workflow.Regex)
 			bMatchedDomain = re.MatchString(result.Domain)
 			bMatchedPTR = re.MatchString(result.PTR)
 		} else {
@@ -106,7 +67,8 @@ func main() {
 		}
 
 		if utils.OptionCmd.Vtscore {
-			result.VtScore = utils.GetVtReport(result.Domain)
+			virustotal := utils.Virustotal{}
+			result.VtScore = virustotal.GetVtReport(result.Domain)
 			if score, err := strconv.ParseUint(utils.OptionCmd.VtscoreValue, 10, 64); err == nil {
 				if result.VtScore < score {
 					continue

cmd/hednsextractor/hednsextractor.go

@@ -3,47 +3,57 @@ module github.com/HuntDownProject/hednsextractor
 go 1.21
 
 require (
-	github.com/PuerkitoBio/goquery v1.8.1
-	github.com/projectdiscovery/goflags v0.1.32
+	github.com/PuerkitoBio/goquery v1.9.1
+	github.com/projectdiscovery/goflags v0.1.46
 	github.com/projectdiscovery/gologger v1.1.12
-	github.com/projectdiscovery/retryablehttp-go v1.0.22
-	github.com/projectdiscovery/utils v0.0.68
-	github.com/tidwall/gjson v1.17.0
+	github.com/projectdiscovery/retryablehttp-go v1.0.54
+	github.com/projectdiscovery/utils v0.0.87
+	github.com/tidwall/gjson v1.17.1
 	gopkg.in/yaml.v2 v2.4.0
 )
 
 require (
+	github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057 // indirect
 	github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809 // indirect
 	github.com/akrylysov/pogreb v0.10.2 // indirect
-	github.com/andybalholm/brotli v1.0.6 // indirect
+	github.com/andybalholm/brotli v1.1.0 // indirect
 	github.com/andybalholm/cascadia v1.3.2 // indirect
 	github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
 	github.com/aymerick/douceur v0.2.0 // indirect
+	github.com/chzyer/readline v1.5.1 // indirect
+	github.com/cloudflare/circl v1.3.7 // indirect
 	github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 // indirect
 	github.com/dimchansky/utfbom v1.1.1 // indirect
+	github.com/docker/go-units v0.5.0 // indirect
 	github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 // indirect
-	github.com/gaukas/godicttls v0.0.3 // indirect
+	github.com/go-logr/logr v1.4.1 // indirect
+	github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
+	github.com/google/go-cmp v0.6.0 // indirect
+	github.com/google/pprof v0.0.0-20240402174815-29b9bb013b0f // indirect
 	github.com/gorilla/css v1.0.1 // indirect
+	github.com/ianlancetaylor/demangle v0.0.0-20240312041847-bd984b5ce465 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
-	github.com/klauspost/compress v1.17.4 // indirect
+	github.com/klauspost/compress v1.17.7 // indirect
 	github.com/klauspost/pgzip v1.2.6 // indirect
 	github.com/logrusorgru/aurora v2.0.3+incompatible // indirect
 	github.com/mholt/archiver/v3 v3.5.1 // indirect
 	github.com/microcosm-cc/bluemonday v1.0.26 // indirect
-	github.com/miekg/dns v1.1.57 // indirect
+	github.com/miekg/dns v1.1.58 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/nwaples/rardecode v1.1.3 // indirect
-	github.com/onsi/gomega v1.22.1 // indirect
-	github.com/pierrec/lz4/v4 v4.1.19 // indirect
+	github.com/onsi/ginkgo/v2 v2.17.1 // indirect
+	github.com/onsi/gomega v1.32.0 // indirect
+	github.com/pierrec/lz4/v4 v4.1.21 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/projectdiscovery/blackrock v0.0.1 // indirect
-	github.com/projectdiscovery/fastdialer v0.0.35 // indirect
-	github.com/projectdiscovery/hmap v0.0.30 // indirect
-	github.com/projectdiscovery/networkpolicy v0.0.6 // indirect
-	github.com/projectdiscovery/retryabledns v1.0.47 // indirect
-	github.com/refraction-networking/utls v1.3.3 // indirect
+	github.com/projectdiscovery/fastdialer v0.0.65 // indirect
+	github.com/projectdiscovery/hmap v0.0.41 // indirect
+	github.com/projectdiscovery/networkpolicy v0.0.8 // indirect
+	github.com/projectdiscovery/retryabledns v1.0.58 // indirect
+	github.com/quic-go/quic-go v0.42.0 // indirect
+	github.com/refraction-networking/utls v1.6.3 // indirect
 	github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect
 	github.com/syndtr/goleveldb v1.0.0 // indirect
 	github.com/tidwall/btree v1.7.0 // indirect
@@ -53,22 +63,23 @@ require (
 	github.com/tidwall/pretty v1.2.1 // indirect
 	github.com/tidwall/rtred v0.1.2 // indirect
 	github.com/tidwall/tinyqueue v0.1.1 // indirect
-	github.com/ulikunitz/xz v0.5.11 // indirect
+	github.com/ulikunitz/xz v0.5.12 // indirect
 	github.com/ulule/deepcopier v0.0.0-20200430083143-45decc6639b6 // indirect
-	github.com/weppos/publicsuffix-go v0.30.2-0.20230730094716-a20f9abcc222 // indirect
+	github.com/weppos/publicsuffix-go v0.30.2 // indirect
 	github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect
 	github.com/yl2chen/cidranger v1.0.2 // indirect
 	github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248 // indirect
 	github.com/zmap/zcrypto v0.0.0-20231219022726-a1f61fb1661c // indirect
-	go.etcd.io/bbolt v1.3.8 // indirect
+	go.etcd.io/bbolt v1.3.9 // indirect
+	go.uber.org/mock v0.4.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
-	golang.org/x/crypto v0.17.0 // indirect
-	golang.org/x/exp v0.0.0-20231214170342-aacd6d4b4611 // indirect
-	golang.org/x/mod v0.14.0 // indirect
-	golang.org/x/net v0.19.0 // indirect
-	golang.org/x/sys v0.15.0 // indirect
+	golang.org/x/crypto v0.21.0 // indirect
+	golang.org/x/exp v0.0.0-20240325151524-a685a6edb6d8 // indirect
+	golang.org/x/mod v0.16.0 // indirect
+	golang.org/x/net v0.23.0 // indirect
+	golang.org/x/sys v0.18.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
-	golang.org/x/tools v0.16.1 // indirect
+	golang.org/x/tools v0.19.0 // indirect
 	gopkg.in/djherbis/times.v1 v1.3.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )