#a98h3 - base functions for limit

Author: droidgod

octopus/core.go

@@ -9,6 +9,7 @@ import (
 func (o *octopus) setupOctopus() {
 	o.setupValidProtocolMap()
 	o.setupTimeToQuit()
+	o.setupMaxLinksCrawled()
 }
 
 func (o *octopus) setupValidProtocolMap() {
@@ -26,6 +27,15 @@ func (o *octopus) setupTimeToQuit() {
 	}
 }
 
+func (o *octopus) setupMaxLinksCrawled() {
+	switch {
+	case o.MaxCrawledUrls == 0:
+		panic("MaxCrawledUrls should either be negative or greater than 0.")
+	case o.MaxCrawledUrls > 0:
+		o.MaxCrawledUrls++ // done for convenience.
+	}
+}
+
 func (o *octopus) SetupSystem() {
 	o.isReady = false
 	o.setupOctopus()
@@ -64,7 +74,7 @@ func (o *octopus) SetupSystem() {
 
 func (o *octopus) BeginCrawling(baseUrlStr string) {
 	if !o.isReady {
-		log.Fatal("Call BuildSystem first to setup Octopus")
+		panic("Call BuildSystem first to setup Octopus")
 	}
 	go func() {
 		o.inputUrlStrChan <- baseUrlStr

octopus/modelfactory.go

@@ -45,9 +45,9 @@ func createNode(parentUrlStr, urlStr string, depth int64) *Node {
 func GetDefaultCrawlOptions() *CrawlOptions {
 	return &CrawlOptions{
 		MaxCrawlDepth:      defaultMaxDepth,
-		MaxCrawlLinks:      defaultCrawlLimit,
+		MaxCrawledUrls:     defaultCrawlLimit,
 		StayWithinBaseHost: false,
-		CrawlRate:    -1,
+		CrawlRate:          -1,
 		RespectRobots:      false,
 		IncludeBody:        true,
 		OpAdapter:          nil,

octopus/models.go

@@ -12,13 +12,14 @@ import (
 // to an instance of the crawler.
 type octopus struct {
 	*CrawlOptions
-	visited         *sync.Map
-	isReady         bool
-	adapterChSet    *NodeChSet
-	isValidProtocol map[string]bool
-	timeToQuit      time.Duration
-	inputUrlStrChan chan string
-	masterQuitCh    chan int
+	visited           *sync.Map
+	isReady           bool
+	adapterChSet      *NodeChSet
+	isValidProtocol   map[string]bool
+	timeToQuit        time.Duration
+	inputUrlStrChan   chan string
+	masterQuitCh      chan int
+	crawledUrlCounter int64
 }
 
 // CrawlOptions is used to house options for crawling.
@@ -29,7 +30,7 @@ type octopus struct {
 // 	MaxCrawlDepth - Indicates the maximum depth that will be crawled,
 // 	for each new link.
 //
-// 	MaxCrawlLinks - Specifies the Maximum Number of Unique Links that will be crawled.
+// 	MaxCrawledUrls - Specifies the Maximum Number of Unique Links that will be crawled.
 // 	Note : When combined with DepthPerLink, it will combine both.
 // 	Use -1 to indicate infinite links to be crawled (only bounded by depth of traversal).
 //
@@ -54,7 +55,7 @@ type octopus struct {
 // 	generated before the crawler quits. This is in seconds.
 type CrawlOptions struct {
 	MaxCrawlDepth      int64
-	MaxCrawlLinks      int64
+	MaxCrawledUrls     int64
 	StayWithinBaseHost bool
 	CrawlRate          int64
 	RespectRobots      bool

octopus/pipe_ctrl_limitcrawl.go

@@ -0,0 +1,18 @@
+package octopus
+
+import (
+	"sync/atomic"
+)
+
+func (o *octopus) makeLimitCrawlPipe(inChSet *NodeChSet) *NodeChSet {
+	return stdLinearNodeFunc(o.checkWithinLimit, inChSet)
+}
+
+func (o *octopus) checkWithinLimit(node *Node, outChSet *NodeChSet) {
+	if v := atomic.AddInt64(&o.crawledUrlCounter,
+		1); v < o.MaxCrawledUrls {
+		outChSet.NodeCh <- node
+	} else {
+		outChSet.QuitCh <- 1
+	}
+}