#a98h3 - working max Links limit

Author: droidgod

adapter/basicadapters.go

@@ -22,10 +22,12 @@ func (s *StdOpAdapter) Consume() *oct.NodeChSet {
 		},
 	}
 	go func() {
+		i := 1
 		for {
 			select {
 			case output := <-listenCh:
-				fmt.Printf("%d - %s\n", output.Depth, output.UrlString)
+				fmt.Printf("%d - %d - %s\n", i, output.Depth, output.UrlString)
+				i++
 			case <-quitCh:
 				return
 			}

main.go

@@ -9,6 +9,7 @@ func main() {
 	opAdapter := &adapter.StdOpAdapter{}
 	options := octopus.GetDefaultCrawlOptions()
 	options.OpAdapter = opAdapter
+	options.MaxCrawledUrls = 150
 	crawler := octopus.New(options)
 	crawler.SetupSystem()
 	crawler.BeginCrawling("https://www.macrumors.com")

octopus/core.go

@@ -28,11 +28,8 @@ func (o *octopus) setupTimeToQuit() {
 }
 
 func (o *octopus) setupMaxLinksCrawled() {
-	switch {
-	case o.MaxCrawledUrls == 0:
+	if o.MaxCrawledUrls == 0 {
 		panic("MaxCrawledUrls should either be negative or greater than 0.")
-	case o.MaxCrawledUrls > 0:
-		o.MaxCrawledUrls++ // done for convenience.
 	}
 }
 
@@ -59,7 +56,15 @@ func (o *octopus) SetupSystem() {
 	pageParseChSet := o.makeParseNodeFromHtmlPipe(ingestChSet)
 	depthLimitChSet := o.makeCrawlDepthFilterPipe(pageParseChSet)
 	maxDelayChSet := o.makeMaxDelayPipe(depthLimitChSet)
-	distributorChSet := o.makeDistributorPipe(maxDelayChSet, outAdapterChSet)
+
+	var distributorChSet *NodeChSet
+	if o.MaxCrawledUrls < 0 {
+		distributorChSet = o.makeDistributorPipe(maxDelayChSet, outAdapterChSet)
+	} else {
+		maxLinksCrawledChSet := o.makeLimitCrawlPipe(outAdapterChSet)
+		distributorChSet = o.makeDistributorPipe(maxDelayChSet, maxLinksCrawledChSet)
+	}
+
 	pageReqChSet := o.makePageRequisitionPipe(distributorChSet)
 	invUrlFilterChSet := o.makeInvalidUrlFilterPipe(pageReqChSet)
 	dupFilterChSet := o.makeDuplicateUrlFilterPipe(invUrlFilterChSet)

octopus/pipe_ctrl_limitcrawl.go

@@ -10,9 +10,10 @@ func (o *octopus) makeLimitCrawlPipe(inChSet *NodeChSet) *NodeChSet {
 
 func (o *octopus) checkWithinLimit(node *Node, outChSet *NodeChSet) {
 	if v := atomic.AddInt64(&o.crawledUrlCounter,
-		1); v < o.MaxCrawledUrls {
+		1); v <= o.MaxCrawledUrls {
 		outChSet.NodeCh <- node
 	} else {
 		outChSet.QuitCh <- 1
+		o.masterQuitCh <- 1
 	}
 }