exp pipeline v9 - depth limited crawling via options 2

Author: rapidclock

experimental/pipeline_composition.go

@@ -3,9 +3,9 @@ package experimental
 func (m *Monster) MakeCompositionPipe(cleanPipe chan<- *Node) chan<- *ReqProp {
 	compositionPipe := make(chan *ReqProp)
 	structFunc := structurize
-	if m.MaxDepth > 0 {
-		structFunc = structurizeWithDepth
-	}
+	//if m.MaxDepth > 0 {
+	//	structFunc = structurizeWithDepth
+	//}
 	go func() {
 		for {
 			select {

experimental/pipeline_requisition.go

@@ -4,16 +4,20 @@ import "net/http"
 
 func (m *Monster) MakeRequisitionPipe(parsePipe chan<- *Node, opAdapterPipe chan<- *Node) chan<- *Node {
 	requisitionPipe := make(chan *Node)
-	go func() {
+	isDepthLimited := false
+	if m.MaxDepth > 0 {
+		isDepthLimited = true
+	}
+	go func(isDepthLimited bool, maxDepth int) {
 		for {
 			select {
 			case node := <-requisitionPipe:
 				{
-					go makeRequest(node, parsePipe, opAdapterPipe)
+					go depthLimitedRequest(isDepthLimited, maxDepth, node, parsePipe, opAdapterPipe)
 				}
 			}
 		}
-	}()
+	}(isDepthLimited, m.MaxDepth)
 	return requisitionPipe
 }
 
@@ -27,3 +31,10 @@ func makeRequest(node *Node, parsePipe chan<- *Node, opAdapterPipe chan<- *Node)
 		}
 	}
 }
+
+func depthLimitedRequest(isDepthLimited bool, maxDepth int,
+	node *Node, parsePipe chan<- *Node, opAdapterPipe chan<- *Node) {
+	if !isDepthLimited || node.Depth <= maxDepth {
+		makeRequest(node, parsePipe, opAdapterPipe)
+	}
+}

main.go

@@ -67,7 +67,7 @@ func runPipeline() {
 
 func runPipelineWithOptions() {
 	opt := &exp.Options{
-		MaxDepth: 3,
+		MaxDepth: 1,
 	}
 	crawler := exp.NewMonsterWithOptions(opt)
 	opAdapterPipe := exp.GetOutputAdapterPipe()