exp pipeline v8 - depth limited crawling via options

Author: rapidclock

experimental/model.go

@@ -13,7 +13,12 @@ type ReqProp struct {
 	Depth     int
 }
 
+type Options struct {
+	MaxDepth int
+}
+
 type Monster struct {
+	*Options
 	listenPipe chan string
 	compPipe   chan<- *ReqProp
 }

experimental/pipeline.go

@@ -1,25 +1,43 @@
 package experimental
 
+func NewMonsterWithOptions(options *Options) *Monster {
+	listenChan := make(chan string)
+	var opt *Options
+	if options != nil {
+		opt = options
+	} else {
+		opt = &Options{
+			-1,
+		}
+	}
+	return &Monster{
+		Options:    opt,
+		listenPipe: listenChan,
+		compPipe:   nil,
+	}
+}
+
 func NewMonster() *Monster {
 	listenChan := make(chan string)
 	return &Monster{
-		listenChan,
-		nil,
+		Options:    nil,
+		listenPipe: listenChan,
+		compPipe:   nil,
 	}
 }
 
 func (m *Monster) BuildSystem(opAdapterPipe chan<- *Node) {
-	parsePipe, compPipeChan := MakeParsingPipe()
+	parsePipe, compPipeChan := m.MakeParsingPipe()
 	var reqPipe chan<- *Node
 	if opAdapterPipe == nil {
-		reqPipe = MakeRequisitionPipe(parsePipe, nil)
+		reqPipe = m.MakeRequisitionPipe(parsePipe, nil)
 	} else {
-		reqPipe = MakeRequisitionPipe(parsePipe, opAdapterPipe)
+		reqPipe = m.MakeRequisitionPipe(parsePipe, opAdapterPipe)
 	}
-	validationPipe := MakeUrlValidationPipe(reqPipe)
-	unduplPipe := MakeUnduplicationPipe(validationPipe)
-	cleanPipe := MakeLinkCleaningPipe(unduplPipe)
-	compPipe := MakeCompositionPipe(cleanPipe)
+	validationPipe := m.MakeUrlValidationPipe(reqPipe)
+	unduplPipe := m.MakeUnduplicationPipe(validationPipe)
+	cleanPipe := m.MakeLinkCleaningPipe(unduplPipe)
+	compPipe := m.MakeCompositionPipe(cleanPipe)
 	compPipeChan <- compPipe
 	m.compPipe = compPipe
 }

experimental/pipeline_adapter.go

@@ -13,6 +13,6 @@ func HandleOutput(opAdapterPipe <-chan *Node) {
 	for {
 		opNode := <-opAdapterPipe
 		count++
-		fmt.Printf("%d - %v\n", count, opNode.UrlStr)
+		fmt.Printf("%d - %d - %v\n", count, opNode.Depth, opNode.UrlStr)
 	}
 }

experimental/pipeline_cleaning.go

@@ -5,7 +5,7 @@ import (
 	"net/url"
 )
 
-func MakeLinkCleaningPipe(undupPipe chan<- *Node) chan<- *Node {
+func (m *Monster) MakeLinkCleaningPipe(undupPipe chan<- *Node) chan<- *Node {
 	cleanPipe := make(chan *Node)
 	go func() {
 		for {

experimental/pipeline_composition.go

@@ -1,22 +1,32 @@
 package experimental
 
-func MakeCompositionPipe(cleanPipe chan<- *Node) chan<- *ReqProp {
+func (m *Monster) MakeCompositionPipe(cleanPipe chan<- *Node) chan<- *ReqProp {
 	compositionPipe := make(chan *ReqProp)
+	structFunc := structurize
+	if m.MaxDepth > 0 {
+		structFunc = structurizeWithDepth
+	}
 	go func() {
 		for {
 			select {
 			case req := <-compositionPipe:
-				go structurize(req, cleanPipe)
+				go structFunc(m.MaxDepth, req, cleanPipe)
 			}
 		}
 	}()
 	return compositionPipe
 }
 
-func structurize(reqProp *ReqProp, cleanPipe chan<- *Node) {
+func structurize(maxDepth int, reqProp *ReqProp, cleanPipe chan<- *Node) {
 	node := &Node{
 		reqProp,
 		nil,
 	}
 	cleanPipe <- node
 }
+
+func structurizeWithDepth(maxDepth int, reqProp *ReqProp, cleanPipe chan<- *Node) {
+	if reqProp.Depth <= maxDepth {
+		structurize(maxDepth, reqProp, cleanPipe)
+	}
+}

experimental/pipeline_parsing.go

@@ -2,7 +2,7 @@ package experimental
 
 import "golang.org/x/net/html"
 
-func MakeParsingPipe() (chan<- *Node, chan<- chan<- *ReqProp) {
+func (m *Monster) MakeParsingPipe() (chan<- *Node, chan<- chan<- *ReqProp) {
 	parsePipe := make(chan *Node)
 	compPipeChan := make(chan chan<- *ReqProp)
 	go func() {

experimental/pipeline_requisition.go

@@ -2,7 +2,7 @@ package experimental
 
 import "net/http"
 
-func MakeRequisitionPipe(parsePipe chan<- *Node, opAdapterPipe chan<- *Node) chan<- *Node {
+func (m *Monster) MakeRequisitionPipe(parsePipe chan<- *Node, opAdapterPipe chan<- *Node) chan<- *Node {
 	requisitionPipe := make(chan *Node)
 	go func() {
 		for {

experimental/pipeline_unduplication.go

@@ -2,7 +2,7 @@ package experimental
 
 import "sync"
 
-func MakeUnduplicationPipe(validationPipe chan<- *Node) chan<- *Node {
+func (m *Monster) MakeUnduplicationPipe(validationPipe chan<- *Node) chan<- *Node {
 	var visitMap sync.Map
 	undupPipe := make(chan *Node)
 	go func() {

experimental/pipeline_urlvalidation.go

@@ -5,7 +5,7 @@ import (
 	"net/http"
 )
 
-func MakeUrlValidationPipe(requisitionPipe chan<- *Node) chan<- *Node {
+func (m *Monster) MakeUrlValidationPipe(requisitionPipe chan<- *Node) chan<- *Node {
 	validationPipe := make(chan *Node)
 	go func() {
 		for {

main.go

@@ -17,7 +17,8 @@ const (
 
 func main() {
 	//exp.Test_makeLinksAbsolute()
-	runPipeline()
+	//runPipeline()
+	runPipelineWithOptions()
 }
 
 func checkPipelineA() {
@@ -63,3 +64,13 @@ func runPipeline() {
 	crawler.BuildSystem(opAdapterPipe)
 	crawler.StartCrawling(LessLinkUrl)
 }
+
+func runPipelineWithOptions() {
+	opt := &exp.Options{
+		MaxDepth: 3,
+	}
+	crawler := exp.NewMonsterWithOptions(opt)
+	opAdapterPipe := exp.GetOutputAdapterPipe()
+	crawler.BuildSystem(opAdapterPipe)
+	crawler.StartCrawling(LessLinkUrl)
+}