exp pipeline v7 - added depth to node

Author: rapidclock

experimental/PIPELINE_ZDESIGN.txt

@@ -1,9 +1,9 @@
 
 Pipeline
 
-url => structure => remove duplicates => validate urls => make request => parse page for urls ||
-			^														   => output adapter	  ||
-			^===================================================================================
+url => structure => absolutify links => remove duplicates => validate urls => make request => parse page for urls ||
+			^											                    			   => output adapter      ||
+			^=====================================================================================================
 
 
 1. Composition

experimental/model.go

@@ -10,6 +10,7 @@ type Node struct {
 type ReqProp struct {
 	ParentUrl string
 	UrlStr    string
+	Depth     int
 }
 
 type Monster struct {

experimental/pipeline.go

@@ -12,9 +12,9 @@ func (m *Monster) BuildSystem(opAdapterPipe chan<- *Node) {
 	parsePipe, compPipeChan := MakeParsingPipe()
 	var reqPipe chan<- *Node
 	if opAdapterPipe == nil {
-		reqPipe = MakeRequistionPipe(parsePipe, nil)
+		reqPipe = MakeRequisitionPipe(parsePipe, nil)
 	} else {
-		reqPipe = MakeRequistionPipe(parsePipe, opAdapterPipe)
+		reqPipe = MakeRequisitionPipe(parsePipe, opAdapterPipe)
 	}
 	validationPipe := MakeUrlValidationPipe(reqPipe)
 	unduplPipe := MakeUnduplicationPipe(validationPipe)
@@ -34,6 +34,7 @@ func (m *Monster) StartCrawling(baseUrlString string) {
 			m.compPipe <- &ReqProp{
 				"",
 				urlStr,
+				0,
 			}
 		}
 	}

experimental/pipeline_cleaning.go

@@ -41,6 +41,7 @@ func Test_makeLinksAbsolute() {
 		&ReqProp{
 			"https://en.wikipedia.org/wiki/Main_Page",
 			"/wiki/Caijia_language",
+			1,
 		},
 		nil,
 	}

experimental/pipeline_parsing.go

@@ -35,6 +35,7 @@ func parsePage(node *Node, compositionPipe chan<- *ReqProp) {
 						compositionPipe <- &ReqProp{
 							ParentUrl: node.UrlStr,
 							UrlStr:    attr.Val,
+							Depth:     node.Depth + 1,
 						}
 					}
 				}

experimental/pipeline_requisition.go

@@ -2,7 +2,7 @@ package experimental
 
 import "net/http"
 
-func MakeRequistionPipe(parsePipe chan<- *Node, opAdapterPipe chan<- *Node) chan<- *Node {
+func MakeRequisitionPipe(parsePipe chan<- *Node, opAdapterPipe chan<- *Node) chan<- *Node {
 	requisitionPipe := make(chan *Node)
 	go func() {
 		for {

octopus/models.go

@@ -12,8 +12,8 @@ type Node struct {
 	Depth           int
 }
 
-// octopus is a concurrent version of webSpider.
-// It has an inbuilt parser based of htmlparser.Parser to collect all links in a web-page.
+// octopus is a concurrent web crawler.
+// It has an inbuilt parser based of html.NewTokenizer to collect all links in a web-page.
 // It also has a CrawlOptions structure to initialize setting specific
 // to an instance of the crawler.
 type octopus struct {