design octopus v1.3 - working design

Author: rapidclock

main.go

@@ -22,35 +22,14 @@ func main() {
 	// exp.Test_makeLinksAbsolute()
 	// runPipeline()
 	// runPipelineWithOptions()
-	// octopusTest()
-	stupid()
+	octopusTest()
 }
 
-func stupid() {
-	// resp, err := http.Head(HomeUrl)
-	// if err == nil && resp == nil {
-	// 	log.Fatal("WOW resp is nill although err is not")
-	// }
-	// if err == nil && resp != nil && resp.StatusCode == 200 {
-	// 	fmt.Printf("%s\n", resp.Status)
-	// }
-	resp, err := http.Head("https://en.wikipedia.org/wiki/Main_Page")
-	fmt.Println("A")
-	if err == nil && resp == nil {
-		log.Fatal("WOW resp is nill although err is not")
-	}
-	fmt.Println("B")
-	if err == nil && resp.StatusCode == 200 {
-		fmt.Printf("\nXX%s\n", resp.Status)
-	}
-	fmt.Println("C")
-}
 func octopusTest() {
 	outputAdapter := &adapter.StdOpAdapter{}
 	// outputAdapter := &adapter.FileWriterAdapter{"crawl_output.txt"}
 
 	crawlOpt := oct.GetDefaultCrawlOptions()
-	crawlOpt.MaxCrawlDepth = 3
 	crawlOpt.OpAdapter = outputAdapter
 
 	octopus := oct.New(crawlOpt)

octopus/core.go

@@ -41,25 +41,23 @@ func (o *octopus) SetupSystem() {
 		ingestQuitCh,
 	}
 
+	o.inputUrlStrChan = ingestStrCh
+	o.masterQuitCh = make(chan int, 1)
+
 	outAdapterChSet := o.OpAdapter.Consume()
 
 	pageParseChSet := o.makeParseNodeFromHtmlPipe(ingestChSet)
 	depthLimitChSet := o.makeCrawlDepthFilterPipe(pageParseChSet)
-	// maxDelayChSet := o.makeMaxDelayPipe(depthLimitChSet)
-	// distributorChSet := o.makeDistributorPipe(maxDelayChSet, outAdapterChSet)
-	distributorChSet := o.makeDistributorPipe(depthLimitChSet, outAdapterChSet)
+	maxDelayChSet := o.makeMaxDelayPipe(depthLimitChSet)
+	distributorChSet := o.makeDistributorPipe(maxDelayChSet, outAdapterChSet)
 	pageReqChSet := o.makePageRequisitionPipe(distributorChSet)
 	invUrlFilterChSet := o.makeInvalidUrlFilterPipe(pageReqChSet)
 	dupFilterChSet := o.makeDuplicateUrlFilterPipe(invUrlFilterChSet)
 	protoFilterChSet := o.makeUrlProtocolFilterPipe(dupFilterChSet)
 	linkAbsChSet := o.makeLinkAbsolutionPipe(protoFilterChSet)
 
-
 	o.makeIngestPipe(inPipeChSet, linkAbsChSet)
 
-	o.inputUrlStrChan = ingestStrCh
-	o.masterQuitCh = make(chan int, 1)
-
 	<-time.After(500 * time.Millisecond)
 	o.isReady = true
 }
@@ -71,19 +69,6 @@ func (o *octopus) BeginCrawling(baseUrlStr string) {
 	go func() {
 		o.inputUrlStrChan <- baseUrlStr
 	}()
-	// for {
-	// 	select {
-	// 	// case urlStr := <-o.inputUrlStrChan:
-	// 	// 	{
-	// 	// 		o.inputUrlStrChan <- urlStr
-	// 	// 	}
-	// 	case <-o.masterQuitCh:
-	// 		{
-	// 			fmt.Println("Master Kill Switch Activated")
-	// 			return
-	// 		}
-	// 	}
-	// }
 	<-o.masterQuitCh
 	fmt.Println("Master Kill Switch Activated")
 }

octopus/modelfactory.go

@@ -6,7 +6,7 @@ const (
 	defaultMaxDepth   int64 = 2
 	anchorTag               = "a"
 	anchorAttrb             = "href"
-	defaultTimeToQuit       = 10
+	defaultTimeToQuit       = 5
 )
 
 // NewWithDefaultOptions - Create an Instance of the Octopus with the default CrawlOptions.

octopus/pipe_filter_urlvalidation.go

@@ -1,7 +1,6 @@
 package octopus
 
 import (
-	"log"
 	"net/http"
 )
 
@@ -11,11 +10,7 @@ func (o *octopus) makeInvalidUrlFilterPipe(outChSet *NodeChSet) *NodeChSet {
 
 func validateUrl(node *Node, outChSet *NodeChSet) {
 	resp, err := http.Head(node.UrlString)
-	if err == nil && resp == nil {
-		log.Fatal("WOW resp is nill although err is not")
-	}
 	if err == nil && resp != nil && resp.StatusCode == 200 {
 		outChSet.NodeCh <- node
 	}
-	// log.Printf("%v\n", err)
 }