recovering from the git fiasco

Author: rapidclock

adapter/basicadapters.go

@@ -73,6 +73,6 @@ func (fw *FileWriterAdapter) writeToFile(listenCh chan *oct.Node,
 }
 
 func (fw *FileWriterAdapter) getFilePointer() (w io.WriteCloser, err error) {
-	w, err = os.OpenFile(fw.FilePath, os.O_RDWR|os.O_CREATE, 0755)
+	w, err = os.OpenFile(fw.FilePath, os.O_RDWR|os.O_CREATE, 0644)
 	return
 }

main.go

@@ -6,7 +6,7 @@ import (
 	"net/http"
 	"time"
 
-	adapter2 "github.com/rapidclock/web-octopus/adapter"
+	"github.com/rapidclock/web-octopus/adapter"
 	exp "github.com/rapidclock/web-octopus/experimental"
 	oct "github.com/rapidclock/web-octopus/octopus"
 )
@@ -22,18 +22,40 @@ func main() {
 	// exp.Test_makeLinksAbsolute()
 	// runPipeline()
 	// runPipelineWithOptions()
-	octopusTest()
+	// octopusTest()
+	stupid()
 }
 
+func stupid() {
+	// resp, err := http.Head(HomeUrl)
+	// if err == nil && resp == nil {
+	// 	log.Fatal("WOW resp is nill although err is not")
+	// }
+	// if err == nil && resp != nil && resp.StatusCode == 200 {
+	// 	fmt.Printf("%s\n", resp.Status)
+	// }
+	resp, err := http.Head("https://en.wikipedia.org/wiki/Main_Page")
+	fmt.Println("A")
+	if err == nil && resp == nil {
+		log.Fatal("WOW resp is nill although err is not")
+	}
+	fmt.Println("B")
+	if err == nil && resp.StatusCode == 200 {
+		fmt.Printf("\nXX%s\n", resp.Status)
+	}
+	fmt.Println("C")
+}
 func octopusTest() {
-	adapter := &adapter2.StdOpAdapter{}
+	outputAdapter := &adapter.StdOpAdapter{}
+	// outputAdapter := &adapter.FileWriterAdapter{"crawl_output.txt"}
 
 	crawlOpt := oct.GetDefaultCrawlOptions()
-	crawlOpt.OpAdapter = adapter
+	crawlOpt.MaxCrawlDepth = 3
+	crawlOpt.OpAdapter = outputAdapter
 
 	octopus := oct.New(crawlOpt)
 	octopus.SetupSystem()
-	octopus.BeginCrawling(Url2)
+	octopus.BeginCrawling(Url1)
 }
 
 func checkPipelineA() {

octopus/core.go

@@ -20,21 +20,23 @@ func (o *octopus) setupValidProtocolMap() {
 
 func (o *octopus) setupTimeToQuit() {
 	if o.TimeToQuit > 0 {
-		o.timeToQuit = time.Duration(o.TimeToQuit)
+		o.timeToQuit = time.Duration(o.TimeToQuit) * time.Second
 	} else {
 		log.Fatalln("TimeToQuit is not greater than 0")
 	}
 }
 
 func (o *octopus) SetupSystem() {
+	o.isReady = false
 	o.setupOctopus()
 
-	ingestCh := make(chan *Node)
+	ingestNodeCh := make(chan *Node)
 	ingestQuitCh := make(chan int, 1)
-	ingestChSet := MakeNodeChSet(ingestCh, ingestQuitCh)
 	ingestStrCh := make(chan string)
+
+	ingestChSet := MakeNodeChSet(ingestNodeCh, ingestQuitCh)
 	inPipeChSet := &ingestPipeChSet{
-		ingestCh,
+		ingestNodeCh,
 		ingestStrCh,
 		ingestQuitCh,
 	}
@@ -43,17 +45,22 @@ func (o *octopus) SetupSystem() {
 
 	pageParseChSet := o.makeParseNodeFromHtmlPipe(ingestChSet)
 	depthLimitChSet := o.makeCrawlDepthFilterPipe(pageParseChSet)
+	// maxDelayChSet := o.makeMaxDelayPipe(depthLimitChSet)
+	// distributorChSet := o.makeDistributorPipe(maxDelayChSet, outAdapterChSet)
 	distributorChSet := o.makeDistributorPipe(depthLimitChSet, outAdapterChSet)
 	pageReqChSet := o.makePageRequisitionPipe(distributorChSet)
 	invUrlFilterChSet := o.makeInvalidUrlFilterPipe(pageReqChSet)
 	dupFilterChSet := o.makeDuplicateUrlFilterPipe(invUrlFilterChSet)
 	protoFilterChSet := o.makeUrlProtocolFilterPipe(dupFilterChSet)
 	linkAbsChSet := o.makeLinkAbsolutionPipe(protoFilterChSet)
 
+
 	o.makeIngestPipe(inPipeChSet, linkAbsChSet)
 
-	o.inpUrlStrChan = ingestStrCh
+	o.inputUrlStrChan = ingestStrCh
 	o.masterQuitCh = make(chan int, 1)
+
+	<-time.After(500 * time.Millisecond)
 	o.isReady = true
 }
 
@@ -62,19 +69,37 @@ func (o *octopus) BeginCrawling(baseUrlStr string) {
 		log.Fatal("Call BuildSystem first to setup Octopus")
 	}
 	go func() {
-		o.inpUrlStrChan <- baseUrlStr
+		o.inputUrlStrChan <- baseUrlStr
 	}()
-	for {
-		select {
-		case urlStr := <-o.inpUrlStrChan:
-			{
-				o.inpUrlStrChan <- urlStr
-			}
-		case <-o.masterQuitCh:
-			{
-				fmt.Println("Master Kill Switch Activated")
-				return
-			}
-		}
+	// for {
+	// 	select {
+	// 	// case urlStr := <-o.inputUrlStrChan:
+	// 	// 	{
+	// 	// 		o.inputUrlStrChan <- urlStr
+	// 	// 	}
+	// 	case <-o.masterQuitCh:
+	// 		{
+	// 			fmt.Println("Master Kill Switch Activated")
+	// 			return
+	// 		}
+	// 	}
+	// }
+	<-o.masterQuitCh
+	fmt.Println("Master Kill Switch Activated")
+}
+
+func (o *octopus) GetInputUrlStrChan() chan<- string {
+	if o.isReady {
+		return o.inputUrlStrChan
+	} else {
+		return nil
+	}
+}
+
+func (o *octopus) GetMasterQuitChan() chan<- int {
+	if o.isReady {
+		return o.masterQuitCh
+	} else {
+		return nil
 	}
 }

octopus/modelfactory.go

@@ -6,7 +6,7 @@ const (
 	defaultMaxDepth   int64 = 2
 	anchorTag               = "a"
 	anchorAttrb             = "href"
-	defaultTimeToQuit       = 5
+	defaultTimeToQuit       = 10
 )
 
 // NewWithDefaultOptions - Create an Instance of the Octopus with the default CrawlOptions.

octopus/models.go

@@ -17,7 +17,7 @@ type octopus struct {
 	adapterChSet    *NodeChSet
 	isValidProtocol map[string]bool
 	timeToQuit      time.Duration
-	inpUrlStrChan   chan string
+	inputUrlStrChan chan string
 	masterQuitCh    chan int
 }
 

octopus/pipe_augment_linkabsolution.go

@@ -1,6 +1,7 @@
 package octopus
 
 import (
+	"log"
 	"net/url"
 )
 
@@ -9,6 +10,10 @@ func (o *octopus) makeLinkAbsolutionPipe(outChSet *NodeChSet) *NodeChSet {
 }
 
 func makeLinkAbsolute(node *Node, outChSet *NodeChSet) {
+	if node == nil || outChSet == nil {
+		log.Fatal("NIL ERROR")
+		return
+	}
 	if node.ParentUrlString != "" {
 		linkUrl, err := url.Parse(node.UrlString)
 		if err != nil {

octopus/pipe_filter_urlvalidation.go

@@ -1,6 +1,7 @@
 package octopus
 
 import (
+	"log"
 	"net/http"
 )
 
@@ -10,7 +11,10 @@ func (o *octopus) makeInvalidUrlFilterPipe(outChSet *NodeChSet) *NodeChSet {
 
 func validateUrl(node *Node, outChSet *NodeChSet) {
 	resp, err := http.Head(node.UrlString)
-	if err == nil && resp.StatusCode == 200 {
+	if err == nil && resp == nil {
+		log.Fatal("WOW resp is nill although err is not")
+	}
+	if err == nil && resp != nil && resp.StatusCode == 200 {
 		outChSet.NodeCh <- node
 	}
 	// log.Printf("%v\n", err)

octopus/pipe_spl_ingest.go

@@ -1,7 +1,6 @@
 package octopus
 
 import (
-	"fmt"
 	"time"
 )
 
@@ -18,33 +17,45 @@ func setupStringIngestPipe(inChSet *ingestPipeChSet, nodeOpChSet *NodeChSet,
 			{
 				nodeOpChSet.NodeCh <- createNode("", str, 1)
 			}
-		// case i := <-inChSet.QuitCh:
-			// 	{
-			// 		nodeOpChSet.QuitCh <- i
-			// 		masterQuitCh <- i
-			// 	}
+		case i := <-inChSet.QuitCh:
+			{
+				nodeOpChSet.QuitCh <- i
+				masterQuitCh <- i
+			}
 		}
 	}
 }
 
 func channelConnector(inChSet *ingestPipeChSet, opChSet *NodeChSet,
 	timeOut time.Duration, masterQuitCh chan int) {
+	// timeOutTimer := time.NewTimer(timeOut)
 	for {
+		// timeOutCh = time.After(timeOut * time.Second)
+		// timeOutCh = time.NewTimer(timeOut)
 		select {
 		case node := <-inChSet.NodeCh:
 			opChSet.NodeCh <- node
-		case i := <-inChSet.QuitCh:
-			{
-				fmt.Println("Quit Received on Ingest Channel")
-				opChSet.QuitCh <- i
-				masterQuitCh <- i
-			}
-		case <-time.After(timeOut * time.Second):
-			{
-				fmt.Println("Timeout Triggered in Ingest Channel")
-				opChSet.QuitCh <- 1
-				return
-			}
+			// if !timeOutTimer.Stop() {
+			// 	<-timeOutTimer.C
+			// }
+			// log.Println("abc")
+			// timeOutTimer.Reset(timeOut)
+			// case i := <-inChSet.QuitCh:
+			// 	{
+			// 		fmt.Println("Quit Received on Ingest Channel")
+			// opChSet.QuitCh <- i
+			// 		masterQuitCh <- i
+			// 		if !timeOutTimer.Stop() {
+			// 			<-timeOutTimer.C
+			// 		}
+			// 		timeOutTimer.Reset(timeOut)
+			// 	}
+			// case <-timeOutTimer.C:
+			// 	fmt.Println("Timeout Triggered in Ingest Channel")
+			// 	opChSet.QuitCh <- 1
+			// 	masterQuitCh <- 1
+			// 	return
+
 		}
 	}
 }

octopus/stdpipefunc.go

@@ -15,9 +15,10 @@ func stdLinearNodeFunc(stdFn stdFunc, outChSet *NodeChSet) *NodeChSet {
 			QuitCh: listenQuitCh,
 		},
 	}
+	// listenChSet := MakeNodeChSet(listenCh, listenQuitCh)
 	go func() {
-		defer close(listenCh)
-		defer close(listenQuitCh)
+		// defer close(listenCh)
+		// defer close(listenQuitCh)
 		for {
 			select {
 			case node := <-listenCh: