m2o - refactoring methods

Author: rapidclock

main.go

@@ -72,5 +72,5 @@ func runPipelineWithOptions() {
 	crawler := exp.NewMonsterWithOptions(opt)
 	opAdapterPipe := exp.GetOutputAdapterPipe()
 	crawler.BuildSystem(opAdapterPipe)
-	crawler.StartCrawling(LessLinkUrl)
+	crawler.StartCrawling(HomeUrl)
 }

octopus/core.go

@@ -1,3 +1,22 @@
 package octopus
 
-func (o *octopus) setup() {}
+func (o *octopus) BuildSystem(opAdapter *OutputAdapter) {
+	//parsePipe, compPipeChan := m.MakeParsingPipe()
+	//var reqPipe chan<- *Node
+	//if opAdapterPipe == nil {
+	//	reqPipe = m.MakeRequisitionPipe(parsePipe, nil)
+	//} else {
+	//	reqPipe = m.MakeRequisitionPipe(parsePipe, opAdapterPipe)
+	//}
+	//validationPipe := m.MakeUrlValidationPipe(reqPipe)
+	//unduplPipe := m.MakeUnduplicationPipe(validationPipe)
+	//cleanPipe := m.MakeLinkCleaningPipe(unduplPipe)
+	//compPipe := m.MakeCompositionPipe(cleanPipe)
+	//compPipeChan <- compPipe
+	//m.compPipe = compPipe
+
+}
+
+func (o *octopus) BeginCrawling() {
+
+}

octopus/modelfactory.go

@@ -1,17 +1,42 @@
 package octopus
 
+import "sync"
+
 const (
 	defaultMaxDepth int16 = 2
 	anchorTag             = "a"
 	anchorAttrb           = "href"
 )
 
-// MakeNew - Creates an Instance of the Octopus Crawler with the given options.
-func MakeNew(opt CrawlOptions) *octopus {
+// NewWithDefaultOptions - Create an Instance of the Octopus with the default CrawlOptions.
+func NewWithDefaultOptions() *octopus {
 	oct := &octopus{
-		CrawlOptions: opt,
-		visited:      make(map[Node]bool),
+		CrawlOptions: getDefaultCrawlOptions(),
+		visited:      new(sync.Map),
+		isBuilt:      false,
 	}
 	oct.setup()
 	return oct
 }
+
+// New - Create an Instance of the Octopus with the given CrawlOptions.
+func New(opt *CrawlOptions) *octopus {
+	oct := &octopus{
+		CrawlOptions: opt,
+		visited:      new(sync.Map),
+		isBuilt:      false,
+	}
+	return oct
+}
+
+func getDefaultCrawlOptions() *CrawlOptions {
+	return &CrawlOptions{
+		MaxDepthCrawled:    -1,
+		MaxLinksCrawled:    -1,
+		StayWithinBaseHost: false,
+		CrawlRatePerSec:    -1,
+		RespectRobots:      false,
+		IncludeBody:        true,
+		OpAdapter:          nil,
+	}
+}

octopus/models.go

@@ -2,7 +2,7 @@ package octopus
 
 import (
 	"io"
-	"time"
+	"sync"
 )
 
 // Node is used to represent each crawled link and its associated depth of crawl.
@@ -17,8 +17,9 @@ type Node struct {
 // It also has a CrawlOptions structure to initialize setting specific
 // to an instance of the crawler.
 type octopus struct {
-	CrawlOptions
-	visited map[Node]bool
+	*CrawlOptions
+	visited *sync.Map
+	isBuilt bool
 }
 
 // CrawlOptions is used to house options for crawling.
@@ -33,18 +34,17 @@ type octopus struct {
 // CrawlRate is the rate at which requests will be made.
 // RespectRobots (unimplemented) choose whether to respect robots.txt or not.
 type CrawlOptions struct {
-	DepthPerLink       int16
+	MaxDepthCrawled    int64
 	MaxLinksCrawled    int64
 	StayWithinBaseHost bool
-	BaseURLString      string
-	CrawlRate          time.Duration
+	CrawlRatePerSec    int64
 	RespectRobots      bool
 	IncludeBody        bool
-	OpAdapter          OutputAdapter
+	OpAdapter          *OutputAdapter
 }
 
 type CrawlOutput struct {
-	Node
+	*Node
 	Body io.ReadCloser
 }
 
@@ -56,5 +56,5 @@ type CrawlOutput struct {
 // Implementers of the interface should listen on this channel for output from
 // the crawler.
 type OutputAdapter interface {
-	Consume(quitCh <-chan bool) chan<- CrawlOutput
+	Consume(quitCh <-chan bool) chan<- *CrawlOutput
 }

octopus/pipe_augment_linkabsolution.go

@@ -0,0 +1 @@
+package octopus

octopus/pipe_composition.go

@@ -0,0 +1 @@
+package octopus

octopus/pipe_filter_duplication.go

@@ -0,0 +1 @@
+package octopus

octopus/pipe_filter_urlvalidation.go

@@ -0,0 +1 @@
+package octopus

octopus/pipe_htmlparsing.go

@@ -0,0 +1 @@
+package octopus

octopus/pipe_pagerequisition.go

@@ -0,0 +1 @@
+package octopus