redesign octopus v1

Author: rapidclock

octopus/models.go

@@ -64,12 +64,7 @@ type NodeChSet struct {
 }
 
 type StringChSet struct {
-	strCh chan<- string
-	*StdChannels
-}
-
-type NodeInfoChSet struct {
-	nodeInfoCh chan<- *NodeInfo
+	StrCh chan<- string
 	*StdChannels
 }
 

octopus/pipe_htmlparsing.go

@@ -4,36 +4,11 @@ import (
 	"golang.org/x/net/html"
 )
 
-func (o *octopus) makeHtmlParsingPipe(outChSet *NodeInfoChSet) *NodeChSet {
-	listenCh := make(chan *Node)
-	listenQuitCh := make(chan int, 1)
-	listenChSet := &NodeChSet{
-		NodeCh: listenCh,
-		StdChannels: &StdChannels{
-			QuitCh: listenQuitCh,
-		},
-	}
-	go func() {
-		defer close(listenCh)
-		defer close(listenQuitCh)
-		for {
-			select {
-			case node := <-listenCh:
-				{
-					go parseHtmlPage(node, outChSet)
-				}
-			case <-listenQuitCh:
-				{
-					outChSet.QuitCh <- 1
-					return
-				}
-			}
-		}
-	}()
-	return listenChSet
+func (o *octopus) makeParseNodeFromHtmlPipe(outChSet *NodeChSet) *NodeChSet {
+	return stdLinearNodeFunc(parseHtmlPage, outChSet)
 }
 
-func parseHtmlPage(node *Node, outChSet *NodeInfoChSet) {
+func parseHtmlPage(node *Node, outChSet *NodeChSet) {
 	defer node.Body.Close()
 	z := html.NewTokenizer(node.Body)
 	for {
@@ -46,10 +21,13 @@ func parseHtmlPage(node *Node, outChSet *NodeInfoChSet) {
 			if "a" == token.Data {
 				for _, attr := range token.Attr {
 					if attr.Key == "href" {
-						outChSet.nodeInfoCh <- &NodeInfo{
-							ParentUrlString: node.UrlString,
-							UrlString:       attr.Val,
-							Depth:           node.Depth + 1,
+						outChSet.NodeCh <- &Node{
+							NodeInfo: &NodeInfo{
+								ParentUrlString: node.UrlString,
+								UrlString:       attr.Val,
+								Depth:           node.Depth + 1,
+							},
+							Body: nil,
 						}
 					}
 				}