Documented the Tokenizer

Author: udan-jayanith

FUTURE-CHANGELOG.md

@@ -3,8 +3,5 @@
 - Serializer bug fix
 
 ## TODO
-* Document Combinator and CombinatorEl struct.
-* Document TokenizeSelectorsAndCombinators and show a example for it.
-* Document Tokenizers and Selectors
 * Update the the readme example and change the CHANGELOG
 

parser.go

@@ -20,7 +20,6 @@ func Decode(r io.Reader) (*Node, error) {
 		nodeTreeBuilder.WriteNodeTree(t.GetCurrentNode(), tt)
 	}
 	return nodeTreeBuilder.GetRootNode(), nil
-
 }
 
 // HTMLToNodeTree return html code as a node-tree. If error were to occur it would be SyntaxError.

selectors.go

@@ -13,6 +13,8 @@ const (
 	Tag
 )
 
+//Selector struct represents a single css selector
+//Ex: .my-class, #video, div
 type Selector struct {
 	selector     string
 	selectorName string
@@ -40,6 +42,8 @@ func matchNode(node *Node, basicSelectorName string, basicSelectorType BasicSele
 	return false
 }
 
+//NewSelector takes a single css selector and returns a Selector struct.
+//Selector string should be only of basic selector.
 func NewSelector(selector string) Selector {
 	selector = strings.TrimSpace(html.EscapeString(selector))
 	selectorStruct := Selector{}
@@ -76,12 +80,14 @@ const (
 	NoneCombinator
 )
 
+//CombinatorEl is used to represent selectors that are around a combinator.
 type CombinatorEl struct {
 	Type      Combinator
 	Selector1 Selector
 	Selector2 Selector
 }
 
+//This takes a selector or combinators and selectors and then returns a slice of CombinatorEl.
 func TokenizeSelectorsAndCombinators(selector string) []CombinatorEl {
 	iter := func(yield func(string) bool) {
 		currentStr := ""

tokenizer.go

@@ -71,7 +71,7 @@ func NewNodeTreeBuilder() NodeTreeBuilder {
 	}
 }
 
-// WriteNodeTree append the node given html.TokenType
+// WriteNodeTree append the node given html.TokenType.
 func (ntb *NodeTreeBuilder) WriteNodeTree(node *Node, tt html.TokenType) {
 	switch tt {
 	case html.EndTagToken:

tokenizer_test.go

@@ -0,0 +1,36 @@
+package GoHtml_test
+
+import (
+	"fmt"
+	"net/http"
+
+	GoHtml "github.com/udan-jayanith/GoHTML"
+	"golang.org/x/net/html"
+)
+
+func ExampleTokenizer() {
+	//Request the html
+	res, err := http.Get("https://go.dev/")
+	if err != nil || res.StatusCode != http.StatusOK {
+		return
+	}
+	defer res.Body.Close()
+
+	//NewTokenizer takes a io.reader that receives UTF-8 encoded html code and returns a Tokenizer.
+	t := GoHtml.NewTokenizer(res.Body)
+	//NewNodeTreeBuilder return a new NodeTreeBuilder that can be used to build a node tree.
+	nodeTreeBuilder := GoHtml.NewNodeTreeBuilder()
+	for {
+		//Advanced scans the next token and returns its type.
+		tt := t.Advanced()
+		if tt == html.ErrorToken {
+			break
+		}
+
+		//WriteNodeTree takes a node and a token type. The node can be nil so if token type is EndTagToken.
+		nodeTreeBuilder.WriteNodeTree(t.GetCurrentNode(), tt)
+	}
+
+	//Prints the root node of the node tree in the nodeTreeBuilder.
+	fmt.Println(nodeTreeBuilder.GetRootNode())
+}

traverser_test.go

@@ -36,7 +36,6 @@ func TestWalkthrough(t *testing.T) {
 			t.Fatal("Expected ", testList[i], "but got ", resList[i], "in index ", i)
 		}
 	}
-	t.Log(GoHtml.NodeTreeToHTML(body))
 }
 
 func ExampleTraverser_Walkthrough() {