Skip to content

Commit 49146d0

Browse files
bugerbuger
authored
Merge pull request #216 from floren/eachkey-fix
Eachkey fix
2 parents cb835d4 + aa3d476 commit 49146d0

File tree

3 files changed

+156
-22
lines changed

3 files changed

+156
-22
lines changed

‎go.mod‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
11
module github.com/buger/jsonparser
22

33
go 1.13
4+

‎parser.go‎

Lines changed: 15 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@ import (
44
"bytes"
55
"errors"
66
"fmt"
7-
"math"
87
"strconv"
98
)
109

@@ -356,14 +355,6 @@ func searchKeys(data []byte, keys ...string) int {
356355
return -1
357356
}
358357

359-
var bitwiseFlags []int64
360-
361-
func init() {
362-
for i := 0; i < 63; i++ {
363-
bitwiseFlags = append(bitwiseFlags, int64(math.Pow(2, float64(i))))
364-
}
365-
}
366-
367358
func sameTree(p1, p2 []string) bool {
368359
minLen := len(p1)
369360
if len(p2) < minLen {
@@ -380,7 +371,8 @@ func sameTree(p1, p2 []string) bool {
380371
}
381372

382373
func EachKey(data []byte, cb func(int, []byte, ValueType, error), paths ...[]string) int {
383-
var pathFlags int64
374+
var x struct{}
375+
pathFlags := make([]bool, len(paths))
384376
var level, pathsMatched, i int
385377
ln := len(data)
386378

@@ -439,15 +431,15 @@ func EachKey(data []byte, cb func(int, []byte, ValueType, error), paths ...[]str
439431

440432
pathsBuf[level-1] = bytesToString(&keyUnesc)
441433
for pi, p := range paths {
442-
if len(p) != level || pathFlags&bitwiseFlags[pi+1] != 0 || !equalStr(&keyUnesc, p[level-1]) || !sameTree(p, pathsBuf[:level]) {
434+
if len(p) != level || pathFlags[pi] || !equalStr(&keyUnesc, p[level-1]) || !sameTree(p, pathsBuf[:level]) {
443435
continue
444436
}
445437

446438
match = pi
447439

448440
i++
449441
pathsMatched++
450-
pathFlags |= bitwiseFlags[pi+1]
442+
pathFlags[pi] = true
451443

452444
v, dt, _, e := Get(data[i:])
453445
cb(pi, v, dt, e)
@@ -485,40 +477,41 @@ func EachKey(data []byte, cb func(int, []byte, ValueType, error), paths ...[]str
485477
case '}':
486478
level--
487479
case '[':
488-
var arrIdxFlags int64
489-
var pIdxFlags int64
480+
var ok bool
481+
arrIdxFlags := make(map[int]struct{})
482+
pIdxFlags := make([]bool, len(paths))
490483

491484
if level < 0 {
492485
cb(-1, nil, Unknown, MalformedJsonError)
493486
return -1
494487
}
495488

496489
for pi, p := range paths {
497-
if len(p) < level+1 || pathFlags&bitwiseFlags[pi+1] != 0 || p[level][0] != '[' || !sameTree(p, pathsBuf[:level]) {
490+
if len(p) < level+1 || pathFlags[pi] || p[level][0] != '[' || !sameTree(p, pathsBuf[:level]) {
498491
continue
499492
}
500493
if len(p[level]) >= 2 {
501494
aIdx, _ := strconv.Atoi(p[level][1 : len(p[level])-1])
502-
arrIdxFlags |= bitwiseFlags[aIdx+1]
503-
pIdxFlags |= bitwiseFlags[pi+1]
495+
arrIdxFlags[aIdx] = x
496+
pIdxFlags[pi] = true
504497
}
505498
}
506499

507-
if arrIdxFlags > 0 {
500+
if len(arrIdxFlags) > 0 {
508501
level++
509502

510503
var curIdx int
511504
arrOff, _ := ArrayEach(data[i:], func(value []byte, dataType ValueType, offset int, err error) {
512-
if arrIdxFlags&bitwiseFlags[curIdx+1] != 0 {
505+
if _, ok = arrIdxFlags[curIdx]; ok {
513506
for pi, p := range paths {
514-
if pIdxFlags&bitwiseFlags[pi+1] != 0 {
507+
if pIdxFlags[pi] {
515508
aIdx, _ := strconv.Atoi(p[level-1][1 : len(p[level-1])-1])
516509

517510
if curIdx == aIdx {
518511
of := searchKeys(value, p[level:]...)
519512

520513
pathsMatched++
521-
pathFlags |= bitwiseFlags[pi+1]
514+
pathFlags[pi] = true
522515

523516
if of != -1 {
524517
v, dt, _, e := Get(value[of:])
@@ -930,7 +923,7 @@ func ArrayEach(data []byte, cb func(value []byte, dataType ValueType, offset int
930923
return -1, MalformedJsonError
931924
}
932925

933-
offset = nT+1
926+
offset = nT + 1
934927

935928
if len(keys) > 0 {
936929
if offset = searchKeys(data, keys...); offset == -1 {

‎parser_error_test.go‎

Lines changed: 140 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package jsonparser
22

33
import (
44
"fmt"
5+
"strings"
56
"testing"
67
)
78

@@ -38,3 +39,142 @@ func TestPanickingErrors(t *testing.T) {
3839
t.Error("Expected error...")
3940
}
4041
}
42+
43+
// check having a very deep key depth
44+
func TestKeyDepth(t *testing.T) {
45+
var sb strings.Builder
46+
var keys []string
47+
//build data
48+
sb.WriteString("{")
49+
for i := 0; i < 128; i++ {
50+
fmt.Fprintf(&sb, `"key%d": %dx,`, i, i)
51+
keys = append(keys, fmt.Sprintf("key%d", i))
52+
}
53+
sb.WriteString("}")
54+
55+
data := []byte(sb.String())
56+
EachKey(data, func(offset int, value []byte, dt ValueType, err error) {
57+
return
58+
}, keys)
59+
}
60+
61+
// check having a bunch of keys in a call to EachKey
62+
func TestKeyCount(t *testing.T) {
63+
var sb strings.Builder
64+
var keys [][]string
65+
//build data
66+
sb.WriteString("{")
67+
for i := 0; i < 128; i++ {
68+
fmt.Fprintf(&sb, `"key%d":"%d"`, i, i)
69+
if i < 127 {
70+
sb.WriteString(",")
71+
}
72+
keys = append(keys, []string{fmt.Sprintf("key%d", i)})
73+
}
74+
sb.WriteString("}")
75+
76+
data := []byte(sb.String())
77+
EachKey(data, func(offset int, value []byte, dt ValueType, err error) {
78+
return
79+
}, keys...)
80+
}
81+
82+
// try pulling lots of keys out of a big array
83+
func TestKeyDepthArray(t *testing.T) {
84+
var sb strings.Builder
85+
var keys []string
86+
//build data
87+
sb.WriteString("[")
88+
for i := 0; i < 128; i++ {
89+
fmt.Fprintf(&sb, `{"key": %d},`, i)
90+
keys = append(keys, fmt.Sprintf("[%d].key", i))
91+
}
92+
sb.WriteString("]")
93+
94+
data := []byte(sb.String())
95+
EachKey(data, func(offset int, value []byte, dt ValueType, err error) {
96+
return
97+
}, keys)
98+
}
99+
100+
// check having a bunch of keys
101+
func TestKeyCountArray(t *testing.T) {
102+
var sb strings.Builder
103+
var keys [][]string
104+
//build data
105+
sb.WriteString("[")
106+
for i := 0; i < 128; i++ {
107+
fmt.Fprintf(&sb, `{"key":"%d"}`, i)
108+
if i < 127 {
109+
sb.WriteString(",")
110+
}
111+
keys = append(keys, []string{fmt.Sprintf("[%d].key", i)})
112+
}
113+
sb.WriteString("]")
114+
115+
data := []byte(sb.String())
116+
EachKey(data, func(offset int, value []byte, dt ValueType, err error) {
117+
return
118+
}, keys...)
119+
}
120+
121+
// check having a bunch of keys in a super deep array
122+
func TestEachKeyArray(t *testing.T) {
123+
var sb strings.Builder
124+
var keys [][]string
125+
//build data
126+
sb.WriteString(`[`)
127+
for i := 0; i < 127; i++ {
128+
fmt.Fprintf(&sb, `%d`, i)
129+
if i < 127 {
130+
sb.WriteString(",")
131+
}
132+
if i < 32 {
133+
keys = append(keys, []string{fmt.Sprintf("[%d]", 128+i)})
134+
}
135+
}
136+
sb.WriteString(`]`)
137+
138+
data := []byte(sb.String())
139+
EachKey(data, func(offset int, value []byte, dt ValueType, err error) {
140+
return
141+
}, keys...)
142+
}
143+
144+
func TestLargeArray(t *testing.T) {
145+
var sb strings.Builder
146+
//build data
147+
sb.WriteString(`[`)
148+
for i := 0; i < 127; i++ {
149+
fmt.Fprintf(&sb, `%d`, i)
150+
if i < 127 {
151+
sb.WriteString(",")
152+
}
153+
}
154+
sb.WriteString(`]`)
155+
keys := [][]string{[]string{`[1]`}}
156+
157+
data := []byte(sb.String())
158+
EachKey(data, func(offset int, value []byte, dt ValueType, err error) {
159+
return
160+
}, keys...)
161+
}
162+
163+
func TestArrayOutOfBounds(t *testing.T) {
164+
var sb strings.Builder
165+
//build data
166+
sb.WriteString(`[`)
167+
for i := 0; i < 61; i++ {
168+
fmt.Fprintf(&sb, `%d`, i)
169+
if i < 61 {
170+
sb.WriteString(",")
171+
}
172+
}
173+
sb.WriteString(`]`)
174+
keys := [][]string{[]string{`[128]`}}
175+
176+
data := []byte(sb.String())
177+
EachKey(data, func(offset int, value []byte, dt ValueType, err error) {
178+
return
179+
}, keys...)
180+
}

0 commit comments

Comments
 (0)