Skip to content

Commit 8c515cf

Browse files
author
mochatek
committed
feat(JsonArrayTransformer): Implement JSOn stream transformer
1 parent cf6beab commit 8c515cf

File tree

2 files changed

+188
-1
lines changed

2 files changed

+188
-1
lines changed

‎src/index.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import JsonArrayStreamer from "./modules/JsonArrayStreamer";
2+
import Transform from "./modules/JsonArrayTransformer";
23
import appendFile from "./modules/JsonArrayAppend";
34

45
const createReadStream = JsonArrayStreamer.create;
56

6-
export { createReadStream, appendFile };
7+
export { createReadStream, appendFile, Transform };

‎src/modules/JsonArrayTransformer.ts‎

Lines changed: 186 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,186 @@
1+
import { Transform, TransformCallback } from "stream";
2+
import { CHARACTER } from "../constants";
3+
4+
class JsonArrayStreamer<T> extends Transform {
5+
private encoding: BufferEncoding;
6+
private rootDetected: boolean;
7+
private elementDetected: boolean;
8+
private elementType: "string" | "array" | "object" | "others";
9+
private elementParser: ((char: string) => void) | null;
10+
private elementEnclosureCount: number;
11+
private isCharInsideQuotes: boolean;
12+
private isCharEscaped: boolean;
13+
private buffer: string;
14+
private parsedElements: T[];
15+
16+
constructor(sourceEncoding?: BufferEncoding) {
17+
super({
18+
readableObjectMode: true,
19+
writableObjectMode: false,
20+
});
21+
if (!sourceEncoding) {
22+
console.warn(
23+
"Warning: Encoding not specified. Defaulting to UTF-8 to prevent issues."
24+
);
25+
}
26+
this.encoding = sourceEncoding || "utf-8";
27+
this.rootDetected = false;
28+
this.elementDetected = false;
29+
this.elementType = "others";
30+
this.elementParser = null;
31+
this.elementEnclosureCount = 0;
32+
this.isCharInsideQuotes = false;
33+
this.isCharEscaped = false;
34+
this.buffer = "";
35+
this.parsedElements = [];
36+
}
37+
38+
private resetParser() {
39+
this.elementDetected = false;
40+
this.elementType = "others";
41+
this.elementParser = null;
42+
this.elementEnclosureCount = 0;
43+
this.isCharInsideQuotes = false;
44+
this.isCharEscaped = false;
45+
this.buffer = "";
46+
}
47+
48+
private parseElement<T>() {
49+
try {
50+
const element: T = JSON.parse(this.buffer);
51+
return element;
52+
} catch (error) {
53+
throw new Error(`Invalid element: ${this.buffer}`);
54+
}
55+
}
56+
57+
private parseStringElement(char: string) {
58+
this.buffer = `${this.buffer}${char}`;
59+
60+
if (char === CHARACTER.QUOTE) {
61+
if (this.isCharInsideQuotes && !this.isCharEscaped) {
62+
const element = this.parseElement<T>();
63+
this.parsedElements.push(element);
64+
this.resetParser();
65+
} else if (this.buffer === CHARACTER.QUOTE) {
66+
this.isCharInsideQuotes = true;
67+
} else if (this.isCharEscaped) {
68+
this.isCharEscaped = false;
69+
}
70+
} else if (char === CHARACTER.ESCAPE) {
71+
this.isCharEscaped = !this.isCharEscaped;
72+
} else if (this.isCharEscaped) {
73+
this.isCharEscaped = false;
74+
}
75+
}
76+
77+
private parsePrimitiveElement(char: string) {
78+
if ([CHARACTER.COMMA, CHARACTER.NEW_LINE, CHARACTER.SPACE].includes(char)) {
79+
const element = this.parseElement<T>();
80+
this.parsedElements.push(element);
81+
this.resetParser();
82+
} else {
83+
this.buffer = `${this.buffer}${char}`;
84+
}
85+
}
86+
87+
private parseContainerElement(char: string) {
88+
const ENCLOSURE =
89+
this.elementType === "array" ? CHARACTER.BRACKET : CHARACTER.BRACE;
90+
91+
if (char === ENCLOSURE.OPEN && !this.isCharInsideQuotes) {
92+
this.buffer = `${this.buffer}${char}`;
93+
this.elementEnclosureCount += 1;
94+
} else if (char === ENCLOSURE.CLOSE && !this.isCharInsideQuotes) {
95+
this.buffer = `${this.buffer}${char}`;
96+
this.elementEnclosureCount -= 1;
97+
98+
if (this.elementEnclosureCount === 0) {
99+
const element = this.parseElement<T>();
100+
this.parsedElements.push(element);
101+
this.resetParser();
102+
}
103+
} else if (this.buffer.length) {
104+
this.buffer = `${this.buffer}${char}`;
105+
106+
if (char === CHARACTER.ESCAPE) {
107+
this.isCharEscaped = !this.isCharEscaped;
108+
} else if (char === CHARACTER.QUOTE && !this.isCharEscaped) {
109+
this.isCharInsideQuotes = !this.isCharInsideQuotes;
110+
} else if (this.isCharEscaped) {
111+
this.isCharEscaped = false;
112+
}
113+
}
114+
}
115+
116+
_transform(
117+
chunk: Buffer,
118+
encoding: BufferEncoding,
119+
callback: TransformCallback
120+
) {
121+
try {
122+
const decodedString = Buffer.from(
123+
chunk as unknown as string,
124+
this.encoding
125+
).toString("utf-8");
126+
for (let char of decodedString) {
127+
if (!this.rootDetected) {
128+
if (
129+
![
130+
CHARACTER.SPACE,
131+
CHARACTER.NEW_LINE,
132+
CHARACTER.BRACKET.OPEN,
133+
].includes(char)
134+
) {
135+
throw new Error("Invalid file structure");
136+
}
137+
138+
this.rootDetected = char === CHARACTER.BRACKET.OPEN;
139+
continue;
140+
}
141+
142+
if (!this.elementDetected) {
143+
if (char === CHARACTER.BRACKET.CLOSE) break;
144+
this.elementDetected = ![
145+
CHARACTER.SPACE,
146+
CHARACTER.COMMA,
147+
CHARACTER.NEW_LINE,
148+
].includes(char);
149+
}
150+
151+
if (this.elementDetected) {
152+
if (!this.elementParser) {
153+
if (char === CHARACTER.BRACKET.OPEN) {
154+
this.elementType = "array";
155+
this.elementParser = this.parseContainerElement.bind(this);
156+
} else if (char === CHARACTER.BRACE.OPEN) {
157+
this.elementType = "object";
158+
this.elementParser = this.parseContainerElement.bind(this);
159+
} else if (char === CHARACTER.QUOTE) {
160+
this.elementType = "string";
161+
this.elementParser = this.parseStringElement.bind(this);
162+
} else {
163+
this.elementType = "others";
164+
this.elementParser = this.parsePrimitiveElement.bind(this);
165+
}
166+
}
167+
168+
this.elementParser(char);
169+
}
170+
}
171+
172+
if (this.parsedElements.length) {
173+
callback(
174+
null,
175+
this.parsedElements.splice(0, this.parsedElements.length)
176+
);
177+
} else {
178+
callback();
179+
}
180+
} catch (error) {
181+
callback(error as Error);
182+
}
183+
}
184+
}
185+
186+
export default JsonArrayStreamer;

0 commit comments

Comments
 (0)