Skip to content

Commit 75ddb84

Browse files
Kos-MKos-M
committed
refactor: core - switch from basic arrays to linked lists to improve performance
1 parent 3f5e0db commit 75ddb84

File tree

5 files changed

+224
-30
lines changed

5 files changed

+224
-30
lines changed

‎__tests__/index.spec.js‎

Lines changed: 16 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,10 @@ describe('pushNewJob method', () => {
2424

2525
expect(encryptedPayload).toHaveProperty('iv');
2626
expect(encryptedPayload).toHaveProperty('encryptedData');
27-
expect(master.jobs).toHaveLength(1);
28-
expect(JSON.parse(master.jobs[0])).toEqual(encryptedPayload);
27+
expect(master.jobs.size).toEqual(1);
28+
expect(JSON.parse(master.jobs.dequeue().value)).toEqual(
29+
encryptedPayload
30+
);
2931
});
3032
it('Push job succesfully and enrcypted , if job is array', async () => {
3133
const encryptedPayload = master.crypt.encrypt(
@@ -35,8 +37,10 @@ describe('pushNewJob method', () => {
3537

3638
expect(encryptedPayload).toHaveProperty('iv');
3739
expect(encryptedPayload).toHaveProperty('encryptedData');
38-
expect(master.jobs).toHaveLength(1);
39-
expect(JSON.parse(master.jobs[0])).toEqual(encryptedPayload);
40+
expect(master.jobs.size).toEqual(1);
41+
expect(JSON.parse(master.jobs.dequeue().value)).toEqual(
42+
encryptedPayload
43+
);
4044
});
4145
it('Push job succesfully and enrcypted , if job is string', async () => {
4246
const jobData = '50';
@@ -45,8 +49,10 @@ describe('pushNewJob method', () => {
4549

4650
expect(encryptedPayload).toHaveProperty('iv');
4751
expect(encryptedPayload).toHaveProperty('encryptedData');
48-
expect(master.jobs).toHaveLength(1);
49-
expect(JSON.parse(master.jobs[0])).toEqual(encryptedPayload);
52+
expect(master.jobs.size).toEqual(1);
53+
expect(JSON.parse(master.jobs.dequeue().value)).toEqual(
54+
encryptedPayload
55+
);
5056
});
5157
it('Push job succesfully and enrcypted , if job is number', async () => {
5258
const jobData = 50;
@@ -55,7 +61,9 @@ describe('pushNewJob method', () => {
5561

5662
expect(encryptedPayload).toHaveProperty('iv');
5763
expect(encryptedPayload).toHaveProperty('encryptedData');
58-
expect(master.jobs).toHaveLength(1);
59-
expect(JSON.parse(master.jobs[0])).toEqual(encryptedPayload);
64+
expect(master.jobs.size).toEqual(1);
65+
expect(JSON.parse(master.jobs.dequeue().value)).toEqual(
66+
encryptedPayload
67+
);
6068
});
6169
});

‎src/Helper.js‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,12 @@ class Helper {
3131
}
3232
});
3333
}
34+
35+
static sleep(ms) {
36+
return new Promise((resolve) => {
37+
setTimeout(resolve, ms);
38+
});
39+
}
3440
}
3541

3642
module.exports = {

‎src/Node.js‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
class Node {
2+
constructor(value) {
3+
this.value = value;
4+
this.next = null;
5+
this.prev = null;
6+
}
7+
}
8+
9+
module.exports = Node;

‎src/Queue.js‎

Lines changed: 156 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,156 @@
1+
const Node = require('./Node');
2+
3+
class Queue {
4+
constructor() {
5+
this.first = null;
6+
this.last = null;
7+
this.size = 0;
8+
}
9+
10+
isEmpty() {
11+
return !this.size;
12+
}
13+
14+
enqueue(item) {
15+
// Create node
16+
const newNode = new Node(item);
17+
/**
18+
* * If our list is empty than both our
19+
* * first item and last item is going to point the new node.
20+
*/
21+
if (this.isEmpty()) {
22+
this.first = newNode;
23+
this.last = newNode;
24+
newNode.prev = null;
25+
} else {
26+
newNode.prev = this.last;
27+
28+
this.last.next = newNode;
29+
this.last = newNode;
30+
}
31+
this.size += 1;
32+
return this;
33+
}
34+
/**
35+
*
36+
* @returns
37+
*/
38+
39+
dequeue() {
40+
//* if our queue is empty we return null
41+
if (this.isEmpty()) return null;
42+
const itemToBeRemoved = this.first;
43+
/**
44+
* * if both our first and last node are pointing the same item
45+
* * we dequeued our last node.
46+
*/
47+
if (this.first === this.last) {
48+
this.last = null;
49+
}
50+
this.first = this.first.next;
51+
this.size -= 1;
52+
return itemToBeRemoved;
53+
}
54+
55+
check() {
56+
let current = this.first;
57+
while (current) {
58+
// while not null
59+
current = current.next;
60+
}
61+
}
62+
63+
/**
64+
* Searches inside list in linear fashion ,
65+
* Time complexity O(n)
66+
* @param {*} item String , Number , Object
67+
* @returns Node , null
68+
*/
69+
search(item) {
70+
let current = this.first;
71+
if (typeof item !== typeof current.value) {
72+
return null;
73+
}
74+
while (current !== null) {
75+
if (current.value === item) {
76+
return current;
77+
}
78+
if (typeof item === 'object') {
79+
const itemKeys = Object.keys(item);
80+
for (let i = 0; i < itemKeys.length; i += 1) {
81+
// check all item properties agains current.value properties
82+
if (
83+
Object.prototype.hasOwnProperty.call(
84+
current.value,
85+
itemKeys[i]
86+
)
87+
) {
88+
if (current.value[itemKeys[i]] === item[itemKeys[i]]) {
89+
return current;
90+
}
91+
}
92+
}
93+
}
94+
current = current.next;
95+
}
96+
return null;
97+
}
98+
99+
bubbleSort() {
100+
let { last } = this;
101+
while (last) {
102+
let node = this.first;
103+
while (node !== last) {
104+
const { next } = node;
105+
if (typeof node.value === 'object') {
106+
// object comparision in first matched (common) property
107+
const itemKeys = Object.keys(node.value);
108+
for (let i = 0; i < itemKeys.length; i += 1) {
109+
if (
110+
Object.prototype.hasOwnProperty.call(
111+
next.value,
112+
itemKeys[i]
113+
)
114+
) {
115+
if (
116+
node.value[itemKeys[i]] >
117+
next.value[itemKeys[i]]
118+
) {
119+
// swap
120+
[node.value, next.value] = [
121+
next.value,
122+
node.value,
123+
];
124+
break;
125+
}
126+
}
127+
}
128+
} else if (typeof node.value === 'number') {
129+
if (node.value > next.value) {
130+
// swap
131+
[node.value, next.value] = [next.value, node.value];
132+
}
133+
} else if (typeof node.value === 'string') {
134+
// Not Implemented yet
135+
}
136+
137+
node = next;
138+
}
139+
last = last.prev; // shorten the range that must be bubbled through
140+
}
141+
}
142+
143+
/**
144+
* * Returns the next element to be dequeued.
145+
* @returns
146+
*/
147+
peek() {
148+
return this.first;
149+
}
150+
151+
tail() {
152+
return this.last;
153+
}
154+
}
155+
156+
module.exports = Queue;

‎src/index.js‎

Lines changed: 37 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ const fs = require('fs');
55
const { Logger } = require('./Logger');
66
const { Helper } = require('./Helper');
77
const Crypt = require('./Crypt');
8+
const Queue = require('./Queue');
89

910
const defaultExecAssets = [
1011
{
@@ -26,7 +27,7 @@ class Master {
2627
transferEncryptToken = null,
2728
} = {}) {
2829
this.availableWorkers = [];
29-
this.jobs = [];
30+
this.jobs = new Queue();
3031

3132
this.event = new events.EventEmitter();
3233
this.log = new Logger({ level: loglevel || process.env.LOG_LEVEL });
@@ -53,6 +54,7 @@ class Master {
5354
// this.peer.on('left', (address)=> );
5455

5556
this.event.addListener('init', this.init);
57+
this.event.addListener('resultsShared', this.onResults);
5658
this.event.emit('init', transferEncryptToken);
5759
}
5860

@@ -102,33 +104,47 @@ class Master {
102104
});
103105
this.peer.register('requestWork', (pk, args, cb) => {
104106
switch (true) {
105-
case this.jobs.length > 1: {
106-
if (args.getBatch && this.jobs.length > args.batchSize) {
107-
args.batchTasks = this.jobs.splice(0, args.batchSize); // splice mutates original array which slices it
107+
case this.jobs.size > 1: {
108+
if (args.getBatch) {
109+
args.batchTasks = [];
110+
let totalJobsSend = args.batchSize;
111+
if (this.jobs.size <= args.batchSize) totalJobsSend = 1; // get only available
112+
for (let i = 0; i < totalJobsSend; i += 1) {
113+
const queuedJob = this.jobs.dequeue();
114+
if (!queuedJob) break;
115+
args.batchTasks.push(queuedJob.value);
116+
}
117+
// args.batchTasks = this.jobs.splice(0, args.batchSize); // splice mutates original array which slices it
108118
this.log.debug(
109-
`task queue reduced:${this.jobs.length} - ${args.batchSize}`
119+
`task queue reduced:${this.jobs.size} - ${args.batchSize}`
110120
);
111121
break;
112122
}
113-
args.task = this.jobs.shift();
114-
this.log.debug(`task queue reduced:${this.jobs.length}`);
123+
const queuedJob = this.jobs.dequeue();
124+
args.task = null;
125+
if (queuedJob) {
126+
args.task = queuedJob.value;
127+
this.log.debug(`task queue reduced:${this.jobs.size}`);
128+
}
115129
break;
116130
}
117-
case this.jobs.length === 1: {
118-
// shift leaves array undefined on last element
119-
[args.task] = this.jobs; // this case is to avoid that
120-
this.jobs = [];
121-
this.log.debug(`task queue finished:${this.jobs.length}`);
131+
case this.jobs.size === 1: {
132+
const queuedJob = this.jobs.dequeue();
133+
args.task = null;
134+
if (queuedJob) {
135+
args.task = queuedJob.value;
136+
this.log.debug(`task queue reduced:${this.jobs.size}`);
137+
}
122138
break;
123139
}
124-
case this.jobs.length === 0: {
140+
case this.jobs.size === 0: {
125141
args.task = null;
126-
this.log.debug(`task queue is empty:${this.jobs.length}`);
142+
this.log.debug(`task queue is empty:${this.jobs.size}`);
127143
break;
128144
}
129145
default: {
130146
args.task = null;
131-
this.log.warning(`task queue is empty:${this.jobs.length}`);
147+
this.log.warning(`task queue is empty:${this.jobs.size}`);
132148

133149
break;
134150
}
@@ -137,7 +153,7 @@ class Master {
137153
});
138154
this.peer.register('shareResults', (pk, args) => {
139155
const results = JSON.parse(this.crypt.decrypt(JSON.parse(args)));
140-
this.onResults(results);
156+
this.event.emit('resultsShared', results);
141157
});
142158
this.peer.register('requestExecAssets', (pk, args, cb) => {
143159
const currentHash = args?.currentHash; // hash of current assets array
@@ -179,14 +195,13 @@ class Master {
179195

180196
const encryptedPayload = this.crypt.encrypt(payloadJson);
181197
this.log.debug('pushNewJob payload: ', payload);
182-
if (this.jobs.length > 20) {
183-
// resolve();
184-
setTimeout(() => {
185-
this.jobs.push(JSON.stringify(encryptedPayload));
198+
if (this.jobs.size >= 1000) {
199+
Helper.sleep(this.jobs.size * 0.3).then(() => {
200+
this.jobs.enqueue(JSON.stringify(encryptedPayload));
186201
resolve();
187-
}, this.jobs.length * 3);
202+
});
188203
} else {
189-
this.jobs.push(JSON.stringify(encryptedPayload));
204+
this.jobs.enqueue(JSON.stringify(encryptedPayload));
190205
resolve();
191206
}
192207
});

0 commit comments

Comments
 (0)