Skip to content

Commit d7a6e80

Browse files
Kos-MKos-M
committed
chore: update example to use local version of Master
1 parent fccfa26 commit d7a6e80

File tree

8 files changed

+3528
-345
lines changed

8 files changed

+3528
-345
lines changed

‎README.md‎

Lines changed: 76 additions & 69 deletions
Original file line numberDiff line numberDiff line change
@@ -1,111 +1,118 @@
1-
2-
31
# Master
42

5-
Master connects to a peer2peer "room" data channel and shares with their peers (Workers)
6-
data about the jobs. Among them is job data( each job/task may have its own data to solve/work on) ,
7-
job code to executed in workers and code dependencies which workers have to install before stat processing tasks.
8-
In other words Master defines the problem , defines the way to solve it, provides all the tools and code needed and adds data (if) needed to solve the problem..
3+
Master connects to a peer2peer "room" data channel and shares with their peers (Workers) data about the jobs. Among them is job data( each job/task may have its own data to solve/work on) , job code to executed in workers and code dependencies which workers have to install before stat processing tasks. In other words Master defines the problem , defines the way to solve it, provides all the tools and code needed and adds data (if) needed to solve the problem..
94

10-
Master and Workers can find each other in any network condition , as long they are connected online.
11-
This is possible from nature of peer to peer networks and [bugout](https://github.com/chr15m/bugout) !
12-
Bugout offers message transfer encryption, but we encrypt all data i/o transfers on top of that , here as well.
5+
Master and Workers can find each other in any network condition , as long they are connected online. This is possible from nature of peer to peer networks and [bugout](https://github.com/chr15m/bugout) ! Bugout offers message transfer encryption, but we encrypt all data i/o transfers on top of that , here as well.
136

147
## Installation
158

169
```bash
1710
git clone https://github.com/queue-xec/master
1811
cd master
19-
yarn # or npm install
12+
yarn # or npm install
2013
```
2114

22-
```bash
15+
```bash
2316
node cli.js --setup
24-
```
17+
```
18+
2519
Will prompt user to enter following info:
26-
- `transferEncryptToken` Enter a 32 length alphanumeric string Or prompt to generate one on the fly
27-
- `token` Enter at least 20 length alphanumeric string Or prompt to generate one on the fly
20+
21+
- `transferEncryptToken` Enter a 32 length alphanumeric string Or prompt to generate one on the fly
22+
- `token` Enter at least 20 length alphanumeric string Or prompt to generate one on the fly
2823

2924
These info will saved and loaded in .env file at root Masters folder.Should be the same on all workers , to be able to communicate.
3025

3126
## How to set job code to run in workers
3227

33-
Edit [taskFile](https://github.com/queue-xec/master/blob/devel/src/task.js) inner `run()` method
34-
⚠️ Beware , this file should never modified in a remote Worker instance because will be overwritten by Master next time Worker instance initiated.
28+
Edit [taskFile](https://github.com/queue-xec/master/blob/devel/src/task.js) inner `run()` method ⚠️ Beware , this file should never modified in a remote Worker instance because will be overwritten by Master next time Worker instance initiated.
29+
3530
```javascript
3631
// Require here libs you passed from master as dependencies
3732
// if needed here.
3833
const { Big } = require('big.js');
3934
const Helper = require('./Helper.js');
40-
class Task { // task.js file will placed by default in WORKER_DIR /workplace/task.js
41-
constructor() {
42-
this.data = null;
43-
// this.helper = new Helper();
44-
}
45-
46-
/**
47-
* Description: This method is the job procces. Receives job data (job)
48-
* Must return job results (Object) , to delivered in master.
49-
* @param {Object} job data from Master about job {id: Number , data: String (needs JSON.parse)}
50-
* @returns {Object} result
51-
*/
52-
async run(job) {
53-
console.dir(job);
54-
const data = JSON.parse(job.data);
55-
// code to solve the problem , in remote Workers
56-
return {};
57-
}
35+
class Task {
36+
// task.js file will placed by default in WORKER_DIR /workplace/task.js
37+
constructor() {
38+
this.data = null;
39+
// this.helper = new Helper();
40+
}
41+
42+
/**
43+
* Description: This method is the job procces. Receives job data (job)
44+
* Must return job results (Object) , to delivered in master.
45+
* @param {Object} job data from Master about job {id: Number , data: String (needs JSON.parse)}
46+
* @returns {Object} result
47+
*/
48+
async run(job) {
49+
console.dir(job);
50+
const data = JSON.parse(job.data);
51+
// code to solve the problem , in remote Workers
52+
return {};
53+
}
5854
}
5955

6056
module.exports = Task;
61-
6257
```
6358

64-
6559
## Example:
60+
6661
```js
67-
const Master = require('queue-xec-master')
68-
function resultCollect(result){ // handle here incoming results from workers..
69-
console.dir(result)
70-
}
71-
async function run(){
72-
const mm = new Master({
73-
onResults: resultCollect,
74-
execAssets: {
75-
dependencies: ['big.js' ,'moment'], // pass Worker dependencies
76-
files: [
77-
{ masterPath: '/src/task.js', name: 'task.js', workerPath: '/workplace/task.js' }, // if task.js file not passed to workers , Master will use default one located in /src/task.js , here you can ovverride it by changing above details
78-
{ masterPath: '/src/Logger.js', name: 'Logger.js', workerPath: '/workplace/Logger.js' },
79-
{ masterPath: '/src/Helper.js', name: 'Helper.js', workerPath: '/workplace/Helper.js' },
80-
],
81-
/* masterPath and workerPath are not absolute , NEVER access files out of their process.cwd() path.
62+
const Master = require('queue-xec-master');
63+
function resultCollect(result) {
64+
// handle here incoming results from workers..
65+
console.dir(result);
66+
}
67+
async function run() {
68+
const mm = new Master({
69+
onResults: resultCollect,
70+
execAssets: {
71+
dependencies: ['big.js', 'moment'], // pass Worker dependencies
72+
files: [
73+
{
74+
masterPath: '/src/task.js',
75+
name: 'task.js',
76+
workerPath: '/workplace/task.js',
77+
}, // if task.js file not passed to workers , Master will use default one located in /src/task.js , here you can ovverride it by changing above details
78+
{
79+
masterPath: '/src/Logger.js',
80+
name: 'Logger.js',
81+
workerPath: '/workplace/Logger.js',
82+
},
83+
{
84+
masterPath: '/src/Helper.js',
85+
name: 'Helper.js',
86+
workerPath: '/workplace/Helper.js',
87+
},
88+
],
89+
/* masterPath and workerPath are not absolute , NEVER access files out of their process.cwd() path.
8290
Are relative to their process current location , respectively */
83-
},
84-
});
85-
const dummy = {
86-
x: 1 ,
87-
y: 2
88-
};
89-
90-
let cnt = 0;
91-
for (let i = 0; i < 5; i++) {
92-
const payload = {
93-
id: cnt,
94-
data: JSON.stringify(dummy),
95-
91+
},
92+
});
93+
const dummy = {
94+
x: 1,
95+
y: 2,
9696
};
97-
await mm.pushNewJob(payload).catch((e) => console.log(e));
98-
cnt += 1;
99-
}
97+
98+
let cnt = 0;
99+
for (let i = 0; i < 5; i++) {
100+
const payload = {
101+
id: cnt,
102+
data: JSON.stringify(dummy),
103+
};
104+
await mm.pushNewJob(payload).catch((e) => console.log(e));
105+
cnt += 1;
106+
}
100107
}
101108
```
102109

103110
### ⚠️ Under development ⚠️
104111

105112
[![MIT License](https://img.shields.io/apm/l/atomic-design-ui.svg?)](https://github.com/queue-xec/master/blob/devel/LICENSE)
106113

107-
108114
## > Contributors <
115+
109116
<a href="https://github.com/queue-xec/master/graphs/contributors">
110117
<img src="https://contrib.rocks/image?repo=queue-xec/master" />
111-
</a>
118+
</a>

‎example/index.js‎

Lines changed: 44 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -1,30 +1,48 @@
1-
const Master = require('queue-xec-master')
1+
/* eslint-disable no-console */
2+
/* eslint-disable import/no-unresolved */
3+
const Master = require('../index');
4+
// const Master = require('queue-xec-master')
25

3-
async function collectResults(response){
4-
console.dir(response)
6+
// set here only for example , transferEncryptToken
7+
process.env.transferEncryptToken = '00000000000000000000000000000000';
8+
process.env.token = 'demoCHannel0';
9+
process.env.LOG_LEVEL = 'debug';
10+
11+
function resultCollect(result) {
12+
// handle here incoming results from workers..
13+
console.dir(result);
514
}
6-
function run(){
7-
const options = {
8-
ip: '127.0.0.1',
9-
port: 8080,
10-
token: 'secretsecretsecretsecretsecrettt',
11-
onResult: collectResults
15+
async function run() {
16+
const mm = new Master({
17+
onResults: resultCollect,
18+
execAssets: {
19+
dependencies: ['big.js', 'moment'], // pass Worker dependencies
20+
files: [
21+
// { masterPath: '/../src/task.js', name: 'task.js', workerPath: '/workplace/task.js' }, // if task.js file not passed to workers , Master will use default one located in /src/task.js , here you can ovverride it by changing above details
22+
{
23+
masterPath: '/task.js',
24+
name: 'task.js',
25+
workerPath: '/workplace/task.js',
26+
}, // if task.js file not passed to workers , Master will use default one located in /src/task.js , here you can ovverride it by changing above details
27+
],
28+
/* masterPath and workerPath are not absolute , NEVER access files out of their process.cwd() path.
29+
Are relative to their process current location , respectively */
30+
},
31+
});
32+
const dummy = {
33+
x: 1,
34+
y: 2,
35+
};
36+
37+
let cnt = 0;
38+
for (let i = 0; i < 500; i += 1) {
39+
const payload = {
40+
id: cnt,
41+
data: JSON.stringify(dummy),
42+
};
43+
// eslint-disable-next-line no-await-in-loop
44+
await mm.pushNewJob(payload).catch((e) => console.log(e));
45+
cnt += 1;
1246
}
13-
const mm = new Master(options)
14-
const dummy = {
15-
test: 'data'
16-
}
17-
setInterval( ()=>{
18-
console.dir(mm.getQueueLength())
19-
var payload = {
20-
data: dummy,
21-
exec:{
22-
file: Master.getBase64('./task.js') ,
23-
name:"exec.js" ,
24-
dependencies: ['big.js', 'technicalindicators' ]} // npm packages to installed at runtime of online worker
25-
};
26-
mm.pushNewJob(JSON.stringify(payload))
27-
},500)
2847
}
29-
30-
run()
48+
run();

0 commit comments

Comments
 (0)