Skip to content

Commit 365c247

Browse files
JackHaegJackHaeg
authored
Merge pull request #2003 from trillium/trilliumsmith/prod-project-clone
feat: Create clear and sync scripts for prod->dev cloning
2 parents 254529a + af5274c commit 365c247

File tree

3 files changed

+332
-1
lines changed

3 files changed

+332
-1
lines changed

‎backend/package.json‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,9 @@
1111
"start": "node server.js",
1212
"dev": "nodemon server.js",
1313
"client": "npm run start --prefix client",
14-
"heroku-postbuild": "cd client && npm install && npm run build"
14+
"heroku-postbuild": "cd client && npm install && npm run build",
15+
"clone-or-sync-projects": "node ./scripts/cloneOrSyncCollections.js",
16+
"clear-dev-collections": "node ./scripts/clearDevCollections.js"
1517
},
1618
"author": "sarL3y",
1719
"license": "ISC",

‎backend/scripts/clearDevCollections.js‎

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
/**
2+
* Script to clear the 'projects' and/or 'recurringevents' collections from the DEV MongoDB database.
3+
*
4+
* Usage:
5+
* node clearDevCollections.js [--projects] [--recurring-events]
6+
*
7+
* Arguments:
8+
* --projects Only clear the 'projects' collection
9+
* --recurring-events Only clear the 'recurringevents' collection
10+
* --events Only clear the 'events' collection
11+
* --checkins Only clear the 'checkins' collection
12+
* --all Clear all supported collections
13+
* --mock Only print what would be deleted, do not delete
14+
* --help Show this help message and exit
15+
* (no flags) Show help (safety mode)
16+
*
17+
* Requires environment variable:
18+
* DEV_DB_URI - MongoDB URI for DEV
19+
*
20+
* Example:
21+
* node clearDevCollections.js --projects
22+
* node clearDevCollections.js --recurring-events
23+
* node clearDevCollections.js
24+
*/
25+
26+
const { MongoClient } = require('mongodb');
27+
require('dotenv').config();
28+
29+
const DEV_DB_NAME = 'vrms-test'; // Match the DEV_DB_NAME from cloneOrSyncProjects.js
30+
const COLLECTIONS = {
31+
projects: 'projects',
32+
recurring_events: 'recurringevents',
33+
events: 'events',
34+
checkins: 'checkins',
35+
};
36+
37+
function printHelp() {
38+
console.log(
39+
`\nUsage: node clearDevCollections.js [options]\n\nOptions:\n --projects Only clear the 'projects' collection\n --recurring-events Only clear the 'recurringevents' collection\n --events Only clear the 'events' collection\n --checkins Only clear the 'checkins' collection\n --all Clear all supported collections\n --mock Only print what would be deleted, do not delete\n --help Show this help message and exit\n\nExamples:\n node clearDevCollections.js --projects\n node clearDevCollections.js --recurring-events\n node clearDevCollections.js --all\n node clearDevCollections.js --mock --all\n`,
40+
);
41+
}
42+
43+
function checkEnv() {
44+
if (!process.env.DEV_DB_URI) {
45+
throw new Error('DEV_DB_URI environment variable must be set.');
46+
}
47+
}
48+
49+
async function clearCollection(devClient, collectionName, isMock) {
50+
const devDb = devClient.db(DEV_DB_NAME);
51+
if (isMock) {
52+
const count = await devDb.collection(collectionName).countDocuments();
53+
console.log(
54+
`[MOCK] Would delete ${count} documents from ${collectionName} in DEV[${DEV_DB_NAME}].`,
55+
);
56+
} else {
57+
const result = await devDb.collection(collectionName).deleteMany({});
58+
console.log(
59+
`[INFO] Cleared ${result.deletedCount} documents from ${collectionName} in DEV[${DEV_DB_NAME}].`,
60+
);
61+
}
62+
}
63+
64+
async function main() {
65+
checkEnv();
66+
const doProjects = process.argv.includes('--projects');
67+
const doRecurring = process.argv.includes('--recurring-events');
68+
const doEvent = process.argv.includes('--events');
69+
const doCheckIn = process.argv.includes('--checkins');
70+
const doAll = process.argv.includes('--all');
71+
const isMock = process.argv.includes('--mock');
72+
const isHelp = process.argv.includes('--help');
73+
const noArgs = process.argv.length <= 2;
74+
if (isHelp || noArgs) {
75+
printHelp();
76+
return;
77+
}
78+
const collectionsToClear = [];
79+
if (doAll) {
80+
collectionsToClear.push('projects', 'recurring_events', 'events', 'checkins');
81+
} else {
82+
if (doProjects) collectionsToClear.push('projects');
83+
if (doRecurring) collectionsToClear.push('recurring_events');
84+
if (doEvent) collectionsToClear.push('events');
85+
if (doCheckIn) collectionsToClear.push('checkins');
86+
}
87+
if (collectionsToClear.length === 0) {
88+
printHelp();
89+
return;
90+
}
91+
92+
let devClient;
93+
try {
94+
devClient = new MongoClient(process.env.DEV_DB_URI);
95+
await devClient.connect();
96+
for (const key of collectionsToClear) {
97+
await clearCollection(devClient, COLLECTIONS[key], isMock);
98+
}
99+
if (!isMock) {
100+
console.log('[SUCCESS] Collections cleared.');
101+
}
102+
} catch (err) {
103+
console.error('[ERROR]', err.message);
104+
process.exitCode = 1;
105+
} finally {
106+
if (devClient) await devClient.close();
107+
}
108+
}
109+
110+
if (require.main === module) {
111+
main();
112+
}

‎backend/scripts/cloneOrSyncCollections.js‎

Lines changed: 217 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,217 @@
1+
/**
2+
* Script to clone or sync collections from PROD to DEV MongoDB.
3+
*
4+
* Usage:
5+
* node cloneOrSyncProjects.js [options]
6+
*
7+
* Options:
8+
* --projects Clone/sync the 'projects' collection
9+
* --recurring-events Clone/sync the 'recurringevents' collection
10+
* --all Clone/sync all supported collections
11+
* --initial Initial clone (insertMany, skip duplicates)
12+
* --mock Only print what would be written, do not write
13+
* --help Show this help message and exit
14+
* (no flags) Show help (safety mode)
15+
*
16+
* Requires environment variables:
17+
* PROD_DB_URI - MongoDB URI for PROD
18+
* DEV_DB_URI - MongoDB URI for DEV
19+
*
20+
* Examples:
21+
* node cloneOrSyncProjects.js --projects
22+
* node cloneOrSyncProjects.js --recurring-events --initial
23+
* node cloneOrSyncProjects.js --all --mock
24+
*/
25+
26+
const { MongoClient } = require('mongodb');
27+
const mongoose = require('mongoose');
28+
require('dotenv').config();
29+
30+
// Print help message for CLI usage
31+
function printHelp() {
32+
console.log(
33+
`\nUsage: node cloneOrSyncProjects.js [options]\n\nOptions:\n --projects Clone/sync the 'projects' collection\n --recurring-events Clone/sync the 'recurringevents' collection\n --all Clone/sync all supported collections\n --initial Initial clone (insertMany, skip duplicates)\n --mock Only print what would be written, do not write\n --help Show this help message and exit\n\nExamples:\n node cloneOrSyncProjects.js --projects\n node cloneOrSyncProjects.js --recurring-events\n node cloneOrSyncProjects.js --all --mock\n`,
34+
);
35+
}
36+
37+
// ---- CONSTANTS ----
38+
const COLLECTIONS = {
39+
projects: {
40+
name: 'projects',
41+
model: 'Project',
42+
},
43+
recurring_events: {
44+
name: 'recurringevents',
45+
model: 'RecurringEvent',
46+
},
47+
};
48+
const PROD_DB_NAME = 'db';
49+
const DEV_DB_NAME = 'vrms-test';
50+
// ---- CONSTANTS ----
51+
// const COLLECTION_NAME = 'projects';
52+
// const PROD_DB_NAME = 'vrms-test';
53+
// const DEV_DB_NAME = 'vrms-test-sync';
54+
55+
/**
56+
* Throws if required environment variables are missing.
57+
*/
58+
function checkEnv() {
59+
if (!process.env.PROD_DB_URI || !process.env.DEV_DB_URI) {
60+
throw new Error('Both PROD_DB_URI and DEV_DB_URI environment variables must be set.');
61+
}
62+
}
63+
64+
/**
65+
* Connects to both PROD (via Mongoose) and DEV (via MongoClient) MongoDB databases.
66+
* @returns {Promise<{prod: typeof mongoose, dev: MongoClient}>}
67+
*/
68+
async function connectDbs() {
69+
// Connect PROD using Mongoose
70+
await mongoose.connect(process.env.PROD_DB_URI, {
71+
dbName: PROD_DB_NAME,
72+
useNewUrlParser: true,
73+
useUnifiedTopology: true,
74+
});
75+
// Connect DEV using MongoClient
76+
const dev = new MongoClient(process.env.DEV_DB_URI);
77+
await dev.connect();
78+
return { prod: mongoose, dev };
79+
}
80+
81+
/**
82+
* Fetches all projects from the PROD database using the Project Mongoose model.
83+
* @returns {Promise<Array<Object>>}
84+
*/
85+
86+
async function fetchProdCollection(collectionKey) {
87+
const { Project, RecurringEvent } = require('../models');
88+
let Model;
89+
if (collectionKey === 'projects') Model = Project;
90+
else if (collectionKey === 'recurring_events') Model = RecurringEvent;
91+
else throw new Error('Unknown collection: ' + collectionKey);
92+
const docs = await Model.find({});
93+
console.log(
94+
`[INFO] Fetched ${docs.length} ${COLLECTIONS[collectionKey].name} from PROD[${PROD_DB_NAME}].`,
95+
);
96+
return docs.map((doc) => doc.toObject());
97+
}
98+
99+
/**
100+
* Performs the initial clone: insertMany, skip duplicates.
101+
* @param {MongoClient} devClient
102+
* @param {Array<Object>} projects
103+
* @returns {Promise<void>}
104+
*/
105+
106+
async function initialClone(devClient, collectionKey, docs) {
107+
const devDb = devClient.db(DEV_DB_NAME);
108+
try {
109+
const result = await devDb
110+
.collection(COLLECTIONS[collectionKey].name)
111+
.insertMany(docs, { ordered: false });
112+
console.log(
113+
`[INFO] Inserted ${result.insertedCount} ${COLLECTIONS[collectionKey].name} into DEV[${DEV_DB_NAME}].`,
114+
);
115+
} catch (err) {
116+
if (err.code === 11000 || (err.writeErrors && err.writeErrors[0]?.code === 11000)) {
117+
// Duplicate key error: some docs inserted, some skipped
118+
const inserted = err.result?.result?.nInserted || err.result?.insertedCount || 0;
119+
console.log(
120+
`[WARN] Duplicate key error. Inserted ${inserted} new ${COLLECTIONS[collectionKey].name}, skipped duplicates.`,
121+
);
122+
} else {
123+
throw err;
124+
}
125+
}
126+
}
127+
128+
/**
129+
* Syncs projects: upsert by _id using bulkWrite.
130+
* @param {MongoClient} devClient
131+
* @param {Array<Object>} projects
132+
* @returns {Promise<void>}
133+
*/
134+
135+
async function syncCollection(devClient, collectionKey, docs) {
136+
const devDb = devClient.db(DEV_DB_NAME);
137+
const ops = docs.map((doc) => ({
138+
updateOne: {
139+
filter: { _id: doc._id },
140+
update: { $set: doc },
141+
upsert: true,
142+
},
143+
}));
144+
const result = await devDb
145+
.collection(COLLECTIONS[collectionKey].name)
146+
.bulkWrite(ops, { ordered: false });
147+
console.log(
148+
`[INFO] Upserted ${result.upsertedCount}, matched ${result.matchedCount}, modified ${result.modifiedCount} ${COLLECTIONS[collectionKey].name} in DEV.`,
149+
);
150+
}
151+
152+
/**
153+
* Main coordinator function.
154+
*/
155+
async function main() {
156+
checkEnv();
157+
const isInitial = process.argv.includes('--initial');
158+
const isMock = process.argv.includes('--mock');
159+
const isHelp = process.argv.includes('--help');
160+
const doProjects = process.argv.includes('--projects');
161+
const doRecurring = process.argv.includes('--recurring-events');
162+
const doAll = process.argv.includes('--all');
163+
const noArgs = process.argv.length <= 2;
164+
if (isHelp || noArgs) {
165+
printHelp();
166+
return;
167+
}
168+
const collectionsToClone = [];
169+
if (doAll) {
170+
collectionsToClone.push('projects', 'recurring_events');
171+
} else {
172+
if (doProjects) collectionsToClone.push('projects');
173+
if (doRecurring) collectionsToClone.push('recurring_events');
174+
}
175+
if (collectionsToClone.length === 0) {
176+
printHelp();
177+
return;
178+
}
179+
180+
let devClient;
181+
try {
182+
const dbs = await connectDbs();
183+
devClient = dbs.dev;
184+
for (const collectionKey of collectionsToClone) {
185+
const docs = await fetchProdCollection(collectionKey);
186+
if (isMock) {
187+
if (isInitial) {
188+
console.log(
189+
`[MOCK] Would insert ${docs.length} ${COLLECTIONS[collectionKey].name} into DEV[${DEV_DB_NAME}].`,
190+
);
191+
} else {
192+
console.log(
193+
`[MOCK] Would upsert ${docs.length} ${COLLECTIONS[collectionKey].name} into DEV[${DEV_DB_NAME}].`,
194+
);
195+
}
196+
} else {
197+
if (isInitial) {
198+
await initialClone(devClient, collectionKey, docs);
199+
} else {
200+
await syncCollection(devClient, collectionKey, docs);
201+
}
202+
}
203+
}
204+
if (!isMock) console.log('[SUCCESS] Operation completed.');
205+
} catch (err) {
206+
console.error('[ERROR]', err.message);
207+
process.exitCode = 1;
208+
} finally {
209+
// Close mongoose connection
210+
await mongoose.disconnect();
211+
if (devClient) await devClient.close();
212+
}
213+
}
214+
215+
if (require.main === module) {
216+
main();
217+
}

0 commit comments

Comments
 (0)