Skip to content

Commit 9a15d47

Browse files
shriyanssshriyanss
committed
refactor: extract URL processing logic into separate function to reduce code duplication
1 parent d8251b3 commit 9a15d47

File tree

1 file changed

+89
-246
lines changed

1 file changed

+89
-246
lines changed

‎src/run/index.ts‎

Lines changed: 89 additions & 246 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,93 @@ import analyze from "../analyze/index.js";
1010
import report from "../report/index.js";
1111
import { clearJsUrls, clearJsonUrls } from "../lazyLoad/globals.js";
1212

13+
const processUrl = async (url, outputDir, workingDir, cmd, isBatch) => {
14+
const targetHost = new URL(url).host.replace(":", "_");
15+
16+
console.log(chalk.bgGreenBright(`[+] Starting analysis for ${url}...`));
17+
18+
if (isBatch) {
19+
clearJsUrls();
20+
clearJsonUrls();
21+
}
22+
23+
console.log(chalk.bgCyan("[1/8] Running lazyload to download JavaScript files..."));
24+
await lazyLoad(url, outputDir, cmd.strictScope, cmd.scope.split(","), cmd.threads, false, "", cmd.insecure);
25+
console.log(chalk.bgGreen("[+] Lazyload complete."));
26+
27+
if (globalsUtil.getTech() === "") {
28+
console.log(chalk.bgRed("[!] Technology not detected. Quitting."));
29+
return;
30+
}
31+
32+
if (globalsUtil.getTech() !== "next") {
33+
console.log(
34+
chalk.bgYellow(
35+
`[!] The tool only supports Next.JS ('next') fully. For ${globalsUtil.getTech()}, only downloading JS files is supported`
36+
)
37+
);
38+
return;
39+
}
40+
41+
const stringsFile = isBatch ? `${workingDir}/strings.json` : "strings.json";
42+
const extractedUrlsFile = isBatch ? `${workingDir}/extracted_urls` : "extracted_urls";
43+
const mappedFile = isBatch ? `${workingDir}/mapped` : "mapped";
44+
const mappedJsonFile = isBatch ? `${workingDir}/mapped.json` : "mapped.json";
45+
const endpointsFile = isBatch ? `${workingDir}/endpoints` : "endpoints";
46+
const openapiFile = isBatch ? `${workingDir}/mapped-openapi.json` : "mapped-openapi.json";
47+
const analyzeFile = isBatch ? `${workingDir}/analyze.json` : "analyze.json";
48+
const reportDbFile = isBatch ? `${workingDir}/js-recon.db` : "js-recon.db";
49+
const reportFile = isBatch ? `${workingDir}/report` : "report";
50+
51+
console.log(chalk.bgCyan("[2/8] Running strings to extract endpoints..."));
52+
await strings(outputDir, stringsFile, true, extractedUrlsFile, false, false, false);
53+
console.log(chalk.bgGreen("[+] Strings complete."));
54+
55+
console.log(chalk.bgCyan("[3/8] Running lazyload with subsequent requests to download JavaScript files..."));
56+
await lazyLoad(
57+
url,
58+
outputDir,
59+
cmd.strictScope,
60+
cmd.scope.split(","),
61+
cmd.threads,
62+
true,
63+
`${extractedUrlsFile}.json`,
64+
cmd.insecure
65+
);
66+
console.log(chalk.bgGreen("[+] Lazyload with subsequent requests complete."));
67+
68+
console.log(chalk.bgCyan("[4/8] Running strings again to extract endpoints..."));
69+
await strings(outputDir, stringsFile, true, extractedUrlsFile, cmd.secrets, true, true);
70+
console.log(chalk.bgGreen("[+] Strings complete."));
71+
72+
console.log(chalk.bgCyan("[5/8] Running map to find functions..."));
73+
globalsUtil.setOpenapi(true);
74+
if (isBatch) {
75+
globalsUtil.setOpenapiOutputFile(openapiFile);
76+
}
77+
await map(outputDir + "/" + targetHost, mappedFile, ["json"], globalsUtil.getTech(), false, false);
78+
console.log(chalk.bgGreen("[+] Map complete."));
79+
80+
console.log(chalk.bgCyan("[6/8] Running endpoints to extract endpoints..."));
81+
if (fs.existsSync(`${outputDir}/${targetHost}/___subsequent_requests`)) {
82+
await endpoints(url, `${outputDir}/${targetHost}/`, endpointsFile, ["json"], "next", false, mappedJsonFile);
83+
} else {
84+
await endpoints(url, undefined, endpointsFile, ["json"], "next", false, mappedJsonFile);
85+
}
86+
console.log(chalk.bgGreen("[+] Endpoints complete."));
87+
88+
console.log(chalk.bgCyan("[7/8] Running analyze to extract endpoints..."));
89+
// @ts-ignore
90+
await analyze("", mappedJsonFile, globalsUtil.getTech(), false, openapiFile, false, analyzeFile);
91+
console.log(chalk.bgGreen("[+] Analyze complete."));
92+
93+
console.log(chalk.bgCyan("[8/8] Running report module..."));
94+
await report(reportDbFile, mappedJsonFile, analyzeFile, `${endpointsFile}.json`, openapiFile, reportFile);
95+
console.log(chalk.bgGreen("[+] Report complete."));
96+
97+
console.log(chalk.bgGreenBright(`[+] Analysis complete for ${url}.`));
98+
};
99+
13100
export default async (cmd) => {
14101
globalsUtil.setApiGatewayConfigFile(cmd.apiGatewayConfig);
15102
globalsUtil.setUseApiGateway(cmd.apiGateway);
@@ -36,106 +123,7 @@ export default async (cmd) => {
36123
return;
37124
}
38125

39-
let targetHost;
40-
try {
41-
targetHost = new URL(cmd.url).host.replace(":", "_");
42-
} catch (e) {
43-
console.log(chalk.red(`[!] Invalid URL: ${cmd.url}`));
44-
process.exit(1);
45-
}
46-
47-
console.log(chalk.bgGreenBright("[+] Starting analysis..."));
48-
49-
console.log(chalk.bgCyan("[1/8] Running lazyload to download JavaScript files..."));
50-
await lazyLoad(
51-
cmd.url,
52-
cmd.output,
53-
cmd.strictScope,
54-
cmd.scope.split(","),
55-
cmd.threads,
56-
false,
57-
"",
58-
cmd.insecure
59-
);
60-
console.log(chalk.bgGreen("[+] Lazyload complete."));
61-
62-
// if tech is undefined, i.e. it can't be detected, quit. Nothing to be done :(
63-
if (globalsUtil.getTech() === "") {
64-
console.log(chalk.bgRed("[!] Technology not detected. Quitting."));
65-
return;
66-
}
67-
68-
// since the app only supports next.js now, move ahead only if the tech is next
69-
if (globalsUtil.getTech() !== "next") {
70-
console.log(
71-
chalk.bgYellow(
72-
`[!] The tool only supports Next.JS ('next') fully. For ${globalsUtil.getTech()}, only downloading JS files is supported`
73-
)
74-
);
75-
return;
76-
}
77-
78-
// run strings
79-
console.log(chalk.bgCyan("[2/8] Running strings to extract endpoints..."));
80-
await strings(cmd.output, "strings.json", true, "extracted_urls", false, false, false);
81-
console.log(chalk.bgGreen("[+] Strings complete."));
82-
83-
// run lazyload with subsequent requests
84-
console.log(chalk.bgCyan("[3/8] Running lazyload with subsequent requests to download JavaScript files..."));
85-
await lazyLoad(
86-
cmd.url,
87-
cmd.output,
88-
cmd.strictScope,
89-
cmd.scope.split(","),
90-
cmd.threads,
91-
true,
92-
"extracted_urls.json",
93-
cmd.insecure
94-
);
95-
console.log(chalk.bgGreen("[+] Lazyload with subsequent requests complete."));
96-
97-
// run strings again to extract endpoints from the files that are downloaded in the previous step
98-
console.log(chalk.bgCyan("[4/8] Running strings again to extract endpoints..."));
99-
await strings(cmd.output, "strings.json", true, "extracted_urls", cmd.secrets, true, true);
100-
console.log(chalk.bgGreen("[+] Strings complete."));
101-
102-
// now, run map
103-
console.log(chalk.bgCyan("[5/8] Running map to find functions..."));
104-
globalsUtil.setOpenapi(true);
105-
await map(cmd.output + "/" + targetHost, "mapped", ["json"], globalsUtil.getTech(), false, false);
106-
console.log(chalk.bgGreen("[+] Map complete."));
107-
108-
// now, run endpoints
109-
console.log(chalk.bgCyan("[6/8] Running endpoints to extract endpoints..."));
110-
// check if the subsequent requests directory exists
111-
if (fs.existsSync(`${cmd.output}/${targetHost}/___subsequent_requests`)) {
112-
await endpoints(
113-
cmd.url,
114-
`${cmd.output}/${targetHost}/`,
115-
"endpoints",
116-
["json"],
117-
"next",
118-
false,
119-
"mapped.json"
120-
);
121-
} else {
122-
await endpoints(cmd.url, undefined, "endpoints", ["json"], "next", false, "mapped.json");
123-
}
124-
console.log(chalk.bgGreen("[+] Endpoints complete."));
125-
126-
// run the analyze module now
127-
console.log(chalk.bgCyan("[7/8] Running analyze to extract endpoints..."));
128-
// since the thirs argument is tech, and it can't be "all", so adding type ignore
129-
// @ts-ignore
130-
await analyze("", "mapped.json", globalsUtil.getTech(), false, "mapped-openapi.json", false, "analyze.json");
131-
console.log(chalk.bgGreen("[+] Analyze complete."));
132-
133-
// run the report module now
134-
console.log(chalk.bgCyan("[8/8] Running report module..."));
135-
await report("js-recon.db", "mapped.json", "analyze.json", "endpoints.json", "mapped-openapi.json", "report");
136-
console.log(chalk.bgGreen("[+] Report complete."));
137-
138-
console.log(chalk.bgGreenBright("[+] Analysis complete."));
126+
await processUrl(cmd.url, cmd.output, ".", cmd, false);
139127
} else {
140128
// since this is a file, we need to first load the URLs in the memory remove empty strings
141129
const urls = fs
@@ -174,156 +162,11 @@ export default async (cmd) => {
174162
}
175163
fs.mkdirSync(toolOutputDir);
176164

177-
// now, run the steps one by one
178165
for (const url of urls) {
179-
// check if output directory exists. If so, ask the user to switch to other directory
180-
// if not done, it might conflict this process
181-
// for devs: run `npm run cleanup` to prepare this directory
182166
const thisTargetWorkingDir = toolOutputDir + "/" + new URL(url).host.replace(":", "_");
183-
// make the directory
184167
fs.mkdirSync(thisTargetWorkingDir);
185168
const outputDir = thisTargetWorkingDir + "/output";
186-
187-
const targetHost = new URL(url).host.replace(":", "_");
188-
189-
console.log(chalk.bgGreenBright("[+] Starting analysis..."));
190-
191-
// first of all, clear the jsUrls/jsonUrls cache
192-
clearJsUrls();
193-
clearJsonUrls();
194-
195-
console.log(chalk.bgCyan("[1/8] Running lazyload to download JavaScript files..."));
196-
await lazyLoad(url, outputDir, cmd.strictScope, cmd.scope.split(","), cmd.threads, false, "", cmd.insecure);
197-
console.log(chalk.bgGreen("[+] Lazyload complete."));
198-
199-
// if tech is "", i.e. it can't be detected, quit. Nothing to be done :(
200-
if (globalsUtil.getTech() === "") {
201-
console.log(chalk.bgRed("[!] Technology not detected. Quitting."));
202-
continue;
203-
}
204-
205-
// since the app only supports next.js now, move ahead only if the tech is next
206-
if (globalsUtil.getTech() !== "next") {
207-
console.log(
208-
chalk.bgYellow(
209-
`[!] The tool only supports Next.JS ('next') fully. For ${globalsUtil.getTech()}, only downloading JS files is supported`
210-
)
211-
);
212-
continue;
213-
}
214-
215-
// run strings
216-
console.log(chalk.bgCyan("[2/8] Running strings to extract endpoints..."));
217-
await strings(
218-
outputDir,
219-
`${thisTargetWorkingDir}/strings.json`,
220-
true,
221-
`${thisTargetWorkingDir}/extracted_urls`,
222-
false,
223-
false,
224-
false
225-
);
226-
console.log(chalk.bgGreen("[+] Strings complete."));
227-
228-
// run lazyload with subsequent requests
229-
console.log(
230-
chalk.bgCyan("[3/8] Running lazyload with subsequent requests to download JavaScript files...")
231-
);
232-
await lazyLoad(
233-
url,
234-
outputDir,
235-
cmd.strictScope,
236-
cmd.scope.split(","),
237-
cmd.threads,
238-
true,
239-
`${thisTargetWorkingDir}/extracted_urls.json`,
240-
cmd.insecure
241-
);
242-
console.log(chalk.bgGreen("[+] Lazyload with subsequent requests complete."));
243-
244-
// run strings again to extract endpoints from the files that are downloaded in the previous step
245-
console.log(chalk.bgCyan("[4/8] Running strings again to extract endpoints..."));
246-
await strings(
247-
outputDir,
248-
`${thisTargetWorkingDir}/strings.json`,
249-
true,
250-
`${thisTargetWorkingDir}/extracted_urls`,
251-
cmd.secrets,
252-
true,
253-
true
254-
);
255-
console.log(chalk.bgGreen("[+] Strings complete."));
256-
257-
// now, run map
258-
console.log(chalk.bgCyan("[5/8] Running map to find functions..."));
259-
globalsUtil.setOpenapi(true);
260-
globalsUtil.setOpenapiOutputFile(`${thisTargetWorkingDir}/mapped-openapi.json`);
261-
await map(
262-
outputDir + "/" + targetHost,
263-
`${thisTargetWorkingDir}/mapped`,
264-
["json"],
265-
globalsUtil.getTech(),
266-
false,
267-
false
268-
);
269-
console.log(chalk.bgGreen("[+] Map complete."));
270-
271-
// now, run endpoints
272-
console.log(chalk.bgCyan("[6/8] Running endpoints to extract endpoints..."));
273-
// check if the subsequent requests directory exists
274-
if (fs.existsSync(`${outputDir}/${targetHost}/___subsequent_requests`)) {
275-
await endpoints(
276-
url,
277-
`${outputDir}/${targetHost}/`,
278-
`${thisTargetWorkingDir}/endpoints`,
279-
["json"],
280-
"next",
281-
false,
282-
`${thisTargetWorkingDir}/mapped.json`
283-
);
284-
} else {
285-
await endpoints(
286-
url,
287-
undefined,
288-
`${thisTargetWorkingDir}/endpoints`,
289-
["json"],
290-
"next",
291-
false,
292-
`${thisTargetWorkingDir}/mapped.json`
293-
);
294-
}
295-
console.log(chalk.bgGreen("[+] Endpoints complete."));
296-
297-
// run the analyze module now
298-
console.log(chalk.bgCyan("[7/8] Running analyze to extract endpoints..."));
299-
// since the thirs argument is tech, and it can't be "all", so adding type ignore
300-
// @ts-ignore
301-
await analyze(
302-
"",
303-
`${thisTargetWorkingDir}/mapped.json`,
304-
// ignoring the below line coz "next" is a string, and the tool won't work if this is incorrect
305-
// @ts-ignore
306-
globalsUtil.getTech(),
307-
false,
308-
`${thisTargetWorkingDir}/mapped-openapi.json`,
309-
false,
310-
`${thisTargetWorkingDir}/analyze.json`
311-
);
312-
console.log(chalk.bgGreen("[+] Analyze complete."));
313-
314-
// run the report module now
315-
console.log(chalk.bgCyan("[8/8] Running report module..."));
316-
await report(
317-
`${thisTargetWorkingDir}/js-recon.db`,
318-
`${thisTargetWorkingDir}/mapped.json`,
319-
`${thisTargetWorkingDir}/analyze.json`,
320-
`${thisTargetWorkingDir}/endpoints.json`,
321-
`${thisTargetWorkingDir}/mapped-openapi.json`,
322-
`${thisTargetWorkingDir}/report`
323-
);
324-
console.log(chalk.bgGreen("[+] Report complete."));
325-
326-
console.log(chalk.bgGreenBright("[+] Analysis complete."));
169+
await processUrl(url, outputDir, thisTargetWorkingDir, cmd, true);
327170
}
328171
}
329172
};

0 commit comments

Comments
 (0)