From 3b72877a658ceb3454aa1434cbd682bbb078ab69 Mon Sep 17 00:00:00 2001 From: pablomendezroyo Date: Wed, 16 Oct 2024 09:17:00 +0200 Subject: [PATCH 1/4] implement indexer api --- .../src/modules/apiServers/indexer/config.ts | 3 + .../src/modules/apiServers/indexer/index.ts | 1 + .../apiServers/indexer/routes/epochs/index.ts | 1 + .../apiServers/indexer/routes/epochs/route.ts | 105 ++++++++++++++++++ .../apiServers/indexer/routes/epochs/types.ts | 22 ++++ .../indexer/routes/epochs/validation.ts | 49 ++++++++ .../apiServers/indexer/routes/index.ts | 1 + .../apiServers/indexer/startIndexerApi.ts | 30 +++++ packages/brain/src/params.ts | 1 + 9 files changed, 213 insertions(+) create mode 100644 packages/brain/src/modules/apiServers/indexer/config.ts create mode 100644 packages/brain/src/modules/apiServers/indexer/index.ts create mode 100644 packages/brain/src/modules/apiServers/indexer/routes/epochs/index.ts create mode 100644 packages/brain/src/modules/apiServers/indexer/routes/epochs/route.ts create mode 100644 packages/brain/src/modules/apiServers/indexer/routes/epochs/types.ts create mode 100644 packages/brain/src/modules/apiServers/indexer/routes/epochs/validation.ts create mode 100644 packages/brain/src/modules/apiServers/indexer/routes/index.ts create mode 100644 packages/brain/src/modules/apiServers/indexer/startIndexerApi.ts diff --git a/packages/brain/src/modules/apiServers/indexer/config.ts b/packages/brain/src/modules/apiServers/indexer/config.ts new file mode 100644 index 00000000..b442066b --- /dev/null +++ b/packages/brain/src/modules/apiServers/indexer/config.ts @@ -0,0 +1,3 @@ +export const corsOptions = { + origin: ["http://csm-report-indexer.dappnode", "http://csm-report-indexer.testnet.dappnode"] // TODO: update with DAppNodePackage-lido-csm.dnp.dappnode.eth domains +}; diff --git a/packages/brain/src/modules/apiServers/indexer/index.ts b/packages/brain/src/modules/apiServers/indexer/index.ts new file mode 100644 index 00000000..b546367b --- /dev/null +++ b/packages/brain/src/modules/apiServers/indexer/index.ts @@ -0,0 +1 @@ +export { startIndexerApi } from "./startIndexerApi.js"; diff --git a/packages/brain/src/modules/apiServers/indexer/routes/epochs/index.ts b/packages/brain/src/modules/apiServers/indexer/routes/epochs/index.ts new file mode 100644 index 00000000..37931b67 --- /dev/null +++ b/packages/brain/src/modules/apiServers/indexer/routes/epochs/index.ts @@ -0,0 +1 @@ +export { createIndexerEpochsRouter } from "./route.js"; diff --git a/packages/brain/src/modules/apiServers/indexer/routes/epochs/route.ts b/packages/brain/src/modules/apiServers/indexer/routes/epochs/route.ts new file mode 100644 index 00000000..74194469 --- /dev/null +++ b/packages/brain/src/modules/apiServers/indexer/routes/epochs/route.ts @@ -0,0 +1,105 @@ +import express from "express"; +import logger from "../../../../logger/index.js"; +import { validateQueryParams } from "./validation.js"; +import { RequestParsed } from "./types.js"; +import { BrainDataBase } from "../../../../db/index.js"; +import { PostgresClient } from "../../../../apiClients/index.js"; +import { Tag } from "@stakingbrain/common"; +import { ValidatorsDataPerEpochMap } from "../../../../apiClients/types.js"; +import { StakingBrainDb } from "../../../../db/types.js"; + +export const createIndexerEpochsRouter = ({ + postgresClient, + brainDb +}: { + postgresClient: PostgresClient; + brainDb: BrainDataBase; +}) => { + const epochsRouter = express.Router(); + const epochsEndpoint = "/api/v0/indexer/epochs"; + + epochsRouter.get(epochsEndpoint, validateQueryParams, async (req: RequestParsed, res) => { + const { start, end, tag } = req.query; + + try { + const brainDbData = brainDb.getData(); + const validatorsTagIndexesMap = getValidatorsTagsIndexesMap({ brainDbData, tag }); + + // If validatorIndexes is empty, return empty object + if (validatorsTagIndexesMap.size === 0) { + res.send({}); + return; + } + + const tagsEpochsMap = await getTagsEpochsMap({ postgresClient, validatorsTagIndexesMap, start, end }); + + res.send(tagsEpochsMap); + } catch (e) { + logger.error(e); + res.status(500).send({ message: "Internal server error" }); + } + }); + + return epochsRouter; +}; + +function getValidatorsTagsIndexesMap({ + brainDbData, + tag +}: { + brainDbData: StakingBrainDb; + tag?: Tag[]; +}): Map { + const validatorsTagIndexesMap = new Map(); + if (tag) { + for (const t of tag) { + if (!brainDbData[t]) continue; + const index = brainDbData[t].index; + if (!index) continue; + if (!validatorsTagIndexesMap.has(t)) validatorsTagIndexesMap.set(t, []); + validatorsTagIndexesMap.get(t)!.push(index); + } + } else { + for (const [_, details] of Object.entries(brainDbData)) { + if (!details.index) continue; + if (!validatorsTagIndexesMap.has(details.tag)) validatorsTagIndexesMap.set(details.tag, []); + validatorsTagIndexesMap.get(details.tag)!.push(details.index); + } + } + + return validatorsTagIndexesMap; +} + +async function getTagsEpochsMap({ + postgresClient, + validatorsTagIndexesMap, + start, + end +}: { + postgresClient: PostgresClient; + validatorsTagIndexesMap: Map; + start: number; + end: number; +}): Promise>> { + const tagsEpochsMap: Map> = new Map(); + + // Get epochs data for each tag + for (const [tag, indexes] of validatorsTagIndexesMap) { + const epochsValidatorsMap = await postgresClient.getEpochsDataMapForEpochRange({ + validatorIndexes: indexes.map((index) => index.toString()), + startEpoch: start, + endEpoch: end + }); + + for (const [epoch, validatorsDataMap] of epochsValidatorsMap) { + if (!tagsEpochsMap.has(epoch)) tagsEpochsMap.set(epoch, new Map()); + + if (!tagsEpochsMap.get(epoch)!.has(tag)) tagsEpochsMap.get(epoch)!.set(tag, new Map()); + + for (const [validatorIndex, data] of validatorsDataMap) + tagsEpochsMap.get(epoch)!.get(tag)!.set(validatorIndex, data); + } + } + + return tagsEpochsMap; +} diff --git a/packages/brain/src/modules/apiServers/indexer/routes/epochs/types.ts b/packages/brain/src/modules/apiServers/indexer/routes/epochs/types.ts new file mode 100644 index 00000000..4a5368c3 --- /dev/null +++ b/packages/brain/src/modules/apiServers/indexer/routes/epochs/types.ts @@ -0,0 +1,22 @@ +import { Request } from "express"; +import { Tag } from "@stakingbrain/common"; // Assuming this is defined somewhere + +// The query parameters before they are parsed or validated +// eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-empty-object-type +export type RequestReceived = Request<{}, any, any, QueryParamsReceived>; + +type QueryParamsReceived = { + start: string | number; + end: string | number; + tag?: Tag[] | Tag; // Can be an array or a single value before validation +}; + +// The query parameters after they are validated and parsed +// eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-empty-object-type +export type RequestParsed = Request<{}, any, any, QueryParamsParsed>; + +type QueryParamsParsed = { + start: number; + end: number; + tag?: Tag[]; // After validation, tag should be an array +}; diff --git a/packages/brain/src/modules/apiServers/indexer/routes/epochs/validation.ts b/packages/brain/src/modules/apiServers/indexer/routes/epochs/validation.ts new file mode 100644 index 00000000..f76bc39a --- /dev/null +++ b/packages/brain/src/modules/apiServers/indexer/routes/epochs/validation.ts @@ -0,0 +1,49 @@ +import { Response, NextFunction } from "express"; +import { Tag, tags } from "@stakingbrain/common"; +import { RequestReceived } from "./types.js"; + +// Validation middleware for query parameters +export function validateQueryParams(req: RequestReceived, res: Response, next: NextFunction): void { + const { start, end, tag } = req.query; + + // validate start and end + if (!start || !end) { + res.status(400).json({ message: "query parameters start and end must be provided" }); + return; + } + if (typeof start !== "string" || typeof end !== "string") { + res.status(400).json({ message: "query parameter start and end must be of type string" }); + return; + } + + // parse start + req.query.start = parseInt(start); + req.query.end = parseInt(end); + // check that start is less or equal to end + if (req.query.start > req.query.end) { + res.status(400).json({ message: "query parameter start must be less than or equal to end" }); + return; + } + + // Validate tag + if (tag) { + // tag may be of type string or array of strings otherwise return 400 + if (typeof tag !== "string" && !Array.isArray(tag)) { + res.status(400).json({ message: "tag must be a string or an array of strings" }); + } + + // if tag is a string, convert it to an array + const tagsArray = Array.isArray(tag) ? tag : [tag]; + const invalidTag = tagsArray.find((t) => !tags.includes(t as Tag)); + + if (invalidTag) { + res.status(400).json({ message: `invalid tag received: ${invalidTag}. Allowed tags are ${tags.join(", ")}` }); + return; + } + + // If validation passed, update req.query.tag to ensure it is always an array for downstream middleware + req.query.tag = tagsArray; + } + + next(); // Continue to the next middleware or route handler +} diff --git a/packages/brain/src/modules/apiServers/indexer/routes/index.ts b/packages/brain/src/modules/apiServers/indexer/routes/index.ts new file mode 100644 index 00000000..fd81a502 --- /dev/null +++ b/packages/brain/src/modules/apiServers/indexer/routes/index.ts @@ -0,0 +1 @@ +export * from "./epochs/index.js"; diff --git a/packages/brain/src/modules/apiServers/indexer/startIndexerApi.ts b/packages/brain/src/modules/apiServers/indexer/startIndexerApi.ts new file mode 100644 index 00000000..e0fa4ed4 --- /dev/null +++ b/packages/brain/src/modules/apiServers/indexer/startIndexerApi.ts @@ -0,0 +1,30 @@ +import express from "express"; +import cors from "cors"; +import logger from "../../logger/index.js"; +import http from "node:http"; +import { params } from "../../../params.js"; +import { corsOptions } from "./config.js"; +import { createIndexerEpochsRouter } from "./routes/index.js"; +import { BrainDataBase } from "../../db/index.js"; +import { PostgresClient } from "../../apiClients/index.js"; + +export function startIndexerApi({ + brainDb, + postgresClient +}: { + brainDb: BrainDataBase; + postgresClient: PostgresClient; +}): http.Server { + const app = express(); + app.use(express.json()); + app.use(cors(corsOptions)); + + app.use(createIndexerEpochsRouter({ brainDb, postgresClient })); + + const server = new http.Server(app); + server.listen(params.indexerPort, () => { + logger.info(`Indexer API listening on port ${params.indexerPort}`); + }); + + return server; +} diff --git a/packages/brain/src/params.ts b/packages/brain/src/params.ts index edc8d520..06779f71 100644 --- a/packages/brain/src/params.ts +++ b/packages/brain/src/params.ts @@ -9,6 +9,7 @@ export const params = { uiPort: 80, launchpadPort: 3000, brainPort: 5000, + indexerPort: 7000, defaultValidatorsMonitorUrl: "https://validators-proofs.dappnode.io", defaultProofsOfValidationCron: 24 * 60 * 60 * 1000 // 1 day in ms }; From 02179abf58a381a063ae77aace65d518c60855d2 Mon Sep 17 00:00:00 2001 From: pablomendezroyo Date: Wed, 16 Oct 2024 10:29:33 +0200 Subject: [PATCH 2/4] start indexer api --- packages/brain/src/index.ts | 3 ++- packages/brain/src/modules/apiServers/index.ts | 6 ++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/brain/src/index.ts b/packages/brain/src/index.ts index 8ca63c26..ba40677d 100644 --- a/packages/brain/src/index.ts +++ b/packages/brain/src/index.ts @@ -51,7 +51,7 @@ reloadValidatorsCronTask.start(); trackValidatorsPerformanceCronTask.start(); // Start server APIs -const { uiServer, launchpadServer, brainApiServer } = getServers({ +const { uiServer, launchpadServer, brainApiServer, indexerApi } = getServers({ brainConfig: config, uiBuildPath: path.resolve(__dirname, params.uiBuildDirName), signerApi, @@ -73,6 +73,7 @@ function handle(signal: string): void { uiServer.close(); launchpadServer.close(); brainApiServer.close(); + indexerApi.close(); logger.debug(`Stopped all cron jobs and closed all connections.`); process.exit(0); } diff --git a/packages/brain/src/modules/apiServers/index.ts b/packages/brain/src/modules/apiServers/index.ts index 7af60b20..2c27a4c9 100644 --- a/packages/brain/src/modules/apiServers/index.ts +++ b/packages/brain/src/modules/apiServers/index.ts @@ -10,6 +10,7 @@ import { BeaconchainApi } from "../apiClients/beaconchain/index.js"; import { CronJob } from "../cron/cron.js"; import { startLaunchpadApi } from "./launchpad/index.js"; import http from "http"; +import { startIndexerApi } from "./indexer/index.js"; export const getServers = ({ brainConfig, @@ -35,6 +36,7 @@ export const getServers = ({ uiServer: http.Server; launchpadServer: http.Server; brainApiServer: http.Server; + indexerApi: http.Server; } => { return { uiServer: startUiServer({ @@ -59,6 +61,10 @@ export const getServers = ({ }), brainApiServer: startBrainApi({ brainDb + }), + indexerApi: startIndexerApi({ + brainDb, + postgresClient }) }; }; From 4f2346df3e8be892136fb236671178deac4abd1e Mon Sep 17 00:00:00 2001 From: pablomendezroyo Date: Thu, 17 Oct 2024 17:23:19 +0200 Subject: [PATCH 3/4] serialize response into json --- .../apiServers/indexer/routes/epochs/route.ts | 78 ++++++++++++++----- 1 file changed, 57 insertions(+), 21 deletions(-) diff --git a/packages/brain/src/modules/apiServers/indexer/routes/epochs/route.ts b/packages/brain/src/modules/apiServers/indexer/routes/epochs/route.ts index 74194469..9ff7ca3a 100644 --- a/packages/brain/src/modules/apiServers/indexer/routes/epochs/route.ts +++ b/packages/brain/src/modules/apiServers/indexer/routes/epochs/route.ts @@ -4,8 +4,8 @@ import { validateQueryParams } from "./validation.js"; import { RequestParsed } from "./types.js"; import { BrainDataBase } from "../../../../db/index.js"; import { PostgresClient } from "../../../../apiClients/index.js"; -import { Tag } from "@stakingbrain/common"; -import { ValidatorsDataPerEpochMap } from "../../../../apiClients/types.js"; +import { Tag, tags } from "@stakingbrain/common"; +import { DataPerEpoch, ValidatorsDataPerEpochMap } from "../../../../apiClients/types.js"; import { StakingBrainDb } from "../../../../db/types.js"; export const createIndexerEpochsRouter = ({ @@ -33,7 +33,8 @@ export const createIndexerEpochsRouter = ({ const tagsEpochsMap = await getTagsEpochsMap({ postgresClient, validatorsTagIndexesMap, start, end }); - res.send(tagsEpochsMap); + // serialize the map to an object + res.send(convertMapIntoObject(tagsEpochsMap)); } catch (e) { logger.error(e); res.status(500).send({ message: "Internal server error" }); @@ -50,20 +51,28 @@ function getValidatorsTagsIndexesMap({ brainDbData: StakingBrainDb; tag?: Tag[]; }): Map { - const validatorsTagIndexesMap = new Map(); - if (tag) { - for (const t of tag) { - if (!brainDbData[t]) continue; - const index = brainDbData[t].index; - if (!index) continue; - if (!validatorsTagIndexesMap.has(t)) validatorsTagIndexesMap.set(t, []); - validatorsTagIndexesMap.get(t)!.push(index); - } - } else { - for (const [_, details] of Object.entries(brainDbData)) { - if (!details.index) continue; - if (!validatorsTagIndexesMap.has(details.tag)) validatorsTagIndexesMap.set(details.tag, []); - validatorsTagIndexesMap.get(details.tag)!.push(details.index); + const validatorsTagIndexesMap = new Map([["solo", [1234]]]); + + // Filter the tags to consider, based on whether the `tag` parameter is provided + const tagsToConsider = tag ? tag : tags; + + // Initialize an empty array for each tag in the map + for (const t of tagsToConsider) validatorsTagIndexesMap.set(t, []); + + // Iterate over brainDbData to populate the map with the indexes + for (const [_, details] of Object.entries(brainDbData)) { + const validatorTag = details.tag; + const validatorIndex = details.index; + + // Check if the validator tag is in the tags to consider and if an index exists + if (tagsToConsider.includes(validatorTag) && validatorIndex !== undefined) { + // Initialize the array if it doesn't exist in the map + if (!validatorsTagIndexesMap.has(validatorTag)) { + validatorsTagIndexesMap.set(validatorTag, []); + } + + // Push the validator's index into the corresponding array + validatorsTagIndexesMap.get(validatorTag)!.push(validatorIndex); } } @@ -94,12 +103,39 @@ async function getTagsEpochsMap({ for (const [epoch, validatorsDataMap] of epochsValidatorsMap) { if (!tagsEpochsMap.has(epoch)) tagsEpochsMap.set(epoch, new Map()); - if (!tagsEpochsMap.get(epoch)!.has(tag)) tagsEpochsMap.get(epoch)!.set(tag, new Map()); - - for (const [validatorIndex, data] of validatorsDataMap) - tagsEpochsMap.get(epoch)!.get(tag)!.set(validatorIndex, data); + tagsEpochsMap.get(epoch)!.set(tag, validatorsDataMap); } } return tagsEpochsMap; } + +function convertMapIntoObject(map: Map>): { + [epoch: number]: { + [tag: string]: { + [validatorIndex: number]: DataPerEpoch; + }; + }; +} { + const object: { + [epoch: number]: { + [tag: string]: { + [validatorIndex: number]: DataPerEpoch; + }; + }; + } = {}; + + for (const [epoch, tagsValidatorsMap] of map) { + object[epoch] = {}; + + for (const [tag, validatorsDataMap] of tagsValidatorsMap) { + object[epoch][tag] = {}; + + for (const [validatorIndex, data] of validatorsDataMap) { + object[epoch][tag][validatorIndex] = data; + } + } + } + + return object; +} From 33b2330fd8db086c9b546ab2a044fe58ed3be3d5 Mon Sep 17 00:00:00 2001 From: pablomendezroyo Date: Fri, 18 Oct 2024 08:59:44 +0200 Subject: [PATCH 4/4] return object instead of Map in postgres client --- .../src/modules/apiClients/postgres/index.ts | 12 ++-- .../src/modules/apiClients/postgres/types.ts | 7 +- .../apiServers/indexer/routes/epochs/route.ts | 67 +++++++------------ .../calls/fetchValidatorsPerformanceData.ts | 4 +- .../src/modules/apiServers/ui/calls/types.ts | 4 +- .../src/modules/validatorsDataIngest/index.ts | 4 +- packages/ui/src/socket/types.ts | 4 +- 7 files changed, 45 insertions(+), 57 deletions(-) diff --git a/packages/brain/src/modules/apiClients/postgres/index.ts b/packages/brain/src/modules/apiClients/postgres/index.ts index 5163947e..9056eec1 100644 --- a/packages/brain/src/modules/apiClients/postgres/index.ts +++ b/packages/brain/src/modules/apiClients/postgres/index.ts @@ -1,6 +1,6 @@ import postgres from "postgres"; import logger from "../../logger/index.js"; -import { EpochsValidatorsMap, DataPerEpoch, Columns, ValidatorsDataPerEpochMap, PostgresDataRow } from "./types.js"; +import { DataPerEpoch, Columns, ValidatorsDataPerEpochMap, PostgresDataRow, EpochsValidatorsData } from "./types.js"; export class PostgresClient { private readonly tableName = "epochs_data"; @@ -99,7 +99,7 @@ ${Columns.error} = EXCLUDED.${Columns.error} validatorIndexes: string[]; startEpoch: number; endEpoch: number; - }): Promise { + }): Promise { const query = ` SELECT * FROM ${this.tableName} WHERE ${Columns.validatorindex} = ANY($1) @@ -109,7 +109,7 @@ AND ${Columns.epoch} <= $3 const result: PostgresDataRow[] = await this.sql.unsafe(query, [validatorIndexes, startEpoch, endEpoch]); - const epochsValidatorsMap: EpochsValidatorsMap = new Map(); + const epochsValidatorData: EpochsValidatorsData = Object.create(null); for (const row of result) { const { validatorindex, epoch, clients, attestation, block, synccommittee, slot, error } = row; @@ -124,12 +124,12 @@ AND ${Columns.epoch} <= $3 [Columns.error]: error }; - if (!epochsValidatorsMap.has(epochNumber)) epochsValidatorsMap.set(epochNumber, new Map()); + if (!epochsValidatorData[epochNumber]) epochsValidatorData[epochNumber] = Object.create(null); - epochsValidatorsMap.get(epochNumber)!.set(validatorIndexNumber, data); + epochsValidatorData[epochNumber][validatorIndexNumber] = data; } - return epochsValidatorsMap; + return epochsValidatorData; } /** diff --git a/packages/brain/src/modules/apiClients/postgres/types.ts b/packages/brain/src/modules/apiClients/postgres/types.ts index f9379586..4a9b6558 100644 --- a/packages/brain/src/modules/apiClients/postgres/types.ts +++ b/packages/brain/src/modules/apiClients/postgres/types.ts @@ -13,8 +13,11 @@ export enum Columns { error = "error" } -// Indexed by epoch number -export type EpochsValidatorsMap = Map; +export interface EpochsValidatorsData { + [epoch: number]: { + [validatorIndex: number]: DataPerEpoch; + }; +} // Indexed by validator index export type ValidatorsDataPerEpochMap = Map; diff --git a/packages/brain/src/modules/apiServers/indexer/routes/epochs/route.ts b/packages/brain/src/modules/apiServers/indexer/routes/epochs/route.ts index 9ff7ca3a..e4b2559b 100644 --- a/packages/brain/src/modules/apiServers/indexer/routes/epochs/route.ts +++ b/packages/brain/src/modules/apiServers/indexer/routes/epochs/route.ts @@ -5,9 +5,17 @@ import { RequestParsed } from "./types.js"; import { BrainDataBase } from "../../../../db/index.js"; import { PostgresClient } from "../../../../apiClients/index.js"; import { Tag, tags } from "@stakingbrain/common"; -import { DataPerEpoch, ValidatorsDataPerEpochMap } from "../../../../apiClients/types.js"; +import { DataPerEpoch } from "../../../../apiClients/types.js"; import { StakingBrainDb } from "../../../../db/types.js"; +interface EpochsTagsValidatorsData { + [epoch: number]: { + [tag: string]: { + [validatorIndex: number]: DataPerEpoch; + }; + }; +} + export const createIndexerEpochsRouter = ({ postgresClient, brainDb @@ -31,10 +39,14 @@ export const createIndexerEpochsRouter = ({ return; } - const tagsEpochsMap = await getTagsEpochsMap({ postgresClient, validatorsTagIndexesMap, start, end }); + const epochsTagsValidatorsData = await getEpochsTagsValidatorsData({ + postgresClient, + validatorsTagIndexesMap, + start, + end + }); - // serialize the map to an object - res.send(convertMapIntoObject(tagsEpochsMap)); + res.send(epochsTagsValidatorsData); } catch (e) { logger.error(e); res.status(500).send({ message: "Internal server error" }); @@ -79,7 +91,7 @@ function getValidatorsTagsIndexesMap({ return validatorsTagIndexesMap; } -async function getTagsEpochsMap({ +async function getEpochsTagsValidatorsData({ postgresClient, validatorsTagIndexesMap, start, @@ -89,53 +101,26 @@ async function getTagsEpochsMap({ validatorsTagIndexesMap: Map; start: number; end: number; -}): Promise>> { - const tagsEpochsMap: Map> = new Map(); +}): Promise { + const epochsTagsValidatorsData: EpochsTagsValidatorsData = Object.create(null); // Get epochs data for each tag for (const [tag, indexes] of validatorsTagIndexesMap) { - const epochsValidatorsMap = await postgresClient.getEpochsDataMapForEpochRange({ + const epochsValidatorsData = await postgresClient.getEpochsDataMapForEpochRange({ validatorIndexes: indexes.map((index) => index.toString()), startEpoch: start, endEpoch: end }); - for (const [epoch, validatorsDataMap] of epochsValidatorsMap) { - if (!tagsEpochsMap.has(epoch)) tagsEpochsMap.set(epoch, new Map()); + // Add the data to the object + for (const [epoch, data] of Object.entries(epochsValidatorsData)) { + const epochNumber = parseInt(epoch); - tagsEpochsMap.get(epoch)!.set(tag, validatorsDataMap); - } - } - - return tagsEpochsMap; -} + if (!epochsTagsValidatorsData[epochNumber]) epochsTagsValidatorsData[epochNumber] = Object.create(null); -function convertMapIntoObject(map: Map>): { - [epoch: number]: { - [tag: string]: { - [validatorIndex: number]: DataPerEpoch; - }; - }; -} { - const object: { - [epoch: number]: { - [tag: string]: { - [validatorIndex: number]: DataPerEpoch; - }; - }; - } = {}; - - for (const [epoch, tagsValidatorsMap] of map) { - object[epoch] = {}; - - for (const [tag, validatorsDataMap] of tagsValidatorsMap) { - object[epoch][tag] = {}; - - for (const [validatorIndex, data] of validatorsDataMap) { - object[epoch][tag][validatorIndex] = data; - } + epochsTagsValidatorsData[epochNumber][tag] = data; } } - return object; + return epochsTagsValidatorsData; } diff --git a/packages/brain/src/modules/apiServers/ui/calls/fetchValidatorsPerformanceData.ts b/packages/brain/src/modules/apiServers/ui/calls/fetchValidatorsPerformanceData.ts index 26b70be4..9a7fdc06 100644 --- a/packages/brain/src/modules/apiServers/ui/calls/fetchValidatorsPerformanceData.ts +++ b/packages/brain/src/modules/apiServers/ui/calls/fetchValidatorsPerformanceData.ts @@ -1,6 +1,6 @@ import { PostgresClient } from "../../../../modules/apiClients/index.js"; import type { NumberOfDaysToQuery } from "../../../../modules/validatorsDataIngest/types.js"; -import type { EpochsValidatorsMap } from "../../../../modules/apiClients/postgres/types.js"; +import type { EpochsValidatorsData } from "../../../../modules/apiClients/postgres/types.js"; import { fetchAndProcessValidatorsData } from "../../../validatorsDataIngest/index.js"; export async function fetchValidatorsPerformanceData({ @@ -15,7 +15,7 @@ export async function fetchValidatorsPerformanceData({ numberOfDaysToQuery?: NumberOfDaysToQuery; minGenesisTime: number; secondsPerSlot: number; -}): Promise { +}): Promise { return await fetchAndProcessValidatorsData({ validatorIndexes, postgresClient, diff --git a/packages/brain/src/modules/apiServers/ui/calls/types.ts b/packages/brain/src/modules/apiServers/ui/calls/types.ts index 43c904cd..4e765e9a 100644 --- a/packages/brain/src/modules/apiServers/ui/calls/types.ts +++ b/packages/brain/src/modules/apiServers/ui/calls/types.ts @@ -7,7 +7,7 @@ import type { BeaconchainPoolVoluntaryExitsPostRequest, Web3signerPostResponse, Web3signerHealthcheckResponse, - EpochsValidatorsMap + EpochsValidatorsData } from "../../../apiClients/types.js"; import { NumberOfDaysToQuery } from "../../../validatorsDataIngest/types.js"; @@ -29,7 +29,7 @@ export interface RpcMethods { fetchValidatorsPerformanceData: ( validatorIndexes: string[], numberOfDaysToQuery?: NumberOfDaysToQuery - ) => Promise; + ) => Promise; } export type ActionRequestOrigin = "ui" | "api"; diff --git a/packages/brain/src/modules/validatorsDataIngest/index.ts b/packages/brain/src/modules/validatorsDataIngest/index.ts index 661f2b7c..d9f15ca0 100644 --- a/packages/brain/src/modules/validatorsDataIngest/index.ts +++ b/packages/brain/src/modules/validatorsDataIngest/index.ts @@ -1,5 +1,5 @@ import { PostgresClient } from "../apiClients/index.js"; -import type { EpochsValidatorsMap } from "../apiClients/postgres/types.js"; +import type { EpochsValidatorsData } from "../apiClients/postgres/types.js"; import logger from "../logger/index.js"; import { getStartAndEndEpochs } from "./getStartAndEndEpochs.js"; import { NumberOfDaysToQuery } from "./types.js"; @@ -26,7 +26,7 @@ export async function fetchAndProcessValidatorsData({ minGenesisTime: number; // import from backend index secondsPerSlot: number; // immport from backend index numberOfDaysToQuery?: NumberOfDaysToQuery; -}): Promise { +}): Promise { logger.info("Processing epochs data"); // Get start timestamp and end timestamp diff --git a/packages/ui/src/socket/types.ts b/packages/ui/src/socket/types.ts index 3b6b14df..ff737e5d 100644 --- a/packages/ui/src/socket/types.ts +++ b/packages/ui/src/socket/types.ts @@ -9,7 +9,7 @@ import type { Web3signerDeleteResponse, Web3signerHealthcheckResponse, Web3signerPostResponse, - EpochsValidatorsMap, + EpochsValidatorsData, NumberOfDaysToQuery } from "@stakingbrain/brain"; import { StakerConfig } from "@stakingbrain/common"; @@ -33,7 +33,7 @@ export interface RpcMethods { }: { validatorIndexes: string[]; numberOfDaysToQuery?: NumberOfDaysToQuery; - }) => Promise; + }) => Promise; // Network getStakerConfig: () => Promise; }