diff --git a/conseilUtil.js b/conseilUtil.js index ae696f9..332ed04 100644 --- a/conseilUtil.js +++ b/conseilUtil.js @@ -1,6 +1,9 @@ const conseiljs = require('conseiljs') const fetch = require('node-fetch') const log = require('loglevel') +const BigNumber = require('bignumber.js') +const pThrottle = require('p-throttle') +// const { performance } = require('perf_hooks'); // NOTE: disabled for prod const logger = log.getLogger('conseiljs') logger.setLevel('error', false) @@ -10,6 +13,8 @@ const conseilServer = 'https://conseil-prod.cryptonomic-infra.tech' const conseilApiKey = 'aa73fa8a-8626-4f43-a605-ff63130f37b1' // signup at nautilus.cloud const tezosNode = '' +const throttleConseil = pThrottle({ limit: 15, interval: 1200 }) + const mainnet = require('./config').networkConfig @@ -36,7 +41,7 @@ const hDAOFeed = async () => { */ const getCollectionForAddress = async (address) => { let collectionQuery = conseiljs.ConseilQueryBuilder.blankQuery(); - collectionQuery = conseiljs.ConseilQueryBuilder.addFields(collectionQuery, 'key', 'value'); + collectionQuery = conseiljs.ConseilQueryBuilder.addFields(collectionQuery, 'key', 'value', 'operation_group_id'); collectionQuery = conseiljs.ConseilQueryBuilder.addPredicate(collectionQuery, 'big_map_id', conseiljs.ConseilOperator.EQ, [mainnet.nftLedger]) collectionQuery = conseiljs.ConseilQueryBuilder.addPredicate(collectionQuery, 'key', conseiljs.ConseilOperator.STARTSWITH, [ `Pair 0x${conseiljs.TezosMessageUtils.writeAddress(address)}`, @@ -46,10 +51,14 @@ const getCollectionForAddress = async (address) => { const collectionResult = await conseiljs.TezosConseilClient.getTezosEntityData({ url: conseilServer, apiKey: conseilApiKey, network: 'mainnet' }, 'mainnet', 'big_map_contents', collectionQuery); let collection = collectionResult.map((i) => { - return { piece: i.key.toString().replace(/.* ([0-9]{1,}$)/, '$1'), amount: Number(i.value) } + return { + piece: i['key'].toString().replace(/.* ([0-9]{1,}$)/, '$1'), + amount: Number(i['value']), + opId: i['operation_group_id'] + } }) - const queryChunks = chunkArray(collection.map(i => i.piece), 20) // NOTE: consider increasing this number somewhat + const queryChunks = chunkArray(collection.map(i => i.piece), 50) // NOTE: consider increasing this number somewhat const makeObjectQuery = (keys) => { let mintedObjectsQuery = conseiljs.ConseilQueryBuilder.blankQuery(); mintedObjectsQuery = conseiljs.ConseilQueryBuilder.addFields(mintedObjectsQuery, 'key_hash', 'value'); @@ -71,17 +80,84 @@ const getCollectionForAddress = async (address) => { objectIpfsMap[objectId] = ipfsHash })))) - collection = collection.map(i => { return { - ipfsHash: objectIpfsMap[i.piece.toString()], - ...i + const operationGroupIds = collectionResult.map((r) => r.operation_group_id) + const priceQueryChunks = chunkArray(operationGroupIds, 30) + const makeLastPriceQuery = (opIds) => { + let lastPriceQuery = conseiljs.ConseilQueryBuilder.blankQuery(); + lastPriceQuery = conseiljs.ConseilQueryBuilder.addFields(lastPriceQuery, 'timestamp', 'amount', 'operation_group_hash', 'parameters_entrypoints', 'parameters'); + lastPriceQuery = conseiljs.ConseilQueryBuilder.addPredicate(lastPriceQuery, 'kind', conseiljs.ConseilOperator.EQ, ['transaction']); + lastPriceQuery = conseiljs.ConseilQueryBuilder.addPredicate(lastPriceQuery, 'status', conseiljs.ConseilOperator.EQ, ['applied']); + lastPriceQuery = conseiljs.ConseilQueryBuilder.addPredicate(lastPriceQuery, 'internal', conseiljs.ConseilOperator.EQ, ['false']); + lastPriceQuery = conseiljs.ConseilQueryBuilder.addPredicate(lastPriceQuery, 'operation_group_hash', opIds.length > 1 ? conseiljs.ConseilOperator.IN : conseiljs.ConseilOperator.EQ, opIds); + lastPriceQuery = conseiljs.ConseilQueryBuilder.setLimit(lastPriceQuery, opIds.length); + + return lastPriceQuery; + } + + const priceQueries = priceQueryChunks.map((c) => makeLastPriceQuery(c)) + const priceMap = {}; + await Promise.all( + priceQueries.map( + async (q) => + await conseiljs.TezosConseilClient.getTezosEntityData({ url: conseilServer, apiKey: conseilApiKey, network: 'mainnet' }, 'mainnet', 'operations', q).then((result) => + result.map((row) => { + let amount = 0; + const action = row.parameters_entrypoints; + + if (action === 'collect') { + amount = Number(row.parameters.toString().replace(/^Pair ([0-9]+) [0-9]+/, '$1')); + } else if (action === 'transfer') { + amount = Number( + row.parameters + .toString() + .replace( + /[{] Pair \"[1-9A-HJ-NP-Za-km-z]{36}\" [{] Pair \"[1-9A-HJ-NP-Za-km-z]{36}\" [(]Pair [0-9]+ [0-9]+[)] [}] [}]/, + '$1' + ) + ); + } + + priceMap[row.operation_group_hash] = { + price: new BigNumber(row.amount), + amount, + timestamp: row.timestamp, + action, + }; + }) + ) + ) + ) + + collection = collection.map(i => { + let price = 0 + let receivedOn = new Date() + let action = '' + + try { + const priceRecord = priceMap[i.opId] + price = priceRecord.price.dividedToIntegerBy(priceRecord.amount).toNumber() + receivedOn = new Date(priceRecord.timestamp) + action = priceRecord.action === 'collect' ? 'Purchased' : 'Received' + } catch { + // + } + + delete i.opId + + return { + price: isNaN(price) ? 0 : price, + receivedOn, + action, + ipfsHash: objectIpfsMap[i.piece.toString()], + ...i }}) - return collection.sort((a, b) => parseInt(b.piece) - parseInt(a.piece)) // sort descending by id – most-recently minted art first + return collection.sort((a, b) => b.receivedOn.getTime() - a.receivedOn.getTime()) // sort descending by date – most-recently acquired art first } const gethDaoBalanceForAddress = async (address) => { - let hDaoBalanceQuery = conseiljs.ConseilQueryBuilder.blankQuery(); - hDaoBalanceQuery = conseiljs.ConseilQueryBuilder.addFields(hDaoBalanceQuery, 'value'); + let hDaoBalanceQuery = conseiljs.ConseilQueryBuilder.blankQuery() + hDaoBalanceQuery = conseiljs.ConseilQueryBuilder.addFields(hDaoBalanceQuery, 'value') hDaoBalanceQuery = conseiljs.ConseilQueryBuilder.addPredicate(hDaoBalanceQuery, 'big_map_id', conseiljs.ConseilOperator.EQ, [mainnet.daoLedger]) hDaoBalanceQuery = conseiljs.ConseilQueryBuilder.addPredicate(hDaoBalanceQuery, 'key', conseiljs.ConseilOperator.EQ, [ `Pair 0x${conseiljs.TezosMessageUtils.writeAddress(address)} 0` @@ -92,7 +168,7 @@ const gethDaoBalanceForAddress = async (address) => { let balance = 0 try { - const balanceResult = await conseiljs.TezosConseilClient.getTezosEntityData({ url: conseilServer, apiKey: conseilApiKey, network: 'mainnet' }, 'mainnet', 'big_map_contents', hDaoBalanceQuery); + const balanceResult = await conseiljs.TezosConseilClient.getTezosEntityData({ url: conseilServer, apiKey: conseilApiKey, network: 'mainnet' }, 'mainnet', 'big_map_contents', hDaoBalanceQuery) balance = balanceResult[0]['value'] // TODO: consider bigNumber here, for the moment there is no reason for it } catch (error) { console.log(`gethDaoBalanceForAddress failed for ${JSON.stringify(hDaoBalanceQuery)} with ${error}`) @@ -108,8 +184,8 @@ const gethDaoBalanceForAddress = async (address) => { * @returns */ const getArtisticOutputForAddress = async (address) => { - let mintOperationQuery = conseiljs.ConseilQueryBuilder.blankQuery(); - mintOperationQuery = conseiljs.ConseilQueryBuilder.addFields(mintOperationQuery, 'operation_group_hash'); + let mintOperationQuery = conseiljs.ConseilQueryBuilder.blankQuery() + mintOperationQuery = conseiljs.ConseilQueryBuilder.addFields(mintOperationQuery, 'operation_group_hash') mintOperationQuery = conseiljs.ConseilQueryBuilder.addPredicate(mintOperationQuery, 'kind', conseiljs.ConseilOperator.EQ, ['transaction']) mintOperationQuery = conseiljs.ConseilQueryBuilder.addPredicate(mintOperationQuery, 'timestamp', conseiljs.ConseilOperator.AFTER, [1612240919000]) // 2021 Feb 1 mintOperationQuery = conseiljs.ConseilQueryBuilder.addPredicate(mintOperationQuery, 'status', conseiljs.ConseilOperator.EQ, ['applied']) @@ -123,14 +199,14 @@ const getArtisticOutputForAddress = async (address) => { { url: conseilServer, apiKey: conseilApiKey, network: 'mainnet' }, 'mainnet', 'operations', - mintOperationQuery); + mintOperationQuery) const operationGroupIds = mintOperationResult.map(r => r['operation_group_hash']) const queryChunks = chunkArray(operationGroupIds, 30) const makeObjectQuery = (opIds) => { - let mintedObjectsQuery = conseiljs.ConseilQueryBuilder.blankQuery(); - mintedObjectsQuery = conseiljs.ConseilQueryBuilder.addFields(mintedObjectsQuery, 'key_hash', 'value'); + let mintedObjectsQuery = conseiljs.ConseilQueryBuilder.blankQuery() + mintedObjectsQuery = conseiljs.ConseilQueryBuilder.addFields(mintedObjectsQuery, 'key_hash', 'value') mintedObjectsQuery = conseiljs.ConseilQueryBuilder.addPredicate(mintedObjectsQuery, 'big_map_id', conseiljs.ConseilOperator.EQ, [mainnet.nftMetadataMap]) mintedObjectsQuery = conseiljs.ConseilQueryBuilder.addPredicate(mintedObjectsQuery, 'operation_group_id', (opIds.length > 1 ? conseiljs.ConseilOperator.IN : conseiljs.ConseilOperator.EQ), opIds) mintedObjectsQuery = conseiljs.ConseilQueryBuilder.setLimit(mintedObjectsQuery, opIds.length) @@ -156,11 +232,11 @@ const getArtisticUniverse = async (max_time) => { let mintOperationQuery = conseiljs.ConseilQueryBuilder.blankQuery(); mintOperationQuery = conseiljs.ConseilQueryBuilder.addFields(mintOperationQuery, 'operation_group_hash'); mintOperationQuery = conseiljs.ConseilQueryBuilder.addPredicate(mintOperationQuery, 'kind', conseiljs.ConseilOperator.EQ, ['transaction']) - mintOperationQuery = conseiljs.ConseilQueryBuilder.addPredicate(mintOperationQuery, 'timestamp', conseiljs.ConseilOperator.AFTER, [1612240919000]) // 2021 Feb 1 + mintOperationQuery = conseiljs.ConseilQueryBuilder.addPredicate(mintOperationQuery, 'timestamp', conseiljs.ConseilOperator.AFTER, [1612240919000]) // after 2021 Feb 1 mintOperationQuery = conseiljs.ConseilQueryBuilder.addPredicate(mintOperationQuery, 'status', conseiljs.ConseilOperator.EQ, ['applied']) mintOperationQuery = conseiljs.ConseilQueryBuilder.addPredicate(mintOperationQuery, 'destination', conseiljs.ConseilOperator.EQ, [mainnet.protocol]) mintOperationQuery = conseiljs.ConseilQueryBuilder.addPredicate(mintOperationQuery, 'parameters_entrypoints', conseiljs.ConseilOperator.EQ, ['mint_OBJKT']) - mintOperationQuery = conseiljs.ConseilQueryBuilder.setLimit(mintOperationQuery, 14_000) + mintOperationQuery = conseiljs.ConseilQueryBuilder.setLimit(mintOperationQuery, 30_000) const mintOperationResult = await conseiljs.TezosConseilClient.getTezosEntityData( { url: conseilServer, apiKey: conseilApiKey, network: 'mainnet' }, @@ -173,7 +249,7 @@ const getArtisticUniverse = async (max_time) => { let royaltiesQuery = conseiljs.ConseilQueryBuilder.blankQuery(); royaltiesQuery = conseiljs.ConseilQueryBuilder.addFields(royaltiesQuery, 'key', 'value'); royaltiesQuery = conseiljs.ConseilQueryBuilder.addPredicate(royaltiesQuery, 'big_map_id', conseiljs.ConseilOperator.EQ, [mainnet.nftRoyaltiesMap]) - royaltiesQuery = conseiljs.ConseilQueryBuilder.setLimit(royaltiesQuery, 10_000) + royaltiesQuery = conseiljs.ConseilQueryBuilder.setLimit(royaltiesQuery, 20_000) const royaltiesResult = await conseiljs.TezosConseilClient.getTezosEntityData({ url: conseilServer, apiKey: conseilApiKey, network: 'mainnet' }, 'mainnet', 'big_map_contents', royaltiesQuery) let artistMap = {} royaltiesResult.forEach(row => { @@ -181,9 +257,9 @@ const getArtisticUniverse = async (max_time) => { }) let swapsQuery = conseiljs.ConseilQueryBuilder.blankQuery(); - swapsQuery = conseiljs.ConseilQueryBuilder.addFields(swapsQuery, 'key', 'value'); + swapsQuery = conseiljs.ConseilQueryBuilder.addFields(swapsQuery, 'key', 'value') swapsQuery = conseiljs.ConseilQueryBuilder.addPredicate(swapsQuery, 'big_map_id', conseiljs.ConseilOperator.EQ, [mainnet.nftSwapMap]) - swapsQuery = conseiljs.ConseilQueryBuilder.setLimit(swapsQuery, 10_000) // NOTE, limited to 10_000 + swapsQuery = conseiljs.ConseilQueryBuilder.setLimit(swapsQuery, 20_000) const swapsResult = await conseiljs.TezosConseilClient.getTezosEntityData({ url: conseilServer, apiKey: conseilApiKey, network: 'mainnet' }, 'mainnet', 'big_map_contents', swapsQuery) @@ -219,12 +295,13 @@ const getArtisticUniverse = async (max_time) => { const objectQueries = queryChunks.map(c => makeObjectQuery(c)) + // const a = performance.now() let universe = [] await Promise.all( objectQueries.map(async (q) => { - const r = [] + let r = [] try { - r = await conseiljs.TezosConseilClient.getTezosEntityData({ url: conseilServer, apiKey: conseilApiKey, network: 'mainnet' }, 'mainnet', 'big_map_contents', q) + r = await throttleConseilQuery('big_map_contents', q) .then(result => result.map(row => { const objectId = row['value'].toString().replace(/^Pair ([0-9]{1,}) .*/, '$1') const objectUrl = row['value'].toString().replace(/.* 0x([0-9a-z]{1,}) \}$/, '$1') @@ -232,10 +309,15 @@ const getArtisticUniverse = async (max_time) => { universe.push({ objectId, ipfsHash, minter: artistMap[objectId], swaps: swapMap[objectId] !== undefined ? swapMap[objectId] : []}) })) // NOTE: it's a work in progress, this will drop failed requests and return a smaller set than expected + } catch (error) { + console.log('failed at query', q, 'with error', error) } finally { return r }})) + // const b = performance.now() + // console.log(`time ${b - a}`) + return universe } @@ -245,7 +327,6 @@ const getArtisticUniverse = async (max_time) => { * @param {number} objectId * @returns */ - const getObjectById = async (objectId) => { let objectQuery = conseiljs.ConseilQueryBuilder.blankQuery(); objectQuery = conseiljs.ConseilQueryBuilder.addFields(objectQuery, 'value'); @@ -298,6 +379,12 @@ const chunkArray = (arr, len) => { // TODO: move to util.js return chunks; } +const throttleConseilQuery = throttleConseil(async (table, query) => { + const result = await conseiljs.TezosConseilClient.getTezosEntityData({ url: conseilServer, apiKey: conseilApiKey, network: 'mainnet' }, 'mainnet', table, query) + + return Promise.resolve(result) +}) + module.exports = { getCollectionForAddress, gethDaoBalanceForAddress, diff --git a/package.json b/package.json index aebde7f..e0010dd 100644 --- a/package.json +++ b/package.json @@ -8,16 +8,18 @@ }, "repository": { "type": "git", - "url": "git+https://github.com/hicetnunc2000/hicetnunc.git" + "url": "git+https://github.com/hicetnunc2000/hicetnunc-api.git" }, "author": "@hicetnunc2000", "license": "MIT", "bugs": { - "url": "https://github.com/hicetnunc2000/hicetnunc/issues" + "url": "https://github.com/hicetnunc2000/hicetnunc-api/issues" }, - "homepage": "https://github.com/hicetnunc2000/hicetnunc#readme", + "homepage": "https://github.com/hicetnunc2000/hicetnunc-api#readme", "dependencies": { "axios": "^0.21.1", + "bignumber.js": "9.0.1", + "cloud-local-storage": "0.0.11", "conseiljs": "5.0.7-2", "cors": "^2.8.5", "dotenv": "^8.2.0", @@ -25,6 +27,8 @@ "lodash": "^4.17.21", "loglevel": "1.7.1", "node-fetch": "2.6.1", + "perf_hooks": "0.0.1", + "p-throttle": "4.1.1", "serverless-dotenv-plugin": "^3.8.1", "serverless-http": "^2.7.0" },