From 65939ea7646ed477d6bfa5a9d935636b67bcf8fd Mon Sep 17 00:00:00 2001 From: Vladimir Svoboda Date: Wed, 26 Mar 2025 16:14:53 +0100 Subject: [PATCH 1/6] feat: magic eden: parametrize listing/offers max limits --- blockapi/v2/api/nft/magic_eden.py | 32 +++++++++++++++++++++++++------ 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/blockapi/v2/api/nft/magic_eden.py b/blockapi/v2/api/nft/magic_eden.py index 202667a2..e1bc7e22 100644 --- a/blockapi/v2/api/nft/magic_eden.py +++ b/blockapi/v2/api/nft/magic_eden.py @@ -46,10 +46,11 @@ class MagicEdenApi(BlockchainApi, INftProvider, INftParser): coin_map = NotImplemented - def __init__(self, sleep_provider): + def __init__(self, sleep_provider, max_listings=15000, max_offers=15000): super().__init__() - self._sleep_provider = sleep_provider + self.max_listings = max_listings + self.max_offers = max_offers def fetch_nfts(self, address: str) -> FetchResult: offset = 0 @@ -192,7 +193,12 @@ def fetch_offers( data_len = len(results) offset += limit - if data.errors or not data.data or data_len < limit or offset > 15000: + if ( + data.errors + or not data.data + or data_len < limit + or offset > self.max_offers + ): return FetchResult( status_code=data.status_code, headers=data.headers, @@ -257,7 +263,11 @@ def _get_offer_price(self, item: dict) -> str: return str(spot_price * (1 - seller_fee - takers_fee - lp_fee)) def fetch_listings( - self, collection: str, cursor: Optional[str] = None + self, + collection: str, + cursor: Optional[str] = None, + sort="updatedAt", + sort_direction="desc", ) -> FetchResult: offset = 0 limit = 100 @@ -266,7 +276,12 @@ def fetch_listings( while True: self._sleep_provider.sleep(self.base_url, self.api_options.rate_limit) data = self.get_data( - 'get_listings', slug=collection, offset=offset, limit=limit + 'get_listings', + slug=collection, + offset=offset, + limit=limit, + sort=sort, + sort_direction=sort_direction, ) if self._should_retry(data): @@ -277,7 +292,12 @@ def fetch_listings( offset += limit # note: if offset is greater than 15000, causes response "offset should be non-negative integer" - if data.errors or not data.data or len(data.data) < limit or offset > 15000: + if ( + data.errors + or not data.data + or len(data.data) < limit + or offset > self.max_listings + ): return FetchResult( status_code=data.status_code, headers=data.headers, From 0d70224a8559ae41059083f8797421599bec203e Mon Sep 17 00:00:00 2001 From: "marek.galvanek" Date: Wed, 6 Aug 2025 11:15:30 +0200 Subject: [PATCH 2/6] fix: adjust max_listings and max_offers logic, add warning for cap limit --- blockapi/v2/api/nft/magic_eden.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/blockapi/v2/api/nft/magic_eden.py b/blockapi/v2/api/nft/magic_eden.py index e1bc7e22..ac7b3db1 100644 --- a/blockapi/v2/api/nft/magic_eden.py +++ b/blockapi/v2/api/nft/magic_eden.py @@ -46,11 +46,18 @@ class MagicEdenApi(BlockchainApi, INftProvider, INftParser): coin_map = NotImplemented - def __init__(self, sleep_provider, max_listings=15000, max_offers=15000): + def __init__(self, sleep_provider, max_listings=200, max_offers=200): super().__init__() self._sleep_provider = sleep_provider - self.max_listings = max_listings + self.max_offers = max_offers + if max_listings > 15000: + logger.warning( + 'Listings cap exceeding 15000 will cause an error. Setting maximum to 15000.' + ) + self.max_listings = 15000 + else: + self.max_listings = max_listings def fetch_nfts(self, address: str) -> FetchResult: offset = 0 @@ -197,7 +204,7 @@ def fetch_offers( data.errors or not data.data or data_len < limit - or offset > self.max_offers + or len(items) >= self.max_offers ): return FetchResult( status_code=data.status_code, @@ -296,7 +303,7 @@ def fetch_listings( data.errors or not data.data or len(data.data) < limit - or offset > self.max_listings + or len(items) > self.max_listings ): return FetchResult( status_code=data.status_code, From b0713c865914350d98cc406731397d38c62f976f Mon Sep 17 00:00:00 2001 From: "marek.galvanek" Date: Thu, 7 Aug 2025 12:16:32 +0200 Subject: [PATCH 3/6] feat: add max_listings and max_offers with dynamic limits, improve pagination logic for OpenSea --- blockapi/v2/api/nft/opensea.py | 56 ++++++++++++++++++++++++++++------ 1 file changed, 47 insertions(+), 9 deletions(-) diff --git a/blockapi/v2/api/nft/opensea.py b/blockapi/v2/api/nft/opensea.py index 0adaa1e4..5703ce71 100644 --- a/blockapi/v2/api/nft/opensea.py +++ b/blockapi/v2/api/nft/opensea.py @@ -3,7 +3,6 @@ from decimal import Decimal from typing import Callable, Iterable, Optional, Tuple -from blockapi.utils.num import raw_to_decimals from blockapi.v2.base import ( ApiException, BlockchainApi, @@ -98,6 +97,8 @@ def __init__( blockchain: Blockchain, sleep_provider: ISleepProvider = None, limit: Optional[int] = None, + max_listings=500, + max_offers=500, ): super().__init__(api_key) @@ -110,6 +111,9 @@ def __init__( self._sleep_provider = sleep_provider or SleepProvider() self._limit = limit + self.max_listings = max_listings + self.max_offers = max_offers + def fetch_nfts(self, address: str, cursor: Optional[str] = None) -> FetchResult: logger.info(f'Fetch nfts from {address}, cursor={cursor}') return self._coalesce(self._yield_nfts(address, cursor)) @@ -256,17 +260,32 @@ def _yield_fetch_data( self, fetch_method: Callable, key: str, cursor: Optional[str] = None ) -> Iterable[Tuple[FetchResult, Optional[str]]]: cursors = set() + page_count = 0 + item_count = 0 - count = 0 while True: self._sleep_provider.sleep(self.base_url, self.api_options.rate_limit) - count += 1 - logger.debug(f'Fetching page {count} of {key} from {cursor}') + page_count += 1 + logger.debug(f'Fetching page {page_count} of {key} from {cursor}') fetched, next_cursor = fetch_method(key, cursor) if self._should_retry(fetched): - count -= 1 + page_count -= 1 continue + # Count items for dynamic limiting + item_limit = None + if fetched.data: + current_items = 0 + if 'offers' in fetched.data: + current_items = len(fetched.data['offers']) + item_limit = self.max_offers + elif 'listings' in fetched.data: + current_items = len(fetched.data['listings']) + item_limit = self.max_listings + # skip `'nfts' in fetched.data`, because it doesn't have a limit + + item_count += current_items + yield fetched, next_cursor if not next_cursor: @@ -278,13 +297,19 @@ def _yield_fetch_data( cursors.add(cursor) - if self._limit and count >= self._limit: + # Check both page and item limits + if self._limit and page_count >= self._limit: + break + + if item_limit and item_count >= item_limit: break def _fetch_offers_page( self, method: str, collection: str, cursor: Optional[str] = None ) -> tuple[FetchResult, Optional[str]]: - params = dict(next=cursor) if cursor else dict() + params = {'limit': 100} # the max limit allowed is 100 items per page + if cursor: + params['next'] = cursor fetched = self.get_data( method, @@ -567,15 +592,17 @@ def _get_type(item_type) -> Optional[OfferItemType]: return OFFER_ITEM_TYPES.get(item_type) - @staticmethod - def _coalesce(fetch_results: Iterable[Tuple[FetchResult, Optional[str]]]): + def _coalesce(self, fetch_results: Iterable[Tuple[FetchResult, Optional[str]]]): + """Simple aggregator - no limiting logic needed here.""" data = [] errors = [] last = None last_cursor = None + for item, cursor in fetch_results: last_cursor = cursor last = item + if item.data: data.append(item.data) @@ -636,3 +663,14 @@ def _should_retry(self, data): return True return False + + +if __name__ == '__main__': + col = 'ever-fragments-of-civitas' + o = OpenSeaApi( + '6c0e52527b124aeeb0bebbcfbaf2e7b6', Blockchain.ETHEREUM, max_listings=100 + ) + # offers = o.fetch_offers(col) + listings = o.fetch_listings(col) + + assert True From 320533ea4900022027c2b6dfe25c85168cca05fe Mon Sep 17 00:00:00 2001 From: "marek.galvanek" Date: Thu, 7 Aug 2025 12:22:03 +0200 Subject: [PATCH 4/6] fix: cleanup OpenSea temporary tests, adjust Magic Eden defaults and sorting logic --- blockapi/v2/api/nft/magic_eden.py | 7 ++++--- blockapi/v2/api/nft/opensea.py | 11 ----------- 2 files changed, 4 insertions(+), 14 deletions(-) diff --git a/blockapi/v2/api/nft/magic_eden.py b/blockapi/v2/api/nft/magic_eden.py index ac7b3db1..691fa182 100644 --- a/blockapi/v2/api/nft/magic_eden.py +++ b/blockapi/v2/api/nft/magic_eden.py @@ -46,14 +46,15 @@ class MagicEdenApi(BlockchainApi, INftProvider, INftParser): coin_map = NotImplemented - def __init__(self, sleep_provider, max_listings=200, max_offers=200): + def __init__(self, sleep_provider, max_listings=500, max_offers=500): super().__init__() self._sleep_provider = sleep_provider self.max_offers = max_offers if max_listings > 15000: logger.warning( - 'Listings cap exceeding 15000 will cause an error. Setting maximum to 15000.' + 'Listings cap exceeding 15000 will cause an error.' + ' Setting maximum to 15000.' ) self.max_listings = 15000 else: @@ -273,7 +274,7 @@ def fetch_listings( self, collection: str, cursor: Optional[str] = None, - sort="updatedAt", + sort="listPrice", sort_direction="desc", ) -> FetchResult: offset = 0 diff --git a/blockapi/v2/api/nft/opensea.py b/blockapi/v2/api/nft/opensea.py index 5703ce71..72657395 100644 --- a/blockapi/v2/api/nft/opensea.py +++ b/blockapi/v2/api/nft/opensea.py @@ -663,14 +663,3 @@ def _should_retry(self, data): return True return False - - -if __name__ == '__main__': - col = 'ever-fragments-of-civitas' - o = OpenSeaApi( - '6c0e52527b124aeeb0bebbcfbaf2e7b6', Blockchain.ETHEREUM, max_listings=100 - ) - # offers = o.fetch_offers(col) - listings = o.fetch_listings(col) - - assert True From 771f34e647a617c9473d7c88627819526acfbbf7 Mon Sep 17 00:00:00 2001 From: "marek.galvanek" Date: Thu, 7 Aug 2025 12:24:28 +0200 Subject: [PATCH 5/6] fix: make _coalesce method static in OpenSea API implementation --- blockapi/v2/api/nft/opensea.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/blockapi/v2/api/nft/opensea.py b/blockapi/v2/api/nft/opensea.py index 72657395..3ea5809f 100644 --- a/blockapi/v2/api/nft/opensea.py +++ b/blockapi/v2/api/nft/opensea.py @@ -592,8 +592,8 @@ def _get_type(item_type) -> Optional[OfferItemType]: return OFFER_ITEM_TYPES.get(item_type) - def _coalesce(self, fetch_results: Iterable[Tuple[FetchResult, Optional[str]]]): - """Simple aggregator - no limiting logic needed here.""" + @staticmethod + def _coalesce(fetch_results: Iterable[Tuple[FetchResult, Optional[str]]]): data = [] errors = [] last = None From d7ffbcb2157033c26d6ffb7017ab61f74b4d103f Mon Sep 17 00:00:00 2001 From: "marek.galvanek" Date: Thu, 7 Aug 2025 12:53:40 +0200 Subject: [PATCH 6/6] fix: clarify warning message for listings cap in Magic Eden API --- blockapi/v2/api/nft/magic_eden.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/blockapi/v2/api/nft/magic_eden.py b/blockapi/v2/api/nft/magic_eden.py index 691fa182..c38fde61 100644 --- a/blockapi/v2/api/nft/magic_eden.py +++ b/blockapi/v2/api/nft/magic_eden.py @@ -53,8 +53,9 @@ def __init__(self, sleep_provider, max_listings=500, max_offers=500): self.max_offers = max_offers if max_listings > 15000: logger.warning( - 'Listings cap exceeding 15000 will cause an error.' - ' Setting maximum to 15000.' + 'Listings cap exceeding 15000 will cause an error: ' + '"offset should be non-negative integer". ' + 'Setting maximum to 15000.' ) self.max_listings = 15000 else: