Skip to content

Commit 3bf91d5

Browse files
committed
fix: rate limit
1 parent 3d01273 commit 3bf91d5

File tree

1 file changed

+18
-3
lines changed

1 file changed

+18
-3
lines changed

src/gateway.py

Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -47,10 +47,25 @@ async def refresh_workers():
4747

4848
def get_next_worker(model: str = "") -> Optional[dict]:
4949
cache = _worker_cache
50-
available = cache["workers"]
51-
if not available:
50+
workers = cache["workers"]
51+
if not workers:
5252
return None
53-
worker = available[cache["index"] % len(available)]
53+
54+
candidates = workers
55+
if model:
56+
current_time = time.time()
57+
candidates = []
58+
for w in workers:
59+
limits = w.get("rate_limited_models", {})
60+
if model in limits:
61+
if limits[model] > current_time:
62+
continue
63+
candidates.append(w)
64+
65+
if not candidates:
66+
return None
67+
68+
worker = candidates[cache["index"] % len(candidates)]
5469
cache["index"] += 1
5570
return worker
5671

0 commit comments

Comments
 (0)