hjnormey/siloqy
Public Access
1
0
Fork 0
Code Issues 7 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

feat(rate-limit-general-check): Added general rate limit check upon startup.

Browse Source
This commit is contained in:
HJ Normey
2025-09-05 14:44:46 +02:00
parent ba220f2380
commit 6d12eb9497
1 changed files with 90 additions and 66 deletions
Download Patch File Download Diff File Expand all files Collapse all files

158
nautilus_actor_test_implementation_5x.py
View File

@@ -412,77 +412,101 @@ class SILOQYSymbolDiscoveryActor(Actor):
self.log.error(f"Nautilus ActorExecutor: Failed to complete symbol discovery: {e}")
# Don't re-raise, let system continue
async def _discover_all_symbols(self):
"""PRESERVED: Original Binance symbol discovery algorithm"""
self.log.info("Starting dynamic symbol discovery from Binance...")
url = "https://api.binance.com/api/v3/exchangeInfo"
async def _discover_all_symbols(self):
"""PRESERVED: Original Binance symbol discovery algorithm"""
self.log.info("Starting dynamic symbol discovery from Binance...")
url = "https://api.binance.com/api/v3/exchangeInfo"
async with httpx.AsyncClient() as client:
self.log.info("Fetching exchange info from Binance API...")
response = await client.get(url, timeout=10)
if response.status_code == 200:
self.log.info("Successfully received exchange info")
data = response.json()
async with httpx.AsyncClient() as client:
# RATE LIMIT CHECK - Before anything else
self.log.info("Checking Binance API rate limit status...")
time_check_url = "https://api.binance.com/api/v3/time"
# STEP 1: Collect all symbol names first
self.log.info("Collecting all available symbols...")
full_symbols = []
for symbol_info in data['symbols']:
if symbol_info['status'] == 'TRADING' and symbol_info['symbol'].endswith('USDT'):
symbol = symbol_info['symbol']
full_symbols.append(symbol)
try:
rate_check_response = await client.get(time_check_url, timeout=5)
# Log the full symbols list obtained from exchange
self.log.info(f"Full symbols discovered from exchange: {len(full_symbols)} total")
self.log.info(f"First 10 symbols: {full_symbols[:10]}")
if rate_check_response.status_code == 200:
# Parse rate limit headers
headers = rate_check_response.headers
used_weight = headers.get('X-MBX-USED-WEIGHT-1M', 'Unknown')
server_time = rate_check_response.json().get('serverTime', 'Unknown')
# STEP 2: Apply throttle mode symbol limiting BEFORE processing
if self.throttle_mode:
symbols_to_process = full_symbols[:self.max_symbols_throttled]
self.log.warning(f"THROTTLE MODE: Will process {len(symbols_to_process)} symbols (from {len(full_symbols)} available)")
self.log.info(f"Rate limit check passed - Used weight: {used_weight}/1200, Server time: {server_time}")
# Check if we're close to rate limit
if used_weight != 'Unknown' and int(used_weight) > 1000:
self.log.warning(f"HIGH RATE LIMIT USAGE: {used_weight}/1200 - Proceeding with caution")
elif rate_check_response.status_code == 429:
retry_after = rate_check_response.headers.get('Retry-After', '60')
self.log.error(f"RATE LIMITED: Must wait {retry_after} seconds before API calls")
raise Exception(f"Binance API rate limited - retry after {retry_after}s")
elif rate_check_response.status_code == 418:
self.log.error("IP BANNED: This IP address has been auto-banned by Binance")
raise Exception("IP address banned by Binance - cannot proceed")
else:
self.log.warning(f"Rate limit check returned status {rate_check_response.status_code}")
self.log.warning("Proceeding anyway, but may encounter issues")
except Exception as e:
if "rate limited" in str(e).lower() or "banned" in str(e).lower():
raise # Re-raise rate limit/ban errors
else:
self.log.warning(f"Rate limit check failed: {e}")
self.log.warning("Proceeding with symbol discovery anyway")
async with httpx.AsyncClient() as client:
self.log.info("Fetching exchange info from Binance API...")
response = await client.get(url, timeout=10)
if response.status_code == 200:
self.log.info("Successfully received exchange info")
data = response.json()
# Combined symbol discovery and tick size extraction
self.log.info("Processing symbols and extracting tick sizes...")
full_symbols = []
for symbol_info in data['symbols']:
if symbol_info['status'] == 'TRADING' and symbol_info['symbol'].endswith('USDT'):
symbol = symbol_info['symbol']
full_symbols.append(symbol)
# Extract tick size while processing # Extract tick size while processing
tick_size = None
for filter_info in symbol_info['filters']:
if filter_info['filterType'] == 'PRICE_FILTER':
tick_size = float(filter_info['tickSize'])
break
# If no PRICE_FILTER found, try other filter types
if tick_size is None:
for filter_info in symbol_info['filters']:
if filter_info['filterType'] == 'TICK_SIZE':
tick_size = float(filter_info['tickSize'])
break
# Fallback to default if still not found
if tick_size is None:
tick_size = 1e-8 # Default fallback
self.log.warning(f"No tick size found for {symbol}, using fallback {tick_size}")
self.tick_sizes[symbol] = tick_size
self.log.info(f"Processed {len(full_symbols)} symbols, extracted {len(self.tick_sizes)} tick sizes")
# Apply throttle mode symbol limiting
if self.throttle_mode:
self.symbols = full_symbols[:self.max_symbols_throttled]
self.log.warning(f"THROTTLE MODE: Limited to {len(self.symbols)} symbols (from {len(full_symbols)} available)")
else:
self.symbols = full_symbols
self.log.info(f"Discovered {len(self.symbols)} trading symbols")
self.log.info(f"First 10 symbols: {self.symbols[:10]}")
else:
symbols_to_process = full_symbols
# STEP 3: Process tick sizes only for selected symbols
self.log.info("Processing symbols and extracting tick sizes...")
symbols_to_process_set = set(symbols_to_process) # For O(1) lookup
for symbol_info in data['symbols']:
symbol = symbol_info['symbol']
# Only process symbols that are in our selected list
if symbol not in symbols_to_process_set:
continue
# Extract tick size while processing
tick_size = None
for filter_info in symbol_info['filters']:
if filter_info['filterType'] == 'PRICE_FILTER':
tick_size = float(filter_info['tickSize'])
break
# If no PRICE_FILTER found, try other filter types
if tick_size is None:
for filter_info in symbol_info['filters']:
if filter_info['filterType'] == 'TICK_SIZE':
tick_size = float(filter_info['tickSize'])
break
# Fallback to default if still not found
if tick_size is None:
tick_size = 1e-8 # Default fallback
self.log.warning(f"No tick size found for {symbol}, using fallback {tick_size}")
self.tick_sizes[symbol] = tick_size
# Set final symbols list
self.symbols = symbols_to_process
self.log.info(f"Processed {len(symbols_to_process)} symbols, extracted {len(self.tick_sizes)} tick sizes")
self.log.info(f"Discovered {len(self.symbols)} trading symbols")
self.log.info(f"First 10 symbols: {self.symbols[:10]}")
else:
self.log.error(f"Failed to fetch exchange info: {response.status_code}")
raise Exception(f"Failed to fetch exchange info: {response.status_code}")
self.log.error(f"Failed to fetch exchange info: {response.status_code}")
raise Exception(f"Failed to fetch exchange info: {response.status_code}")
async def _fetch_stats_and_reconstruct_candles(self):
"""PRESERVED: All original rate limiting with Nautilus async patterns"""
@@ -1446,7 +1470,7 @@ def test_siloqy_actors_with_nautilus_process_management():
"candle_interval_ms": 15 * 60 * 1000,
"throttle_mode": True, # ENABLED: Safe for dual instance testing
"throttle_rate_limit_seconds": 10.0, # 10s between batches (vs 2.5s)
"max_symbols_throttled": 400 # Only 100 symbols (vs 2000+)
"max_symbols_throttled": 414 # Only 100 symbols (vs 2000+)
}
)