Skip to content

Improves the performance of wallet import and address discovery by optimizing rate limiting! #528

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jamespepper81 merged 2 commits into from
Jan 29, 2026
Merged

Improves the performance of wallet import and address discovery by optimizing rate limiting! #528

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
155be0f
perf: optimize wallet import transaction fetching speed
claude Jan 28, 2026
d1c516a
Merge pull request #527 from BitSleuthAI/claude/optimize-wallet-impor…
jamespepper81 Jan 29, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 8 additions & 7 deletions services/esplora-service.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,13 +19,14 @@ const MEMPOOL_SPACE_API_BASE = 'https://mempool.space/api';

const ESPLORA_BASES = [BLOCKSTREAM_API_BASE, MEMPOOL_SPACE_API_BASE];

// Rate limiting configuration - PRODUCTION OPTIMIZED
// Based on Blockstream Green (1000ms), Trust Wallet (1500ms), and Bluewallet (800ms) best practices
// Blockstream public API allows ~10 req/sec = 100ms, but bursts cause 429s
// Conservative approach: 1000ms base delay with exponential backoff for retries
const RATE_LIMIT_DELAY_MS = 1000; // Increased from 400ms to 1000ms - more conservative
const MAX_CONCURRENT_REQUESTS = 1; // Reduced to 1 to completely avoid race conditions and 429s
const RATE_LIMIT_JITTER_MS = 200; // Add random jitter to avoid thundering herd
// Rate limiting configuration - OPTIMIZED FOR WALLET IMPORT PERFORMANCE
// Blockstream public API allows ~10 req/sec = 100ms, mempool.space is similar
// Previous: 1000ms delay + 1 concurrent = ~1 req/sec (too slow for wallet import)
// New: 350ms delay + 3 concurrent = ~8.5 req/sec (within limits, much faster import)
// Circuit breaker + exponential backoff handles any 429 errors gracefully
const RATE_LIMIT_DELAY_MS = 350; // Reduced from 1000ms - balance of speed and reliability
const MAX_CONCURRENT_REQUESTS = 3; // Increased from 1 - allows parallel requests for faster discovery
const RATE_LIMIT_JITTER_MS = 100; // Reduced jitter for more predictable timing

// Request queue for rate limiting with deduplication
class RequestQueue {
Expand Down
113 changes: 63 additions & 50 deletions services/wallet-service.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -160,78 +160,88 @@ export async function discoverUsedAddresses(xpub: string, returnMetadata: boolea
throw new Error('Upstream data provider is temporarily unavailable. Please try again in a moment.');
}

// Check both external (0) and internal (1) chains
for (const chain of [0, 1]) {
console.log(`🔍 Checking ${chain === 0 ? 'external' : 'internal'} chain...`);
// Helper function to discover addresses for a single chain
async function discoverChain(chain: number): Promise<{
usedAddresses: string[];
metadata: Array<{ address: string; index: number; chain: number; isUsed: boolean }>;
}> {
const chainName = chain === 0 ? 'external' : 'internal';
console.log(`🔍 Checking ${chainName} chain...`);

const chainUsedAddresses: string[] = [];
const chainMetadata: Array<{ address: string; index: number; chain: number; isUsed: boolean }> = [];
let gap = 0;
let index = 0;

while (gap < GAP_LIMIT) {
const batch = await deriveAddressBatch(node, chain, index, index + GAP_LIMIT);
console.log(`🔍 Checking batch ${index}-${index + GAP_LIMIT - 1} (${batch.length} addresses)`);

// OPTIMIZED: Sequential requests with aggressive rate limiting (1000ms+ per request)
// Based on best practices from Blockstream Green, Trust Wallet, and Bluewallet
// Using sequential requests to completely avoid 429 errors
// The request queue in esplora-service handles deduplication and rate limiting
const addressTxs: any[] = new Array(batch.length);

// Process addresses sequentially (not in parallel)
for (let i = 0; i < batch.length; i++) {
const addr = batch[i];
console.log(`🔍 [${chainName}] Checking batch ${index}-${index + GAP_LIMIT - 1} (${batch.length} addresses)`);

// OPTIMIZED: Fetch transaction counts for batch in parallel
// Rate limiting is handled at the request queue level
const addressTxsPromises = batch.map(async (addr, i) => {
const addressIndex = index + i;

try {
const result = await esploraGet(`/address/${addr}/txs`, 900000, xpub);
const txCount = Array.isArray(result) ? result.length : 0;
if (txCount > 0) {
console.log(`✅ Address ${addressIndex}: ${txCount} transactions`);
console.log(`✅ [${chainName}] Address ${addressIndex}: ${txCount} transactions`);
}
addressTxs[i] = result;
return result;
} catch (error) {
console.warn(`⚠️ Failed to check address ${addressIndex}:`, error);
addressTxs[i] = [];
console.warn(`⚠️ [${chainName}] Failed to check address ${addressIndex}:`, error);
return [];
}

// No additional delay needed - rate limiting is handled in esplora-service
// The request queue enforces 1000-1200ms between requests automatically (1000ms base + 0-200ms jitter)
}

});

const addressTxs = await Promise.all(addressTxsPromises);

// Process addresses in order and track gap correctly
for (let i = 0; i < addressTxs.length; i++) {
const addressTxsResult = addressTxs[i];
const isUsed = addressTxsResult && Array.isArray(addressTxsResult) && addressTxsResult.length > 0;
const addressIndex = index + i;

if (returnMetadata) {
allAddressMetadata.push({
address: batch[i],
index: addressIndex,
chain,
isUsed
});
}


chainMetadata.push({
address: batch[i],
index: addressIndex,
chain,
isUsed
});

if (isUsed) {
allUsedAddresses.push(batch[i]);
chainUsedAddresses.push(batch[i]);
gap = 0; // Reset gap when we find a used address
console.log(`✅ Found used address at index ${addressIndex}: ${batch[i].substring(0, 10)}... (${addressTxsResult.length} txs)`);
console.log(`✅ [${chainName}] Found used address at index ${addressIndex}: ${batch[i].substring(0, 10)}... (${addressTxsResult.length} txs)`);
} else {
gap++; // Increment gap for unused address
}
}

// Check if we've reached the gap limit
if (gap >= GAP_LIMIT) {
console.log(`🔍 Gap limit reached for ${chain === 0 ? 'external' : 'internal'} chain at index ${index + gap - 1}`);
console.log(`🔍 [${chainName}] Gap limit reached at index ${index + gap - 1}`);
break;
}

index += GAP_LIMIT;

// No additional delay needed between batches
// The request queue in esplora-service handles all rate limiting (1000-1200ms per request)
}

return { usedAddresses: chainUsedAddresses, metadata: chainMetadata };
}

// OPTIMIZATION: Discover both chains in parallel
// External chain (0) for receiving addresses, internal chain (1) for change addresses
console.log(`🔍 Starting parallel chain discovery...`);
const [externalResult, internalResult] = await Promise.all([
discoverChain(0),
discoverChain(1)
]);

// Merge results from both chains
allUsedAddresses = [...externalResult.usedAddresses, ...internalResult.usedAddresses];
if (returnMetadata) {
allAddressMetadata = [...externalResult.metadata, ...internalResult.metadata];
}

// Cache the metadata for future use
Expand Down Expand Up @@ -303,18 +313,21 @@ export async function getWalletData(xpub: string): Promise<{ data: any | null; e
const utxos: any[] = [];
const addressInfos: any[] = [];

// Process addresses sequentially (not in parallel)
// Process addresses with parallelized API calls per address
// Rate limiting is handled at the request queue level in esplora-service
for (let idx = 0; idx < usedAddresses.length; idx++) {
const address = usedAddresses[idx];
try {
console.log(`📊 Processing address ${idx + 1}/${usedAddresses.length}: ${address.substring(0, 10)}...`);

// Fetch UTXOs first (most important for balance)
// Then fetch transactions and stats
// Sequential to respect rate limits
const utxosResult = await getAddressUTXOs(address, xpub);
const txsResult = await getAddressTransactions(address, xpub);
const statsResult = await getAddressStats(address, xpub);

// OPTIMIZATION: Fetch UTXOs, transactions, and stats in parallel
// These are independent requests for the same address
// Rate limiting is enforced by the request queue, not here
const [utxosResult, txsResult, statsResult] = await Promise.all([
getAddressUTXOs(address, xpub),
getAddressTransactions(address, xpub),
getAddressStats(address, xpub)
]);

if (txsResult.data && Array.isArray(txsResult.data)) {
// Add all transactions to the map
Expand Down
Loading