# Check latency by route
curl -G 'http://prometheus:9090/api/v1/query' \
--data-urlencode 'query=histogram_quantile(0.95, sum(rate(http_request_duration_seconds_bucket[5m])) by (le, route))'
# CPU usage
docker stats nova-backend --no-stream
# Memory usage
docker exec -it nova-backend free -h
# Disk I/O
iostat -x 1 5
# Active queries
docker exec -it nova-postgres psql -U nova -d nova_rewards -c "
SELECT pid, now() - query_start as duration, query
FROM pg_stat_activity
WHERE state = 'active'
ORDER BY duration DESC
LIMIT 10;"
# Slow queries
docker exec -it nova-postgres psql -U nova -d nova_rewards -c "
SELECT query, calls, total_time, mean_time
FROM pg_stat_statements
ORDER BY mean_time DESC
LIMIT 10;"
# Connection count
docker exec -it nova-postgres psql -U nova -d nova_rewards -c "
SELECT count(*) FROM pg_stat_activity;"
# Redis latency
docker exec -it nova-redis redis-cli --latency
# Slow log
docker exec -it nova-redis redis-cli SLOWLOG GET 10
# Memory usage
docker exec -it nova-redis redis-cli INFO memory
# Test database connection
docker exec -it nova-backend time node -e "const {pool} = require('./db'); pool.query('SELECT 1').then(() => console.log('Done'))"
# Test Redis connection
docker exec -it nova-backend time node -e "const redis = require('./lib/redis'); redis.ping().then(() => console.log('Done'))"
# Test external APIs (Stellar)
docker exec -it nova-backend time curl -I https://horizon-testnet.stellar.org/
Symptoms: High database query times, connection pool exhaustion
Solution:
# Identify missing indexes
docker exec -it nova-postgres psql -U nova -d nova_rewards -c "
SELECT schemaname, tablename, attname, n_distinct, correlation
FROM pg_stats
WHERE schemaname = 'public'
ORDER BY abs(correlation) DESC;"
# Add indexes for frequently queried columns
# Example: CREATE INDEX idx_users_wallet ON users(wallet_address);
# Analyze tables
docker exec -it nova-postgres psql -U nova -d nova_rewards -c "ANALYZE;"
Symptoms: Increased database load, slow leaderboard queries
Solution:
# Check cache hit rate
docker exec -it nova-redis redis-cli INFO stats | grep keyspace
# Warm up cache
curl -X POST http://backend:4000/api/admin/cache/warm
# Increase cache TTL if appropriate
Symptoms: All endpoints slow, high CPU usage
Solution:
# Scale up instances (AWS)
aws autoscaling set-desired-capacity \
--auto-scaling-group-name nova-rewards-asg \
--desired-capacity 4
# Or scale Docker Compose
docker-compose up -d --scale backend=3
# Enable rate limiting if not already active
Symptoms: Gradually increasing memory, eventual OOM
Solution:
# Take heap snapshot
docker exec -it nova-backend node --expose-gc -e "
const v8 = require('v8');
const fs = require('fs');
const heapSnapshot = v8.writeHeapSnapshot();
console.log('Heap snapshot written to', heapSnapshot);
"
# Restart service to free memory
docker restart nova-backend
# Investigate memory leak in code
Symptoms: Stellar-related endpoints slow
Solution:
# Check Stellar Horizon status
curl https://horizon-testnet.stellar.org/
# Implement timeout and retry logic
# Add circuit breaker pattern
# Consider caching Stellar responses
// In backend code
const timeout = process.env.REQUEST_TIMEOUT || 30000; // Increase from 5000
# Increase Redis cache TTL
docker exec -it nova-redis redis-cli CONFIG SET maxmemory-policy allkeys-lru
// Add stricter rate limiting
const expensiveEndpointLimiter = rateLimit({
windowMs: 60 * 1000,
max: 10 // Reduce from 100
});