Event ID 4999: High RPC Latency

# Check RPC latency using Performance Counters
$server = $env:COMPUTERNAME
$counters = @(
    "\$server\MSExchangeIS Store(*)RPC Latency average (msec)",
    "\$server\MSExchangeIS Store(*)RPC Requests",
    "\$server\MSExchangeIS Store(*)RPC Operations/sec",
    "\$server\MSExchange RpcClientAccessRPC Averaged Latency",
    "\$server\MSExchange RpcClientAccessRPC Operations/sec"
)

Get-Counter -Counter $counters -SampleInterval 2 -MaxSamples 5 |
    ForEach-Object { $_.CounterSamples } |
    Format-Table Path, CookedValue -AutoSize

# Quick one-liner for current latency
Get-Counter "\$server\MSExchangeIS Store(*)RPC Latency average (msec)" |
    Select-Object -ExpandProperty CounterSamples | Format-Table InstanceName, CookedValue

# Check database I/O latency
$counters = @(
    "\$server\MSExchange Database ==> Instances(*)I/O Database Reads (Attached) Average Latency",
    "\$server\MSExchange Database ==> Instances(*)I/O Database Writes (Attached) Average Latency",
    "\$server\MSExchange Database ==> Instances(*)I/O Log Writes Average Latency",
    "\$server\MSExchange Database ==> Instances(*)Database Page Fault Stalls/sec"
)

Get-Counter -Counter $counters -SampleInterval 2 -MaxSamples 3 |
    ForEach-Object { $_.CounterSamples } |
    Where-Object { $_.CookedValue -gt 0 } |
    Format-Table Path, @{N="Latency(ms)";E={[math]::Round($_.CookedValue,2)}} -AutoSize

# Check for database copy status (DAG environments)
Get-MailboxDatabaseCopyStatus * | Format-Table Name, Status, CopyQueueLength, ReplayQueueLength, ContentIndexState

# Check CPU and Memory usage
Get-Counter @(
    "\$server\Processor(_Total)\% Processor Time",
    "\$server\Memory\Available MBytes",
    "\$server\Memory\Pages/sec",
    "\$server\Paging File(_Total)\% Usage"
) | ForEach-Object { $_.CounterSamples } | Format-Table Path, CookedValue

# Check Exchange process memory usage
Get-Process -Name MSExchangeIS, Microsoft.Exchange.Store.Worker, w3wp |
    Select-Object Name, @{N='CPU';E={$_.CPU}}, @{N='Memory(MB)';E={[math]::Round($_.WorkingSet64/1MB,2)}},
    @{N='Handles';E={$_.HandleCount}}, @{N='Threads';E={$_.Threads.Count}} |
    Format-Table -AutoSize

# Check database cache size
$cacheCounter = "\$server\MSExchange Database(*)Database Cache Size (MB)"
Get-Counter $cacheCounter | Select-Object -ExpandProperty CounterSamples |
    Format-Table InstanceName, @{N="CacheMB";E={[math]::Round($_.CookedValue,0)}}

# Find mailboxes with most RPC operations
Get-MailboxStatistics -Server $server |
    Sort-Object LastLogonTime -Descending |
    Select-Object DisplayName, ItemCount, TotalItemSize, LastLogonTime, DatabaseName |
    Select-Object -First 20 | Format-Table -AutoSize

# Check for large mailboxes that might cause issues
Get-MailboxStatistics -Server $server |
    Where-Object {$_.TotalItemSize.Value.ToMB() -gt 10000} |
    Select-Object DisplayName, ItemCount, @{N='SizeGB';E={[math]::Round($_.TotalItemSize.Value.ToGB(),2)}} |
    Sort-Object SizeGB -Descending | Format-Table

# Find users with excessive folder counts
Get-MailboxFolderStatistics -Identity "user@domain.com" |
    Measure-Object | Select-Object -ExpandProperty Count

# Check RPC client connections
Get-Counter "\$server\MSExchange RpcClientAccess\Connection Count" |
    Select-Object -ExpandProperty CounterSamples

# Check current disk performance
Get-Counter @(
    "\$server\PhysicalDisk(*)Avg. Disk sec/Read",
    "\$server\PhysicalDisk(*)Avg. Disk sec/Write",
    "\$server\PhysicalDisk(*)Current Disk Queue Length",
    "\$server\PhysicalDisk(*)Disk Reads/sec",
    "\$server\PhysicalDisk(*)Disk Writes/sec"
) -SampleInterval 5 -MaxSamples 6 |
    ForEach-Object { $_.CounterSamples } |
    Where-Object {$_.InstanceName -ne "_total"} |
    Format-Table Path, CookedValue -AutoSize

# Disk latency targets:
# - Database reads: < 20ms
# - Database writes: < 20ms
# - Log writes: < 10ms

# If latency is high, consider:
# 1. Migrate databases to SSD/NVMe storage
# 2. Separate database and log files to different volumes
# 3. Enable storage tiering if using hybrid arrays
# 4. Check RAID controller cache settings (ensure write-back with BBU)-back with BBU)

# Move database to faster storage
$db = Get-MailboxDatabase "Mailbox Database 01"
Move-DatabasePath -Identity $db -EdbFilePath "D:ExchangeDatabasesDB01DB01.edb" -LogFolderPath "L:ExchangeLogsDB01"

# Check current memory allocation
Get-Counter @(
    "\$server\MSExchange Database(*)Database Cache Size (MB)",
    "\$server\MSExchange Database(*)Database Cache % Hit",
    "\$server\Memory\Available MBytes",
    "\$server\Memory\% Committed Bytes In Use"
) | ForEach-Object { $_.CounterSamples } | Format-Table Path, CookedValue

# Memory recommendations:
# - Exchange 2019: Minimum 128GB, recommended 256GB for large deployments
# - Database cache should be 25-50% of total RAM-50% of total RAM
# - Cache hit ratio should be > 98%

# Check if memory pressure is limiting cache
$cacheHit = Get-Counter "\$server\MSExchange Database(*)Database Cache % Hit"
$cacheHitValue = ($cacheHit.CounterSamples | Measure-Object CookedValue -Average).Average
if ($cacheHitValue -lt 98) {
    Write-Host "WARNING: Cache hit ratio is $cacheHitValue%. Consider adding RAM." -ForegroundColor Red
}

# Configure page file appropriately
# Rule: Page file = RAM + 10MB (for memory dumps)
# Place on fast SSD, not database drives

# Check for memory-hogging processes
Get-Process | Sort-Object WorkingSet64 -Descending |
    Select-Object -First 10 Name, @{N='MemoryGB';E={[math]::Round($_.WorkingSet64/1GB,2)}}

# Check database white space (indicates fragmentation level)
Get-MailboxDatabase -Status | Select-Object Name, DatabaseSize, AvailableNewMailboxSpace |
    ForEach-Object {
        $percentFree = [math]::Round(($_.AvailableNewMailboxSpace.ToMB() / $_.DatabaseSize.ToMB()) * 100, 2)
        [PSCustomObject]@{
            Database = $_.Name
            SizeGB = [math]::Round($_.DatabaseSize.ToGB(), 2)
            FreeSpaceGB = [math]::Round($_.AvailableNewMailboxSpace.ToGB(), 2)
            PercentFree = $percentFree
        }
    } | Format-Table -AutoSize

# Schedule online maintenance (ensure it's enabled)
Get-MailboxDatabase | Set-MailboxDatabase -MaintenanceSchedule "Sun.1:00 AM-Sun.5:00 AM"00 AM-Sun.5:00 AM"

# For severe fragmentation, perform offline defragmentation
# WARNING: Requires database dismount - plan maintenance window!

# 1. Dismount database
Dismount-Database "Mailbox Database 01" -Confirm:$false

# 2. Run ESEUTIL defragmentation
# From command prompt (not PowerShell):
# eseutil /d "D:ExchangeDatabasesDB01DB01.edb" /t "D:Temp	empdfrg.edb""D:Temp	empdfrg.edb"

# 3. Remount database
Mount-Database "Mailbox Database 01"

# Alternative: Create new database and move mailboxes (no downtime per mailbox)
New-MailboxDatabase -Name "Mailbox Database 01-New"-New" -Server EXCH01 -EdbFilePath "D:ExDBDB01New.edb" -LogFolderPath "L:ExLogsDB01New"
Get-Mailbox -Database "Mailbox Database 01" | New-MoveRequest -TargetDatabase "Mailbox Database 01-New"-New"

# Check current RPC connection count
Get-Counter "\$server\MSExchange RpcClientAccess\Connection Count" |
    Select-Object -ExpandProperty CounterSamples

# Set RPC throttling policies
# Create a new throttling policy for heavy users
New-ThrottlingPolicy -Name "HighVolumeUsersPolicy" -RcaMaxConcurrency 20 -EwsMaxConcurrency 20 -CpaMaxConcurrency 20

# Apply to specific users
Set-Mailbox -Identity "heavyuser@domain.com" -ThrottlingPolicy "HighVolumeUsersPolicy"

# Configure RPC Client Access settings
Get-RpcClientAccess | Set-RpcClientAccess -MaximumConnections 65535 -EncryptionRequired $true

# Implement connection limits per user (registry - requires restart)
# HKEY_LOCAL_MACHINESYSTEMCurrentControlSetServicesMSExchangeRPCParametersSystem
# MaxConnections REG_DWORD = 5000

# Force Cached Exchange Mode for Outlook clients
# This reduces server load significantly
# Use Group Policy: User Configuration > Administrative Templates > Microsoft Outlook > Exchange