# Error code: 0x800423f3
# Name: VSS_E_WRITERERROR_RETRYABLE

# Common Event Log entries:
Event ID: 12290
Source: VSS
The writer experienced a transient error.
Writer: Microsoft Exchange Writer
Error: VSS_E_WRITERERROR_RETRYABLE

Event ID: 2005
Source: MSExchangeRepl
Backup encountered a retryable error.
Operation will be attempted again.

# vssadmin output when writer is failing:
Writer name: 'Microsoft Exchange Writer'
   Writer Id: {76fe1ac4-15f7-4bcd-987e-8e1acb462fb7}
   State: [8] Failed
   Last error: Retryable error

# Check current writer state
$output = vssadmin list writers 2>&1
$output | Select-String -Pattern "Microsoft Exchange" -Context 0,4

# Check for any failed writers
$output | Select-String -Pattern "State: \[8\] Failed|Last error: Retryable"

# Get recent VSS events
Get-EventLog -LogName Application -Source VSS -Newest 50 | Where-Object { $_.EntryType -eq "Error" } | Select-Object TimeGenerated, EventID, Message | Format-List

# Check disk queue length (should be < 2)
Get-Counter -Counter "\PhysicalDisk(*)\Current Disk Queue Length" -SampleInterval 5 -MaxSamples 3

# Check disk latency (should be < 20ms for Exchange)
Get-Counter -Counter "\PhysicalDisk(*)\Avg. Disk sec/Read", "\PhysicalDisk(*)\Avg. Disk sec/Write" -SampleInterval 5 -MaxSamples 3

# Convert to milliseconds for readability
Get-Counter -Counter "\PhysicalDisk(*)\Avg. Disk sec/Read" | ForEach-Object {
    $_.CounterSamples | ForEach-Object {
        [PSCustomObject]@{
            Disk = $_.InstanceName
            ReadLatencyMs = [math]::Round($_.CookedValue * 1000, 2)
        }
    }
} | Format-Table

# List current shadow copies being created
vssadmin list shadows /for=D:

# Check for running VSS processes
Get-Process -Name VSSVC -ErrorAction SilentlyContinue | Format-List Id, CPU, WorkingSet

# Check Windows backup state
Get-WBJob -Previous 1 | Select-Object JobState, StartTime, EndTime, JobType

# Check if other applications are using VSS
Get-EventLog -LogName Application -Source VSS -Newest 20 | Select-Object TimeGenerated, Message | Format-List

# Stop any pending VSS operations
vssadmin delete shadows /for=D: /oldest /quiet

# Reset VSS writer by restarting services
Write-Host "Resetting VSS writers..."
Restart-Service VSS -Force
Start-Sleep -Seconds 10

# Reset Exchange writer
Restart-Service MSExchangeRepl -Force
Start-Sleep -Seconds 20

# Verify writer is now stable
vssadmin list writers | Select-String -Pattern "Microsoft Exchange" -Context 0,4

Write-Host "VSS reset complete. Retry backup operation."

# Increase VSS timeout for freeze operation
# Default: 60 seconds, increase to 300 for large databases300 for large databases

$regPath = "HKLM:\SOFTWARE\Microsoft\Windows NT\CurrentVersion\SPP\CreateTimeout"
if (!(Test-Path $regPath)) { New-Item -Path $regPath -Force }
Set-ItemProperty -Path $regPath -Name "(Default)" -Value 300 -Type DWord

# Increase Exchange-specific backup timeout
$exchPath = "HKLM:\SOFTWARE\Microsoft\ExchangeServer\v15\Replay\Parameters"
Set-ItemProperty -Path $exchPath -Name "VSSBackupTimeout" -Value 600000 -Type DWord  # Milliseconds

# Increase COM timeout for VSS operations
$comPath = "HKLM:\SOFTWARE\Microsoft\COM3\Timeout"
if (!(Test-Path $comPath)) { New-Item -Path $comPath -Force }
Set-ItemProperty -Path $comPath -Name "Timeout" -Value 300000 -Type DWord

Write-Host "Timeouts configured. Restart server for all changes to take effect."

# Schedule backup during off-peak hours
# Example: 2 AM when email activity is lowest

# Reduce database maintenance during backup window
# Disable online defragmentation temporarily
$db = Get-MailboxDatabase -Identity "MailboxDB01"
Set-MailboxDatabase -Identity $db -MaintenanceSchedule "Sun.1:00 AM-Sun.5:00 AM"00 AM-Sun.5:00 AM"

# If using DAG, backup from passive copy to reduce active server load
# Configure backup software to target passive database copies

# Spread backups across multiple windows
# Don't backup all databases simultaneously

# Check current maintenance schedules
Get-MailboxDatabase | Select-Object Name, MaintenanceSchedule | Format-Table

# PowerShell wrapper for backup with retry logic
function Start-ExchangeBackupWithRetry {
    param(
        [string]$Database,
        [int]$MaxRetries = 3,
        [int]$RetryDelaySeconds = 300
    )

    $attempt = 1
    $success = $false

    while ($attempt -le $MaxRetries -and -not $success) {
        Write-Host "Backup attempt $attempt of $MaxRetries for $Database"$MaxRetries for $Database"

        # Check VSS writer state before attempting
        $writers = vssadmin list writers 2>&1
        if ($writers -match "State: \[8\] Failed") {
            Write-Host "VSS writer in failed state, resetting..."
            Restart-Service VSS, MSExchangeRepl -Force
            Start-Sleep -Seconds 60
        }

        # Attempt backup (replace with your backup command)
        try {
            # Your backup command here
            # Example: wbadmin start backup -backupTarget:E: -include:D: -vssFull -quiet-include:D: -vssFull -quiet
            $success = $true
            Write-Host "Backup successful on attempt $attempt"
        }
        catch {
            Write-Host "Backup failed: $_"
            if ($attempt -lt $MaxRetries) {
                Write-Host "Waiting $RetryDelaySeconds seconds before retry..."
                Start-Sleep -Seconds $RetryDelaySeconds
            }
        }
        $attempt++
    }
    return $success
}

# Check current memory usage
$os = Get-WmiObject Win32_OperatingSystem
$freeMemGB = [math]::Round($os.FreePhysicalMemory/1MB, 2)
$totalMemGB = [math]::Round($os.TotalVisibleMemorySize/1MB, 2)
Write-Host "Memory: $freeMemGB GB free of $totalMemGB GB total"$totalMemGB GB total"

# Exchange should have minimum 8GB, recommended 16GB+
# VSS operations need additional headroom

# Check if memory is being consumed by other processes
Get-Process | Sort-Object WorkingSet -Descending | Select-Object Name, @{N='MemoryMB';E={[math]::Round($_.WorkingSet/1MB,0)}} -First 15

# If memory is low, consider:
# 1. Reducing database cache size temporarily during backup
# 2. Scheduling backup when fewer users are active
# 3. Adding more RAM to the server

# Check Exchange process memory
Get-Process -Name *Exchange*, *EdgeTransport*, *Store* -ErrorAction SilentlyContinue | Select-Object Name, @{N='MemoryGB';E={[math]::Round($_.WorkingSet/1GB,2)}} | Format-Table

# Run test backup
Write-Host "Testing VSS backup..."

# Verify writer is stable
$writers = vssadmin list writers 2>&1
if ($writers -match "Microsoft Exchange Writer.*State: \[1\] Stable") {
    Write-Host "Exchange VSS writer is stable" -ForegroundColor Green
} else {
    Write-Host "Exchange VSS writer may not be ready" -ForegroundColor Yellow
}

# Check I/O before backup
$queueLength = (Get-Counter -Counter "\PhysicalDisk(_Total)\Current Disk Queue Length" -SampleInterval 1 -MaxSamples 5).CounterSamples.CookedValue | Measure-Object -Average
Write-Host "Average disk queue: $([math]::Round($queueLength.Average, 2))"2))"

# Run actual backup test (Windows Server Backup example)
# wbadmin start backup -backupTarget:E: -include:D: -vssFull -quiet-include:D: -vssFull -quiet

# Check backup result
Get-WBJob -Previous 1 | Format-List JobState, StartTime, EndTime, HResult

# Monitor for retryable errors in next 24 hours
Write-Host ""
Write-Host "Monitor these events for next 24 hours:"
Write-Host "- Event ID 12290 (VSS)"
Write-Host "- Event ID 2005 (MSExchangeRepl)"