Event IDs 5300/1106: Scan Engine Failed - Fix Guide 2025

Log Name:      Application
Source:        MSExchange Antimalware
Event ID:      5300
Level:         Error
Description:   The antimalware engine encountered an error trying
               to scan this message. The message was not scanned.
               Error: 0x80004005

Log Name:      Application
Source:        Microsoft-Windows-CAPI2
Event ID:      1106
Level:         Error
Description:   Cryptographic Services failed while processing the
               OnIdentity() call. Error: This implementation is
               not part of the Windows Platform FIPS validated
               cryptographic algorithms.

# Check current FIPS status
$fipsKey = "HKLM:\SYSTEM\CurrentControlSet\Control\Lsa\FipsAlgorithmPolicy"
Get-ItemProperty $fipsKey

# Disable FIPS mode via registry (requires reboot)
Set-ItemProperty -Path $fipsKey -Name "Enabled" -Value 0

# Alternative: Disable via Group Policy
# Computer Configuration > Windows Settings > Security Settings >
# Local Policies > Security Options >
# "System cryptography: Use FIPS compliant algorithms" = Disabled

# Reboot server for change to take effect
# Restart-Computer -Force-Force

# Restart Transport service (reloads malware agent)
Restart-Service MSExchangeTransport

# On Edge Transport servers
Restart-Service MSExchangeTransport
Restart-Service MSExchangeAntimalwareUpdateService -ErrorAction SilentlyContinue

# Verify malware agent is running
Get-TransportAgent "Malware Agent" | Select-Object Identity, Enabled, Priority

# Run the update script
& "C:\Program Files\Microsoft\Exchange Server\V15\Scripts\Update-MalwareFilteringServer.ps1"

# If update fails due to network, try with explicit proxy
$proxy = "http://proxy.company.com:8080"
[System.Net.WebRequest]::DefaultWebProxy = New-Object System.Net.WebProxy($proxy)

# Check engine version after update
Get-MalwareFilteringServer | Select-Object Name, EngineVersion, UpdateVersion

# Restart Transport to load new engine
Restart-Service MSExchangeTransport

# Disable the malware agent while troubleshooting
Disable-TransportAgent "Malware Agent"

# Restart Transport
Restart-Service MSExchangeTransport

# IMPORTANT: Re-enable after fixing root cause!
# Enable-TransportAgent "Malware Agent""Malware Agent"
# Restart-Service MSExchangeTransport

# Alternative: Configure to allow mail through without scanning
Set-MalwareFilteringServer -BypassFiltering $true

# Option 1: Use Exchange Online Protection instead of local scanning
# Configure hybrid or EOP standalone for malware filtering

# Option 2: Disable only Exchange's malware agent
Disable-TransportAgent "Malware Agent"
Restart-Service MSExchangeTransport

# Then configure third-party FIPS-compliant malware scanning-compliant malware scanning
# Examples: Symantec Mail Security, McAfee for Exchange

# Option 3: Request Microsoft support for FIPS-compliant engine-compliant engine
# Some Exchange versions have FIPS-compatible scan engines

# Required exclusions for Exchange Server:

# Process exclusions:
# - EdgeTransport.exe
# - MSExchangeTransport.exe
# - Microsoft.Exchange.*.exe
# - W3wp.exe

# Folder exclusions:
$exclusions = @(
    "C:\Program Files\Microsoft\Exchange Server\V15\",
    "C:\Program Files\Microsoft\Exchange Server\V15\FIP-FS\",
    "C:\Program Files\Microsoft\Exchange Server\V15\TransportRoles\",
    "C:\Program Files\Microsoft\Exchange Server\V15\Mailbox\"
)

# For Windows Defender, add exclusions:
foreach ($path in $exclusions) {
    Add-MpPreference -ExclusionPath $path
}

# For third-party AV, configure via their management console
# After adding exclusions, restart Transport
Restart-Service MSExchangeTransport

# Stop Transport service
Stop-Service MSExchangeTransport

# Rename existing engine folder (backup)
$enginePath = "C:\Program Files\Microsoft\Exchange Server\V15\FIP-FS"
Rename-Item $enginePath "$enginePath.bak"

# Run Exchange Setup to repair (this recreates FIP-FS)
$setupPath = "C:\ExchangeSetup\Setup.exe"
& $setupPath /Mode:Install /Role:Mailbox

# Or download fresh engine files from Microsoft Update
# Force full update
& "C:\Program Files\Microsoft\Exchange Server\V15\Scripts\Update-MalwareFilteringServer.ps1" -EngineUpdatePath "C:\Temp\EngineUpdates"

# Start Transport
Start-Service MSExchangeTransport

# Check current setting
Get-MalwareFilteringServer | Select-Object Name, DeferWhenScanErrorDetected

# Enable deferral when scan fails (safer, but causes delays)
Set-MalwareFilteringServer -DeferWhenScanErrorDetected $true

# Configure scan timeout
Set-MalwareFilteringServer -ScanTimeout 300

# Set primary and secondary scan actions
Set-MalwareFilterPolicy "Default" -Action DeleteMessage
# Or use: DeleteAttachmentAndUseDefaultAlertText, DeleteMessage, Block

# Restart Transport for changes
Restart-Service MSExchangeTransport

# 1. Check malware filtering server status
Get-MalwareFilteringServer | Select-Object Name, EngineVersion, Enabled

# 2. Verify malware agent is enabled and working
Get-TransportAgent "Malware Agent" | Select-Object Identity, Enabled

# 3. Check for new scan errors
Get-EventLog -LogName Application -Source "MSExchange Antimalware" -Newest 10 |
  Where-Object {$_.EntryType -eq "Error"}

# 4. Send test message with EICAR test file (safe malware test)
# Create EICAR test string (NOT actual malware)
$eicar = "X5O!P%@AP[4PZX54(P^)7CC)7}" + '$EICAR-STANDARD-ANTIVIRUS-TEST-FILE!$H+H*'-STANDARD-ANTIVIRUS-TEST-FILE!$H+H*'
$eicarPath = "C:\Temp\eicar.txt"
$eicar | Out-File $eicarPath

# Send as attachment - should be blocked
Send-MailMessage -From "admin@company.com" -To "test@company.com" `
  -Subject "Malware Scan Test $(Get-Date)" -Body "Testing malware scanning" `
  -Attachments $eicarPath -SmtpServer localhost

# 5. Verify message was blocked
Get-MessageTrackingLog -Start (Get-Date).AddMinutes(-5) -MessageSubject "Malware Scan Test*" |
  Select-Object Timestamp, EventId, Source, Recipients

# Create scheduled task for daily updates
$action = New-ScheduledTaskAction -Execute "PowerShell.exe" `
  -Argument "-NoProfile -ExecutionPolicy Bypass -File 'C:\Program Files\Microsoft\Exchange Server\V15\Scripts\Update-MalwareFilteringServer.ps1'"-ExecutionPolicy Bypass -File 'C:\Program Files\Microsoft\Exchange Server\V15\Scripts\Update-MalwareFilteringServer.ps1'"

$trigger = New-ScheduledTaskTrigger -Daily -At "2:00AM"

Register-ScheduledTask -TaskName "Exchange Malware Engine Update" `
  -Action $action -Trigger $trigger -User "SYSTEM" -RunLevel Highest

# Run hourly via scheduled task
$events = Get-EventLog -LogName Application -Source "MSExchange Antimalware" `
  -After (Get-Date).AddHours(-1) -EntryType Error -ErrorAction SilentlyContinue

if ($events) {
    $body = "Malware scan engine errors detected:\n"
    $body += ($events | Select-Object TimeGenerated, EventID, Message | Out-String)

    Send-MailMessage -To "admin@company.com" -From "alerts@company.com" `
      -Subject "Exchange Malware Engine Alert" -Body $body `
      -SmtpServer "backup-smtp.company.com"
}