fix: retry busy queue requests

setup.ps1

@@ -440,14 +440,24 @@ function Write-QueueState {
 }
 
 function Invoke-WithQueueStateLock {
-    param([Parameter(Mandatory)][scriptblock]$ScriptBlock)
+    param(
+        [Parameter(Mandatory)][scriptblock]$ScriptBlock,
+        [int]$TimeoutSeconds = 30
+    )
 
     $mutex = New-Object System.Threading.Mutex($false, $QueueStateMutexName)
     $hasLock = $false
     try {
-        $hasLock = $mutex.WaitOne([TimeSpan]::FromSeconds(30))
+        try {
+            $hasLock = $mutex.WaitOne([TimeSpan]::FromSeconds($TimeoutSeconds))
+        }
+        catch [System.Threading.AbandonedMutexException] {
+            $hasLock = $true
+            Write-Log "Recovered abandoned queue state lock '$QueueStateMutexName'." 'WARN'
+        }
+
         if (-not $hasLock) {
-            throw "Timed out waiting for queue state lock '$QueueStateMutexName'."
+            throw (New-Object System.TimeoutException -ArgumentList "Timed out waiting for queue state lock '$QueueStateMutexName'.")
         }
 
         & $ScriptBlock
@@ -459,7 +469,10 @@ function Invoke-WithQueueStateLock {
 }
 
 function Update-QueueState {
-    param([Parameter(Mandatory)][scriptblock]$ScriptBlock)
+    param(
+        [Parameter(Mandatory)][scriptblock]$ScriptBlock,
+        [int]$TimeoutSeconds = 30
+    )
 
     Invoke-WithQueueStateLock -ScriptBlock {
         $state = Read-QueueState
@@ -468,11 +481,41 @@ function Update-QueueState {
         $state.UpdatedAt = (Get-Date).ToString('o')
         Write-QueueState -State $state
         return $state
-    }
+    } -TimeoutSeconds $TimeoutSeconds
 }
 
 function Get-QueueStateSnapshot {
-    Invoke-WithQueueStateLock -ScriptBlock { Read-QueueState }
+    param([int]$TimeoutSeconds = 30)
+
+    Invoke-WithQueueStateLock -ScriptBlock { Read-QueueState } -TimeoutSeconds $TimeoutSeconds
+}
+
+function Test-IsQueueStateLockTimeout {
+    param([Parameter(Mandatory)]$ErrorObject)
+
+    if ($ErrorObject -is [System.Management.Automation.ErrorRecord]) {
+        $exception = $ErrorObject.Exception
+    }
+    elseif ($ErrorObject -is [System.Exception]) {
+        $exception = $ErrorObject
+    }
+    else {
+        return $false
+    }
+
+    while ($exception) {
+        if ($exception -is [System.TimeoutException] -and $exception.Message -like "*$QueueStateMutexName*") {
+            return $true
+        }
+
+        if ($exception.Message -like "*Timed out waiting for queue state lock '$QueueStateMutexName'*") {
+            return $true
+        }
+
+        $exception = $exception.InnerException
+    }
+
+    return $false
 }
 
 function New-QueueItem {
@@ -568,7 +611,10 @@ function Get-RequiredAliasSet {
 }
 
 function Add-QueueItem {
-    param([Parameter(Mandatory)]$Item)
+    param(
+        [Parameter(Mandatory)]$Item,
+        [int]$TimeoutSeconds = 30
+    )
 
     $result = $null
     $null = Update-QueueState -ScriptBlock {
@@ -585,7 +631,7 @@ function Add-QueueItem {
 
         $state.Queue = @($queue + $Item)
         $script:QueueAddResult = [pscustomobject]@{ Added = $true; Item = $Item }
-    }
+    } -TimeoutSeconds $TimeoutSeconds
 
     $result = $script:QueueAddResult
     $script:QueueAddResult = $null
@@ -598,7 +644,8 @@ function Add-SoftwareRequestToQueue {
         [Parameter(Mandatory)][string]$Alias,
         [string]$RequestedBy,
         [string]$RequestedBySid,
-        [ValidateSet('User','Required')][string]$Source = 'User'
+        [ValidateSet('User','Required')][string]$Source = 'User',
+        [int]$TimeoutSeconds = 30
     )
 
     $normalizedAlias = $Alias.Trim().ToLowerInvariant()
@@ -623,10 +670,12 @@ function Add-SoftwareRequestToQueue {
         -RequestedBy $RequestedBy `
         -RequestedBySid $RequestedBySid
 
-    return Add-QueueItem -Item $item
+    return Add-QueueItem -Item $item -TimeoutSeconds $TimeoutSeconds
 }
 
 function Ensure-RequiredSoftwareQueued {
+    param([int]$TimeoutSeconds = 30)
+
     $addedCount = 0
     if (-not (Test-SoftwareCatalogPolicyEnabled)) { return 0 }
     if (-not (Test-RequiredEnforcementEnabled)) { return 0 }
@@ -643,7 +692,7 @@ function Ensure-RequiredSoftwareQueued {
             continue
         }
 
-        $result = Add-SoftwareRequestToQueue -Action Install -Alias $alias -RequestedBy 'Policy' -RequestedBySid 'Policy' -Source Required
+        $result = Add-SoftwareRequestToQueue -Action Install -Alias $alias -RequestedBy 'Policy' -RequestedBySid 'Policy' -Source Required -TimeoutSeconds $TimeoutSeconds
         if ($result -and $result.Added) {
             $addedCount++
             Write-Log "Queued required software alias '$alias' package '$($app.PackageId)'."
@@ -694,6 +743,29 @@ function Test-WingetLockAvailable {
     }
 }
 
+function Test-QueueStateLockAvailable {
+    $mutex = New-Object System.Threading.Mutex($false, $QueueStateMutexName)
+    $hasLock = $false
+
+    try {
+        try {
+            $hasLock = $mutex.WaitOne(0)
+        }
+        catch [System.Threading.AbandonedMutexException] {
+            $hasLock = $true
+            Write-Log "Recovered abandoned queue state lock '$QueueStateMutexName' during availability check." 'WARN'
+        }
+
+        return $hasLock
+    }
+    finally {
+        if ($hasLock) {
+            $mutex.ReleaseMutex()
+        }
+        $mutex.Dispose()
+    }
+}
+
 function ConvertTo-ProcessArgument {
     param([Parameter(Mandatory)][string]$Value)
 
@@ -753,6 +825,21 @@ function Start-QueueWorker {
     Start-SoftwarekatalogWorker -WorkerMode ProcessQueue
 }
 
+function Start-QueueWorkerIfAvailable {
+    if (-not (Test-WingetLockAvailable)) {
+        Write-Log 'Queue worker not started because winget lock is already held.'
+        return $false
+    }
+
+    if (-not (Test-QueueStateLockAvailable)) {
+        Write-Log 'Queue worker not started because queue state lock is already held.' 'WARN'
+        return $false
+    }
+
+    Start-QueueWorker
+    return $true
+}
+
 function Get-WingetPath {
     $candidates = @()
 
@@ -2097,9 +2184,7 @@ function Unregister-LegacyQueueTask {
     }
 }
 
-function Register-OrUpdateTasks {
+function Unregister-ObsoleteDaemonTask {
-    Unregister-LegacyQueueTask
-
     try {
         $existingDaemon = Get-ScheduledTask -TaskName $DaemonTaskName -ErrorAction SilentlyContinue
         if ($existingDaemon) {
@@ -2111,7 +2196,86 @@ function Register-OrUpdateTasks {
     catch {
         Write-Log "Could not remove daemon task '$DaemonTaskName': $($_.Exception.Message)" 'WARN'
     }
+}
 
+function Test-CommandLineHasWorkerMode {
+    param(
+        [Parameter(Mandatory)][string]$CommandLine,
+        [Parameter(Mandatory)][string]$Mode
+    )
+
+    $pattern = ('(?i)(^|\s)-Mode\s+["'']?{0}["'']?(\s|$)' -f [regex]::Escape($Mode))
+    return [bool]($CommandLine -match $pattern)
+}
+
+function Stop-ObsoleteDaemonProcesses {
+    $currentWorkerModes = @(
+        'Install',
+        'ProcessRequests',
+        'ProcessQueue',
+        'UpgradeAll',
+        'ApplyPolicy',
+        'ProcessRequest'
+    )
+    $daemonMarkers = @(
+        '-Mode Daemon',
+        '-Mode "Daemon"',
+        "-Mode 'Daemon'",
+        'Mode=Daemon',
+        'NamedPipe',
+        'Named pipe',
+        '\\.\pipe\Softwarekatalog'
+    )
+
+    try {
+        $processes = @(Get-CimInstance -ClassName Win32_Process -ErrorAction Stop)
+    }
+    catch {
+        Write-Log "Could not enumerate processes for obsolete daemon cleanup: $($_.Exception.Message)" 'WARN'
+        return
+    }
+
+    foreach ($process in $processes) {
+        if (-not $process) { continue }
+        if ([int]$process.ProcessId -eq $PID) { continue }
+        if ([string]$process.Name -notin @('powershell.exe','pwsh.exe')) { continue }
+
+        $commandLine = [string]$process.CommandLine
+        if ([string]::IsNullOrWhiteSpace($commandLine)) { continue }
+
+        $matchesScriptPath = $commandLine.IndexOf($LocalScriptPath, [System.StringComparison]::OrdinalIgnoreCase) -ge 0
+        $matchesScriptName = ($commandLine.IndexOf('SelfServiceWinget.ps1', [System.StringComparison]::OrdinalIgnoreCase) -ge 0 -and $commandLine.IndexOf($BaseDir, [System.StringComparison]::OrdinalIgnoreCase) -ge 0)
+        if (-not ($matchesScriptPath -or $matchesScriptName)) { continue }
+
+        $hasCurrentWorkerMode = $false
+        foreach ($mode in $currentWorkerModes) {
+            if (Test-CommandLineHasWorkerMode -CommandLine $commandLine -Mode $mode) {
+                $hasCurrentWorkerMode = $true
+                break
+            }
+        }
+
+        $hasDaemonMarker = $false
+        foreach ($marker in $daemonMarkers) {
+            if ($commandLine.IndexOf($marker, [System.StringComparison]::OrdinalIgnoreCase) -ge 0) {
+                $hasDaemonMarker = $true
+                break
+            }
+        }
+
+        if ($hasCurrentWorkerMode -and -not $hasDaemonMarker) { continue }
+
+        try {
+            Write-Log "Stopping obsolete Softwarekatalog process pid=$($process.ProcessId)."
+            Stop-Process -Id ([int]$process.ProcessId) -Force -ErrorAction Stop
+        }
+        catch {
+            Write-Log "Could not stop obsolete daemon process pid=$($process.ProcessId): $($_.Exception.Message)" 'WARN'
+        }
+    }
+}
+
+function Register-OrUpdateTasks {
     $principal = New-ScheduledTaskPrincipal `
         -UserId 'SYSTEM' `
         -LogonType ServiceAccount `
@@ -2444,9 +2608,19 @@ function Invoke-RequiredSoftwarePolicy {
         }
     }
 
-    $queued = Ensure-RequiredSoftwareQueued
+    try {
+        $queued = Ensure-RequiredSoftwareQueued -TimeoutSeconds 5
+    }
+    catch {
+        if (Test-IsQueueStateLockTimeout -ErrorObject $_) {
+            Write-Log "Required software queueing skipped because the queue state lock is busy: $($_.Exception.Message)" 'WARN'
+            return [pscustomobject]@{ Installed = 0; Failed = 0; Skipped = 0; Queued = 0 }
+        }
+        throw
+    }
+
     if ($queued -gt 0) {
-        Start-QueueWorker
+        [void](Start-QueueWorkerIfAvailable)
     }
 
     return [pscustomobject]@{ Installed = 0; Failed = 0; Skipped = 0; Queued = $queued }
@@ -2622,14 +2796,25 @@ function Invoke-UpgradeAllMode {
     Ensure-Directory -Path $LogDir
 
     $item = New-QueueItem -Action UpgradeAll -DisplayName 'winget upgrade --all' -Source System -RequestedBy 'System' -RequestedBySid 'System'
-    $result = Add-QueueItem -Item $item
+    try {
+        $result = Add-QueueItem -Item $item -TimeoutSeconds 5
+    }
+    catch {
+        if (Test-IsQueueStateLockTimeout -ErrorObject $_) {
+            Write-Log "Skipped 2-hour winget upgrade queueing because the queue state lock is busy: $($_.Exception.Message)" 'WARN'
+            return
+        }
+        throw
+    }
+
     if ($result.Added) {
         Write-Log 'Queued 2-hour winget upgrade.'
     }
     else {
         Write-Log '2-hour winget upgrade is already queued or running.'
     }
-    Start-QueueWorker
+
+    [void](Start-QueueWorkerIfAvailable)
 }
 
 function Invoke-ApplyPolicyMode {
@@ -2666,9 +2851,19 @@ function Invoke-ProcessRequestMode {
         return
     }
 
-    $result = Add-SoftwareRequestToQueue -Action $RequestAction -Alias $alias -RequestedBy $RequestedBy -RequestedBySid $RequestedBySid -Source User
+    try {
+        $result = Add-SoftwareRequestToQueue -Action $RequestAction -Alias $alias -RequestedBy $RequestedBy -RequestedBySid $RequestedBySid -Source User -TimeoutSeconds 5
+    }
+    catch {
+        if (Test-IsQueueStateLockTimeout -ErrorObject $_) {
+            Write-Log "Worker request action=$RequestAction alias='$alias' could not be queued because the queue state lock is busy: $($_.Exception.Message)" 'WARN'
+            return
+        }
+        throw
+    }
+
     Write-Log "Queued worker request action=$RequestAction alias='$alias' added=$($result.Added) RequestedBy='$RequestedBy' Sid='$RequestedBySid'"
-    Start-QueueWorker
+    [void](Start-QueueWorkerIfAvailable)
 }
 
 function Get-RequestFileOwnerInfo {
@@ -2699,6 +2894,98 @@ function Get-RequestFileOwnerInfo {
     }
 }
 
+function Assert-RequestJsonDepth {
+    param(
+        [Parameter(Mandatory)][string]$Raw,
+        [int]$MaxDepth = 8
+    )
+
+    $depth = 0
+    $inString = $false
+    $escaped = $false
+
+    for ($i = 0; $i -lt $Raw.Length; $i++) {
+        $code = [int][char]$Raw[$i]
+
+        if ($inString) {
+            if ($escaped) {
+                $escaped = $false
+                continue
+            }
+
+            if ($code -eq 92) {
+                $escaped = $true
+                continue
+            }
+
+            if ($code -eq 34) {
+                $inString = $false
+            }
+            continue
+        }
+
+        if ($code -eq 34) {
+            $inString = $true
+            continue
+        }
+
+        if ($code -eq 123 -or $code -eq 91) {
+            $depth++
+            if ($depth -gt $MaxDepth) {
+                throw "Request JSON nesting exceeds maximum depth $MaxDepth."
+            }
+            continue
+        }
+
+        if ($code -eq 125 -or $code -eq 93) {
+            if ($depth -lt 1) {
+                throw 'Request JSON has an unexpected closing bracket.'
+            }
+            $depth--
+        }
+    }
+
+    if ($inString) {
+        throw 'Request JSON string is unterminated.'
+    }
+
+    if ($depth -ne 0) {
+        throw 'Request JSON is incomplete.'
+    }
+}
+
+function Get-RequestScalarValue {
+    param(
+        [Parameter(Mandatory)]$Request,
+        [Parameter(Mandatory)][string[]]$Names,
+        [Parameter(Mandatory)][string]$DisplayName,
+        [int]$MaxLength = 256
+    )
+
+    foreach ($name in $Names) {
+        $property = $Request.PSObject.Properties[$name]
+        if (-not $property) { continue }
+
+        $value = $property.Value
+        if ($null -eq $value) { continue }
+
+        if ($value -is [System.Array] -or $value -is [System.Collections.IDictionary] -or $value -is [pscustomobject]) {
+            throw "Request property '$name' must be a scalar value."
+        }
+
+        $text = ([string]$value).Trim()
+        if ([string]::IsNullOrWhiteSpace($text)) { continue }
+
+        if ($text.Length -gt $MaxLength) {
+            throw "Request property '$name' is too long ($($text.Length) characters; max=$MaxLength)."
+        }
+
+        return $text
+    }
+
+    throw "Request is missing $DisplayName."
+}
+
 function Move-RequestFile {
     param(
         [Parameter(Mandatory)][string]$Path,
@@ -2713,12 +3000,18 @@ function Move-RequestFile {
         $destination = Join-Path $DestinationDir ("{0}-{1}.json" -f [IO.Path]::GetFileNameWithoutExtension($leaf), [Guid]::NewGuid().ToString('N'))
     }
 
-    Move-Item -LiteralPath $Path -Destination $destination -Force
+    Microsoft.PowerShell.Management\Move-Item -LiteralPath $Path -Destination $destination -Force -ErrorAction Stop
     Set-FileAclAdminsOnly -Path $destination
 
     if (-not [string]::IsNullOrWhiteSpace($ErrorMessage)) {
-        Set-Content -LiteralPath ($destination + '.error.txt') -Value $ErrorMessage -Encoding UTF8
+        $errorPath = $destination + '.error.txt'
-        Set-FileAclAdminsOnly -Path ($destination + '.error.txt')
+        try {
+            Microsoft.PowerShell.Management\Set-Content -LiteralPath $errorPath -Value $ErrorMessage -Encoding UTF8 -ErrorAction Stop
+            Set-FileAclAdminsOnly -Path $errorPath
+        }
+        catch {
+            Write-Log "Could not write request error file '$errorPath': $($_.Exception.Message)" 'WARN'
+        }
     }
 }
 
@@ -2747,19 +3040,13 @@ function Invoke-ProcessRequestsMode {
                 throw 'Request file is empty.'
             }
 
+            Assert-RequestJsonDepth -Raw $raw -MaxDepth 8
             $request = $raw | ConvertFrom-Json
-            $commandValue = $null
+            if ($request -is [System.Array] -or $request -isnot [pscustomobject]) {
-            if ($request.PSObject.Properties['Command'] -and $request.Command) {
+                throw 'Request JSON must be an object.'
-                $commandValue = [string]$request.Command
-            }
-            elseif ($request.PSObject.Properties['Action'] -and $request.Action) {
-                $commandValue = [string]$request.Action
-            }
-
-            if ([string]::IsNullOrWhiteSpace($commandValue)) {
-                throw 'Request is missing Command.'
             }
 
+            $commandValue = Get-RequestScalarValue -Request $request -Names @('Command','Action') -DisplayName 'Command' -MaxLength 32
             $command = $commandValue.Trim().ToLowerInvariant()
             if ($command -in @('refreshstatus','getstatus')) {
                 $owner = Get-RequestFileOwnerInfo -Path $file.FullName
@@ -2785,27 +3072,21 @@ function Invoke-ProcessRequestsMode {
                 throw "Unsupported request command '$commandValue'."
             }
 
-            $aliasValue = $null
+            $aliasValue = Get-RequestScalarValue -Request $request -Names @('AppAlias','Alias') -DisplayName 'AppAlias' -MaxLength 128
-            if ($request.PSObject.Properties['AppAlias'] -and $request.AppAlias) {
-                $aliasValue = [string]$request.AppAlias
-            }
-            elseif ($request.PSObject.Properties['Alias'] -and $request.Alias) {
-                $aliasValue = [string]$request.Alias
-            }
-
-            if ([string]::IsNullOrWhiteSpace($aliasValue)) {
-                throw 'Request is missing AppAlias.'
-            }
-
             $alias = $aliasValue.Trim().ToLowerInvariant()
             $owner = Get-RequestFileOwnerInfo -Path $file.FullName
-            $result = Add-SoftwareRequestToQueue -Action $action -Alias $alias -RequestedBy $owner.Name -RequestedBySid $owner.Sid -Source User
+            $result = Add-SoftwareRequestToQueue -Action $action -Alias $alias -RequestedBy $owner.Name -RequestedBySid $owner.Sid -Source User -TimeoutSeconds 5
             Write-Log "Accepted request file '$($file.Name)' action=$action alias='$alias' added=$($result.Added) owner='$($owner.Name)' sid='$($owner.Sid)'."
             Move-RequestFile -Path $file.FullName -DestinationDir $ProcessedRequestsDir
             $shouldStartQueue = $true
         }
         catch {
             $message = $_.Exception.Message
+            if (Test-IsQueueStateLockTimeout -ErrorObject $_) {
+                Write-Log "Request file '$($file.Name)' will be retried because the queue state lock is busy: $message" 'WARN'
+                break
+            }
+
             Write-Log "Rejected request file '$($file.Name)': $message" 'WARN'
             try {
                 Move-RequestFile -Path $file.FullName -DestinationDir $FailedRequestsDir -ErrorMessage $message
@@ -2816,26 +3097,42 @@ function Invoke-ProcessRequestsMode {
         }
     }
 
-    $requiredQueued = Ensure-RequiredSoftwareQueued
+    try {
-    if ($requiredQueued -gt 0) {
+        $requiredQueued = Ensure-RequiredSoftwareQueued -TimeoutSeconds 5
-        Write-Log "Queued required software from request worker count=$requiredQueued."
+        if ($requiredQueued -gt 0) {
-        $shouldStartQueue = $true
+            Write-Log "Queued required software from request worker count=$requiredQueued."
+            $shouldStartQueue = $true
+        }
+    }
+    catch {
+        if (Test-IsQueueStateLockTimeout -ErrorObject $_) {
+            Write-Log "Required software queueing deferred because the queue state lock is busy: $($_.Exception.Message)" 'WARN'
+        }
+        else {
+            throw
+        }
     }
 
-    foreach ($queuedItem in @((Get-QueueStateSnapshot).Queue)) {
+    try {
-        if ($queuedItem -and $queuedItem.State -eq 'Queued') {
+        $queueState = Get-QueueStateSnapshot -TimeoutSeconds 5
-            $shouldStartQueue = $true
+        foreach ($queuedItem in @($queueState.Queue)) {
-            break
+            if ($queuedItem -and $queuedItem.State -eq 'Queued') {
+                $shouldStartQueue = $true
+                break
+            }
+        }
+    }
+    catch {
+        if (Test-IsQueueStateLockTimeout -ErrorObject $_) {
+            Write-Log "Queued work inspection skipped because the queue state lock is busy: $($_.Exception.Message)" 'WARN'
+        }
+        else {
+            throw
         }
     }
 
     if ($shouldStartQueue) {
-        if (Test-WingetLockAvailable) {
+        [void](Start-QueueWorkerIfAvailable)
-            Start-QueueWorker
-        }
-        else {
-            Write-Log 'Queue worker not started because winget lock is already held.'
-        }
     }
 }
 
@@ -2846,6 +3143,10 @@ function Install-Service {
     Ensure-Directory -Path $ProcessedRequestsDir
     Ensure-Directory -Path $FailedRequestsDir
 
+    Unregister-LegacyQueueTask
+    Unregister-ObsoleteDaemonTask
+    Stop-ObsoleteDaemonProcesses
+
     Set-DirectoryAcl -Path $BaseDir -UsersCanModify:$false
     Set-DirectoryAcl -Path $LogDir -UsersCanModify:$false
     Set-RequestDirectoryAcl -Path $RequestsDir