(hopefully) finally fixed installation queueing

.github/workflows/release.yml

@@ -20,6 +20,8 @@ jobs:
 
       - name: Build dist
         run: make dist
+        env:
+          SOFTKAT_BUILD_COMMIT: ${{ github.sha }}
 
       - name: Package release zip
         run: |

scripts/build-dist.sh

@@ -4,16 +4,31 @@ set -euo pipefail
 src="${1:-setup.ps1}"
 out="${2:-dist/setup.ps1}"
 out_dir="$(dirname "$out")"
+build_commit="${SOFTKAT_BUILD_COMMIT:-${GITHUB_SHA:-}}"
+
+if [ -z "$build_commit" ] && git rev-parse --verify HEAD >/dev/null 2>&1; then
+  build_commit="$(git rev-parse HEAD)"
+fi
+
+if [ -z "$build_commit" ]; then
+  build_commit="unknown"
+fi
 
 mkdir -p "$out_dir"
 
+payload_src="$(mktemp)"
+trap 'rm -f "$payload_src"' EXIT
+printf '# Softwarekatalog build commit: %s\n' "$build_commit" > "$payload_src"
+cat "$src" >> "$payload_src"
+
 if base64 --help 2>&1 | grep -q -- '-w'; then
-  payload="$(base64 -w 0 "$src")"
+  payload="$(base64 -w 0 "$payload_src")"
 else
-  payload="$(base64 "$src" | tr -d '\n')"
+  payload="$(base64 "$payload_src" | tr -d '\n')"
 fi
 
 cat > "$out" <<EOF
+# Softwarekatalog build commit: $build_commit
 # ASCII-only bootstrap generated from setup.ps1. Do not edit.
 [CmdletBinding()]
 param(

setup.ps1

@@ -46,6 +46,7 @@ $CatalogGuiPath         = Join-Path $BaseDir 'Software-Catalog.ps1'
 $CatalogLauncherPath    = Join-Path $BaseDir 'Launch-Software-Catalog.vbs'
 $InstalledPackagesPath  = Join-Path $BaseDir 'Installed-Packages.json'
 $QueueStatePath         = Join-Path $BaseDir 'Queue-State.json'
+$QueueLockInfoPath      = Join-Path $BaseDir 'Queue-Lock.json'
 $RequestsDir            = Join-Path $BaseDir 'Requests'
 $ProcessedRequestsDir   = Join-Path $BaseDir 'Processed'
 $FailedRequestsDir      = Join-Path $BaseDir 'Failed'
@@ -58,6 +59,7 @@ $PolicyMaxEntries       = 500
 $LegacyQueueTaskName    = 'Softwarekatalog - Winget Queue'
 $DaemonTaskName         = 'Softwarekatalog - Daemon'
 $RequestTaskName        = 'Softwarekatalog - Requests'
+$QueueTaskName          = 'Softwarekatalog - Queue'
 $UpgradeTaskName        = 'Softwarekatalog - Winget Upgrade'
 $WingetMutexName        = 'Global\Softwarekatalog-Winget'
 $QueueStateMutexName    = 'Global\Softwarekatalog-QueueState'
@@ -68,6 +70,8 @@ $CommonDesktopDir       = [Environment]::GetFolderPath('CommonDesktopDirectory')
 $CommonProgramsDir      = [Environment]::GetFolderPath('CommonPrograms')
 $DesktopShortcutPath    = Join-Path $CommonDesktopDir "$CatalogDisplayName.lnk"
 $StartMenuShortcutPath  = Join-Path $CommonProgramsDir "$CatalogDisplayName.lnk"
+$script:QueueProgressThrottle = @{}
+$script:SoftwarekatalogBuildCommit = $null
 
 # ----------------------------
 # Utility functions
@@ -91,6 +95,88 @@ function Write-Log {
     Add-Content -LiteralPath $LogFile -Value $line -Encoding UTF8
 }
 
+function Get-SoftwarekatalogBuildCommit {
+    if (-not [string]::IsNullOrWhiteSpace($script:SoftwarekatalogBuildCommit)) {
+        return $script:SoftwarekatalogBuildCommit
+    }
+
+    $paths = @()
+    if ($PSCommandPath) {
+        $paths += $PSCommandPath
+    }
+    elseif ($MyInvocation.MyCommand.Path) {
+        $paths += $MyInvocation.MyCommand.Path
+    }
+    elseif ($LocalScriptPath) {
+        $paths += $LocalScriptPath
+    }
+
+    foreach ($path in @($paths | Select-Object -Unique)) {
+        if ([string]::IsNullOrWhiteSpace($path)) { continue }
+        if (-not (Test-Path -LiteralPath $path)) { continue }
+
+        try {
+            foreach ($line in @(Get-Content -LiteralPath $path -TotalCount 5 -Encoding UTF8 -ErrorAction Stop)) {
+                $match = [regex]::Match([string]$line, '^#\s*Softwarekatalog build commit:\s*(.+?)\s*$')
+                if ($match.Success) {
+                    $script:SoftwarekatalogBuildCommit = $match.Groups[1].Value
+                    return $script:SoftwarekatalogBuildCommit
+                }
+            }
+        }
+        catch {
+        }
+    }
+
+    $script:SoftwarekatalogBuildCommit = 'unknown'
+    return $script:SoftwarekatalogBuildCommit
+}
+
+function Write-QueueLockOwnerInfo {
+    param([Parameter(Mandatory)][string]$Operation)
+
+    try {
+        Ensure-Directory -Path $BaseDir
+        $owner = [ordered]@{
+            Pid         = $PID
+            Mode        = $Mode
+            Operation   = $Operation
+            StartedAt   = (Get-Date).ToString('o')
+            ScriptPath  = $PSCommandPath
+            BuildCommit = Get-SoftwarekatalogBuildCommit
+        }
+        $owner | ConvertTo-Json -Depth 4 | Set-Content -LiteralPath $QueueLockInfoPath -Encoding UTF8 -ErrorAction Stop
+    }
+    catch {
+    }
+}
+
+function Clear-QueueLockOwnerInfo {
+    try {
+        if (Test-Path -LiteralPath $QueueLockInfoPath) {
+            Remove-Item -LiteralPath $QueueLockInfoPath -Force -ErrorAction Stop
+        }
+    }
+    catch {
+    }
+}
+
+function Get-QueueLockOwnerSummary {
+    if (-not (Test-Path -LiteralPath $QueueLockInfoPath)) {
+        return 'owner=unknown'
+    }
+
+    try {
+        $raw = Get-Content -LiteralPath $QueueLockInfoPath -Raw -Encoding UTF8 -ErrorAction Stop
+        if ([string]::IsNullOrWhiteSpace($raw)) { return 'owner=unknown' }
+        $owner = $raw | ConvertFrom-Json
+        return "ownerPid=$($owner.Pid) ownerMode='$($owner.Mode)' operation='$($owner.Operation)' startedAt='$($owner.StartedAt)' build='$($owner.BuildCommit)' script='$($owner.ScriptPath)'"
+    }
+    catch {
+        return "owner=unreadable ($($_.Exception.Message))"
+    }
+}
+
 function Set-DirectoryAcl {
     param(
         [Parameter(Mandatory)][string]$Path,
@@ -394,6 +480,75 @@ function New-QueueStateObject {
     }
 }
 
+function Backup-CorruptQueueState {
+    param([string]$Reason)
+
+    if (-not (Test-Path -LiteralPath $QueueStatePath)) { return }
+
+    try {
+        $backupPath = Join-Path $BaseDir ("Queue-State.corrupt-{0}.json" -f (Get-Date -Format 'yyyyMMddHHmmss'))
+        Copy-Item -LiteralPath $QueueStatePath -Destination $backupPath -Force -ErrorAction Stop
+        Write-Log "Backed up corrupt queue state to '$backupPath': $Reason" 'WARN'
+    }
+    catch {
+        Write-Log "Could not back up corrupt queue state: $($_.Exception.Message)" 'WARN'
+    }
+}
+
+function Normalize-QueueState {
+    param($State)
+
+    if (-not $State -or $State -is [System.Array] -or $State -isnot [pscustomobject]) {
+        return New-QueueStateObject
+    }
+
+    if (-not $State.PSObject.Properties['Version']) { $State | Add-Member -NotePropertyName Version -NotePropertyValue 0 }
+    if (-not $State.PSObject.Properties['UpdatedAt']) { $State | Add-Member -NotePropertyName UpdatedAt -NotePropertyValue (Get-Date).ToString('o') }
+    if (-not $State.PSObject.Properties['Current']) { $State | Add-Member -NotePropertyName Current -NotePropertyValue $null }
+    if (-not $State.PSObject.Properties['Queue']) {
+        $State | Add-Member -NotePropertyName Queue -NotePropertyValue @()
+    }
+    elseif (-not $State.Queue) {
+        $State.Queue = @()
+    }
+    else {
+        $State.Queue = @(@($State.Queue) | Where-Object { $null -ne $_ })
+    }
+
+    return $State
+}
+
+function Get-QueueItemSortTimestamp {
+    param($Item)
+
+    foreach ($propertyName in @('FinishedAt','StartedAt','CreatedAt')) {
+        if ($Item -and $Item.PSObject.Properties[$propertyName] -and $Item.$propertyName) {
+            try { return [DateTime]::Parse([string]$Item.$propertyName) }
+            catch { }
+        }
+    }
+
+    return [DateTime]::MinValue
+}
+
+function Compact-QueueState {
+    param($State)
+
+    $State = Normalize-QueueState -State $State
+    $active = New-Object 'System.Collections.Generic.List[object]'
+    $finished = New-Object 'System.Collections.Generic.List[object]'
+
+    foreach ($item in @($State.Queue)) {
+        if (-not $item) { continue }
+        if ($item.State -in @('Queued','Running')) { [void]$active.Add($item) }
+        else { [void]$finished.Add($item) }
+    }
+
+    $keptFinished = @($finished | Sort-Object { Get-QueueItemSortTimestamp -Item $_ } -Descending | Select-Object -First 100)
+    $State.Queue = @(@($active | Sort-Object CreatedAt) + @($keptFinished | Sort-Object { Get-QueueItemSortTimestamp -Item $_ }))
+    return $State
+}
+
 function Read-QueueState {
     if (-not (Test-Path -LiteralPath $QueueStatePath)) {
         return New-QueueStateObject
@@ -406,19 +561,16 @@ function Read-QueueState {
         }
 
         $state = $raw | ConvertFrom-Json
-        if (-not $state.PSObject.Properties['Version']) { $state | Add-Member -NotePropertyName Version -NotePropertyValue 0 }
+        if ($state -is [System.Array] -or $state -isnot [pscustomobject]) {
-        if (-not $state.PSObject.Properties['UpdatedAt']) { $state | Add-Member -NotePropertyName UpdatedAt -NotePropertyValue (Get-Date).ToString('o') }
+            Backup-CorruptQueueState -Reason "Queue state root is not a JSON object."
-        if (-not $state.PSObject.Properties['Current']) { $state | Add-Member -NotePropertyName Current -NotePropertyValue $null }
+            return New-QueueStateObject
-        if (-not $state.PSObject.Properties['Queue']) {
-            $state | Add-Member -NotePropertyName Queue -NotePropertyValue @()
         }
-        elseif (-not $state.Queue) {
+
-            $state.Queue = @()
+        return Normalize-QueueState -State $state
-        }
-        return $state
     }
     catch {
         Write-Log "Failed to read queue state: $($_.Exception.Message)" 'WARN'
+        Backup-CorruptQueueState -Reason $_.Exception.Message
         return New-QueueStateObject
     }
 }
@@ -427,6 +579,7 @@ function Write-QueueState {
     param([Parameter(Mandatory)]$State)
 
     Ensure-Directory -Path $BaseDir
+    $State = Compact-QueueState -State $State
     $tempPath = Join-Path $BaseDir ("Queue-State-{0}.json.tmp" -f [Guid]::NewGuid().ToString('N'))
     try {
         $State | ConvertTo-Json -Depth 8 | Set-Content -LiteralPath $tempPath -Encoding UTF8
@@ -440,26 +593,46 @@ function Write-QueueState {
 }
 
 function Invoke-WithQueueStateLock {
-    param([Parameter(Mandatory)][scriptblock]$ScriptBlock)
+    param(
+        [Parameter(Mandatory)][scriptblock]$ScriptBlock,
+        [int]$TimeoutSeconds = 30,
+        [string]$Operation = 'QueueState'
+    )
 
     $mutex = New-Object System.Threading.Mutex($false, $QueueStateMutexName)
     $hasLock = $false
     try {
-        $hasLock = $mutex.WaitOne([TimeSpan]::FromSeconds(30))
+        try {
-        if (-not $hasLock) {
+            $hasLock = $mutex.WaitOne([TimeSpan]::FromSeconds($TimeoutSeconds))
-            throw "Timed out waiting for queue state lock '$QueueStateMutexName'."
+        }
+        catch [System.Threading.AbandonedMutexException] {
+            $hasLock = $true
+            Write-Log "Recovered abandoned queue state lock '$QueueStateMutexName'." 'WARN'
         }
 
+        if (-not $hasLock) {
+            $owner = Get-QueueLockOwnerSummary
+            throw (New-Object System.TimeoutException -ArgumentList "Timed out waiting for queue state lock '$QueueStateMutexName' ($owner).")
+        }
+
+        Write-QueueLockOwnerInfo -Operation $Operation
         & $ScriptBlock
     }
     finally {
-        if ($hasLock) { $mutex.ReleaseMutex() }
+        if ($hasLock) {
+            Clear-QueueLockOwnerInfo
+            $mutex.ReleaseMutex()
+        }
         $mutex.Dispose()
     }
 }
 
 function Update-QueueState {
-    param([Parameter(Mandatory)][scriptblock]$ScriptBlock)
+    param(
+        [Parameter(Mandatory)][scriptblock]$ScriptBlock,
+        [int]$TimeoutSeconds = 30,
+        [string]$Operation = 'UpdateQueueState'
+    )
 
     Invoke-WithQueueStateLock -ScriptBlock {
         $state = Read-QueueState
@@ -468,11 +641,41 @@ function Update-QueueState {
         $state.UpdatedAt = (Get-Date).ToString('o')
         Write-QueueState -State $state
         return $state
-    }
+    } -TimeoutSeconds $TimeoutSeconds -Operation $Operation
 }
 
 function Get-QueueStateSnapshot {
-    Invoke-WithQueueStateLock -ScriptBlock { Read-QueueState }
+    param([int]$TimeoutSeconds = 30)
+
+    Invoke-WithQueueStateLock -ScriptBlock { Read-QueueState } -TimeoutSeconds $TimeoutSeconds -Operation 'GetQueueStateSnapshot'
+}
+
+function Test-IsQueueStateLockTimeout {
+    param([Parameter(Mandatory)]$ErrorObject)
+
+    if ($ErrorObject -is [System.Management.Automation.ErrorRecord]) {
+        $exception = $ErrorObject.Exception
+    }
+    elseif ($ErrorObject -is [System.Exception]) {
+        $exception = $ErrorObject
+    }
+    else {
+        return $false
+    }
+
+    while ($exception) {
+        if ($exception -is [System.TimeoutException] -and $exception.Message -like "*$QueueStateMutexName*") {
+            return $true
+        }
+
+        if ($exception.Message -like "*Timed out waiting for queue state lock '$QueueStateMutexName'*") {
+            return $true
+        }
+
+        $exception = $exception.InnerException
+    }
+
+    return $false
 }
 
 function New-QueueItem {
@@ -532,7 +735,28 @@ function Update-QueueItemProgress {
         [bool]$IsIndeterminate = $true
     )
 
-    $null = Update-QueueState -ScriptBlock {
+    if (-not $script:QueueProgressThrottle) { $script:QueueProgressThrottle = @{} }
+    $now = Get-Date
+    $last = $script:QueueProgressThrottle[$Id]
+    if ($last) {
+        $sameStage = [string]::Equals([string]$last.Stage, [string]$Stage, [System.StringComparison]::Ordinal)
+        $sameText = [string]::Equals([string]$last.StageText, [string]$StageText, [System.StringComparison]::Ordinal)
+        $samePercent = ([string]$last.ProgressPercent) -eq ([string]$ProgressPercent)
+        $sameIndeterminate = ([bool]$last.IsIndeterminate) -eq ([bool]$IsIndeterminate)
+        if ($sameStage -and $sameText -and $samePercent -and $sameIndeterminate -and (($now - $last.UpdatedAt).TotalSeconds -lt 5)) {
+            return
+        }
+    }
+
+    $script:QueueProgressThrottle[$Id] = [pscustomobject]@{
+        Stage = $Stage
+        StageText = $StageText
+        ProgressPercent = $ProgressPercent
+        IsIndeterminate = $IsIndeterminate
+        UpdatedAt = $now
+    }
+
+    $null = Update-QueueState -Operation "Progress:$Id" -ScriptBlock {
         param($state)
         foreach ($item in @($state.Queue)) {
             if (-not $item) { continue }
@@ -568,7 +792,10 @@ function Get-RequiredAliasSet {
 }
 
 function Add-QueueItem {
-    param([Parameter(Mandatory)]$Item)
+    param(
+        [Parameter(Mandatory)]$Item,
+        [int]$TimeoutSeconds = 30
+    )
 
     $result = $null
     $null = Update-QueueState -ScriptBlock {
@@ -585,7 +812,7 @@ function Add-QueueItem {
 
         $state.Queue = @($queue + $Item)
         $script:QueueAddResult = [pscustomobject]@{ Added = $true; Item = $Item }
-    }
+    } -TimeoutSeconds $TimeoutSeconds -Operation "AddQueueItem:$($Item.Action):$($Item.Alias)"
 
     $result = $script:QueueAddResult
     $script:QueueAddResult = $null
@@ -598,7 +825,8 @@ function Add-SoftwareRequestToQueue {
         [Parameter(Mandatory)][string]$Alias,
         [string]$RequestedBy,
         [string]$RequestedBySid,
-        [ValidateSet('User','Required')][string]$Source = 'User'
+        [ValidateSet('User','Required')][string]$Source = 'User',
+        [int]$TimeoutSeconds = 30
     )
 
     $normalizedAlias = $Alias.Trim().ToLowerInvariant()
@@ -623,10 +851,12 @@ function Add-SoftwareRequestToQueue {
         -RequestedBy $RequestedBy `
         -RequestedBySid $RequestedBySid
 
-    return Add-QueueItem -Item $item
+    return Add-QueueItem -Item $item -TimeoutSeconds $TimeoutSeconds
 }
 
 function Ensure-RequiredSoftwareQueued {
+    param([int]$TimeoutSeconds = 30)
+
     $addedCount = 0
     if (-not (Test-SoftwareCatalogPolicyEnabled)) { return 0 }
     if (-not (Test-RequiredEnforcementEnabled)) { return 0 }
@@ -643,7 +873,7 @@ function Ensure-RequiredSoftwareQueued {
             continue
         }
 
-        $result = Add-SoftwareRequestToQueue -Action Install -Alias $alias -RequestedBy 'Policy' -RequestedBySid 'Policy' -Source Required
+        $result = Add-SoftwareRequestToQueue -Action Install -Alias $alias -RequestedBy 'Policy' -RequestedBySid 'Policy' -Source Required -TimeoutSeconds $TimeoutSeconds
         if ($result -and $result.Added) {
             $addedCount++
             Write-Log "Queued required software alias '$alias' package '$($app.PackageId)'."
@@ -694,6 +924,29 @@ function Test-WingetLockAvailable {
     }
 }
 
+function Test-QueueStateLockAvailable {
+    $mutex = New-Object System.Threading.Mutex($false, $QueueStateMutexName)
+    $hasLock = $false
+
+    try {
+        try {
+            $hasLock = $mutex.WaitOne(0)
+        }
+        catch [System.Threading.AbandonedMutexException] {
+            $hasLock = $true
+            Write-Log "Recovered abandoned queue state lock '$QueueStateMutexName' during availability check." 'WARN'
+        }
+
+        return $hasLock
+    }
+    finally {
+        if ($hasLock) {
+            $mutex.ReleaseMutex()
+        }
+        $mutex.Dispose()
+    }
+}
+
 function ConvertTo-ProcessArgument {
     param([Parameter(Mandatory)][string]$Value)
 
@@ -750,7 +1003,41 @@ function Start-SoftwarekatalogWorker {
 }
 
 function Start-QueueWorker {
+    try {
+        $task = Get-ScheduledTask -TaskName $QueueTaskName -ErrorAction SilentlyContinue
+        if ($task) {
+            Start-ScheduledTask -TaskName $QueueTaskName -ErrorAction Stop
+            Write-Log "Started scheduled queue task '$QueueTaskName'."
+            return $true
+        }
+    }
+    catch {
+        Write-Log "Could not start scheduled queue task '$QueueTaskName': $($_.Exception.Message)" 'WARN'
+    }
+
+    try {
         Start-SoftwarekatalogWorker -WorkerMode ProcessQueue
+        Write-Log 'Started queue worker process fallback.' 'WARN'
+        return $true
+    }
+    catch {
+        Write-Log "Could not start queue worker process fallback: $($_.Exception.Message)" 'ERROR'
+        return $false
+    }
+}
+
+function Start-QueueWorkerIfAvailable {
+    if (-not (Test-WingetLockAvailable)) {
+        Write-Log 'Queue worker not started because winget lock is already held.'
+        return $false
+    }
+
+    if (-not (Test-QueueStateLockAvailable)) {
+        Write-Log 'Queue worker not started because queue state lock is already held.' 'WARN'
+        return $false
+    }
+
+    return (Start-QueueWorker)
 }
 
 function Get-WingetPath {
@@ -2097,9 +2384,7 @@ function Unregister-LegacyQueueTask {
     }
 }
 
-function Register-OrUpdateTasks {
+function Unregister-ObsoleteDaemonTask {
-    Unregister-LegacyQueueTask
-
     try {
         $existingDaemon = Get-ScheduledTask -TaskName $DaemonTaskName -ErrorAction SilentlyContinue
         if ($existingDaemon) {
@@ -2111,7 +2396,62 @@ function Register-OrUpdateTasks {
     catch {
         Write-Log "Could not remove daemon task '$DaemonTaskName': $($_.Exception.Message)" 'WARN'
     }
+}
 
+function Stop-SoftwarekatalogScheduledTasksForUpdate {
+    foreach ($taskName in @($RequestTaskName, $QueueTaskName, $UpgradeTaskName, $LegacyQueueTaskName, $DaemonTaskName)) {
+        if ([string]::IsNullOrWhiteSpace($taskName)) { continue }
+
+        try {
+            $task = Get-ScheduledTask -TaskName $taskName -ErrorAction SilentlyContinue
+            if ($task) {
+                Write-Log "Stopping scheduled task '$taskName' before update."
+                Stop-ScheduledTask -TaskName $taskName -ErrorAction SilentlyContinue
+                for ($attempt = 0; $attempt -lt 10; $attempt++) {
+                    $currentTask = Get-ScheduledTask -TaskName $taskName -ErrorAction SilentlyContinue
+                    if (-not $currentTask -or [string]$currentTask.State -ne "Running") { break }
+                    Start-Sleep -Milliseconds 500
+                }
+            }
+        }
+        catch {
+            Write-Log "Could not stop scheduled task '$taskName': $($_.Exception.Message)" 'WARN'
+        }
+    }
+}
+
+function Stop-ObsoleteDaemonProcesses {
+    try {
+        $processes = @(Get-CimInstance -ClassName Win32_Process -ErrorAction Stop)
+    }
+    catch {
+        Write-Log "Could not enumerate processes for Softwarekatalog worker cleanup: $($_.Exception.Message)" 'WARN'
+        return
+    }
+
+    foreach ($process in $processes) {
+        if (-not $process) { continue }
+        if ([int]$process.ProcessId -eq $PID) { continue }
+        if ([string]$process.Name -notin @('powershell.exe','pwsh.exe')) { continue }
+
+        $commandLine = [string]$process.CommandLine
+        if ([string]::IsNullOrWhiteSpace($commandLine)) { continue }
+
+        $matchesScriptPath = $commandLine.IndexOf($LocalScriptPath, [System.StringComparison]::OrdinalIgnoreCase) -ge 0
+        $matchesScriptName = ($commandLine.IndexOf('SelfServiceWinget.ps1', [System.StringComparison]::OrdinalIgnoreCase) -ge 0 -and $commandLine.IndexOf($BaseDir, [System.StringComparison]::OrdinalIgnoreCase) -ge 0)
+        if (-not ($matchesScriptPath -or $matchesScriptName)) { continue }
+
+        try {
+            Write-Log "Stopping existing Softwarekatalog process pid=$($process.ProcessId)."
+            Stop-Process -Id ([int]$process.ProcessId) -Force -ErrorAction Stop
+        }
+        catch {
+            Write-Log "Could not stop Softwarekatalog process pid=$($process.ProcessId): $($_.Exception.Message)" 'WARN'
+        }
+    }
+}
+
+function Register-OrUpdateTasks {
     $principal = New-ScheduledTaskPrincipal `
         -UserId 'SYSTEM' `
         -LogonType ServiceAccount `
@@ -2141,6 +2481,23 @@ function Register-OrUpdateTasks {
 
     Register-ScheduledTask -TaskName $RequestTaskName -InputObject $requestTask -Force | Out-Null
 
+    Write-Log "Registering/updating scheduled task '$QueueTaskName'."
+    $queueArgs = "-NoProfile -ExecutionPolicy Bypass -File `"$LocalScriptPath`" -Mode ProcessQueue"
+    $queueAction = New-ScheduledTaskAction -Execute $PowerShellExe -Argument $queueArgs
+    $queueSettings = New-ScheduledTaskSettingsSet `
+        -AllowStartIfOnBatteries `
+        -DontStopIfGoingOnBatteries `
+        -StartWhenAvailable `
+        -MultipleInstances IgnoreNew `
+        -ExecutionTimeLimit (New-TimeSpan -Hours 3)
+
+    $queueTask = New-ScheduledTask `
+        -Action $queueAction `
+        -Principal $principal `
+        -Settings $queueSettings
+
+    Register-ScheduledTask -TaskName $QueueTaskName -InputObject $queueTask -Force | Out-Null
+
     Write-Log "Registering/updating scheduled task '$UpgradeTaskName'."
     $upgradeArgs = "-NoProfile -ExecutionPolicy Bypass -File `"$LocalScriptPath`" -Mode UpgradeAll"
     $upgradeAction = New-ScheduledTaskAction -Execute $PowerShellExe -Argument $upgradeArgs
@@ -2444,16 +2801,26 @@ function Invoke-RequiredSoftwarePolicy {
         }
     }
 
-    $queued = Ensure-RequiredSoftwareQueued
+    try {
+        $queued = Ensure-RequiredSoftwareQueued -TimeoutSeconds 5
+    }
+    catch {
+        if (Test-IsQueueStateLockTimeout -ErrorObject $_) {
+            Write-Log "Required software queueing skipped because the queue state lock is busy: $($_.Exception.Message)" 'WARN'
+            return [pscustomobject]@{ Installed = 0; Failed = 0; Skipped = 0; Queued = 0 }
+        }
+        throw
+    }
+
     if ($queued -gt 0) {
-        Start-QueueWorker
+        [void](Start-QueueWorkerIfAvailable)
     }
 
     return [pscustomobject]@{ Installed = 0; Failed = 0; Skipped = 0; Queued = $queued }
 }
 
 function Reset-RunningQueueItems {
-    $null = Update-QueueState -ScriptBlock {
+    $null = Update-QueueState -Operation 'ResetRunningQueueItems' -ScriptBlock {
         param($state)
         foreach ($item in @($state.Queue)) {
             if (-not $item) { continue }
@@ -2472,7 +2839,7 @@ function Reset-RunningQueueItems {
 
 function Get-NextQueuedItem {
     $script:NextQueueItem = $null
-    $null = Update-QueueState -ScriptBlock {
+    $null = Update-QueueState -Operation 'GetNextQueuedItem' -ScriptBlock {
         param($state)
         foreach ($item in (@($state.Queue) | Sort-Object CreatedAt)) {
             if (-not $item) { continue }
@@ -2512,7 +2879,7 @@ function Complete-QueueItem {
         [string]$Message
     )
 
-    $null = Update-QueueState -ScriptBlock {
+    $null = Update-QueueState -Operation "CompleteQueueItem:${Id}:${State}" -ScriptBlock {
         param($queueState)
         foreach ($item in @($queueState.Queue)) {
             if (-not $item) { continue }
@@ -2613,7 +2980,7 @@ function Invoke-ProcessQueueMode {
             }
         }
 
-        $null = Update-QueueState -ScriptBlock { param($state) $state.Current = $null }
+        $null = Update-QueueState -Operation 'ClearCurrentQueueItem' -ScriptBlock { param($state) $state.Current = $null }
     }
 }
 
@@ -2622,14 +2989,25 @@ function Invoke-UpgradeAllMode {
     Ensure-Directory -Path $LogDir
 
     $item = New-QueueItem -Action UpgradeAll -DisplayName 'winget upgrade --all' -Source System -RequestedBy 'System' -RequestedBySid 'System'
-    $result = Add-QueueItem -Item $item
+    try {
+        $result = Add-QueueItem -Item $item -TimeoutSeconds 5
+    }
+    catch {
+        if (Test-IsQueueStateLockTimeout -ErrorObject $_) {
+            Write-Log "Skipped 2-hour winget upgrade queueing because the queue state lock is busy: $($_.Exception.Message)" 'WARN'
+            return
+        }
+        throw
+    }
+
     if ($result.Added) {
         Write-Log 'Queued 2-hour winget upgrade.'
     }
     else {
         Write-Log '2-hour winget upgrade is already queued or running.'
     }
-    Start-QueueWorker
+
+    [void](Start-QueueWorkerIfAvailable)
 }
 
 function Invoke-ApplyPolicyMode {
@@ -2666,9 +3044,19 @@ function Invoke-ProcessRequestMode {
         return
     }
 
-    $result = Add-SoftwareRequestToQueue -Action $RequestAction -Alias $alias -RequestedBy $RequestedBy -RequestedBySid $RequestedBySid -Source User
+    try {
+        $result = Add-SoftwareRequestToQueue -Action $RequestAction -Alias $alias -RequestedBy $RequestedBy -RequestedBySid $RequestedBySid -Source User -TimeoutSeconds 5
+    }
+    catch {
+        if (Test-IsQueueStateLockTimeout -ErrorObject $_) {
+            Write-Log "Worker request action=$RequestAction alias='$alias' could not be queued because the queue state lock is busy: $($_.Exception.Message)" 'WARN'
+            return
+        }
+        throw
+    }
+
     Write-Log "Queued worker request action=$RequestAction alias='$alias' added=$($result.Added) RequestedBy='$RequestedBy' Sid='$RequestedBySid'"
-    Start-QueueWorker
+    [void](Start-QueueWorkerIfAvailable)
 }
 
 function Get-RequestFileOwnerInfo {
@@ -2699,6 +3087,98 @@ function Get-RequestFileOwnerInfo {
     }
 }
 
+function Assert-RequestJsonDepth {
+    param(
+        [Parameter(Mandatory)][string]$Raw,
+        [int]$MaxDepth = 8
+    )
+
+    $depth = 0
+    $inString = $false
+    $escaped = $false
+
+    for ($i = 0; $i -lt $Raw.Length; $i++) {
+        $code = [int][char]$Raw[$i]
+
+        if ($inString) {
+            if ($escaped) {
+                $escaped = $false
+                continue
+            }
+
+            if ($code -eq 92) {
+                $escaped = $true
+                continue
+            }
+
+            if ($code -eq 34) {
+                $inString = $false
+            }
+            continue
+        }
+
+        if ($code -eq 34) {
+            $inString = $true
+            continue
+        }
+
+        if ($code -eq 123 -or $code -eq 91) {
+            $depth++
+            if ($depth -gt $MaxDepth) {
+                throw "Request JSON nesting exceeds maximum depth $MaxDepth."
+            }
+            continue
+        }
+
+        if ($code -eq 125 -or $code -eq 93) {
+            if ($depth -lt 1) {
+                throw 'Request JSON has an unexpected closing bracket.'
+            }
+            $depth--
+        }
+    }
+
+    if ($inString) {
+        throw 'Request JSON string is unterminated.'
+    }
+
+    if ($depth -ne 0) {
+        throw 'Request JSON is incomplete.'
+    }
+}
+
+function Get-RequestScalarValue {
+    param(
+        [Parameter(Mandatory)]$Request,
+        [Parameter(Mandatory)][string[]]$Names,
+        [Parameter(Mandatory)][string]$DisplayName,
+        [int]$MaxLength = 256
+    )
+
+    foreach ($name in $Names) {
+        $property = $Request.PSObject.Properties[$name]
+        if (-not $property) { continue }
+
+        $value = $property.Value
+        if ($null -eq $value) { continue }
+
+        if ($value -is [System.Array] -or $value -is [System.Collections.IDictionary] -or $value -is [pscustomobject]) {
+            throw "Request property '$name' must be a scalar value."
+        }
+
+        $text = ([string]$value).Trim()
+        if ([string]::IsNullOrWhiteSpace($text)) { continue }
+
+        if ($text.Length -gt $MaxLength) {
+            throw "Request property '$name' is too long ($($text.Length) characters; max=$MaxLength)."
+        }
+
+        return $text
+    }
+
+    throw "Request is missing $DisplayName."
+}
+
 function Move-RequestFile {
     param(
         [Parameter(Mandatory)][string]$Path,
@@ -2713,12 +3193,61 @@ function Move-RequestFile {
         $destination = Join-Path $DestinationDir ("{0}-{1}.json" -f [IO.Path]::GetFileNameWithoutExtension($leaf), [Guid]::NewGuid().ToString('N'))
     }
 
-    Move-Item -LiteralPath $Path -Destination $destination -Force
+    Microsoft.PowerShell.Management\Move-Item -LiteralPath $Path -Destination $destination -Force -ErrorAction Stop
     Set-FileAclAdminsOnly -Path $destination
 
     if (-not [string]::IsNullOrWhiteSpace($ErrorMessage)) {
-        Set-Content -LiteralPath ($destination + '.error.txt') -Value $ErrorMessage -Encoding UTF8
+        $errorPath = $destination + '.error.txt'
-        Set-FileAclAdminsOnly -Path ($destination + '.error.txt')
+        try {
+            Microsoft.PowerShell.Management\Set-Content -LiteralPath $errorPath -Value $ErrorMessage -Encoding UTF8 -ErrorAction Stop
+            Set-FileAclAdminsOnly -Path $errorPath
+        }
+        catch {
+            Write-Log "Could not write request error file '$errorPath': $($_.Exception.Message)" 'WARN'
+        }
+    }
+}
+
+function Test-IsTransientRequestFailureMessage {
+    param([string]$Message)
+
+    if ([string]::IsNullOrWhiteSpace($Message)) { return $false }
+
+    return (
+        $Message -like "*$QueueStateMutexName*" -or
+        $Message -like "*queue state lock*" -or
+        $Message -like "*Aufruftiefe*" -or
+        $Message -like "*call depth*" -or
+        $Message -like "*recursion*"
+    )
+}
+
+function Restore-QueueLockFailedRequests {
+    Ensure-Directory -Path $RequestsDir
+    Ensure-Directory -Path $FailedRequestsDir
+
+    $errorFiles = @(Get-ChildItem -LiteralPath $FailedRequestsDir -Filter '*.json.error.txt' -File -ErrorAction SilentlyContinue)
+    foreach ($errorFile in $errorFiles) {
+        try {
+            $message = Get-Content -LiteralPath $errorFile.FullName -Raw -Encoding UTF8 -ErrorAction Stop
+            if (-not (Test-IsTransientRequestFailureMessage -Message $message)) { continue }
+
+            $requestName = $errorFile.Name.Substring(0, $errorFile.Name.Length - '.error.txt'.Length)
+            $requestPath = Join-Path $FailedRequestsDir $requestName
+            if (-not (Test-Path -LiteralPath $requestPath)) { continue }
+
+            $destination = Join-Path $RequestsDir $requestName
+            if (Test-Path -LiteralPath $destination) {
+                $destination = Join-Path $RequestsDir ("{0}-{1}.json" -f [IO.Path]::GetFileNameWithoutExtension($requestName), [Guid]::NewGuid().ToString('N'))
+            }
+
+            Move-Item -LiteralPath $requestPath -Destination $destination -Force -ErrorAction Stop
+            Remove-Item -LiteralPath $errorFile.FullName -Force -ErrorAction SilentlyContinue
+            Write-Log "Restored failed request '$requestName' for retry after queue lock timeout."
+        }
+        catch {
+            Write-Log "Could not restore failed request '$($errorFile.Name)': $($_.Exception.Message)" 'WARN'
+        }
     }
 }
 
@@ -2747,19 +3276,13 @@ function Invoke-ProcessRequestsMode {
                 throw 'Request file is empty.'
             }
 
+            Assert-RequestJsonDepth -Raw $raw -MaxDepth 8
             $request = $raw | ConvertFrom-Json
-            $commandValue = $null
+            if ($request -is [System.Array] -or $request -isnot [pscustomobject]) {
-            if ($request.PSObject.Properties['Command'] -and $request.Command) {
+                throw 'Request JSON must be an object.'
-                $commandValue = [string]$request.Command
-            }
-            elseif ($request.PSObject.Properties['Action'] -and $request.Action) {
-                $commandValue = [string]$request.Action
-            }
-
-            if ([string]::IsNullOrWhiteSpace($commandValue)) {
-                throw 'Request is missing Command.'
             }
 
+            $commandValue = Get-RequestScalarValue -Request $request -Names @('Command','Action') -DisplayName 'Command' -MaxLength 32
             $command = $commandValue.Trim().ToLowerInvariant()
             if ($command -in @('refreshstatus','getstatus')) {
                 $owner = Get-RequestFileOwnerInfo -Path $file.FullName
@@ -2785,27 +3308,21 @@ function Invoke-ProcessRequestsMode {
                 throw "Unsupported request command '$commandValue'."
             }
 
-            $aliasValue = $null
+            $aliasValue = Get-RequestScalarValue -Request $request -Names @('AppAlias','Alias') -DisplayName 'AppAlias' -MaxLength 128
-            if ($request.PSObject.Properties['AppAlias'] -and $request.AppAlias) {
-                $aliasValue = [string]$request.AppAlias
-            }
-            elseif ($request.PSObject.Properties['Alias'] -and $request.Alias) {
-                $aliasValue = [string]$request.Alias
-            }
-
-            if ([string]::IsNullOrWhiteSpace($aliasValue)) {
-                throw 'Request is missing AppAlias.'
-            }
-
             $alias = $aliasValue.Trim().ToLowerInvariant()
             $owner = Get-RequestFileOwnerInfo -Path $file.FullName
-            $result = Add-SoftwareRequestToQueue -Action $action -Alias $alias -RequestedBy $owner.Name -RequestedBySid $owner.Sid -Source User
+            $result = Add-SoftwareRequestToQueue -Action $action -Alias $alias -RequestedBy $owner.Name -RequestedBySid $owner.Sid -Source User -TimeoutSeconds 5
             Write-Log "Accepted request file '$($file.Name)' action=$action alias='$alias' added=$($result.Added) owner='$($owner.Name)' sid='$($owner.Sid)'."
             Move-RequestFile -Path $file.FullName -DestinationDir $ProcessedRequestsDir
             $shouldStartQueue = $true
         }
         catch {
             $message = $_.Exception.Message
+            if (Test-IsQueueStateLockTimeout -ErrorObject $_) {
+                Write-Log "Request file '$($file.Name)' will be retried because the queue state lock is busy: $message" 'WARN'
+                break
+            }
+
             Write-Log "Rejected request file '$($file.Name)': $message" 'WARN'
             try {
                 Move-RequestFile -Path $file.FullName -DestinationDir $FailedRequestsDir -ErrorMessage $message
@@ -2816,26 +3333,24 @@ function Invoke-ProcessRequestsMode {
         }
     }
 
-    $requiredQueued = Ensure-RequiredSoftwareQueued
+    try {
+        $requiredQueued = Ensure-RequiredSoftwareQueued -TimeoutSeconds 5
         if ($requiredQueued -gt 0) {
             Write-Log "Queued required software from request worker count=$requiredQueued."
             $shouldStartQueue = $true
         }
-
+    }
-    foreach ($queuedItem in @((Get-QueueStateSnapshot).Queue)) {
+    catch {
-        if ($queuedItem -and $queuedItem.State -eq 'Queued') {
+        if (Test-IsQueueStateLockTimeout -ErrorObject $_) {
-            $shouldStartQueue = $true
+            Write-Log "Required software queueing deferred because the queue state lock is busy: $($_.Exception.Message)" 'WARN'
-            break
+        }
+        else {
+            throw
         }
     }
 
     if ($shouldStartQueue) {
-        if (Test-WingetLockAvailable) {
+        [void](Start-QueueWorkerIfAvailable)
-            Start-QueueWorker
-        }
-        else {
-            Write-Log 'Queue worker not started because winget lock is already held.'
-        }
     }
 }
 
@@ -2846,6 +3361,15 @@ function Install-Service {
     Ensure-Directory -Path $ProcessedRequestsDir
     Ensure-Directory -Path $FailedRequestsDir
 
+    Write-Log "Softwarekatalog build commit: $(Get-SoftwarekatalogBuildCommit)."
+    Stop-SoftwarekatalogScheduledTasksForUpdate
+    Stop-ObsoleteDaemonProcesses
+    Unregister-LegacyQueueTask
+    Unregister-ObsoleteDaemonTask
+    Restore-QueueLockFailedRequests
+    try { Reset-RunningQueueItems }
+    catch { Write-Log "Could not reset running queue items during install/update: $($_.Exception.Message)" 'WARN' }
+
     Set-DirectoryAcl -Path $BaseDir -UsersCanModify:$false
     Set-DirectoryAcl -Path $LogDir -UsersCanModify:$false
     Set-RequestDirectoryAcl -Path $RequestsDir
@@ -2866,6 +3390,24 @@ function Install-Service {
     New-Shortcut -ShortcutPath $StartMenuShortcutPath
     Register-OrUpdateTasks
 
+    try {
+        $queueState = Get-QueueStateSnapshot -TimeoutSeconds 5
+        foreach ($queuedItem in @($queueState.Queue)) {
+            if ($queuedItem -and $queuedItem.State -eq 'Queued') {
+                [void](Start-QueueWorkerIfAvailable)
+                break
+            }
+        }
+    }
+    catch {
+        if (Test-IsQueueStateLockTimeout -ErrorObject $_) {
+            Write-Log "Could not inspect queued work during install/update because the queue state lock is busy: $($_.Exception.Message)" 'WARN'
+        }
+        else {
+            Write-Log "Could not inspect queued work during install/update: $($_.Exception.Message)" 'WARN'
+        }
+    }
+
     Write-Log 'Install/update completed successfully.'
 }
 
@@ -2885,6 +3427,11 @@ try {
 }
 catch {
     try {
+        if ($Mode -in @('ProcessRequests','UpgradeAll') -and (Test-IsQueueStateLockTimeout -ErrorObject $_)) {
+            Write-Log "Mode '$Mode' deferred because the queue state lock is busy: $($_.Exception.Message)" 'WARN'
+            return
+        }
+
         Write-Log "Fatal error in mode '$Mode': $($_.Exception.Message)" 'ERROR'
     }
     catch {