(hopefully) finally fixed installation queueing

.github/workflows/release.yml

@@ -20,6 +20,8 @@ jobs:
 
       - name: Build dist
         run: make dist
+        env:
+          SOFTKAT_BUILD_COMMIT: ${{ github.sha }}
 
       - name: Package release zip
         run: |

scripts/build-dist.sh

@@ -4,16 +4,31 @@ set -euo pipefail
 src="${1:-setup.ps1}"
 out="${2:-dist/setup.ps1}"
 out_dir="$(dirname "$out")"
+build_commit="${SOFTKAT_BUILD_COMMIT:-${GITHUB_SHA:-}}"
+
+if [ -z "$build_commit" ] && git rev-parse --verify HEAD >/dev/null 2>&1; then
+  build_commit="$(git rev-parse HEAD)"
+fi
+
+if [ -z "$build_commit" ]; then
+  build_commit="unknown"
+fi
 
 mkdir -p "$out_dir"
 
+payload_src="$(mktemp)"
+trap 'rm -f "$payload_src"' EXIT
+printf '# Softwarekatalog build commit: %s\n' "$build_commit" > "$payload_src"
+cat "$src" >> "$payload_src"
+
 if base64 --help 2>&1 | grep -q -- '-w'; then
-  payload="$(base64 -w 0 "$src")"
+  payload="$(base64 -w 0 "$payload_src")"
 else
-  payload="$(base64 "$src" | tr -d '\n')"
+  payload="$(base64 "$payload_src" | tr -d '\n')"
 fi
 
 cat > "$out" <<EOF
+# Softwarekatalog build commit: $build_commit
 # ASCII-only bootstrap generated from setup.ps1. Do not edit.
 [CmdletBinding()]
 param(

setup.ps1

@@ -46,6 +46,7 @@ $CatalogGuiPath         = Join-Path $BaseDir 'Software-Catalog.ps1'
 $CatalogLauncherPath    = Join-Path $BaseDir 'Launch-Software-Catalog.vbs'
 $InstalledPackagesPath  = Join-Path $BaseDir 'Installed-Packages.json'
 $QueueStatePath         = Join-Path $BaseDir 'Queue-State.json'
+$QueueLockInfoPath      = Join-Path $BaseDir 'Queue-Lock.json'
 $RequestsDir            = Join-Path $BaseDir 'Requests'
 $ProcessedRequestsDir   = Join-Path $BaseDir 'Processed'
 $FailedRequestsDir      = Join-Path $BaseDir 'Failed'
@@ -58,6 +59,7 @@ $PolicyMaxEntries       = 500
 $LegacyQueueTaskName    = 'Softwarekatalog - Winget Queue'
 $DaemonTaskName         = 'Softwarekatalog - Daemon'
 $RequestTaskName        = 'Softwarekatalog - Requests'
+$QueueTaskName          = 'Softwarekatalog - Queue'
 $UpgradeTaskName        = 'Softwarekatalog - Winget Upgrade'
 $WingetMutexName        = 'Global\Softwarekatalog-Winget'
 $QueueStateMutexName    = 'Global\Softwarekatalog-QueueState'
@@ -68,6 +70,8 @@ $CommonDesktopDir       = [Environment]::GetFolderPath('CommonDesktopDirectory')
 $CommonProgramsDir      = [Environment]::GetFolderPath('CommonPrograms')
 $DesktopShortcutPath    = Join-Path $CommonDesktopDir "$CatalogDisplayName.lnk"
 $StartMenuShortcutPath  = Join-Path $CommonProgramsDir "$CatalogDisplayName.lnk"
+$script:QueueProgressThrottle = @{}
+$script:SoftwarekatalogBuildCommit = $null
 
 # ----------------------------
 # Utility functions
@@ -91,6 +95,88 @@ function Write-Log {
     Add-Content -LiteralPath $LogFile -Value $line -Encoding UTF8
 }
 
+function Get-SoftwarekatalogBuildCommit {
+    if (-not [string]::IsNullOrWhiteSpace($script:SoftwarekatalogBuildCommit)) {
+        return $script:SoftwarekatalogBuildCommit
+    }
+
+    $paths = @()
+    if ($PSCommandPath) {
+        $paths += $PSCommandPath
+    }
+    elseif ($MyInvocation.MyCommand.Path) {
+        $paths += $MyInvocation.MyCommand.Path
+    }
+    elseif ($LocalScriptPath) {
+        $paths += $LocalScriptPath
+    }
+
+    foreach ($path in @($paths | Select-Object -Unique)) {
+        if ([string]::IsNullOrWhiteSpace($path)) { continue }
+        if (-not (Test-Path -LiteralPath $path)) { continue }
+
+        try {
+            foreach ($line in @(Get-Content -LiteralPath $path -TotalCount 5 -Encoding UTF8 -ErrorAction Stop)) {
+                $match = [regex]::Match([string]$line, '^#\s*Softwarekatalog build commit:\s*(.+?)\s*$')
+                if ($match.Success) {
+                    $script:SoftwarekatalogBuildCommit = $match.Groups[1].Value
+                    return $script:SoftwarekatalogBuildCommit
+                }
+            }
+        }
+        catch {
+        }
+    }
+
+    $script:SoftwarekatalogBuildCommit = 'unknown'
+    return $script:SoftwarekatalogBuildCommit
+}
+
+function Write-QueueLockOwnerInfo {
+    param([Parameter(Mandatory)][string]$Operation)
+
+    try {
+        Ensure-Directory -Path $BaseDir
+        $owner = [ordered]@{
+            Pid         = $PID
+            Mode        = $Mode
+            Operation   = $Operation
+            StartedAt   = (Get-Date).ToString('o')
+            ScriptPath  = $PSCommandPath
+            BuildCommit = Get-SoftwarekatalogBuildCommit
+        }
+        $owner | ConvertTo-Json -Depth 4 | Set-Content -LiteralPath $QueueLockInfoPath -Encoding UTF8 -ErrorAction Stop
+    }
+    catch {
+    }
+}
+
+function Clear-QueueLockOwnerInfo {
+    try {
+        if (Test-Path -LiteralPath $QueueLockInfoPath) {
+            Remove-Item -LiteralPath $QueueLockInfoPath -Force -ErrorAction Stop
+        }
+    }
+    catch {
+    }
+}
+
+function Get-QueueLockOwnerSummary {
+    if (-not (Test-Path -LiteralPath $QueueLockInfoPath)) {
+        return 'owner=unknown'
+    }
+
+    try {
+        $raw = Get-Content -LiteralPath $QueueLockInfoPath -Raw -Encoding UTF8 -ErrorAction Stop
+        if ([string]::IsNullOrWhiteSpace($raw)) { return 'owner=unknown' }
+        $owner = $raw | ConvertFrom-Json
+        return "ownerPid=$($owner.Pid) ownerMode='$($owner.Mode)' operation='$($owner.Operation)' startedAt='$($owner.StartedAt)' build='$($owner.BuildCommit)' script='$($owner.ScriptPath)'"
+    }
+    catch {
+        return "owner=unreadable ($($_.Exception.Message))"
+    }
+}
+
 function Set-DirectoryAcl {
     param(
         [Parameter(Mandatory)][string]$Path,
@@ -394,6 +480,75 @@ function New-QueueStateObject {
     }
 }
 
+function Backup-CorruptQueueState {
+    param([string]$Reason)
+
+    if (-not (Test-Path -LiteralPath $QueueStatePath)) { return }
+
+    try {
+        $backupPath = Join-Path $BaseDir ("Queue-State.corrupt-{0}.json" -f (Get-Date -Format 'yyyyMMddHHmmss'))
+        Copy-Item -LiteralPath $QueueStatePath -Destination $backupPath -Force -ErrorAction Stop
+        Write-Log "Backed up corrupt queue state to '$backupPath': $Reason" 'WARN'
+    }
+    catch {
+        Write-Log "Could not back up corrupt queue state: $($_.Exception.Message)" 'WARN'
+    }
+}
+
+function Normalize-QueueState {
+    param($State)
+
+    if (-not $State -or $State -is [System.Array] -or $State -isnot [pscustomobject]) {
+        return New-QueueStateObject
+    }
+
+    if (-not $State.PSObject.Properties['Version']) { $State | Add-Member -NotePropertyName Version -NotePropertyValue 0 }
+    if (-not $State.PSObject.Properties['UpdatedAt']) { $State | Add-Member -NotePropertyName UpdatedAt -NotePropertyValue (Get-Date).ToString('o') }
+    if (-not $State.PSObject.Properties['Current']) { $State | Add-Member -NotePropertyName Current -NotePropertyValue $null }
+    if (-not $State.PSObject.Properties['Queue']) {
+        $State | Add-Member -NotePropertyName Queue -NotePropertyValue @()
+    }
+    elseif (-not $State.Queue) {
+        $State.Queue = @()
+    }
+    else {
+        $State.Queue = @(@($State.Queue) | Where-Object { $null -ne $_ })
+    }
+
+    return $State
+}
+
+function Get-QueueItemSortTimestamp {
+    param($Item)
+
+    foreach ($propertyName in @('FinishedAt','StartedAt','CreatedAt')) {
+        if ($Item -and $Item.PSObject.Properties[$propertyName] -and $Item.$propertyName) {
+            try { return [DateTime]::Parse([string]$Item.$propertyName) }
+            catch { }
+        }
+    }
+
+    return [DateTime]::MinValue
+}
+
+function Compact-QueueState {
+    param($State)
+
+    $State = Normalize-QueueState -State $State
+    $active = New-Object 'System.Collections.Generic.List[object]'
+    $finished = New-Object 'System.Collections.Generic.List[object]'
+
+    foreach ($item in @($State.Queue)) {
+        if (-not $item) { continue }
+        if ($item.State -in @('Queued','Running')) { [void]$active.Add($item) }
+        else { [void]$finished.Add($item) }
+    }
+
+    $keptFinished = @($finished | Sort-Object { Get-QueueItemSortTimestamp -Item $_ } -Descending | Select-Object -First 100)
+    $State.Queue = @(@($active | Sort-Object CreatedAt) + @($keptFinished | Sort-Object { Get-QueueItemSortTimestamp -Item $_ }))
+    return $State
+}
+
 function Read-QueueState {
     if (-not (Test-Path -LiteralPath $QueueStatePath)) {
         return New-QueueStateObject
@@ -406,19 +561,16 @@ function Read-QueueState {
         }
 
         $state = $raw | ConvertFrom-Json
-        if (-not $state.PSObject.Properties['Version']) { $state | Add-Member -NotePropertyName Version -NotePropertyValue 0 }
-        if (-not $state.PSObject.Properties['UpdatedAt']) { $state | Add-Member -NotePropertyName UpdatedAt -NotePropertyValue (Get-Date).ToString('o') }
-        if (-not $state.PSObject.Properties['Current']) { $state | Add-Member -NotePropertyName Current -NotePropertyValue $null }
-        if (-not $state.PSObject.Properties['Queue']) {
-            $state | Add-Member -NotePropertyName Queue -NotePropertyValue @()
+        if ($state -is [System.Array] -or $state -isnot [pscustomobject]) {
+            Backup-CorruptQueueState -Reason "Queue state root is not a JSON object."
+            return New-QueueStateObject
         }
-        elseif (-not $state.Queue) {
-            $state.Queue = @()
-        }
-        return $state
+
+        return Normalize-QueueState -State $state
     }
     catch {
         Write-Log "Failed to read queue state: $($_.Exception.Message)" 'WARN'
+        Backup-CorruptQueueState -Reason $_.Exception.Message
         return New-QueueStateObject
     }
 }
@@ -427,6 +579,7 @@ function Write-QueueState {
     param([Parameter(Mandatory)]$State)
 
     Ensure-Directory -Path $BaseDir
+    $State = Compact-QueueState -State $State
     $tempPath = Join-Path $BaseDir ("Queue-State-{0}.json.tmp" -f [Guid]::NewGuid().ToString('N'))
     try {
         $State | ConvertTo-Json -Depth 8 | Set-Content -LiteralPath $tempPath -Encoding UTF8
@@ -442,7 +595,8 @@ function Write-QueueState {
 function Invoke-WithQueueStateLock {
     param(
         [Parameter(Mandatory)][scriptblock]$ScriptBlock,
-        [int]$TimeoutSeconds = 30
+        [int]$TimeoutSeconds = 30,
+        [string]$Operation = 'QueueState'
     )
 
     $mutex = New-Object System.Threading.Mutex($false, $QueueStateMutexName)
@@ -457,13 +611,18 @@ function Invoke-WithQueueStateLock {
         }
 
         if (-not $hasLock) {
-            throw (New-Object System.TimeoutException -ArgumentList "Timed out waiting for queue state lock '$QueueStateMutexName'.")
+            $owner = Get-QueueLockOwnerSummary
+            throw (New-Object System.TimeoutException -ArgumentList "Timed out waiting for queue state lock '$QueueStateMutexName' ($owner).")
         }
 
+        Write-QueueLockOwnerInfo -Operation $Operation
         & $ScriptBlock
     }
     finally {
-        if ($hasLock) { $mutex.ReleaseMutex() }
+        if ($hasLock) {
+            Clear-QueueLockOwnerInfo
+            $mutex.ReleaseMutex()
+        }
         $mutex.Dispose()
     }
 }
@@ -471,7 +630,8 @@ function Invoke-WithQueueStateLock {
 function Update-QueueState {
     param(
         [Parameter(Mandatory)][scriptblock]$ScriptBlock,
-        [int]$TimeoutSeconds = 30
+        [int]$TimeoutSeconds = 30,
+        [string]$Operation = 'UpdateQueueState'
     )
 
     Invoke-WithQueueStateLock -ScriptBlock {
@@ -481,13 +641,41 @@ function Update-QueueState {
         $state.UpdatedAt = (Get-Date).ToString('o')
         Write-QueueState -State $state
         return $state
-    } -TimeoutSeconds $TimeoutSeconds
+    } -TimeoutSeconds $TimeoutSeconds -Operation $Operation
 }
 
 function Get-QueueStateSnapshot {
     param([int]$TimeoutSeconds = 30)
 
-    Invoke-WithQueueStateLock -ScriptBlock { Read-QueueState } -TimeoutSeconds $TimeoutSeconds
+    Invoke-WithQueueStateLock -ScriptBlock { Read-QueueState } -TimeoutSeconds $TimeoutSeconds -Operation 'GetQueueStateSnapshot'
+}
+
+function Test-IsQueueStateLockTimeout {
+    param([Parameter(Mandatory)]$ErrorObject)
+
+    if ($ErrorObject -is [System.Management.Automation.ErrorRecord]) {
+        $exception = $ErrorObject.Exception
+    }
+    elseif ($ErrorObject -is [System.Exception]) {
+        $exception = $ErrorObject
+    }
+    else {
+        return $false
+    }
+
+    while ($exception) {
+        if ($exception -is [System.TimeoutException] -and $exception.Message -like "*$QueueStateMutexName*") {
+            return $true
+        }
+
+        if ($exception.Message -like "*Timed out waiting for queue state lock '$QueueStateMutexName'*") {
+            return $true
+        }
+
+        $exception = $exception.InnerException
+    }
+
+    return $false
 }
 
 function New-QueueItem {
@@ -547,7 +735,28 @@ function Update-QueueItemProgress {
         [bool]$IsIndeterminate = $true
     )
 
-    $null = Update-QueueState -ScriptBlock {
+    if (-not $script:QueueProgressThrottle) { $script:QueueProgressThrottle = @{} }
+    $now = Get-Date
+    $last = $script:QueueProgressThrottle[$Id]
+    if ($last) {
+        $sameStage = [string]::Equals([string]$last.Stage, [string]$Stage, [System.StringComparison]::Ordinal)
+        $sameText = [string]::Equals([string]$last.StageText, [string]$StageText, [System.StringComparison]::Ordinal)
+        $samePercent = ([string]$last.ProgressPercent) -eq ([string]$ProgressPercent)
+        $sameIndeterminate = ([bool]$last.IsIndeterminate) -eq ([bool]$IsIndeterminate)
+        if ($sameStage -and $sameText -and $samePercent -and $sameIndeterminate -and (($now - $last.UpdatedAt).TotalSeconds -lt 5)) {
+            return
+        }
+    }
+
+    $script:QueueProgressThrottle[$Id] = [pscustomobject]@{
+        Stage = $Stage
+        StageText = $StageText
+        ProgressPercent = $ProgressPercent
+        IsIndeterminate = $IsIndeterminate
+        UpdatedAt = $now
+    }
+
+    $null = Update-QueueState -Operation "Progress:$Id" -ScriptBlock {
         param($state)
         foreach ($item in @($state.Queue)) {
             if (-not $item) { continue }
@@ -603,7 +812,7 @@ function Add-QueueItem {
 
         $state.Queue = @($queue + $Item)
         $script:QueueAddResult = [pscustomobject]@{ Added = $true; Item = $Item }
-    } -TimeoutSeconds $TimeoutSeconds
+    } -TimeoutSeconds $TimeoutSeconds -Operation "AddQueueItem:$($Item.Action):$($Item.Alias)"
 
     $result = $script:QueueAddResult
     $script:QueueAddResult = $null
@@ -616,7 +825,8 @@ function Add-SoftwareRequestToQueue {
         [Parameter(Mandatory)][string]$Alias,
         [string]$RequestedBy,
         [string]$RequestedBySid,
-        [ValidateSet('User','Required')][string]$Source = 'User'
+        [ValidateSet('User','Required')][string]$Source = 'User',
+        [int]$TimeoutSeconds = 30
     )
 
     $normalizedAlias = $Alias.Trim().ToLowerInvariant()
@@ -641,10 +851,12 @@ function Add-SoftwareRequestToQueue {
         -RequestedBy $RequestedBy `
         -RequestedBySid $RequestedBySid
 
-    return Add-QueueItem -Item $item
+    return Add-QueueItem -Item $item -TimeoutSeconds $TimeoutSeconds
 }
 
 function Ensure-RequiredSoftwareQueued {
+    param([int]$TimeoutSeconds = 30)
+
     $addedCount = 0
     if (-not (Test-SoftwareCatalogPolicyEnabled)) { return 0 }
     if (-not (Test-RequiredEnforcementEnabled)) { return 0 }
@@ -661,7 +873,7 @@ function Ensure-RequiredSoftwareQueued {
             continue
         }
 
-        $result = Add-SoftwareRequestToQueue -Action Install -Alias $alias -RequestedBy 'Policy' -RequestedBySid 'Policy' -Source Required
+        $result = Add-SoftwareRequestToQueue -Action Install -Alias $alias -RequestedBy 'Policy' -RequestedBySid 'Policy' -Source Required -TimeoutSeconds $TimeoutSeconds
         if ($result -and $result.Added) {
             $addedCount++
             Write-Log "Queued required software alias '$alias' package '$($app.PackageId)'."
@@ -712,6 +924,29 @@ function Test-WingetLockAvailable {
     }
 }
 
+function Test-QueueStateLockAvailable {
+    $mutex = New-Object System.Threading.Mutex($false, $QueueStateMutexName)
+    $hasLock = $false
+
+    try {
+        try {
+            $hasLock = $mutex.WaitOne(0)
+        }
+        catch [System.Threading.AbandonedMutexException] {
+            $hasLock = $true
+            Write-Log "Recovered abandoned queue state lock '$QueueStateMutexName' during availability check." 'WARN'
+        }
+
+        return $hasLock
+    }
+    finally {
+        if ($hasLock) {
+            $mutex.ReleaseMutex()
+        }
+        $mutex.Dispose()
+    }
+}
+
 function ConvertTo-ProcessArgument {
     param([Parameter(Mandatory)][string]$Value)
 
@@ -768,7 +1003,41 @@ function Start-SoftwarekatalogWorker {
 }
 
 function Start-QueueWorker {
+    try {
+        $task = Get-ScheduledTask -TaskName $QueueTaskName -ErrorAction SilentlyContinue
+        if ($task) {
+            Start-ScheduledTask -TaskName $QueueTaskName -ErrorAction Stop
+            Write-Log "Started scheduled queue task '$QueueTaskName'."
+            return $true
+        }
+    }
+    catch {
+        Write-Log "Could not start scheduled queue task '$QueueTaskName': $($_.Exception.Message)" 'WARN'
+    }
+
+    try {
         Start-SoftwarekatalogWorker -WorkerMode ProcessQueue
+        Write-Log 'Started queue worker process fallback.' 'WARN'
+        return $true
+    }
+    catch {
+        Write-Log "Could not start queue worker process fallback: $($_.Exception.Message)" 'ERROR'
+        return $false
+    }
+}
+
+function Start-QueueWorkerIfAvailable {
+    if (-not (Test-WingetLockAvailable)) {
+        Write-Log 'Queue worker not started because winget lock is already held.'
+        return $false
+    }
+
+    if (-not (Test-QueueStateLockAvailable)) {
+        Write-Log 'Queue worker not started because queue state lock is already held.' 'WARN'
+        return $false
+    }
+
+    return (Start-QueueWorker)
 }
 
 function Get-WingetPath {
@@ -2129,22 +2398,34 @@ function Unregister-ObsoleteDaemonTask {
     }
 }
 
-function Stop-ObsoleteDaemonProcesses {
-    $daemonMarkers = @(
-        '-Mode Daemon',
-        '-Mode "Daemon"',
-        "-Mode 'Daemon'",
-        'Mode=Daemon',
-        'NamedPipe',
-        'Named pipe',
-        '\\.\pipe\Softwarekatalog'
-    )
+function Stop-SoftwarekatalogScheduledTasksForUpdate {
+    foreach ($taskName in @($RequestTaskName, $QueueTaskName, $UpgradeTaskName, $LegacyQueueTaskName, $DaemonTaskName)) {
+        if ([string]::IsNullOrWhiteSpace($taskName)) { continue }
 
+        try {
+            $task = Get-ScheduledTask -TaskName $taskName -ErrorAction SilentlyContinue
+            if ($task) {
+                Write-Log "Stopping scheduled task '$taskName' before update."
+                Stop-ScheduledTask -TaskName $taskName -ErrorAction SilentlyContinue
+                for ($attempt = 0; $attempt -lt 10; $attempt++) {
+                    $currentTask = Get-ScheduledTask -TaskName $taskName -ErrorAction SilentlyContinue
+                    if (-not $currentTask -or [string]$currentTask.State -ne "Running") { break }
+                    Start-Sleep -Milliseconds 500
+                }
+            }
+        }
+        catch {
+            Write-Log "Could not stop scheduled task '$taskName': $($_.Exception.Message)" 'WARN'
+        }
+    }
+}
+
+function Stop-ObsoleteDaemonProcesses {
     try {
         $processes = @(Get-CimInstance -ClassName Win32_Process -ErrorAction Stop)
     }
     catch {
-        Write-Log "Could not enumerate processes for obsolete daemon cleanup: $($_.Exception.Message)" 'WARN'
+        Write-Log "Could not enumerate processes for Softwarekatalog worker cleanup: $($_.Exception.Message)" 'WARN'
         return
     }
 
@@ -2160,21 +2441,12 @@ function Stop-ObsoleteDaemonProcesses {
         $matchesScriptName = ($commandLine.IndexOf('SelfServiceWinget.ps1', [System.StringComparison]::OrdinalIgnoreCase) -ge 0 -and $commandLine.IndexOf($BaseDir, [System.StringComparison]::OrdinalIgnoreCase) -ge 0)
         if (-not ($matchesScriptPath -or $matchesScriptName)) { continue }
 
-        $hasDaemonMarker = $false
-        foreach ($marker in $daemonMarkers) {
-            if ($commandLine.IndexOf($marker, [System.StringComparison]::OrdinalIgnoreCase) -ge 0) {
-                $hasDaemonMarker = $true
-                break
-            }
-        }
-        if (-not $hasDaemonMarker) { continue }
-
         try {
-            Write-Log "Stopping obsolete daemon process pid=$($process.ProcessId)."
+            Write-Log "Stopping existing Softwarekatalog process pid=$($process.ProcessId)."
             Stop-Process -Id ([int]$process.ProcessId) -Force -ErrorAction Stop
         }
         catch {
-            Write-Log "Could not stop obsolete daemon process pid=$($process.ProcessId): $($_.Exception.Message)" 'WARN'
+            Write-Log "Could not stop Softwarekatalog process pid=$($process.ProcessId): $($_.Exception.Message)" 'WARN'
         }
     }
 }
@@ -2209,6 +2481,23 @@ function Register-OrUpdateTasks {
 
     Register-ScheduledTask -TaskName $RequestTaskName -InputObject $requestTask -Force | Out-Null
 
+    Write-Log "Registering/updating scheduled task '$QueueTaskName'."
+    $queueArgs = "-NoProfile -ExecutionPolicy Bypass -File `"$LocalScriptPath`" -Mode ProcessQueue"
+    $queueAction = New-ScheduledTaskAction -Execute $PowerShellExe -Argument $queueArgs
+    $queueSettings = New-ScheduledTaskSettingsSet `
+        -AllowStartIfOnBatteries `
+        -DontStopIfGoingOnBatteries `
+        -StartWhenAvailable `
+        -MultipleInstances IgnoreNew `
+        -ExecutionTimeLimit (New-TimeSpan -Hours 3)
+
+    $queueTask = New-ScheduledTask `
+        -Action $queueAction `
+        -Principal $principal `
+        -Settings $queueSettings
+
+    Register-ScheduledTask -TaskName $QueueTaskName -InputObject $queueTask -Force | Out-Null
+
     Write-Log "Registering/updating scheduled task '$UpgradeTaskName'."
     $upgradeArgs = "-NoProfile -ExecutionPolicy Bypass -File `"$LocalScriptPath`" -Mode UpgradeAll"
     $upgradeAction = New-ScheduledTaskAction -Execute $PowerShellExe -Argument $upgradeArgs
@@ -2512,16 +2801,26 @@ function Invoke-RequiredSoftwarePolicy {
         }
     }
 
-    $queued = Ensure-RequiredSoftwareQueued
+    try {
+        $queued = Ensure-RequiredSoftwareQueued -TimeoutSeconds 5
+    }
+    catch {
+        if (Test-IsQueueStateLockTimeout -ErrorObject $_) {
+            Write-Log "Required software queueing skipped because the queue state lock is busy: $($_.Exception.Message)" 'WARN'
+            return [pscustomobject]@{ Installed = 0; Failed = 0; Skipped = 0; Queued = 0 }
+        }
+        throw
+    }
+
     if ($queued -gt 0) {
-        Start-QueueWorker
+        [void](Start-QueueWorkerIfAvailable)
     }
 
     return [pscustomobject]@{ Installed = 0; Failed = 0; Skipped = 0; Queued = $queued }
 }
 
 function Reset-RunningQueueItems {
-    $null = Update-QueueState -ScriptBlock {
+    $null = Update-QueueState -Operation 'ResetRunningQueueItems' -ScriptBlock {
         param($state)
         foreach ($item in @($state.Queue)) {
             if (-not $item) { continue }
@@ -2540,7 +2839,7 @@ function Reset-RunningQueueItems {
 
 function Get-NextQueuedItem {
     $script:NextQueueItem = $null
-    $null = Update-QueueState -ScriptBlock {
+    $null = Update-QueueState -Operation 'GetNextQueuedItem' -ScriptBlock {
         param($state)
         foreach ($item in (@($state.Queue) | Sort-Object CreatedAt)) {
             if (-not $item) { continue }
@@ -2580,7 +2879,7 @@ function Complete-QueueItem {
         [string]$Message
     )
 
-    $null = Update-QueueState -ScriptBlock {
+    $null = Update-QueueState -Operation "CompleteQueueItem:${Id}:${State}" -ScriptBlock {
         param($queueState)
         foreach ($item in @($queueState.Queue)) {
             if (-not $item) { continue }
@@ -2681,7 +2980,7 @@ function Invoke-ProcessQueueMode {
             }
         }
 
-        $null = Update-QueueState -ScriptBlock { param($state) $state.Current = $null }
+        $null = Update-QueueState -Operation 'ClearCurrentQueueItem' -ScriptBlock { param($state) $state.Current = $null }
     }
 }
 
@@ -2693,10 +2992,13 @@ function Invoke-UpgradeAllMode {
     try {
         $result = Add-QueueItem -Item $item -TimeoutSeconds 5
     }
-    catch [System.TimeoutException] {
+    catch {
+        if (Test-IsQueueStateLockTimeout -ErrorObject $_) {
             Write-Log "Skipped 2-hour winget upgrade queueing because the queue state lock is busy: $($_.Exception.Message)" 'WARN'
             return
         }
+        throw
+    }
 
     if ($result.Added) {
         Write-Log 'Queued 2-hour winget upgrade.'
@@ -2705,12 +3007,7 @@ function Invoke-UpgradeAllMode {
         Write-Log '2-hour winget upgrade is already queued or running.'
     }
 
-    if (Test-WingetLockAvailable) {
-        Start-QueueWorker
-    }
-    else {
-        Write-Log 'Queue worker not started because winget lock is already held; queued upgrade will be processed by the active worker.'
-    }
+    [void](Start-QueueWorkerIfAvailable)
 }
 
 function Invoke-ApplyPolicyMode {
@@ -2747,9 +3044,19 @@ function Invoke-ProcessRequestMode {
         return
     }
 
-    $result = Add-SoftwareRequestToQueue -Action $RequestAction -Alias $alias -RequestedBy $RequestedBy -RequestedBySid $RequestedBySid -Source User
+    try {
+        $result = Add-SoftwareRequestToQueue -Action $RequestAction -Alias $alias -RequestedBy $RequestedBy -RequestedBySid $RequestedBySid -Source User -TimeoutSeconds 5
+    }
+    catch {
+        if (Test-IsQueueStateLockTimeout -ErrorObject $_) {
+            Write-Log "Worker request action=$RequestAction alias='$alias' could not be queued because the queue state lock is busy: $($_.Exception.Message)" 'WARN'
+            return
+        }
+        throw
+    }
+
     Write-Log "Queued worker request action=$RequestAction alias='$alias' added=$($result.Added) RequestedBy='$RequestedBy' Sid='$RequestedBySid'"
-    Start-QueueWorker
+    [void](Start-QueueWorkerIfAvailable)
 }
 
 function Get-RequestFileOwnerInfo {
@@ -2901,6 +3208,49 @@ function Move-RequestFile {
     }
 }
 
+function Test-IsTransientRequestFailureMessage {
+    param([string]$Message)
+
+    if ([string]::IsNullOrWhiteSpace($Message)) { return $false }
+
+    return (
+        $Message -like "*$QueueStateMutexName*" -or
+        $Message -like "*queue state lock*" -or
+        $Message -like "*Aufruftiefe*" -or
+        $Message -like "*call depth*" -or
+        $Message -like "*recursion*"
+    )
+}
+
+function Restore-QueueLockFailedRequests {
+    Ensure-Directory -Path $RequestsDir
+    Ensure-Directory -Path $FailedRequestsDir
+
+    $errorFiles = @(Get-ChildItem -LiteralPath $FailedRequestsDir -Filter '*.json.error.txt' -File -ErrorAction SilentlyContinue)
+    foreach ($errorFile in $errorFiles) {
+        try {
+            $message = Get-Content -LiteralPath $errorFile.FullName -Raw -Encoding UTF8 -ErrorAction Stop
+            if (-not (Test-IsTransientRequestFailureMessage -Message $message)) { continue }
+
+            $requestName = $errorFile.Name.Substring(0, $errorFile.Name.Length - '.error.txt'.Length)
+            $requestPath = Join-Path $FailedRequestsDir $requestName
+            if (-not (Test-Path -LiteralPath $requestPath)) { continue }
+
+            $destination = Join-Path $RequestsDir $requestName
+            if (Test-Path -LiteralPath $destination) {
+                $destination = Join-Path $RequestsDir ("{0}-{1}.json" -f [IO.Path]::GetFileNameWithoutExtension($requestName), [Guid]::NewGuid().ToString('N'))
+            }
+
+            Move-Item -LiteralPath $requestPath -Destination $destination -Force -ErrorAction Stop
+            Remove-Item -LiteralPath $errorFile.FullName -Force -ErrorAction SilentlyContinue
+            Write-Log "Restored failed request '$requestName' for retry after queue lock timeout."
+        }
+        catch {
+            Write-Log "Could not restore failed request '$($errorFile.Name)': $($_.Exception.Message)" 'WARN'
+        }
+    }
+}
+
 function Invoke-ProcessRequestsMode {
     Ensure-Directory -Path $BaseDir
     Ensure-Directory -Path $LogDir
@@ -2961,13 +3311,18 @@ function Invoke-ProcessRequestsMode {
             $aliasValue = Get-RequestScalarValue -Request $request -Names @('AppAlias','Alias') -DisplayName 'AppAlias' -MaxLength 128
             $alias = $aliasValue.Trim().ToLowerInvariant()
             $owner = Get-RequestFileOwnerInfo -Path $file.FullName
-            $result = Add-SoftwareRequestToQueue -Action $action -Alias $alias -RequestedBy $owner.Name -RequestedBySid $owner.Sid -Source User
+            $result = Add-SoftwareRequestToQueue -Action $action -Alias $alias -RequestedBy $owner.Name -RequestedBySid $owner.Sid -Source User -TimeoutSeconds 5
             Write-Log "Accepted request file '$($file.Name)' action=$action alias='$alias' added=$($result.Added) owner='$($owner.Name)' sid='$($owner.Sid)'."
             Move-RequestFile -Path $file.FullName -DestinationDir $ProcessedRequestsDir
             $shouldStartQueue = $true
         }
         catch {
             $message = $_.Exception.Message
+            if (Test-IsQueueStateLockTimeout -ErrorObject $_) {
+                Write-Log "Request file '$($file.Name)' will be retried because the queue state lock is busy: $message" 'WARN'
+                break
+            }
+
             Write-Log "Rejected request file '$($file.Name)': $message" 'WARN'
             try {
                 Move-RequestFile -Path $file.FullName -DestinationDir $FailedRequestsDir -ErrorMessage $message
@@ -2978,26 +3333,24 @@ function Invoke-ProcessRequestsMode {
         }
     }
 
-    $requiredQueued = Ensure-RequiredSoftwareQueued
+    try {
+        $requiredQueued = Ensure-RequiredSoftwareQueued -TimeoutSeconds 5
         if ($requiredQueued -gt 0) {
             Write-Log "Queued required software from request worker count=$requiredQueued."
             $shouldStartQueue = $true
         }
-
-    foreach ($queuedItem in @((Get-QueueStateSnapshot).Queue)) {
-        if ($queuedItem -and $queuedItem.State -eq 'Queued') {
-            $shouldStartQueue = $true
-            break
+    }
+    catch {
+        if (Test-IsQueueStateLockTimeout -ErrorObject $_) {
+            Write-Log "Required software queueing deferred because the queue state lock is busy: $($_.Exception.Message)" 'WARN'
+        }
+        else {
+            throw
         }
     }
 
     if ($shouldStartQueue) {
-        if (Test-WingetLockAvailable) {
-            Start-QueueWorker
-        }
-        else {
-            Write-Log 'Queue worker not started because winget lock is already held.'
-        }
+        [void](Start-QueueWorkerIfAvailable)
     }
 }
 
@@ -3008,9 +3361,14 @@ function Install-Service {
     Ensure-Directory -Path $ProcessedRequestsDir
     Ensure-Directory -Path $FailedRequestsDir
 
+    Write-Log "Softwarekatalog build commit: $(Get-SoftwarekatalogBuildCommit)."
+    Stop-SoftwarekatalogScheduledTasksForUpdate
+    Stop-ObsoleteDaemonProcesses
     Unregister-LegacyQueueTask
     Unregister-ObsoleteDaemonTask
-    Stop-ObsoleteDaemonProcesses
+    Restore-QueueLockFailedRequests
+    try { Reset-RunningQueueItems }
+    catch { Write-Log "Could not reset running queue items during install/update: $($_.Exception.Message)" 'WARN' }
 
     Set-DirectoryAcl -Path $BaseDir -UsersCanModify:$false
     Set-DirectoryAcl -Path $LogDir -UsersCanModify:$false
@@ -3032,6 +3390,24 @@ function Install-Service {
     New-Shortcut -ShortcutPath $StartMenuShortcutPath
     Register-OrUpdateTasks
 
+    try {
+        $queueState = Get-QueueStateSnapshot -TimeoutSeconds 5
+        foreach ($queuedItem in @($queueState.Queue)) {
+            if ($queuedItem -and $queuedItem.State -eq 'Queued') {
+                [void](Start-QueueWorkerIfAvailable)
+                break
+            }
+        }
+    }
+    catch {
+        if (Test-IsQueueStateLockTimeout -ErrorObject $_) {
+            Write-Log "Could not inspect queued work during install/update because the queue state lock is busy: $($_.Exception.Message)" 'WARN'
+        }
+        else {
+            Write-Log "Could not inspect queued work during install/update: $($_.Exception.Message)" 'WARN'
+        }
+    }
+
     Write-Log 'Install/update completed successfully.'
 }
 
@@ -3051,6 +3427,11 @@ try {
 }
 catch {
     try {
+        if ($Mode -in @('ProcessRequests','UpgradeAll') -and (Test-IsQueueStateLockTimeout -ErrorObject $_)) {
+            Write-Log "Mode '$Mode' deferred because the queue state lock is busy: $($_.Exception.Message)" 'WARN'
+            return
+        }
+
         Write-Log "Fatal error in mode '$Mode': $($_.Exception.Message)" 'ERROR'
     }
     catch {