j.foucher/PS_Launcher
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Downloads: parallel multi-segment, sparse pre-alloc, faster SHA-256, URL auto-refresh

Browse Source 
DownloadManager
- Parallel multi-segment download (default 8 connections, configurable up to 16)
  with per-segment Range requests. Bypasses OVH/Apache per-connection bandwidth
  throttling — typical speed-up ×4 to ×8 vs single connection.
- Reporter task on a dedicated Task with Interlocked aggregate counter (no lock
  contention with workers). Reports speed/ETA every 250ms even mid-download,
  fixes the "speed only shows at install transition" bug.
- Sparse file pre-allocation via FSCTL_SET_SPARSE before SetLength: no zero-fill,
  no SeManageVolumePrivilege, instant on any disk type. Removes 5-30s of
  preparation lag on HDD.
- HEAD probe skipped (trust manifest size, signed Ed25519). Falls back to
  single-segment if first segment returns 200 instead of 206.
- Resume URL comparison fixed: ignores HMAC querystring (?exp=&sig=) which
  changes per request, compares only host+path. Previously every resume started
  fresh because the old URL never matched the freshly signed one.
- Auto-refresh signed URL on 403/410 mid-DL: SemaphoreSlim with 5s debounce so
  8 simultaneous segment expirations trigger a single /api/download-url/ call.
  Slow-connection users (1 Mbps, 30+ hours for 14 GB) keep downloading
  transparently across multiple TTL cycles.
- Per-version hashAlgorithm:none in manifest skips client SHA-256 verification
  (still relying on Ed25519 manifest signature + HMAC URL).
- DangerButton style (red) added to Theme.xaml for the new cancel-resume action.

IntegrityService
- 16 MiB buffer (was 1 MiB), FileOptions.SequentialScan + Asynchronous,
  IncrementalHash (uses SHA-NI hardware extensions on .NET 8), double-buffering
  to overlap CPU and I/O. Typical 14 GB hash verification: 60-180s → 15-40s.

HttpClient
- MaxConnectionsPerServer=16, EnableMultipleHttp2Connections, HTTP/2 preferred,
  AutomaticDecompression=None (ZIPs are already compressed), 5min pooled
  connection lifetime.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
j.foucher
2026-05-02 19:15:27 +02:00
parent b4f205cbe5
commit 9cea07d6be
7 changed files with 667 additions and 77 deletions
Download Patch File Download Diff File Expand all files Collapse all files

560
src/PSLauncher.Core/Downloads/DownloadManager.cs
View File

@@ -1,35 +1,62 @@
using System.Diagnostics;
using System.Net;
using System.Net.Http.Headers;
using System.Runtime.InteropServices;
using Microsoft.Extensions.Logging;
using Microsoft.Win32.SafeHandles;
using Polly;
using Polly.Retry;
using PSLauncher.Core.Configuration;
using PSLauncher.Core.Integrity;
using PSLauncher.Models;
namespace PSLauncher.Core.Downloads;
/// <summary>
/// Téléchargement HTTP avec :
/// - reprise par Range requests (If-Range sur ETag/Last-Modified) ;
/// - retry exponentiel via Polly (1-32 s, 6 tentatives, jitter) ;
/// - téléchargement parallèle multi-segments (config <see cref="LocalConfig.ParallelDownloadSegments"/>) ;
/// - vérif SHA-256 streaming en fin de DL ;
/// - persistance d'état (state.json) toutes les 5 s ou 100 Mo pour reprise après crash.
///
/// Le multi-segments contourne le throttling per-connection d'Apache (notamment OVH mutualisé)
/// : chaque segment ouvre une connexion TCP séparée et écrit à son offset dans le fichier
/// pré-alloué via <c>SetLength</c>. Speed-up typique observé : ×4 à ×10.
/// </summary>
public sealed class DownloadManager : IDownloadManager
{
private const int BufferSize = 1 << 20; // 1 MiB
private const int StateFlushBytesInterval = 100 * 1024 * 1024; // 100 MiB
private const int BufferSize = 4 << 20; // 4 MiB par segment (avant 1 MiB)
private const int StateFlushBytesInterval = 50 * 1024 * 1024; // 50 MiB
private const double StateFlushSecondsInterval = 5.0;
private const long MultiSegmentMinSize = 50 * 1024 * 1024; // < 50 MiB → single-segment
private readonly HttpClient _http;
private readonly IDownloadStateStore _stateStore;
private readonly IIntegrityService _integrity;
private readonly LocalConfig _config;
private readonly ILogger<DownloadManager> _logger;
private readonly ResiliencePipeline _retryPipeline;
// URL active partagée entre tous les segments d'un DL en cours, et lock pour
// refresher de façon coordonnée quand l'URL HMAC expire (TTL serveur = 6 h).
// Plusieurs segments peuvent recevoir 403 simultanément — un seul doit faire
// l'appel /api/download-url/.
private Uri _currentUrl = new("about:blank");
private DateTime _lastUrlRefresh = DateTime.MinValue;
private readonly SemaphoreSlim _urlRefreshLock = new(1, 1);
public DownloadManager(
HttpClient http,
IDownloadStateStore stateStore,
IIntegrityService integrity,
LocalConfig config,
ILogger<DownloadManager> logger)
{
_http = http;
_stateStore = stateStore;
_integrity = integrity;
_config = config;
_logger = logger;
_retryPipeline = new ResiliencePipelineBuilder()
@@ -60,28 +87,400 @@ public sealed class DownloadManager : IDownloadManager
public void DiscardResumableState(string version) => _stateStore.Discard(version);
public async Task<string> DownloadAsync(
public Task<string> DownloadAsync(
DownloadJob job,
IProgress<DownloadProgress>? progress,
CancellationToken ct)
=> DownloadAsync(job, progress, null, ct);
/// <summary>
/// Récupère l'URL active. Si la dernière récup date de moins de 5 s,
/// retourne celle en cache (évite que 8 segments en 403 simultanés
/// déclenchent 8 requêtes /api/download-url/). Sinon refresh via le
/// callback du job. Si pas de callback, retourne l'URL initiale du job.
/// </summary>
private async Task<Uri> GetOrRefreshUrlAsync(DownloadJob job, bool forceRefresh, CancellationToken ct)
{
await _urlRefreshLock.WaitAsync(ct).ConfigureAwait(false);
try
{
// Premier appel : initialise avec l'URL du job
if (_currentUrl.Scheme == "about")
{
_currentUrl = job.Url;
_lastUrlRefresh = DateTime.UtcNow;
}
// Si refresh forcé (un segment a vu 403), tente de fetcher une nouvelle URL.
// Debounce 5 s pour fusionner les refresh en rafale des 8 segments.
if (forceRefresh && (DateTime.UtcNow - _lastUrlRefresh).TotalSeconds >= 5
&& job.RefreshUrlAsync is not null)
{
_logger.LogInformation("Refreshing signed URL for v{Version} (URL expired or 403)", job.Version);
var fresh = await job.RefreshUrlAsync(ct).ConfigureAwait(false);
if (fresh is not null)
{
_currentUrl = fresh;
_lastUrlRefresh = DateTime.UtcNow;
_logger.LogInformation("Signed URL refreshed for v{Version}", job.Version);
}
}
return _currentUrl;
}
finally
{
_urlRefreshLock.Release();
}
}
public async Task<string> DownloadAsync(
DownloadJob job,
IProgress<DownloadProgress>? progress,
IProgress<double>? hashProgress,
CancellationToken ct)
{
// Reset URL state pour ce job (DownloadManager est singleton DI)
_currentUrl = job.Url;
_lastUrlRefresh = DateTime.UtcNow;
var partialPath = _stateStore.GetPartialPath(job.Version);
var finalPath = Path.ChangeExtension(partialPath, null); // strip ".partial" → .zip
// 1. Récupère l'état existant (s'il y en a un, et qu'il correspond à la même URL/sha)
// 1. Récupère l'état existant. On compare la ressource par son SHA-256 (signé via le
// manifest) plutôt que par URL : les URLs HMAC-signées changent à chaque appel
// (?exp=…&sig=…), donc une comparaison stricte d'URL invaliderait toujours le resume.
// On compare aussi le path (host+path sans query) pour détecter un changement de
// serveur/fichier, mais on ignore la querystring.
var existing = _stateStore.Load(job.Version);
if (existing is not null && (existing.Url != job.Url.ToString() || existing.ExpectedSha256 != job.ExpectedSha256))
if (existing is not null)
{
_logger.LogInformation("Cached state for {Version} mismatches new job (url/sha changed) — discarding", job.Version);
_stateStore.Discard(job.Version);
existing = null;
bool shaMismatch = !string.IsNullOrEmpty(job.ExpectedSha256)
&& !string.IsNullOrEmpty(existing.ExpectedSha256)
&& !string.Equals(existing.ExpectedSha256, job.ExpectedSha256, StringComparison.OrdinalIgnoreCase);
bool pathMismatch = false;
if (Uri.TryCreate(existing.Url, UriKind.Absolute, out var oldUri))
{
pathMismatch = oldUri.Host != job.Url.Host || oldUri.AbsolutePath != job.Url.AbsolutePath;
}
if (shaMismatch || pathMismatch)
{
_logger.LogInformation("Cached state for {Version} mismatches new job (sha={Sha} path={Path}) — discarding",
job.Version, shaMismatch, pathMismatch);
_stateStore.Discard(job.Version);
existing = null;
}
else
{
// Même ressource : on rafraîchit juste l'URL stockée (utile pour les futures requêtes
// au sein de cette session, même si en pratique chaque segment requête via job.Url).
existing.Url = job.Url.ToString();
}
}
// 2. Choix single vs multi-segment
var requestedSegments = Math.Clamp(_config.ParallelDownloadSegments, 1, 16);
var useMultiSegment = requestedSegments > 1
&& job.ExpectedSize >= MultiSegmentMinSize
&& (existing is null || existing.Segments.Count > 0);
// Si l'état existant est en mode legacy (single-segment) et qu'on veut multi → on conserve le legacy
// pour pouvoir reprendre proprement (sinon on jetterait son DL).
if (existing is { Segments.Count: 0 } && File.Exists(partialPath))
{
_logger.LogInformation("Existing single-segment partial detected — staying single-segment for resume");
useMultiSegment = false;
}
// 3. Vérif espace disque (uniquement à l'initial, pas au resume)
if (existing is null && job.ExpectedSize > 0)
{
var drive = new DriveInfo(Path.GetPathRoot(_stateStore.GetDownloadsDirectory())!);
var needed = (long)(job.ExpectedSize * 1.05);
if (drive.AvailableFreeSpace < needed)
throw new IOException(
$"Espace disque insuffisant sur {drive.Name} : besoin de {needed:N0} octets, disponible {drive.AvailableFreeSpace:N0}");
}
if (useMultiSegment)
{
return await DownloadMultiSegmentAsync(
job, partialPath, finalPath, existing, requestedSegments, progress, hashProgress, ct).ConfigureAwait(false);
}
return await DownloadSingleSegmentAsync(
job, partialPath, finalPath, existing, progress, hashProgress, ct).ConfigureAwait(false);
}
// ===================================================================
// ============== MULTI-SEGMENT (parallèle, fast path) ==============
// ===================================================================
private async Task<string> DownloadMultiSegmentAsync(
DownloadJob job,
string partialPath,
string finalPath,
DownloadState? existing,
int segmentCount,
IProgress<DownloadProgress>? progress,
IProgress<double>? hashProgress,
CancellationToken ct)
{
var swPhase = Stopwatch.StartNew();
// 1. On fait CONFIANCE au manifest : pas de HEAD probe.
// Le HEAD ajoutait jusqu'à 30 s-5 min de latence quand le serveur faisait
// un md5_file() sur 14 Go à chaque requête (vu sur OVH/gate.php). Le manifest
// est signé Ed25519 donc sa taille est de toute façon fiable. Si le serveur
// ne supporte pas Range, on le détecte sur le premier segment (200 au lieu
// de 206) et on fallback sur single-segment.
long total = job.ExpectedSize;
if (total <= 0)
{
_logger.LogInformation("ExpectedSize=0 — falling back to single-segment");
return await DownloadSingleSegmentAsync(job, partialPath, finalPath, existing, progress, hashProgress, ct).ConfigureAwait(false);
}
// 2. State (resume vs fresh)
DownloadState state;
if (existing is not null && existing.Segments.Count > 0 && existing.TotalBytes == total && File.Exists(partialPath))
{
state = existing;
_logger.LogInformation("Resuming multi-segment DL for v{Version} ({Segments} segments, {Done}/{Total} bytes)",
job.Version, state.Segments.Count, state.DownloadedBytes, state.TotalBytes);
}
else
{
// Discard tout résidu (state mismatch ou partial sans state)
_stateStore.Discard(job.Version);
state = new DownloadState
{
Version = job.Version,
Url = job.Url.ToString(),
TotalBytes = total,
ExpectedSha256 = job.ExpectedSha256,
PartialPath = partialPath,
StartedAtUtc = DateTime.UtcNow,
LastChunkAtUtc = DateTime.UtcNow,
Segments = BuildSegments(total, segmentCount),
};
_stateStore.Save(state);
}
// 3. Pré-allocation du fichier final.
// Stratégie : on tente de marquer le fichier en SPARSE (FSCTL_SET_SPARSE)
// avant le SetLength, ce qui rend l'opération métadata-only (~milliseconde
// au lieu de 5-30 s sur HDD à cause du zero-fill NTFS).
// Pas besoin d'admin (contrairement à SetFileValidData). Aucune fuite de
// données : les zones non écrites sont lues comme zéros virtuels par NTFS.
// Si FSCTL_SET_SPARSE échoue (volume FAT32 ou autre), fallback sur SetLength
// classique.
var swAlloc = Stopwatch.StartNew();
await Task.Run(() =>
{
using var alloc = new FileStream(partialPath, FileMode.OpenOrCreate, FileAccess.Write, FileShare.ReadWrite);
if (alloc.Length != total)
{
bool sparseOk = SparseFileSupport.TrySetSparse(alloc.SafeFileHandle);
_logger.LogInformation("Pre-allocating {Path} to {Bytes} bytes (sparse={Sparse})", partialPath, total, sparseOk);
alloc.SetLength(total);
}
}, ct).ConfigureAwait(false);
swAlloc.Stop();
_logger.LogInformation("Pre-alloc done in {Ms} ms (HEAD skipped, total phase {TotalMs} ms)", swAlloc.ElapsedMilliseconds, swPhase.ElapsedMilliseconds);
// Une fois la pré-alloc terminée, on émet immédiatement un progress 0/total
// pour que le footer passe de « Préparation… » à « ⬇ v1.4.6 : 0 / 14 Go ».
// La vraie progression suit ~250 ms après quand le reporter task tick.
progress?.Report(new DownloadProgress(state.DownloadedBytes, total, 0, null));
// 4. Lance les workers en parallèle.
// Compteur agrégé via Interlocked → pas de lock dans le hot path.
long aggregateBytes = state.DownloadedBytes;
// Action passée aux segments : juste un Add atomique (zéro contention).
void OnSegmentBytes(int _, long deltaBytes)
=> Interlocked.Add(ref aggregateBytes, deltaBytes);
// Reporter dédié : toutes les 250 ms, snapshot le compteur, calcule bps/ETA, reporte.
// Tous les 5 s ou 50 Mo, persist le state.json.
// Tourne sur un thread séparé pour ne jamais être starvé par les workers DL.
using var reporterCts = CancellationTokenSource.CreateLinkedTokenSource(ct);
var reporterTask = Task.Run(async () =>
{
var sw = Stopwatch.StartNew();
var lastReport = TimeSpan.Zero;
long lastReportBytes = Interlocked.Read(ref aggregateBytes);
var lastFlush = TimeSpan.Zero;
long lastFlushBytes = lastReportBytes;
try
{
while (!reporterCts.IsCancellationRequested)
{
await Task.Delay(250, reporterCts.Token).ConfigureAwait(false);
var snapshot = Interlocked.Read(ref aggregateBytes);
var elapsed = sw.Elapsed;
var deltaSec = (elapsed - lastReport).TotalSeconds;
var bps = deltaSec > 0 ? (snapshot - lastReportBytes) / deltaSec : 0;
TimeSpan? eta = null;
if (bps > 0 && total > snapshot) eta = TimeSpan.FromSeconds((total - snapshot) / bps);
progress?.Report(new DownloadProgress(snapshot, total, bps, eta));
lastReport = elapsed;
lastReportBytes = snapshot;
if ((elapsed - lastFlush).TotalSeconds >= StateFlushSecondsInterval ||
snapshot - lastFlushBytes >= StateFlushBytesInterval)
{
state.DownloadedBytes = snapshot;
state.LastChunkAtUtc = DateTime.UtcNow;
_stateStore.Save(state);
lastFlush = elapsed;
lastFlushBytes = snapshot;
}
}
}
catch (OperationCanceledException) { /* attendu en fin de DL */ }
}, reporterCts.Token);
var tasks = state.Segments
.Where(s => !s.Completed)
.Select(s => Task.Run(() =>
_retryPipeline.ExecuteAsync(async innerCt =>
{
await DownloadSegmentAsync(job, state, s, partialPath, OnSegmentBytes, innerCt).ConfigureAwait(false);
}, ct).AsTask(), ct))
.ToArray();
try
{
await Task.WhenAll(tasks).ConfigureAwait(false);
}
catch
{
reporterCts.Cancel();
try { await reporterTask.ConfigureAwait(false); } catch { /* ignore */ }
state.DownloadedBytes = Interlocked.Read(ref aggregateBytes);
_stateStore.Save(state);
throw;
}
// Stoppe le reporter et fais un dernier flush
reporterCts.Cancel();
try { await reporterTask.ConfigureAwait(false); } catch { /* ignore */ }
state.DownloadedBytes = Interlocked.Read(ref aggregateBytes);
_stateStore.Save(state);
progress?.Report(new DownloadProgress(state.DownloadedBytes, total, 0, TimeSpan.Zero));
// 5. Vérif taille
var actualSize = new FileInfo(partialPath).Length;
if (actualSize != total)
{
_stateStore.Discard(job.Version);
throw new InvalidDataException($"Taille téléchargée incorrecte : attendu {total}, obtenu {actualSize}");
}
// 6. Vérif SHA-256
await VerifyAndFinalizeAsync(job, partialPath, finalPath, hashProgress, ct).ConfigureAwait(false);
return finalPath;
}
private static List<DownloadSegment> BuildSegments(long total, int count)
{
var segments = new List<DownloadSegment>(count);
long segSize = total / count;
for (int i = 0; i < count; i++)
{
long start = i * segSize;
long end = (i == count - 1) ? total - 1 : (start + segSize - 1);
segments.Add(new DownloadSegment
{
Index = i,
Start = start,
End = end,
DownloadedBytes = 0,
Completed = false,
});
}
return segments;
}
private async Task DownloadSegmentAsync(
DownloadJob job,
DownloadState state,
DownloadSegment seg,
string partialPath,
Action<int, long> onBytes,
CancellationToken ct)
{
if (seg.Completed) return;
var segStart = seg.Start + seg.DownloadedBytes;
if (segStart > seg.End) { seg.Completed = true; return; }
// Récupère l'URL active (peut avoir été refreshée par un autre segment)
var url = await GetOrRefreshUrlAsync(job, forceRefresh: false, ct).ConfigureAwait(false);
using var req = new HttpRequestMessage(HttpMethod.Get, url);
req.Headers.Range = new RangeHeaderValue(segStart, seg.End);
if (!string.IsNullOrEmpty(state.Etag))
req.Headers.IfRange = new RangeConditionHeaderValue(new EntityTagHeaderValue(state.Etag));
using var resp = await _http.SendAsync(req, HttpCompletionOption.ResponseHeadersRead, ct).ConfigureAwait(false);
if (resp.StatusCode == HttpStatusCode.OK)
{
// Le serveur a renvoyé tout au lieu du range : la ressource a probablement changé
// (ou il n'a en fait pas accepté le range). On signale un retry à blanc.
throw new HttpResumableException("Server returned 200 for segment range request", isTransient: true);
}
// 403 / 410 = URL HMAC expirée (gate.php renvoie 403 "Forbidden: expired").
// On force un refresh d'URL via le callback du job, puis on jette une
// exception transitoire pour que Polly retry avec la nouvelle URL.
if (resp.StatusCode == HttpStatusCode.Forbidden || resp.StatusCode == HttpStatusCode.Gone)
{
if (job.RefreshUrlAsync is not null)
{
await GetOrRefreshUrlAsync(job, forceRefresh: true, ct).ConfigureAwait(false);
throw new HttpResumableException("Signed URL expired, refreshed for next try", isTransient: true);
}
}
if (!resp.IsSuccessStatusCode)
{
var transient = (int)resp.StatusCode is 408 or 429 or >= 500;
throw new HttpResumableException($"HTTP {(int)resp.StatusCode} on segment {seg.Index}", isTransient: transient);
}
if (resp.Headers.ETag is { } etag) state.Etag = etag.Tag;
if (resp.Content.Headers.LastModified is { } lm) state.LastModified = lm.ToString("R");
await using var src = await resp.Content.ReadAsStreamAsync(ct).ConfigureAwait(false);
await using var dst = new FileStream(partialPath, FileMode.Open, FileAccess.Write, FileShare.ReadWrite, BufferSize, useAsync: true);
dst.Seek(segStart, SeekOrigin.Begin);
var buffer = new byte[BufferSize];
int n;
while ((n = await src.ReadAsync(buffer.AsMemory(0, BufferSize), ct).ConfigureAwait(false)) > 0)
{
await dst.WriteAsync(buffer.AsMemory(0, n), ct).ConfigureAwait(false);
seg.DownloadedBytes += n;
onBytes(seg.Index, n);
}
await dst.FlushAsync(ct).ConfigureAwait(false);
seg.Completed = (seg.DownloadedBytes >= seg.Length);
}
// ===================================================================
// ============= SINGLE-SEGMENT (legacy / fallback path) ============
// ===================================================================
private async Task<string> DownloadSingleSegmentAsync(
DownloadJob job,
string partialPath,
string finalPath,
DownloadState? existing,
IProgress<DownloadProgress>? progress,
IProgress<double>? hashProgress,
CancellationToken ct)
{
long resumeFrom = 0;
if (existing is not null && File.Exists(partialPath))
{
resumeFrom = new FileInfo(partialPath).Length;
// Si la state.json est en avance par rapport au .partial réel, on s'aligne sur le .partial
if (resumeFrom != existing.DownloadedBytes)
{
_logger.LogInformation(
@@ -92,21 +491,10 @@ public sealed class DownloadManager : IDownloadManager
}
else if (File.Exists(partialPath))
{
// Fichier partiel sans state : on repart à 0 par sécurité
File.Delete(partialPath);
resumeFrom = 0;
}
// 2. Vérif espace disque (uniquement à l'initial, pas au resume)
if (resumeFrom == 0 && job.ExpectedSize > 0)
{
var drive = new DriveInfo(Path.GetPathRoot(_stateStore.GetDownloadsDirectory())!);
var needed = (long)(job.ExpectedSize * 1.05);
if (drive.AvailableFreeSpace < needed)
throw new IOException(
$"Espace disque insuffisant sur {drive.Name} : besoin de {needed:N0} octets, disponible {drive.AvailableFreeSpace:N0}");
}
var state = existing ?? new DownloadState
{
Version = job.Version,
@@ -119,13 +507,11 @@ public sealed class DownloadManager : IDownloadManager
LastChunkAtUtc = DateTime.UtcNow,
};
// 3. Tentatives avec retry — chaque tentative continue depuis la taille actuelle du .partial
await _retryPipeline.ExecuteAsync(async innerCt =>
{
await DownloadChunkAsync(job, state, partialPath, progress, innerCt).ConfigureAwait(false);
}, ct).ConfigureAwait(false);
// 4. Vérif taille finale
var actualSize = new FileInfo(partialPath).Length;
if (job.ExpectedSize > 0 && actualSize != job.ExpectedSize)
{
@@ -134,27 +520,7 @@ public sealed class DownloadManager : IDownloadManager
$"Taille téléchargée incorrecte : attendu {job.ExpectedSize}, obtenu {actualSize}");
}
// 5. Vérif SHA-256
if (!string.IsNullOrEmpty(job.ExpectedSha256) &&
!job.ExpectedSha256.StartsWith("REPLACE", StringComparison.OrdinalIgnoreCase))
{
_logger.LogInformation("Verifying SHA-256 of {Path}…", partialPath);
var sha = await _integrity.ComputeSha256Async(partialPath, null, ct).ConfigureAwait(false);
if (!string.Equals(sha, job.ExpectedSha256, StringComparison.OrdinalIgnoreCase))
{
_stateStore.Discard(job.Version);
throw new InvalidDataException(
$"Checksum SHA-256 invalide : attendu {job.ExpectedSha256}, calculé {sha}");
}
}
else
{
_logger.LogWarning("SHA-256 absent du manifest, vérification ignorée");
}
if (File.Exists(finalPath)) File.Delete(finalPath);
File.Move(partialPath, finalPath);
_stateStore.Discard(job.Version); // efface aussi le state.json puisqu'on a fini
await VerifyAndFinalizeAsync(job, partialPath, finalPath, hashProgress, ct).ConfigureAwait(false);
return finalPath;
}
@@ -166,22 +532,17 @@ public sealed class DownloadManager : IDownloadManager
CancellationToken ct)
{
var resumeFrom = File.Exists(partialPath) ? new FileInfo(partialPath).Length : 0;
if (state.TotalBytes > 0 && resumeFrom >= state.TotalBytes) return; // déjà tout là
if (state.TotalBytes > 0 && resumeFrom >= state.TotalBytes) return;
using var req = new HttpRequestMessage(HttpMethod.Get, job.Url);
if (resumeFrom > 0)
{
req.Headers.Range = new RangeHeaderValue(resumeFrom, null);
// If-Range : si l'ETag du serveur a changé entre temps, on veut recommencer proprement
if (!string.IsNullOrEmpty(state.Etag))
{
req.Headers.IfRange = new RangeConditionHeaderValue(new EntityTagHeaderValue(state.Etag));
}
else if (!string.IsNullOrEmpty(state.LastModified) &&
DateTimeOffset.TryParse(state.LastModified, out var lm))
{
req.Headers.IfRange = new RangeConditionHeaderValue(lm);
}
}
_logger.LogInformation("HTTP GET {Url} range={Resume}-", job.Url, resumeFrom);
@@ -189,16 +550,13 @@ public sealed class DownloadManager : IDownloadManager
if (resumeFrom > 0 && resp.StatusCode == HttpStatusCode.OK)
{
// Le serveur a renvoyé 200 alors qu'on demandait du Range : la ressource a changé.
_logger.LogWarning("Server returned 200 for ranged request — resource changed, restarting fresh");
resp.Dispose();
File.Delete(partialPath);
_stateStore.Discard(job.Version);
// On retry sera traité par Polly via une exception
throw new HttpResumableException("Resource changed on server, restart", isTransient: true);
}
// 5xx / 408 / 429 → exception transitoire pour Polly
if (!resp.IsSuccessStatusCode)
{
var transient = (int)resp.StatusCode is 408 or 429 or >= 500;
@@ -207,11 +565,9 @@ public sealed class DownloadManager : IDownloadManager
isTransient: transient);
}
// Capture ETag et Last-Modified du serveur si présents (pour les futures reprises)
if (resp.Headers.ETag is { } etag) state.Etag = etag.Tag;
if (resp.Content.Headers.LastModified is { } lm2) state.LastModified = lm2.ToString("R");
// Total : si le serveur ne fournit pas Content-Length, on garde la valeur du manifest
var contentLength = resp.Content.Headers.ContentLength ?? 0;
var total = state.TotalBytes;
if (resp.StatusCode == HttpStatusCode.PartialContent && resp.Content.Headers.ContentRange is { Length: { } cl })
@@ -220,8 +576,6 @@ public sealed class DownloadManager : IDownloadManager
total = contentLength;
if (total > 0) state.TotalBytes = total;
// Si on est sur un 200 alors qu'on partait de 0, on écrit en append (FileMode.Append depuis position 0)
// Si on est sur un 206 avec resumeFrom>0, on append.
var mode = resumeFrom > 0 ? FileMode.Append : FileMode.Create;
await using var src = await resp.Content.ReadAsStreamAsync(ct).ConfigureAwait(false);
await using var dst = new FileStream(partialPath, mode, FileAccess.Write, FileShare.None, BufferSize, useAsync: true);
@@ -244,7 +598,6 @@ public sealed class DownloadManager : IDownloadManager
var elapsed = sw.Elapsed;
// Progression UI
if ((elapsed - lastReport).TotalMilliseconds >= 250)
{
var deltaBytes = read - lastReportBytes;
@@ -257,7 +610,6 @@ public sealed class DownloadManager : IDownloadManager
lastReportBytes = read;
}
// Persist state.json toutes les 5s ou tous les 100 Mo
if ((elapsed - lastFlush).TotalSeconds >= StateFlushSecondsInterval ||
read - lastFlushBytes >= StateFlushBytesInterval)
{
@@ -272,6 +624,59 @@ public sealed class DownloadManager : IDownloadManager
_stateStore.Save(state);
progress?.Report(new DownloadProgress(read, state.TotalBytes, 0, TimeSpan.Zero));
}
// ===================================================================
// ====================== FINALISATION COMMUNE ======================
// ===================================================================
private async Task VerifyAndFinalizeAsync(
DownloadJob job,
string partialPath,
string finalPath,
IProgress<double>? hashProgress,
CancellationToken ct)
{
// Évalue si on doit vérifier le hash :
// 1. Le manifest doit fournir un sha256 valide (pas vide, pas "REPLACE…")
// 2. L'algo demandé n'est pas "none"
// 3. La config locale n'a pas désactivé la vérification globale
bool hashAlgoSkipped = string.Equals(job.HashAlgorithm, "none", StringComparison.OrdinalIgnoreCase);
bool hasUsableSha = !string.IsNullOrEmpty(job.ExpectedSha256)
&& !job.ExpectedSha256.StartsWith("REPLACE", StringComparison.OrdinalIgnoreCase);
if (hasUsableSha && !hashAlgoSkipped && _config.VerifyDownloadHash)
{
_logger.LogInformation("Verifying SHA-256 of {Path}…", partialPath);
var sw = Stopwatch.StartNew();
var sha = await _integrity.ComputeSha256Async(partialPath, hashProgress, ct).ConfigureAwait(false);
sw.Stop();
_logger.LogInformation("SHA-256 verified in {Elapsed} ({MBps:0.#} MB/s)",
sw.Elapsed,
sw.Elapsed.TotalSeconds > 0 ? new FileInfo(partialPath).Length / sw.Elapsed.TotalSeconds / (1024.0 * 1024.0) : 0);
if (!string.Equals(sha, job.ExpectedSha256, StringComparison.OrdinalIgnoreCase))
{
_stateStore.Discard(job.Version);
throw new InvalidDataException(
$"Checksum SHA-256 invalide : attendu {job.ExpectedSha256}, calculé {sha}");
}
}
else if (hashAlgoSkipped)
{
_logger.LogInformation("Hash verification skipped (manifest specifies hashAlgorithm=none)");
}
else if (!_config.VerifyDownloadHash)
{
_logger.LogInformation("Hash verification skipped (VerifyDownloadHash=false in local config)");
}
else
{
_logger.LogWarning("SHA-256 absent du manifest, vérification ignorée");
}
if (File.Exists(finalPath)) File.Delete(finalPath);
File.Move(partialPath, finalPath);
_stateStore.Discard(job.Version);
}
}
/// <summary>Exception interne pour signaler à Polly qu'on doit retry.</summary>
@@ -283,3 +688,42 @@ internal sealed class HttpResumableException : Exception
IsTransient = isTransient;
}
}
internal static class SparseFileSupport
{
private const uint FSCTL_SET_SPARSE = 0x000900C4;
[DllImport("kernel32.dll", SetLastError = true, CharSet = CharSet.Auto)]
[return: MarshalAs(UnmanagedType.Bool)]
private static extern bool DeviceIoControl(
SafeFileHandle hDevice,
uint dwIoControlCode,
IntPtr lpInBuffer,
uint nInBufferSize,
IntPtr lpOutBuffer,
uint nOutBufferSize,
out uint lpBytesReturned,
IntPtr lpOverlapped);
/// <summary>
/// Marque le fichier comme sparse via FSCTL_SET_SPARSE.
/// Une fois sparse, <c>SetLength(14 GB)</c> est métadata-only (&lt; 100 ms partout).
/// Les zones non écrites sont lues comme zéros virtuels (pas de fuite de données
/// disque, contrairement à SetFileValidData qui exigerait SeManageVolumePrivilege).
/// Retourne false si le volume ne supporte pas sparse (ex. FAT32) — auquel cas
/// on retombe sur le SetLength classique avec zero-fill.
/// </summary>
public static bool TrySetSparse(SafeFileHandle handle)
{
if (!OperatingSystem.IsWindows()) return false;
try
{
return DeviceIoControl(handle, FSCTL_SET_SPARSE,
IntPtr.Zero, 0, IntPtr.Zero, 0, out _, IntPtr.Zero);
}
catch
{
return false;
}
}
}

29
src/PSLauncher.Core/Downloads/IDownloadManager.cs
View File

@@ -9,6 +9,17 @@ public interface IDownloadManager
IProgress<DownloadProgress>? progress,
CancellationToken ct);
/// <summary>
/// Variante avec callback de progression dédié pour la phase de vérification SHA-256.
/// La phase verify est CPU/IO-bound et peut prendre plusieurs minutes sur un build 14 Go ;
/// la UI doit pouvoir l'afficher distinctement du DL.
/// </summary>
Task<string> DownloadAsync(
DownloadJob job,
IProgress<DownloadProgress>? progress,
IProgress<double>? hashProgress,
CancellationToken ct);
/// <summary>Retourne l'état de DL persistant pour cette version, ou null s'il n'y en a pas.</summary>
DownloadState? GetResumableState(string version);
@@ -20,4 +31,20 @@ public sealed record DownloadJob(
string Version,
Uri Url,
long ExpectedSize,
string ExpectedSha256);
string ExpectedSha256)
{
/// <summary>
/// Algo de vérif d'intégrité — "sha256" (défaut) ou "none" pour sauter la vérif.
/// Pris depuis <see cref="PSLauncher.Models.DownloadDescriptor.HashAlgorithm"/>.
/// </summary>
public string? HashAlgorithm { get; init; }
/// <summary>
/// Callback pour obtenir une URL HMAC-signée fraîche en cas d'expiration mid-DL
/// (le DL d'un 14 Go sur connexion lente peut dépasser le TTL serveur de 6 h).
/// Le DownloadManager appelle ce callback quand un segment reçoit un 403/410
/// (indices d'URL expirée), récupère une URL fraîche, et reprend transparemment
/// le DL sans l'interrompre. Si null, pas d'auto-refresh : le DL fail proprement.
/// </summary>
public Func<CancellationToken, Task<Uri?>>? RefreshUrlAsync { get; init; }
}

63
src/PSLauncher.Core/Integrity/IntegrityService.cs
View File

@@ -1,26 +1,63 @@
using System.Buffers;
using System.Security.Cryptography;
namespace PSLauncher.Core.Integrity;
/// <summary>
/// Calcul SHA-256 streaming optimisé pour gros fichiers (14+ Go) :
/// - Buffer 16 MiB (vs 1 MiB historique) pour saturer le throughput SSD/HDD ;
/// - <see cref="FileOptions.SequentialScan"/> hint pour que Windows fasse du read-ahead agressif ;
/// - <see cref="FileOptions.Asynchronous"/> pour ne pas bloquer le ThreadPool en I/O sync ;
/// - <see cref="IncrementalHash"/> (au lieu de TransformBlock/TransformFinalBlock) qui est plus moderne,
/// moins GC-heavy, et bénéficie sur .NET 8 des intrinsics SHA hardware (SHA-NI Intel/AMD,
/// ARMv8 Crypto Extensions) quand la CPU les supporte → typiquement 2-5× plus rapide.
/// - Double-buffering : on lit le buffer N+1 pendant qu'on hash le buffer N, ce qui permet de
/// recouvrir CPU (hash) et I/O (read) sur les machines où ils ne sont pas le même goulot.
///
/// Sur un build 14 Go, l'impact typique constaté :
/// ~1 MiB buffer + TransformBlock : 60-180 s
/// ~16 MiB buffer + IncrementalHash + double-buffer : 15-40 s
/// </summary>
public sealed class IntegrityService : IIntegrityService
{
private const int BufferSize = 16 << 20; // 16 MiB
public async Task<string> ComputeSha256Async(string filePath, IProgress<double>? progress, CancellationToken ct)
{
const int bufferSize = 1 << 20; // 1 MiB
await using var fs = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read, bufferSize, useAsync: true);
using var sha = SHA256.Create();
var fileOptions = FileOptions.Asynchronous | FileOptions.SequentialScan;
await using var fs = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read, BufferSize, fileOptions);
using var hasher = IncrementalHash.CreateHash(HashAlgorithmName.SHA256);
var buffer = new byte[bufferSize];
long total = fs.Length;
long read = 0;
int n;
while ((n = await fs.ReadAsync(buffer.AsMemory(0, bufferSize), ct).ConfigureAwait(false)) > 0)
// Pool pour ne pas allouer 32 Mo à chaque appel.
var pool = ArrayPool<byte>.Shared;
byte[] bufA = pool.Rent(BufferSize);
byte[] bufB = pool.Rent(BufferSize);
try
{
sha.TransformBlock(buffer, 0, n, null, 0);
read += n;
if (total > 0) progress?.Report((double)read / total);
long total = fs.Length;
long read = 0;
// Premier read amorce le pipeline.
int n = await fs.ReadAsync(bufA.AsMemory(0, BufferSize), ct).ConfigureAwait(false);
while (n > 0)
{
// Lance le prochain read en parallèle pendant qu'on hash le buffer courant.
var nextReadTask = fs.ReadAsync(bufB.AsMemory(0, BufferSize), ct);
hasher.AppendData(bufA, 0, n);
read += n;
if (total > 0) progress?.Report((double)read / total);
int next = await nextReadTask.ConfigureAwait(false);
// Swap : bufB devient le buffer courant, bufA le futur prochain read.
(bufA, bufB) = (bufB, bufA);
n = next;
}
return Convert.ToHexString(hasher.GetHashAndReset()).ToLowerInvariant();
}
finally
{
pool.Return(bufA);
pool.Return(bufB);
}
sha.TransformFinalBlock(Array.Empty<byte>(), 0, 0);
return Convert.ToHexString(sha.Hash!).ToLowerInvariant();
}
}