elwig/Elwig/Helpers/DatabaseBackupService.cs

using Microsoft.Data.Sqlite;
using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;

namespace Elwig.Helpers {
    internal sealed class DatabaseBackupService {

        private const int DailyBackupCount = 7;
        private const int WeeklyBackupCount = 5;

        private readonly string DatabaseFile;
        private readonly string BackupDirectory;
        private readonly string DatabaseName;
        private readonly string BackupPrefix;
        private readonly string LockFile;
        private readonly Func<DateTime> Now;
        private readonly SemaphoreSlim Gate = new(1, 1);

        internal DatabaseBackupService(string databaseFile, Func<DateTime>? now = null) {
            DatabaseFile = Path.GetFullPath(databaseFile);
            BackupDirectory = Path.Combine(Path.GetDirectoryName(DatabaseFile)!, "backup");
            DatabaseName = Path.GetFileNameWithoutExtension(DatabaseFile);
            BackupPrefix = $"{DatabaseName}_";
            LockFile = Path.Combine(BackupDirectory, $".{DatabaseName}.backup.lock");
            Now = now ?? (() => DateTime.Now);
        }

        internal async Task RunIfNeededAsync() {
            if (!await Gate.WaitAsync(0)) return;
            try {
                var slotDate = GetSlotDate(Now());
                await Task.Run(() => Run(slotDate));
            } catch {
                // Automatic backups dont prevent Elwig from starting or running.
            } finally {
                Gate.Release();
            }
        }

        private void Run(DateOnly slotDate) {
            Directory.CreateDirectory(BackupDirectory);

            FileStream lockStream;
            try {
                lockStream = new FileStream(LockFile, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None);
            } catch (IOException) {
                // Another Elwig instance is currently creating/pruning backups.
                return;
            }

            using (lockStream) {
                CleanupTemporaryFiles();

                var backupFile = GetBackupFile(slotDate);
                if (!File.Exists(backupFile)) {
                    CreateBackup(backupFile);
                }

                if (File.Exists(backupFile)) {
                    PruneBackups(slotDate);
                }
            }
        }

        private void CreateBackup(string backupFile) {
            var id = Guid.NewGuid().ToString("N");
            var temporaryDatabase = Path.Combine(BackupDirectory, $".{DatabaseName}.backup-{id}.sqlite3.tmp");
            var temporaryArchive = Path.Combine(BackupDirectory, $".{DatabaseName}.backup-{id}.zip.tmp");

            try {
                var sourceConnectionString = new SqliteConnectionStringBuilder {
                    DataSource = DatabaseFile,
                    Mode = SqliteOpenMode.ReadOnly,
                    Cache = SqliteCacheMode.Default,
                    Pooling = false,
                }.ToString();
                var destinationConnectionString = new SqliteConnectionStringBuilder {
                    DataSource = temporaryDatabase,
                    Mode = SqliteOpenMode.ReadWriteCreate,
                    Cache = SqliteCacheMode.Default,
                    Pooling = false,
                }.ToString();

                using (var source = new SqliteConnection(sourceConnectionString))
                using (var destination = new SqliteConnection(destinationConnectionString)) {
                    source.Open();
                    destination.Open();
                    source.BackupDatabase(destination);

                    using var check = destination.CreateCommand();
                    check.CommandText = "PRAGMA quick_check";
                    if (!string.Equals(check.ExecuteScalar()?.ToString(), "ok", StringComparison.OrdinalIgnoreCase)) {
                        throw new InvalidDataException("The automatic database backup failed its SQLite integrity check.");
                    }
                }

                using (var archive = ZipFile.Open(temporaryArchive, ZipArchiveMode.Create)) {
                    archive.CreateEntryFromFile(temporaryDatabase, "database.sqlite3", CompressionLevel.SmallestSize);
                }

                try {
                    File.Move(temporaryArchive, backupFile, false);
                } catch (IOException) when (File.Exists(backupFile)) {
                    // A completed backup won a publish race. Its time slot is satisfied.
                }
            } finally {
                DeleteFile(temporaryDatabase);
                DeleteFile(temporaryArchive);
            }
        }

        private void CleanupTemporaryFiles() {
            foreach (var file in Directory.EnumerateFiles(BackupDirectory, $".{DatabaseName}.backup-*.tmp")) {
                DeleteFile(file);
            }
        }

        internal void PruneBackups(DateOnly slotDate) {
            var backups = GetManagedBackups().ToArray();
            var retained = new HashSet<string>(StringComparer.OrdinalIgnoreCase);

            var dailyStart = slotDate.AddDays(-(DailyBackupCount - 1));
            foreach (var backup in backups.Where(b => b.Date >= dailyStart && b.Date <= slotDate)) {
                retained.Add(backup.File);
            }

            var daysSinceMonday = ((int)slotDate.DayOfWeek + 6) % 7;
            var currentWeekStart = slotDate.AddDays(-daysSinceMonday);
            for (var i = 1; i <= WeeklyBackupCount; i++) {
                var weekStart = currentWeekStart.AddDays(-7 * i);
                RetainNewest(backups, retained, weekStart, weekStart.AddDays(7));
            }

            var currentMonthStart = new DateOnly(slotDate.Year, slotDate.Month, 1);
            foreach (var month in backups
                .Where(b => b.Date < currentMonthStart)
                .GroupBy(b => (b.Date.Year, b.Date.Month))) {
                retained.Add(month.OrderByDescending(b => b.Date).First().File);
            }

            // Clock corrections should not cause future-dated backups to be deleted.
            foreach (var backup in backups.Where(b => b.Date > slotDate)) {
                retained.Add(backup.File);
            }

            foreach (var backup in backups.Where(b => !retained.Contains(b.File))) {
                DeleteFile(backup.File);
            }
        }

        private static void RetainNewest(
            IEnumerable<(string File, DateOnly Date)> backups,
            ISet<string> retained,
            DateOnly start,
            DateOnly end
        ) {
            var newest = backups
                .Where(b => b.Date >= start && b.Date < end)
                .OrderByDescending(b => b.Date)
                .FirstOrDefault();
            if (newest.File != null) retained.Add(newest.File);
        }

        private IEnumerable<(string File, DateOnly Date)> GetManagedBackups() {
            foreach (var file in Directory.EnumerateFiles(BackupDirectory, $"{BackupPrefix}*.sqlite3.zip")) {
                var name = Path.GetFileName(file);
                if (!name.StartsWith(BackupPrefix, StringComparison.OrdinalIgnoreCase) ||
                    !name.EndsWith(".sqlite3.zip", StringComparison.OrdinalIgnoreCase)) {
                    continue;
                }

                var dateText = name[BackupPrefix.Length..^".sqlite3.zip".Length];
                if (DateOnly.TryParseExact(dateText, "yyyy-MM-dd", CultureInfo.InvariantCulture, DateTimeStyles.None, out var date)) {
                    yield return (file, date);
                }
            }
        }

        private string GetBackupFile(DateOnly date) {
            return Path.Combine(BackupDirectory, $"{BackupPrefix}{date:yyyy-MM-dd}.sqlite3.zip");
        }

        private static void DeleteFile(string file) {
            try {
                File.Delete(file);
            } catch {
                // Cleanup and retention are best-effort and will be retried later.
            }
        }

        internal static DateOnly GetSlotDate(DateTime timestamp) {
            return DateOnly.FromDateTime(timestamp.Hour >= 3 ? timestamp : timestamp.AddDays(-1));
        }
    }
}