Compare commits

...

4 Commits

Author SHA1 Message Date
00a0b3e14f Add --reimport and --reprocess-reactions CLI modes
Add flags to reimport all data from archived files or selectively
reprocess reactions. Also replace EnsureCreatedAsync with explicit
database/table existence checks for safer initialization.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-05 23:20:25 -05:00
a281f7f1e7 Use export date from filename for archive subdirectory
Extract the date from filenames like "2026-01-20.json" instead of
using the current date, so archives are organized by export date
rather than processing date.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-05 23:20:17 -05:00
6f63f36df0 Improve import resilience with per-message saves and duplicate handling
Save after each message to isolate failures, catch and skip duplicate
key violations (SQL error 2601), and clear change tracker on rollback
to prevent cascading failures.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-05 23:20:08 -05:00
fbe52f72d6 Refactor reactions to track per-user instead of aggregate count
Replace Reaction.Count with UserId foreign key to User entity,
add unique index on (MessageId, EmojiName, UserId) to prevent
duplicates, and configure the User-Reactions relationship.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-05 23:19:59 -05:00
6 changed files with 131 additions and 40 deletions

View File

@@ -111,7 +111,13 @@ public class DiscordArchiveContext : DbContext
.WithMany(m => m.Reactions) .WithMany(m => m.Reactions)
.HasForeignKey(e => e.MessageId) .HasForeignKey(e => e.MessageId)
.OnDelete(DeleteBehavior.Cascade); .OnDelete(DeleteBehavior.Cascade);
entity.HasOne(e => e.User)
.WithMany(u => u.Reactions)
.HasForeignKey(e => e.UserId)
.OnDelete(DeleteBehavior.Restrict);
entity.HasIndex(e => e.MessageId); entity.HasIndex(e => e.MessageId);
entity.HasIndex(e => e.UserId);
entity.HasIndex(e => new { e.MessageId, e.EmojiName, e.UserId }).IsUnique();
}); });
// Mention configuration // Mention configuration

View File

@@ -19,9 +19,14 @@ public class Reaction
[MaxLength(256)] [MaxLength(256)]
public string EmojiName { get; set; } = null!; public string EmojiName { get; set; } = null!;
public int Count { get; set; } [Required]
[MaxLength(32)]
public string UserId { get; set; } = null!;
// Navigation properties // Navigation properties
[ForeignKey(nameof(MessageId))] [ForeignKey(nameof(MessageId))]
public Message Message { get; set; } = null!; public Message Message { get; set; } = null!;
[ForeignKey(nameof(UserId))]
public User User { get; set; } = null!;
} }

View File

@@ -22,4 +22,5 @@ public class User
public ICollection<UserSnapshot> Snapshots { get; set; } = new List<UserSnapshot>(); public ICollection<UserSnapshot> Snapshots { get; set; } = new List<UserSnapshot>();
public ICollection<Message> Messages { get; set; } = new List<Message>(); public ICollection<Message> Messages { get; set; } = new List<Message>();
public ICollection<Mention> Mentions { get; set; } = new List<Mention>(); public ICollection<Mention> Mentions { get; set; } = new List<Mention>();
public ICollection<Reaction> Reactions { get; set; } = new List<Reaction>();
} }

View File

@@ -1,6 +1,8 @@
using DiscordArchiveManager.Data; using DiscordArchiveManager.Data;
using DiscordArchiveManager.Services; using DiscordArchiveManager.Services;
using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Storage;
using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Hosting;
@@ -32,11 +34,23 @@ var inputDirectory = config["Paths:InputDirectory"] ?? "/app/input";
var archiveDirectory = config["Paths:ArchiveDirectory"] ?? "/app/archive"; var archiveDirectory = config["Paths:ArchiveDirectory"] ?? "/app/archive";
var imageDirectory = config["Paths:ImageDirectory"] ?? "/app/images"; var imageDirectory = config["Paths:ImageDirectory"] ?? "/app/images";
var reprocessReactions = args.Contains("--reprocess-reactions");
var reimport = args.Contains("--reimport");
logger.LogInformation("Discord Archive Manager starting..."); logger.LogInformation("Discord Archive Manager starting...");
logger.LogInformation("Input directory: {Path}", inputDirectory); logger.LogInformation("Input directory: {Path}", inputDirectory);
logger.LogInformation("Archive directory: {Path}", archiveDirectory); logger.LogInformation("Archive directory: {Path}", archiveDirectory);
logger.LogInformation("Image directory: {Path}", imageDirectory); logger.LogInformation("Image directory: {Path}", imageDirectory);
if (reimport)
{
logger.LogInformation("Mode: Reimport all data from archive");
}
else if (reprocessReactions)
{
logger.LogInformation("Mode: Reprocess reactions from archive");
}
// Ensure directories exist // Ensure directories exist
Directory.CreateDirectory(inputDirectory); Directory.CreateDirectory(inputDirectory);
Directory.CreateDirectory(archiveDirectory); Directory.CreateDirectory(archiveDirectory);
@@ -46,17 +60,46 @@ Directory.CreateDirectory(imageDirectory);
using (var scope = host.Services.CreateScope()) using (var scope = host.Services.CreateScope())
{ {
var context = scope.ServiceProvider.GetRequiredService<DiscordArchiveContext>(); var context = scope.ServiceProvider.GetRequiredService<DiscordArchiveContext>();
logger.LogInformation("Ensuring database exists and applying migrations..."); logger.LogInformation("Ensuring database schema exists...");
await context.Database.EnsureCreatedAsync();
// Get the database creator for more control
var creator = context.GetService<IRelationalDatabaseCreator>()!;
if (!await creator.ExistsAsync())
{
logger.LogError("Database does not exist. Please create it first.");
return;
}
if (!await creator.HasTablesAsync())
{
logger.LogInformation("Creating database tables...");
await creator.CreateTablesAsync();
}
logger.LogInformation("Database ready."); logger.LogInformation("Database ready.");
} }
// Process files // Process files or reprocess reactions
using (var scope = host.Services.CreateScope()) using (var scope = host.Services.CreateScope())
{ {
var importService = scope.ServiceProvider.GetRequiredService<JsonImportService>(); var importService = scope.ServiceProvider.GetRequiredService<JsonImportService>();
var archiveService = scope.ServiceProvider.GetRequiredService<ArchiveService>(); var archiveService = scope.ServiceProvider.GetRequiredService<ArchiveService>();
if (reimport)
{
logger.LogInformation("Reimporting all data from archived files...");
var (success, errors) = await importService.ReimportFromArchiveAsync(archiveDirectory, imageDirectory);
logger.LogInformation("Reimport complete. Imported: {Success}, Errors: {Errors}", success, errors);
}
else if (reprocessReactions)
{
logger.LogInformation("Reprocessing reactions from archived files...");
var added = await importService.ReprocessReactionsAsync(archiveDirectory);
logger.LogInformation("Reprocessing complete. Added {Count} reactions.", added);
}
else
{
var files = archiveService.GetExportFiles(inputDirectory).ToList(); var files = archiveService.GetExportFiles(inputDirectory).ToList();
if (files.Count == 0) if (files.Count == 0)
@@ -96,5 +139,6 @@ using (var scope = host.Services.CreateScope())
successCount, skipCount, errorCount); successCount, skipCount, errorCount);
} }
} }
}
logger.LogInformation("Discord Archive Manager finished."); logger.LogInformation("Discord Archive Manager finished.");

View File

@@ -28,8 +28,8 @@ public class ArchiveService
var jsonDirectory = Path.GetDirectoryName(jsonFilePath)!; var jsonDirectory = Path.GetDirectoryName(jsonFilePath)!;
var filesDirectory = GetFilesDirectoryPath(jsonFilePath); var filesDirectory = GetFilesDirectoryPath(jsonFilePath);
// Create archive subdirectory based on date // Create archive subdirectory based on export date from filename
var archiveSubdir = DateTime.Now.ToString("yyyy-MM-dd"); var archiveSubdir = GetExportDateFromFilename(jsonFileName) ?? DateTime.Now.ToString("yyyy-MM-dd");
var archivePath = Path.Combine(archiveRoot, archiveSubdir); var archivePath = Path.Combine(archiveRoot, archiveSubdir);
Directory.CreateDirectory(archivePath); Directory.CreateDirectory(archivePath);
@@ -152,6 +152,26 @@ public class ArchiveService
} }
} }
/// <summary>
/// Extracts the export date from a filename like "2026-01-20.json".
/// Returns null if the filename doesn't match the expected pattern.
/// </summary>
private static string? GetExportDateFromFilename(string filename)
{
var nameWithoutExtension = Path.GetFileNameWithoutExtension(filename);
if (DateTime.TryParseExact(nameWithoutExtension, "yyyy-MM-dd", null, System.Globalization.DateTimeStyles.None, out _))
{
return nameWithoutExtension;
}
// Handle filenames with suffix like "2026-01-20_1.json"
var parts = nameWithoutExtension.Split('_');
if (parts.Length > 0 && DateTime.TryParseExact(parts[0], "yyyy-MM-dd", null, System.Globalization.DateTimeStyles.None, out _))
{
return parts[0];
}
return null;
}
/// <summary> /// <summary>
/// Moves a file, falling back to copy+delete for cross-device moves. /// Moves a file, falling back to copy+delete for cross-device moves.
/// </summary> /// </summary>

View File

@@ -72,20 +72,32 @@ public class JsonImportService
// Upsert Channel // Upsert Channel
await UpsertChannelAsync(export.Channel, export.Guild.Id); await UpsertChannelAsync(export.Channel, export.Guild.Id);
// Process messages // Process messages - save after each to isolate any issues
var processedCount = 0; var processedCount = 0;
foreach (var message in export.Messages) foreach (var message in export.Messages)
{
try
{ {
if (await ProcessMessageAsync(message, export.Channel.Id, jsonFilePath, imageRoot)) if (await ProcessMessageAsync(message, export.Channel.Id, jsonFilePath, imageRoot))
{ {
await _context.SaveChangesAsync();
processedCount++; processedCount++;
} }
} }
catch (DbUpdateException ex) when (ex.InnerException is Microsoft.Data.SqlClient.SqlException sqlEx && sqlEx.Number == 2601)
{
// Duplicate key - log and continue
_logger.LogWarning("Duplicate key error for message {MessageId}, skipping: {Error}",
message.Id, sqlEx.Message);
_context.ChangeTracker.Clear();
// Re-upsert guild and channel as they were cleared
await UpsertGuildAsync(export.Guild);
await UpsertChannelAsync(export.Channel, export.Guild.Id);
}
}
_logger.LogInformation("Processed {Count} new messages", processedCount); _logger.LogInformation("Processed {Count} new messages", processedCount);
await _context.SaveChangesAsync();
// Archive the file // Archive the file
var archivePath = _archiveService.ArchiveExport(jsonFilePath, archiveRoot); var archivePath = _archiveService.ArchiveExport(jsonFilePath, archiveRoot);
@@ -106,6 +118,9 @@ public class JsonImportService
catch (Exception ex) catch (Exception ex)
{ {
await transaction.RollbackAsync(); await transaction.RollbackAsync();
_context.ChangeTracker.Clear(); // Clear tracked entities to prevent cascading failures
_processedMessages.Clear(); // Clear the session tracking
_addedReactions.Clear();
_logger.LogError(ex, "Error processing file, rolled back transaction: {Path}", jsonFilePath); _logger.LogError(ex, "Error processing file, rolled back transaction: {Path}", jsonFilePath);
throw; throw;
} }