Compare commits

...

4 Commits

Author SHA1 Message Date
00a0b3e14f Add --reimport and --reprocess-reactions CLI modes
Add flags to reimport all data from archived files or selectively
reprocess reactions. Also replace EnsureCreatedAsync with explicit
database/table existence checks for safer initialization.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-05 23:20:25 -05:00
a281f7f1e7 Use export date from filename for archive subdirectory
Extract the date from filenames like "2026-01-20.json" instead of
using the current date, so archives are organized by export date
rather than processing date.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-05 23:20:17 -05:00
6f63f36df0 Improve import resilience with per-message saves and duplicate handling
Save after each message to isolate failures, catch and skip duplicate
key violations (SQL error 2601), and clear change tracker on rollback
to prevent cascading failures.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-05 23:20:08 -05:00
fbe52f72d6 Refactor reactions to track per-user instead of aggregate count
Replace Reaction.Count with UserId foreign key to User entity,
add unique index on (MessageId, EmojiName, UserId) to prevent
duplicates, and configure the User-Reactions relationship.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-05 23:19:59 -05:00
6 changed files with 131 additions and 40 deletions

View File

@@ -111,7 +111,13 @@ public class DiscordArchiveContext : DbContext
.WithMany(m => m.Reactions)
.HasForeignKey(e => e.MessageId)
.OnDelete(DeleteBehavior.Cascade);
entity.HasOne(e => e.User)
.WithMany(u => u.Reactions)
.HasForeignKey(e => e.UserId)
.OnDelete(DeleteBehavior.Restrict);
entity.HasIndex(e => e.MessageId);
entity.HasIndex(e => e.UserId);
entity.HasIndex(e => new { e.MessageId, e.EmojiName, e.UserId }).IsUnique();
});
// Mention configuration

View File

@@ -19,9 +19,14 @@ public class Reaction
[MaxLength(256)]
public string EmojiName { get; set; } = null!;
public int Count { get; set; }
[Required]
[MaxLength(32)]
public string UserId { get; set; } = null!;
// Navigation properties
[ForeignKey(nameof(MessageId))]
public Message Message { get; set; } = null!;
[ForeignKey(nameof(UserId))]
public User User { get; set; } = null!;
}

View File

@@ -22,4 +22,5 @@ public class User
public ICollection<UserSnapshot> Snapshots { get; set; } = new List<UserSnapshot>();
public ICollection<Message> Messages { get; set; } = new List<Message>();
public ICollection<Mention> Mentions { get; set; } = new List<Mention>();
public ICollection<Reaction> Reactions { get; set; } = new List<Reaction>();
}

View File

@@ -1,6 +1,8 @@
using DiscordArchiveManager.Data;
using DiscordArchiveManager.Services;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Storage;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
@@ -32,11 +34,23 @@ var inputDirectory = config["Paths:InputDirectory"] ?? "/app/input";
var archiveDirectory = config["Paths:ArchiveDirectory"] ?? "/app/archive";
var imageDirectory = config["Paths:ImageDirectory"] ?? "/app/images";
var reprocessReactions = args.Contains("--reprocess-reactions");
var reimport = args.Contains("--reimport");
logger.LogInformation("Discord Archive Manager starting...");
logger.LogInformation("Input directory: {Path}", inputDirectory);
logger.LogInformation("Archive directory: {Path}", archiveDirectory);
logger.LogInformation("Image directory: {Path}", imageDirectory);
if (reimport)
{
logger.LogInformation("Mode: Reimport all data from archive");
}
else if (reprocessReactions)
{
logger.LogInformation("Mode: Reprocess reactions from archive");
}
// Ensure directories exist
Directory.CreateDirectory(inputDirectory);
Directory.CreateDirectory(archiveDirectory);
@@ -46,17 +60,46 @@ Directory.CreateDirectory(imageDirectory);
using (var scope = host.Services.CreateScope())
{
var context = scope.ServiceProvider.GetRequiredService<DiscordArchiveContext>();
logger.LogInformation("Ensuring database exists and applying migrations...");
await context.Database.EnsureCreatedAsync();
logger.LogInformation("Ensuring database schema exists...");
// Get the database creator for more control
var creator = context.GetService<IRelationalDatabaseCreator>()!;
if (!await creator.ExistsAsync())
{
logger.LogError("Database does not exist. Please create it first.");
return;
}
if (!await creator.HasTablesAsync())
{
logger.LogInformation("Creating database tables...");
await creator.CreateTablesAsync();
}
logger.LogInformation("Database ready.");
}
// Process files
// Process files or reprocess reactions
using (var scope = host.Services.CreateScope())
{
var importService = scope.ServiceProvider.GetRequiredService<JsonImportService>();
var archiveService = scope.ServiceProvider.GetRequiredService<ArchiveService>();
if (reimport)
{
logger.LogInformation("Reimporting all data from archived files...");
var (success, errors) = await importService.ReimportFromArchiveAsync(archiveDirectory, imageDirectory);
logger.LogInformation("Reimport complete. Imported: {Success}, Errors: {Errors}", success, errors);
}
else if (reprocessReactions)
{
logger.LogInformation("Reprocessing reactions from archived files...");
var added = await importService.ReprocessReactionsAsync(archiveDirectory);
logger.LogInformation("Reprocessing complete. Added {Count} reactions.", added);
}
else
{
var files = archiveService.GetExportFiles(inputDirectory).ToList();
if (files.Count == 0)
@@ -95,6 +138,7 @@ using (var scope = host.Services.CreateScope())
logger.LogInformation("Processing complete. Processed: {Success}, Skipped: {Skip}, Errors: {Error}",
successCount, skipCount, errorCount);
}
}
}
logger.LogInformation("Discord Archive Manager finished.");

View File

@@ -28,8 +28,8 @@ public class ArchiveService
var jsonDirectory = Path.GetDirectoryName(jsonFilePath)!;
var filesDirectory = GetFilesDirectoryPath(jsonFilePath);
// Create archive subdirectory based on date
var archiveSubdir = DateTime.Now.ToString("yyyy-MM-dd");
// Create archive subdirectory based on export date from filename
var archiveSubdir = GetExportDateFromFilename(jsonFileName) ?? DateTime.Now.ToString("yyyy-MM-dd");
var archivePath = Path.Combine(archiveRoot, archiveSubdir);
Directory.CreateDirectory(archivePath);
@@ -152,6 +152,26 @@ public class ArchiveService
}
}
/// <summary>
/// Extracts the export date from a filename like "2026-01-20.json".
/// Returns null if the filename doesn't match the expected pattern.
/// </summary>
private static string? GetExportDateFromFilename(string filename)
{
var nameWithoutExtension = Path.GetFileNameWithoutExtension(filename);
if (DateTime.TryParseExact(nameWithoutExtension, "yyyy-MM-dd", null, System.Globalization.DateTimeStyles.None, out _))
{
return nameWithoutExtension;
}
// Handle filenames with suffix like "2026-01-20_1.json"
var parts = nameWithoutExtension.Split('_');
if (parts.Length > 0 && DateTime.TryParseExact(parts[0], "yyyy-MM-dd", null, System.Globalization.DateTimeStyles.None, out _))
{
return parts[0];
}
return null;
}
/// <summary>
/// Moves a file, falling back to copy+delete for cross-device moves.
/// </summary>

View File

@@ -72,20 +72,32 @@ public class JsonImportService
// Upsert Channel
await UpsertChannelAsync(export.Channel, export.Guild.Id);
// Process messages
// Process messages - save after each to isolate any issues
var processedCount = 0;
foreach (var message in export.Messages)
{
try
{
if (await ProcessMessageAsync(message, export.Channel.Id, jsonFilePath, imageRoot))
{
await _context.SaveChangesAsync();
processedCount++;
}
}
catch (DbUpdateException ex) when (ex.InnerException is Microsoft.Data.SqlClient.SqlException sqlEx && sqlEx.Number == 2601)
{
// Duplicate key - log and continue
_logger.LogWarning("Duplicate key error for message {MessageId}, skipping: {Error}",
message.Id, sqlEx.Message);
_context.ChangeTracker.Clear();
// Re-upsert guild and channel as they were cleared
await UpsertGuildAsync(export.Guild);
await UpsertChannelAsync(export.Channel, export.Guild.Id);
}
}
_logger.LogInformation("Processed {Count} new messages", processedCount);
await _context.SaveChangesAsync();
// Archive the file
var archivePath = _archiveService.ArchiveExport(jsonFilePath, archiveRoot);
@@ -106,6 +118,9 @@ public class JsonImportService
catch (Exception ex)
{
await transaction.RollbackAsync();
_context.ChangeTracker.Clear(); // Clear tracked entities to prevent cascading failures
_processedMessages.Clear(); // Clear the session tracking
_addedReactions.Clear();
_logger.LogError(ex, "Error processing file, rolled back transaction: {Path}", jsonFilePath);
throw;
}