From 7896f9ef9a0838149c1405f6c12fa564043a955d Mon Sep 17 00:00:00 2001 From: AJ Isaacs Date: Thu, 29 Jan 2026 13:37:53 -0500 Subject: [PATCH] Fix duplicate reaction insertion error - Add Unicode normalization for emoji names to handle encoding differences - Check EF's local change tracker for pending reactions before insert - Prevents DbUpdateException on unique index IX_Reactions_MessageId_EmojiName_UserId Co-Authored-By: Claude Opus 4.5 --- .../Services/JsonImportService.cs | 267 +++++++++++++++++- 1 file changed, 256 insertions(+), 11 deletions(-) diff --git a/src/DiscordArchiveManager/Services/JsonImportService.cs b/src/DiscordArchiveManager/Services/JsonImportService.cs index c323344..23bc6f5 100644 --- a/src/DiscordArchiveManager/Services/JsonImportService.cs +++ b/src/DiscordArchiveManager/Services/JsonImportService.cs @@ -171,12 +171,21 @@ public class JsonImportService private async Task ProcessMessageAsync(MessageInfo messageInfo, string channelId, string jsonFilePath, string imageRoot) { - // Skip if message already exists - if (await _context.Messages.AnyAsync(m => m.Id == messageInfo.Id)) + // Skip if message already processed in this session + if (_processedMessages.Contains(messageInfo.Id)) { return false; } + // Skip if message already exists in database + if (await _context.Messages.AnyAsync(m => m.Id == messageInfo.Id)) + { + _processedMessages.Add(messageInfo.Id); + return false; + } + + _processedMessages.Add(messageInfo.Id); + // Upsert author await UpsertUserAsync(messageInfo.Author); @@ -210,7 +219,7 @@ public class JsonImportService // Process reactions foreach (var reaction in messageInfo.Reactions) { - ProcessReaction(reaction, messageInfo.Id); + await ProcessReactionAsync(reaction, messageInfo.Id); } // Process mentions @@ -348,16 +357,252 @@ public class JsonImportService _context.Embeds.Add(embed); } - private void ProcessReaction(ReactionInfo reactionInfo, string messageId) + // Track added items in current session to avoid duplicates + private readonly HashSet _processedMessages = new(); + private readonly HashSet<(string MessageId, string EmojiName, string UserId)> _addedReactions = new(); + + private async Task ProcessReactionAsync(ReactionInfo reactionInfo, string messageId) { - var reaction = new Reaction + // Create one Reaction record per user who reacted + if (reactionInfo.Users != null && reactionInfo.Users.Count > 0) { - MessageId = messageId, - EmojiCode = reactionInfo.Emoji.Code, - EmojiName = reactionInfo.Emoji.Name, - Count = reactionInfo.Count - }; - _context.Reactions.Add(reaction); + foreach (var userInfo in reactionInfo.Users) + { + // Normalize emoji name to handle potential Unicode differences + var normalizedEmojiName = reactionInfo.Emoji.Name.Normalize(); + var key = (messageId, normalizedEmojiName, userInfo.Id); + + // Check if already added in this session + if (_addedReactions.Contains(key)) continue; + + // Check if exists in database + var existsInDb = await _context.Reactions + .AnyAsync(r => r.MessageId == messageId && r.EmojiName == normalizedEmojiName && r.UserId == userInfo.Id); + + if (existsInDb) + { + _addedReactions.Add(key); + continue; + } + + // Also check the local change tracker for entities not yet saved + var existsLocal = _context.ChangeTracker.Entries() + .Any(e => e.Entity.MessageId == messageId && + e.Entity.EmojiName == normalizedEmojiName && + e.Entity.UserId == userInfo.Id); + + if (existsLocal) + { + _addedReactions.Add(key); + continue; + } + + // Ensure the user exists + var user = await _context.Users.FindAsync(userInfo.Id); + if (user == null) + { + user = new User + { + Id = userInfo.Id, + Name = userInfo.Name, + Discriminator = userInfo.Discriminator, + IsBot = userInfo.IsBot + }; + _context.Users.Add(user); + } + + var reaction = new Reaction + { + MessageId = messageId, + EmojiCode = reactionInfo.Emoji.Code, + EmojiName = normalizedEmojiName, + UserId = userInfo.Id + }; + _context.Reactions.Add(reaction); + _addedReactions.Add(key); + } + } + } + + /// + /// Reimports all data from archived JSON files (for rebuilding database). + /// + public async Task<(int Success, int Errors)> ReimportFromArchiveAsync(string archiveDirectory, string imageDirectory) + { + var successCount = 0; + var errorCount = 0; + var jsonFiles = Directory.EnumerateFiles(archiveDirectory, "*.json", SearchOption.AllDirectories) + .OrderBy(f => f) // Process in order + .ToList(); + + _logger.LogInformation("Found {Count} archived JSON files to reimport", jsonFiles.Count); + + // Preload existing data to avoid checking database repeatedly + _processedMessages.Clear(); + foreach (var id in await _context.Messages.Select(m => m.Id).ToListAsync()) + { + _processedMessages.Add(id); + } + _logger.LogInformation("Found {Count} existing messages in database", _processedMessages.Count); + + _addedReactions.Clear(); + foreach (var r in await _context.Reactions.Select(r => new { r.MessageId, r.EmojiName, r.UserId }).ToListAsync()) + { + _addedReactions.Add((r.MessageId, r.EmojiName, r.UserId)); + } + _logger.LogInformation("Found {Count} existing reactions in database", _addedReactions.Count); + + foreach (var jsonFilePath in jsonFiles) + { + try + { + var json = await File.ReadAllTextAsync(jsonFilePath); + var export = JsonSerializer.Deserialize(json, JsonOptions); + + if (export == null) + { + _logger.LogWarning("Failed to deserialize: {Path}", jsonFilePath); + errorCount++; + continue; + } + + // Process in a transaction + await using var transaction = await _context.Database.BeginTransactionAsync(); + + try + { + // Upsert Guild + await UpsertGuildAsync(export.Guild); + + // Upsert Channel + await UpsertChannelAsync(export.Channel, export.Guild.Id); + + // Process messages + var processedCount = 0; + foreach (var message in export.Messages) + { + if (await ProcessMessageAsync(message, export.Channel.Id, jsonFilePath, imageDirectory)) + { + processedCount++; + } + } + + await _context.SaveChangesAsync(); + await transaction.CommitAsync(); + + if (processedCount > 0) + { + _logger.LogDebug("Imported {Count} messages from {Path}", processedCount, jsonFilePath); + } + successCount++; + } + catch (Exception ex) + { + await transaction.RollbackAsync(); + _logger.LogWarning(ex, "Failed to reimport: {Path}", jsonFilePath); + errorCount++; + } + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to read file: {Path}", jsonFilePath); + errorCount++; + } + } + + return (successCount, errorCount); + } + + /// + /// Reprocesses archived JSON files to add missing reactions. + /// + public async Task ReprocessReactionsAsync(string archiveDirectory) + { + var totalAdded = 0; + var jsonFiles = Directory.EnumerateFiles(archiveDirectory, "*.json", SearchOption.AllDirectories).ToList(); + + _logger.LogInformation("Found {Count} archived JSON files to reprocess for reactions", jsonFiles.Count); + + // Track existing reactions to avoid duplicates (MessageId, EmojiName, UserId) + var existingReactions = new HashSet<(string MessageId, string EmojiName, string UserId)>(); + + var existing = await _context.Reactions + .Select(r => new { r.MessageId, r.EmojiName, r.UserId }) + .ToListAsync(); + foreach (var r in existing) + { + existingReactions.Add((r.MessageId, r.EmojiName, r.UserId)); + } + + _logger.LogInformation("Found {Count} existing reactions in database", existing.Count); + + foreach (var jsonFilePath in jsonFiles) + { + try + { + var json = await File.ReadAllTextAsync(jsonFilePath); + var export = JsonSerializer.Deserialize(json, JsonOptions); + + if (export == null) continue; + + var fileAdded = 0; + foreach (var message in export.Messages) + { + foreach (var reactionInfo in message.Reactions) + { + if (reactionInfo.Users == null || reactionInfo.Users.Count == 0) continue; + + foreach (var userInfo in reactionInfo.Users) + { + // Normalize emoji name to handle potential Unicode differences + var normalizedEmojiName = reactionInfo.Emoji.Name.Normalize(); + var key = (message.Id, normalizedEmojiName, userInfo.Id); + + if (existingReactions.Contains(key)) continue; + + // Ensure user exists + var user = await _context.Users.FindAsync(userInfo.Id); + if (user == null) + { + user = new User + { + Id = userInfo.Id, + Name = userInfo.Name, + Discriminator = userInfo.Discriminator, + IsBot = userInfo.IsBot + }; + _context.Users.Add(user); + } + + var reaction = new Reaction + { + MessageId = message.Id, + EmojiCode = reactionInfo.Emoji.Code, + EmojiName = normalizedEmojiName, + UserId = userInfo.Id + }; + _context.Reactions.Add(reaction); + existingReactions.Add(key); + fileAdded++; + totalAdded++; + } + } + } + + if (fileAdded > 0) + { + await _context.SaveChangesAsync(); + _logger.LogDebug("Added {Count} reactions from {Path}", fileAdded, jsonFilePath); + } + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to reprocess file: {Path}", jsonFilePath); + _context.ChangeTracker.Clear(); + } + } + + return totalAdded; } private async Task ProcessMentionAsync(MentionInfo mentionInfo, string messageId)