Compare commits
4 Commits
7896f9ef9a
...
00a0b3e14f
| Author | SHA1 | Date | |
|---|---|---|---|
| 00a0b3e14f | |||
| a281f7f1e7 | |||
| 6f63f36df0 | |||
| fbe52f72d6 |
@@ -111,7 +111,13 @@ public class DiscordArchiveContext : DbContext
|
||||
.WithMany(m => m.Reactions)
|
||||
.HasForeignKey(e => e.MessageId)
|
||||
.OnDelete(DeleteBehavior.Cascade);
|
||||
entity.HasOne(e => e.User)
|
||||
.WithMany(u => u.Reactions)
|
||||
.HasForeignKey(e => e.UserId)
|
||||
.OnDelete(DeleteBehavior.Restrict);
|
||||
entity.HasIndex(e => e.MessageId);
|
||||
entity.HasIndex(e => e.UserId);
|
||||
entity.HasIndex(e => new { e.MessageId, e.EmojiName, e.UserId }).IsUnique();
|
||||
});
|
||||
|
||||
// Mention configuration
|
||||
|
||||
@@ -19,9 +19,14 @@ public class Reaction
|
||||
[MaxLength(256)]
|
||||
public string EmojiName { get; set; } = null!;
|
||||
|
||||
public int Count { get; set; }
|
||||
[Required]
|
||||
[MaxLength(32)]
|
||||
public string UserId { get; set; } = null!;
|
||||
|
||||
// Navigation properties
|
||||
[ForeignKey(nameof(MessageId))]
|
||||
public Message Message { get; set; } = null!;
|
||||
|
||||
[ForeignKey(nameof(UserId))]
|
||||
public User User { get; set; } = null!;
|
||||
}
|
||||
|
||||
@@ -22,4 +22,5 @@ public class User
|
||||
public ICollection<UserSnapshot> Snapshots { get; set; } = new List<UserSnapshot>();
|
||||
public ICollection<Message> Messages { get; set; } = new List<Message>();
|
||||
public ICollection<Mention> Mentions { get; set; } = new List<Mention>();
|
||||
public ICollection<Reaction> Reactions { get; set; } = new List<Reaction>();
|
||||
}
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
using DiscordArchiveManager.Data;
|
||||
using DiscordArchiveManager.Services;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.EntityFrameworkCore.Infrastructure;
|
||||
using Microsoft.EntityFrameworkCore.Storage;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
@@ -32,11 +34,23 @@ var inputDirectory = config["Paths:InputDirectory"] ?? "/app/input";
|
||||
var archiveDirectory = config["Paths:ArchiveDirectory"] ?? "/app/archive";
|
||||
var imageDirectory = config["Paths:ImageDirectory"] ?? "/app/images";
|
||||
|
||||
var reprocessReactions = args.Contains("--reprocess-reactions");
|
||||
var reimport = args.Contains("--reimport");
|
||||
|
||||
logger.LogInformation("Discord Archive Manager starting...");
|
||||
logger.LogInformation("Input directory: {Path}", inputDirectory);
|
||||
logger.LogInformation("Archive directory: {Path}", archiveDirectory);
|
||||
logger.LogInformation("Image directory: {Path}", imageDirectory);
|
||||
|
||||
if (reimport)
|
||||
{
|
||||
logger.LogInformation("Mode: Reimport all data from archive");
|
||||
}
|
||||
else if (reprocessReactions)
|
||||
{
|
||||
logger.LogInformation("Mode: Reprocess reactions from archive");
|
||||
}
|
||||
|
||||
// Ensure directories exist
|
||||
Directory.CreateDirectory(inputDirectory);
|
||||
Directory.CreateDirectory(archiveDirectory);
|
||||
@@ -46,17 +60,46 @@ Directory.CreateDirectory(imageDirectory);
|
||||
using (var scope = host.Services.CreateScope())
|
||||
{
|
||||
var context = scope.ServiceProvider.GetRequiredService<DiscordArchiveContext>();
|
||||
logger.LogInformation("Ensuring database exists and applying migrations...");
|
||||
await context.Database.EnsureCreatedAsync();
|
||||
logger.LogInformation("Ensuring database schema exists...");
|
||||
|
||||
// Get the database creator for more control
|
||||
var creator = context.GetService<IRelationalDatabaseCreator>()!;
|
||||
|
||||
if (!await creator.ExistsAsync())
|
||||
{
|
||||
logger.LogError("Database does not exist. Please create it first.");
|
||||
return;
|
||||
}
|
||||
|
||||
if (!await creator.HasTablesAsync())
|
||||
{
|
||||
logger.LogInformation("Creating database tables...");
|
||||
await creator.CreateTablesAsync();
|
||||
}
|
||||
|
||||
logger.LogInformation("Database ready.");
|
||||
}
|
||||
|
||||
// Process files
|
||||
// Process files or reprocess reactions
|
||||
using (var scope = host.Services.CreateScope())
|
||||
{
|
||||
var importService = scope.ServiceProvider.GetRequiredService<JsonImportService>();
|
||||
var archiveService = scope.ServiceProvider.GetRequiredService<ArchiveService>();
|
||||
|
||||
if (reimport)
|
||||
{
|
||||
logger.LogInformation("Reimporting all data from archived files...");
|
||||
var (success, errors) = await importService.ReimportFromArchiveAsync(archiveDirectory, imageDirectory);
|
||||
logger.LogInformation("Reimport complete. Imported: {Success}, Errors: {Errors}", success, errors);
|
||||
}
|
||||
else if (reprocessReactions)
|
||||
{
|
||||
logger.LogInformation("Reprocessing reactions from archived files...");
|
||||
var added = await importService.ReprocessReactionsAsync(archiveDirectory);
|
||||
logger.LogInformation("Reprocessing complete. Added {Count} reactions.", added);
|
||||
}
|
||||
else
|
||||
{
|
||||
var files = archiveService.GetExportFiles(inputDirectory).ToList();
|
||||
|
||||
if (files.Count == 0)
|
||||
@@ -96,5 +139,6 @@ using (var scope = host.Services.CreateScope())
|
||||
successCount, skipCount, errorCount);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
logger.LogInformation("Discord Archive Manager finished.");
|
||||
|
||||
@@ -28,8 +28,8 @@ public class ArchiveService
|
||||
var jsonDirectory = Path.GetDirectoryName(jsonFilePath)!;
|
||||
var filesDirectory = GetFilesDirectoryPath(jsonFilePath);
|
||||
|
||||
// Create archive subdirectory based on date
|
||||
var archiveSubdir = DateTime.Now.ToString("yyyy-MM-dd");
|
||||
// Create archive subdirectory based on export date from filename
|
||||
var archiveSubdir = GetExportDateFromFilename(jsonFileName) ?? DateTime.Now.ToString("yyyy-MM-dd");
|
||||
var archivePath = Path.Combine(archiveRoot, archiveSubdir);
|
||||
Directory.CreateDirectory(archivePath);
|
||||
|
||||
@@ -152,6 +152,26 @@ public class ArchiveService
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Extracts the export date from a filename like "2026-01-20.json".
|
||||
/// Returns null if the filename doesn't match the expected pattern.
|
||||
/// </summary>
|
||||
private static string? GetExportDateFromFilename(string filename)
|
||||
{
|
||||
var nameWithoutExtension = Path.GetFileNameWithoutExtension(filename);
|
||||
if (DateTime.TryParseExact(nameWithoutExtension, "yyyy-MM-dd", null, System.Globalization.DateTimeStyles.None, out _))
|
||||
{
|
||||
return nameWithoutExtension;
|
||||
}
|
||||
// Handle filenames with suffix like "2026-01-20_1.json"
|
||||
var parts = nameWithoutExtension.Split('_');
|
||||
if (parts.Length > 0 && DateTime.TryParseExact(parts[0], "yyyy-MM-dd", null, System.Globalization.DateTimeStyles.None, out _))
|
||||
{
|
||||
return parts[0];
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Moves a file, falling back to copy+delete for cross-device moves.
|
||||
/// </summary>
|
||||
|
||||
@@ -72,20 +72,32 @@ public class JsonImportService
|
||||
// Upsert Channel
|
||||
await UpsertChannelAsync(export.Channel, export.Guild.Id);
|
||||
|
||||
// Process messages
|
||||
// Process messages - save after each to isolate any issues
|
||||
var processedCount = 0;
|
||||
foreach (var message in export.Messages)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (await ProcessMessageAsync(message, export.Channel.Id, jsonFilePath, imageRoot))
|
||||
{
|
||||
await _context.SaveChangesAsync();
|
||||
processedCount++;
|
||||
}
|
||||
}
|
||||
catch (DbUpdateException ex) when (ex.InnerException is Microsoft.Data.SqlClient.SqlException sqlEx && sqlEx.Number == 2601)
|
||||
{
|
||||
// Duplicate key - log and continue
|
||||
_logger.LogWarning("Duplicate key error for message {MessageId}, skipping: {Error}",
|
||||
message.Id, sqlEx.Message);
|
||||
_context.ChangeTracker.Clear();
|
||||
// Re-upsert guild and channel as they were cleared
|
||||
await UpsertGuildAsync(export.Guild);
|
||||
await UpsertChannelAsync(export.Channel, export.Guild.Id);
|
||||
}
|
||||
}
|
||||
|
||||
_logger.LogInformation("Processed {Count} new messages", processedCount);
|
||||
|
||||
await _context.SaveChangesAsync();
|
||||
|
||||
// Archive the file
|
||||
var archivePath = _archiveService.ArchiveExport(jsonFilePath, archiveRoot);
|
||||
|
||||
@@ -106,6 +118,9 @@ public class JsonImportService
|
||||
catch (Exception ex)
|
||||
{
|
||||
await transaction.RollbackAsync();
|
||||
_context.ChangeTracker.Clear(); // Clear tracked entities to prevent cascading failures
|
||||
_processedMessages.Clear(); // Clear the session tracking
|
||||
_addedReactions.Clear();
|
||||
_logger.LogError(ex, "Error processing file, rolled back transaction: {Path}", jsonFilePath);
|
||||
throw;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user