Compare commits
4 Commits
7896f9ef9a
...
00a0b3e14f
| Author | SHA1 | Date | |
|---|---|---|---|
| 00a0b3e14f | |||
| a281f7f1e7 | |||
| 6f63f36df0 | |||
| fbe52f72d6 |
@@ -111,7 +111,13 @@ public class DiscordArchiveContext : DbContext
|
|||||||
.WithMany(m => m.Reactions)
|
.WithMany(m => m.Reactions)
|
||||||
.HasForeignKey(e => e.MessageId)
|
.HasForeignKey(e => e.MessageId)
|
||||||
.OnDelete(DeleteBehavior.Cascade);
|
.OnDelete(DeleteBehavior.Cascade);
|
||||||
|
entity.HasOne(e => e.User)
|
||||||
|
.WithMany(u => u.Reactions)
|
||||||
|
.HasForeignKey(e => e.UserId)
|
||||||
|
.OnDelete(DeleteBehavior.Restrict);
|
||||||
entity.HasIndex(e => e.MessageId);
|
entity.HasIndex(e => e.MessageId);
|
||||||
|
entity.HasIndex(e => e.UserId);
|
||||||
|
entity.HasIndex(e => new { e.MessageId, e.EmojiName, e.UserId }).IsUnique();
|
||||||
});
|
});
|
||||||
|
|
||||||
// Mention configuration
|
// Mention configuration
|
||||||
|
|||||||
@@ -19,9 +19,14 @@ public class Reaction
|
|||||||
[MaxLength(256)]
|
[MaxLength(256)]
|
||||||
public string EmojiName { get; set; } = null!;
|
public string EmojiName { get; set; } = null!;
|
||||||
|
|
||||||
public int Count { get; set; }
|
[Required]
|
||||||
|
[MaxLength(32)]
|
||||||
|
public string UserId { get; set; } = null!;
|
||||||
|
|
||||||
// Navigation properties
|
// Navigation properties
|
||||||
[ForeignKey(nameof(MessageId))]
|
[ForeignKey(nameof(MessageId))]
|
||||||
public Message Message { get; set; } = null!;
|
public Message Message { get; set; } = null!;
|
||||||
|
|
||||||
|
[ForeignKey(nameof(UserId))]
|
||||||
|
public User User { get; set; } = null!;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -22,4 +22,5 @@ public class User
|
|||||||
public ICollection<UserSnapshot> Snapshots { get; set; } = new List<UserSnapshot>();
|
public ICollection<UserSnapshot> Snapshots { get; set; } = new List<UserSnapshot>();
|
||||||
public ICollection<Message> Messages { get; set; } = new List<Message>();
|
public ICollection<Message> Messages { get; set; } = new List<Message>();
|
||||||
public ICollection<Mention> Mentions { get; set; } = new List<Mention>();
|
public ICollection<Mention> Mentions { get; set; } = new List<Mention>();
|
||||||
|
public ICollection<Reaction> Reactions { get; set; } = new List<Reaction>();
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,6 +1,8 @@
|
|||||||
using DiscordArchiveManager.Data;
|
using DiscordArchiveManager.Data;
|
||||||
using DiscordArchiveManager.Services;
|
using DiscordArchiveManager.Services;
|
||||||
using Microsoft.EntityFrameworkCore;
|
using Microsoft.EntityFrameworkCore;
|
||||||
|
using Microsoft.EntityFrameworkCore.Infrastructure;
|
||||||
|
using Microsoft.EntityFrameworkCore.Storage;
|
||||||
using Microsoft.Extensions.Configuration;
|
using Microsoft.Extensions.Configuration;
|
||||||
using Microsoft.Extensions.DependencyInjection;
|
using Microsoft.Extensions.DependencyInjection;
|
||||||
using Microsoft.Extensions.Hosting;
|
using Microsoft.Extensions.Hosting;
|
||||||
@@ -32,11 +34,23 @@ var inputDirectory = config["Paths:InputDirectory"] ?? "/app/input";
|
|||||||
var archiveDirectory = config["Paths:ArchiveDirectory"] ?? "/app/archive";
|
var archiveDirectory = config["Paths:ArchiveDirectory"] ?? "/app/archive";
|
||||||
var imageDirectory = config["Paths:ImageDirectory"] ?? "/app/images";
|
var imageDirectory = config["Paths:ImageDirectory"] ?? "/app/images";
|
||||||
|
|
||||||
|
var reprocessReactions = args.Contains("--reprocess-reactions");
|
||||||
|
var reimport = args.Contains("--reimport");
|
||||||
|
|
||||||
logger.LogInformation("Discord Archive Manager starting...");
|
logger.LogInformation("Discord Archive Manager starting...");
|
||||||
logger.LogInformation("Input directory: {Path}", inputDirectory);
|
logger.LogInformation("Input directory: {Path}", inputDirectory);
|
||||||
logger.LogInformation("Archive directory: {Path}", archiveDirectory);
|
logger.LogInformation("Archive directory: {Path}", archiveDirectory);
|
||||||
logger.LogInformation("Image directory: {Path}", imageDirectory);
|
logger.LogInformation("Image directory: {Path}", imageDirectory);
|
||||||
|
|
||||||
|
if (reimport)
|
||||||
|
{
|
||||||
|
logger.LogInformation("Mode: Reimport all data from archive");
|
||||||
|
}
|
||||||
|
else if (reprocessReactions)
|
||||||
|
{
|
||||||
|
logger.LogInformation("Mode: Reprocess reactions from archive");
|
||||||
|
}
|
||||||
|
|
||||||
// Ensure directories exist
|
// Ensure directories exist
|
||||||
Directory.CreateDirectory(inputDirectory);
|
Directory.CreateDirectory(inputDirectory);
|
||||||
Directory.CreateDirectory(archiveDirectory);
|
Directory.CreateDirectory(archiveDirectory);
|
||||||
@@ -46,17 +60,46 @@ Directory.CreateDirectory(imageDirectory);
|
|||||||
using (var scope = host.Services.CreateScope())
|
using (var scope = host.Services.CreateScope())
|
||||||
{
|
{
|
||||||
var context = scope.ServiceProvider.GetRequiredService<DiscordArchiveContext>();
|
var context = scope.ServiceProvider.GetRequiredService<DiscordArchiveContext>();
|
||||||
logger.LogInformation("Ensuring database exists and applying migrations...");
|
logger.LogInformation("Ensuring database schema exists...");
|
||||||
await context.Database.EnsureCreatedAsync();
|
|
||||||
|
// Get the database creator for more control
|
||||||
|
var creator = context.GetService<IRelationalDatabaseCreator>()!;
|
||||||
|
|
||||||
|
if (!await creator.ExistsAsync())
|
||||||
|
{
|
||||||
|
logger.LogError("Database does not exist. Please create it first.");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!await creator.HasTablesAsync())
|
||||||
|
{
|
||||||
|
logger.LogInformation("Creating database tables...");
|
||||||
|
await creator.CreateTablesAsync();
|
||||||
|
}
|
||||||
|
|
||||||
logger.LogInformation("Database ready.");
|
logger.LogInformation("Database ready.");
|
||||||
}
|
}
|
||||||
|
|
||||||
// Process files
|
// Process files or reprocess reactions
|
||||||
using (var scope = host.Services.CreateScope())
|
using (var scope = host.Services.CreateScope())
|
||||||
{
|
{
|
||||||
var importService = scope.ServiceProvider.GetRequiredService<JsonImportService>();
|
var importService = scope.ServiceProvider.GetRequiredService<JsonImportService>();
|
||||||
var archiveService = scope.ServiceProvider.GetRequiredService<ArchiveService>();
|
var archiveService = scope.ServiceProvider.GetRequiredService<ArchiveService>();
|
||||||
|
|
||||||
|
if (reimport)
|
||||||
|
{
|
||||||
|
logger.LogInformation("Reimporting all data from archived files...");
|
||||||
|
var (success, errors) = await importService.ReimportFromArchiveAsync(archiveDirectory, imageDirectory);
|
||||||
|
logger.LogInformation("Reimport complete. Imported: {Success}, Errors: {Errors}", success, errors);
|
||||||
|
}
|
||||||
|
else if (reprocessReactions)
|
||||||
|
{
|
||||||
|
logger.LogInformation("Reprocessing reactions from archived files...");
|
||||||
|
var added = await importService.ReprocessReactionsAsync(archiveDirectory);
|
||||||
|
logger.LogInformation("Reprocessing complete. Added {Count} reactions.", added);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
var files = archiveService.GetExportFiles(inputDirectory).ToList();
|
var files = archiveService.GetExportFiles(inputDirectory).ToList();
|
||||||
|
|
||||||
if (files.Count == 0)
|
if (files.Count == 0)
|
||||||
@@ -95,6 +138,7 @@ using (var scope = host.Services.CreateScope())
|
|||||||
logger.LogInformation("Processing complete. Processed: {Success}, Skipped: {Skip}, Errors: {Error}",
|
logger.LogInformation("Processing complete. Processed: {Success}, Skipped: {Skip}, Errors: {Error}",
|
||||||
successCount, skipCount, errorCount);
|
successCount, skipCount, errorCount);
|
||||||
}
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
logger.LogInformation("Discord Archive Manager finished.");
|
logger.LogInformation("Discord Archive Manager finished.");
|
||||||
|
|||||||
@@ -28,8 +28,8 @@ public class ArchiveService
|
|||||||
var jsonDirectory = Path.GetDirectoryName(jsonFilePath)!;
|
var jsonDirectory = Path.GetDirectoryName(jsonFilePath)!;
|
||||||
var filesDirectory = GetFilesDirectoryPath(jsonFilePath);
|
var filesDirectory = GetFilesDirectoryPath(jsonFilePath);
|
||||||
|
|
||||||
// Create archive subdirectory based on date
|
// Create archive subdirectory based on export date from filename
|
||||||
var archiveSubdir = DateTime.Now.ToString("yyyy-MM-dd");
|
var archiveSubdir = GetExportDateFromFilename(jsonFileName) ?? DateTime.Now.ToString("yyyy-MM-dd");
|
||||||
var archivePath = Path.Combine(archiveRoot, archiveSubdir);
|
var archivePath = Path.Combine(archiveRoot, archiveSubdir);
|
||||||
Directory.CreateDirectory(archivePath);
|
Directory.CreateDirectory(archivePath);
|
||||||
|
|
||||||
@@ -152,6 +152,26 @@ public class ArchiveService
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Extracts the export date from a filename like "2026-01-20.json".
|
||||||
|
/// Returns null if the filename doesn't match the expected pattern.
|
||||||
|
/// </summary>
|
||||||
|
private static string? GetExportDateFromFilename(string filename)
|
||||||
|
{
|
||||||
|
var nameWithoutExtension = Path.GetFileNameWithoutExtension(filename);
|
||||||
|
if (DateTime.TryParseExact(nameWithoutExtension, "yyyy-MM-dd", null, System.Globalization.DateTimeStyles.None, out _))
|
||||||
|
{
|
||||||
|
return nameWithoutExtension;
|
||||||
|
}
|
||||||
|
// Handle filenames with suffix like "2026-01-20_1.json"
|
||||||
|
var parts = nameWithoutExtension.Split('_');
|
||||||
|
if (parts.Length > 0 && DateTime.TryParseExact(parts[0], "yyyy-MM-dd", null, System.Globalization.DateTimeStyles.None, out _))
|
||||||
|
{
|
||||||
|
return parts[0];
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Moves a file, falling back to copy+delete for cross-device moves.
|
/// Moves a file, falling back to copy+delete for cross-device moves.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
|
|||||||
@@ -72,20 +72,32 @@ public class JsonImportService
|
|||||||
// Upsert Channel
|
// Upsert Channel
|
||||||
await UpsertChannelAsync(export.Channel, export.Guild.Id);
|
await UpsertChannelAsync(export.Channel, export.Guild.Id);
|
||||||
|
|
||||||
// Process messages
|
// Process messages - save after each to isolate any issues
|
||||||
var processedCount = 0;
|
var processedCount = 0;
|
||||||
foreach (var message in export.Messages)
|
foreach (var message in export.Messages)
|
||||||
|
{
|
||||||
|
try
|
||||||
{
|
{
|
||||||
if (await ProcessMessageAsync(message, export.Channel.Id, jsonFilePath, imageRoot))
|
if (await ProcessMessageAsync(message, export.Channel.Id, jsonFilePath, imageRoot))
|
||||||
{
|
{
|
||||||
|
await _context.SaveChangesAsync();
|
||||||
processedCount++;
|
processedCount++;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
catch (DbUpdateException ex) when (ex.InnerException is Microsoft.Data.SqlClient.SqlException sqlEx && sqlEx.Number == 2601)
|
||||||
|
{
|
||||||
|
// Duplicate key - log and continue
|
||||||
|
_logger.LogWarning("Duplicate key error for message {MessageId}, skipping: {Error}",
|
||||||
|
message.Id, sqlEx.Message);
|
||||||
|
_context.ChangeTracker.Clear();
|
||||||
|
// Re-upsert guild and channel as they were cleared
|
||||||
|
await UpsertGuildAsync(export.Guild);
|
||||||
|
await UpsertChannelAsync(export.Channel, export.Guild.Id);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
_logger.LogInformation("Processed {Count} new messages", processedCount);
|
_logger.LogInformation("Processed {Count} new messages", processedCount);
|
||||||
|
|
||||||
await _context.SaveChangesAsync();
|
|
||||||
|
|
||||||
// Archive the file
|
// Archive the file
|
||||||
var archivePath = _archiveService.ArchiveExport(jsonFilePath, archiveRoot);
|
var archivePath = _archiveService.ArchiveExport(jsonFilePath, archiveRoot);
|
||||||
|
|
||||||
@@ -106,6 +118,9 @@ public class JsonImportService
|
|||||||
catch (Exception ex)
|
catch (Exception ex)
|
||||||
{
|
{
|
||||||
await transaction.RollbackAsync();
|
await transaction.RollbackAsync();
|
||||||
|
_context.ChangeTracker.Clear(); // Clear tracked entities to prevent cascading failures
|
||||||
|
_processedMessages.Clear(); // Clear the session tracking
|
||||||
|
_addedReactions.Clear();
|
||||||
_logger.LogError(ex, "Error processing file, rolled back transaction: {Path}", jsonFilePath);
|
_logger.LogError(ex, "Error processing file, rolled back transaction: {Path}", jsonFilePath);
|
||||||
throw;
|
throw;
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user