From 3d6b47d537b9f2a19f385cfbdbafa79b4f33c3e1 Mon Sep 17 00:00:00 2001 From: AJ Date: Sat, 11 Oct 2025 22:17:37 -0400 Subject: [PATCH] Optimize transaction preview duplicate checking for large imports MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Improve performance when uploading thousands of transactions by loading all existing transactions into memory once for duplicate detection, rather than running individual database queries for each transaction. This reduces database calls from O(n) to O(1) for n transactions. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- MoneyMap/Pages/Upload.cshtml.cs | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/MoneyMap/Pages/Upload.cshtml.cs b/MoneyMap/Pages/Upload.cshtml.cs index 33231d0..1dc8858 100644 --- a/MoneyMap/Pages/Upload.cshtml.cs +++ b/MoneyMap/Pages/Upload.cshtml.cs @@ -255,6 +255,11 @@ namespace MoneyMap.Pages var previewItems = new List(); var addedInThisBatch = new HashSet(); + // Load all existing transactions into memory for fast duplicate checking + var existingTransactions = await _db.Transactions + .Select(t => new TransactionKey(t.Date, t.Amount, t.Name, t.Memo, t.AccountId, t.CardId)) + .ToHashSetAsync(); + using var reader = new StreamReader(csvStream); using var csv = new CsvReader(reader, new CsvConfiguration(CultureInfo.InvariantCulture) { @@ -279,7 +284,8 @@ namespace MoneyMap.Pages var transaction = MapToTransaction(row, paymentResolution); var key = new TransactionKey(transaction); - bool isDuplicate = addedInThisBatch.Contains(key) || await IsDuplicate(transaction); + // Fast in-memory duplicate checking + bool isDuplicate = addedInThisBatch.Contains(key) || existingTransactions.Contains(key); previewItems.Add(new TransactionPreview { @@ -415,7 +421,7 @@ namespace MoneyMap.Pages return PaymentResolutionResult.SuccessAccount(account.Id, account.Last4); } - private async Task ResolveAutomaticallyAsync(string? memo, ImportContext context) + private Task ResolveAutomaticallyAsync(string? memo, ImportContext context) { // Extract last4 from both memo and filename var last4FromFile = CardIdentifierExtractor.FromFileName(context.FileName); @@ -425,26 +431,26 @@ namespace MoneyMap.Pages if (!string.IsNullOrWhiteSpace(last4FromMemo)) { var result = TryResolveByLast4(last4FromMemo, context); - if (result != null) return result; + if (result != null) return Task.FromResult(result); } // PRIORITY 2: Fall back to filename (for account-level CSVs or when memo has no card) if (!string.IsNullOrWhiteSpace(last4FromFile)) { var result = TryResolveByLast4(last4FromFile, context); - if (result != null) return result; + if (result != null) return Task.FromResult(result); } // Nothing found - error var searchedLast4 = last4FromMemo ?? last4FromFile; if (string.IsNullOrWhiteSpace(searchedLast4)) { - return PaymentResolutionResult.Failure( - "Couldn't determine card or account from memo or file name. Choose an account manually."); + return Task.FromResult(PaymentResolutionResult.Failure( + "Couldn't determine card or account from memo or file name. Choose an account manually.")); } - return PaymentResolutionResult.Failure( - $"Couldn't find account or card with last4 '{searchedLast4}'. Choose an account manually."); + return Task.FromResult(PaymentResolutionResult.Failure( + $"Couldn't find account or card with last4 '{searchedLast4}'. Choose an account manually.")); } private PaymentResolutionResult? TryResolveByLast4(string last4, ImportContext context)