aj/MoneyMap
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Optimize transaction preview duplicate checking for large imports

Browse Source 
Improve performance when uploading thousands of transactions by loading all existing transactions into memory once for duplicate detection, rather than running individual database queries for each transaction. This reduces database calls from O(n) to O(1) for n transactions.

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
AJ
2025-10-11 22:17:37 -04:00
parent 40135ab6d6
commit 3d6b47d537
1 changed files with 14 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
MoneyMap/Pages/Upload.cshtml.cs
View File

@@ -255,6 +255,11 @@ namespace MoneyMap.Pages
var previewItems = new List<TransactionPreview>(); var previewItems = new List<TransactionPreview>();
var addedInThisBatch = new HashSet<TransactionKey>(); var addedInThisBatch = new HashSet<TransactionKey>();
// Load all existing transactions into memory for fast duplicate checking
var existingTransactions = await _db.Transactions
.Select(t => new TransactionKey(t.Date, t.Amount, t.Name, t.Memo, t.AccountId, t.CardId))
.ToHashSetAsync();
using var reader = new StreamReader(csvStream); using var reader = new StreamReader(csvStream);
using var csv = new CsvReader(reader, new CsvConfiguration(CultureInfo.InvariantCulture) using var csv = new CsvReader(reader, new CsvConfiguration(CultureInfo.InvariantCulture)
{ {
@@ -279,7 +284,8 @@ namespace MoneyMap.Pages
var transaction = MapToTransaction(row, paymentResolution); var transaction = MapToTransaction(row, paymentResolution);
var key = new TransactionKey(transaction); var key = new TransactionKey(transaction);
bool isDuplicate = addedInThisBatch.Contains(key) || await IsDuplicate(transaction); // Fast in-memory duplicate checking
bool isDuplicate = addedInThisBatch.Contains(key) || existingTransactions.Contains(key);
previewItems.Add(new TransactionPreview previewItems.Add(new TransactionPreview
{ {
@@ -415,7 +421,7 @@ namespace MoneyMap.Pages
return PaymentResolutionResult.SuccessAccount(account.Id, account.Last4); return PaymentResolutionResult.SuccessAccount(account.Id, account.Last4);
} }
private async Task<PaymentResolutionResult> ResolveAutomaticallyAsync(string? memo, ImportContext context) private Task<PaymentResolutionResult> ResolveAutomaticallyAsync(string? memo, ImportContext context)
{ {
// Extract last4 from both memo and filename // Extract last4 from both memo and filename
var last4FromFile = CardIdentifierExtractor.FromFileName(context.FileName); var last4FromFile = CardIdentifierExtractor.FromFileName(context.FileName);
@@ -425,26 +431,26 @@ namespace MoneyMap.Pages
if (!string.IsNullOrWhiteSpace(last4FromMemo)) if (!string.IsNullOrWhiteSpace(last4FromMemo))
{ {
var result = TryResolveByLast4(last4FromMemo, context); var result = TryResolveByLast4(last4FromMemo, context);
if (result != null) return result; if (result != null) return Task.FromResult(result);
} }
// PRIORITY 2: Fall back to filename (for account-level CSVs or when memo has no card) // PRIORITY 2: Fall back to filename (for account-level CSVs or when memo has no card)
if (!string.IsNullOrWhiteSpace(last4FromFile)) if (!string.IsNullOrWhiteSpace(last4FromFile))
{ {
var result = TryResolveByLast4(last4FromFile, context); var result = TryResolveByLast4(last4FromFile, context);
if (result != null) return result; if (result != null) return Task.FromResult(result);
} }
// Nothing found - error // Nothing found - error
var searchedLast4 = last4FromMemo ?? last4FromFile; var searchedLast4 = last4FromMemo ?? last4FromFile;
if (string.IsNullOrWhiteSpace(searchedLast4)) if (string.IsNullOrWhiteSpace(searchedLast4))
{ {
return PaymentResolutionResult.Failure( return Task.FromResult(PaymentResolutionResult.Failure(
"Couldn't determine card or account from memo or file name. Choose an account manually."); "Couldn't determine card or account from memo or file name. Choose an account manually."));
} }
return PaymentResolutionResult.Failure( return Task.FromResult(PaymentResolutionResult.Failure(
$"Couldn't find account or card with last4 '{searchedLast4}'. Choose an account manually."); $"Couldn't find account or card with last4 '{searchedLast4}'. Choose an account manually."));
} }
private PaymentResolutionResult? TryResolveByLast4(string last4, ImportContext context) private PaymentResolutionResult? TryResolveByLast4(string last4, ImportContext context)