Feature: Save AI parsing notes with receipt

Store user-provided parsing notes in the database so they persist
across parsing attempts. Notes are displayed in Receipt Information
and pre-populated in the textarea for future parses.

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
2026-01-18 20:49:31 -05:00
parent c5fad34658
commit 865195ad16
7 changed files with 893 additions and 126 deletions

View File

@@ -0,0 +1,661 @@
// <auto-generated />
using System;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Metadata;
using Microsoft.EntityFrameworkCore.Migrations;
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
using MoneyMap.Data;
#nullable disable
namespace MoneyMap.Migrations
{
[DbContext(typeof(MoneyMapContext))]
[Migration("20260119005858_AddReceiptParsingNotes")]
partial class AddReceiptParsingNotes
{
/// <inheritdoc />
protected override void BuildTargetModel(ModelBuilder modelBuilder)
{
#pragma warning disable 612, 618
modelBuilder
.HasAnnotation("ProductVersion", "9.0.9")
.HasAnnotation("Relational:MaxIdentifierLength", 128);
SqlServerModelBuilderExtensions.UseIdentityColumns(modelBuilder);
modelBuilder.Entity("MoneyMap.Models.Account", b =>
{
b.Property<int>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("int");
SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property<int>("Id"));
b.Property<int>("AccountType")
.HasColumnType("int");
b.Property<string>("Institution")
.IsRequired()
.HasMaxLength(100)
.HasColumnType("nvarchar(100)");
b.Property<string>("Last4")
.IsRequired()
.HasMaxLength(4)
.HasColumnType("nvarchar(4)");
b.Property<string>("Nickname")
.HasMaxLength(50)
.HasColumnType("nvarchar(50)");
b.Property<string>("Owner")
.IsRequired()
.HasMaxLength(100)
.HasColumnType("nvarchar(100)");
b.HasKey("Id");
b.HasIndex("Institution", "Last4", "Owner");
b.ToTable("Accounts");
});
modelBuilder.Entity("MoneyMap.Models.Budget", b =>
{
b.Property<int>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("int");
SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property<int>("Id"));
b.Property<decimal>("Amount")
.HasColumnType("decimal(18,2)");
b.Property<string>("Category")
.HasMaxLength(100)
.HasColumnType("nvarchar(100)");
b.Property<bool>("IsActive")
.HasColumnType("bit");
b.Property<string>("Notes")
.HasMaxLength(500)
.HasColumnType("nvarchar(500)");
b.Property<int>("Period")
.HasColumnType("int");
b.Property<DateTime>("StartDate")
.HasColumnType("datetime2");
b.HasKey("Id");
b.HasIndex("Category", "Period")
.IsUnique()
.HasFilter("[IsActive] = 1");
b.ToTable("Budgets");
});
modelBuilder.Entity("MoneyMap.Models.Card", b =>
{
b.Property<int>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("int");
SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property<int>("Id"));
b.Property<int?>("AccountId")
.HasColumnType("int");
b.Property<string>("Issuer")
.IsRequired()
.HasMaxLength(100)
.HasColumnType("nvarchar(100)");
b.Property<string>("Last4")
.IsRequired()
.HasMaxLength(4)
.HasColumnType("nvarchar(4)");
b.Property<string>("Nickname")
.HasMaxLength(50)
.HasColumnType("nvarchar(50)");
b.Property<string>("Owner")
.IsRequired()
.HasMaxLength(100)
.HasColumnType("nvarchar(100)");
b.HasKey("Id");
b.HasIndex("AccountId");
b.HasIndex("Issuer", "Last4", "Owner");
b.ToTable("Cards");
});
modelBuilder.Entity("MoneyMap.Models.CategoryMapping", b =>
{
b.Property<int>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("int");
SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property<int>("Id"));
b.Property<string>("Category")
.IsRequired()
.HasMaxLength(100)
.HasColumnType("nvarchar(100)");
b.Property<decimal?>("Confidence")
.HasColumnType("decimal(5,4)");
b.Property<DateTime?>("CreatedAt")
.HasColumnType("datetime2");
b.Property<string>("CreatedBy")
.HasMaxLength(50)
.HasColumnType("nvarchar(50)");
b.Property<int?>("MerchantId")
.HasColumnType("int");
b.Property<string>("Pattern")
.IsRequired()
.HasMaxLength(200)
.HasColumnType("nvarchar(200)");
b.Property<int>("Priority")
.HasColumnType("int");
b.HasKey("Id");
b.HasIndex("MerchantId");
b.ToTable("CategoryMappings");
});
modelBuilder.Entity("MoneyMap.Models.Merchant", b =>
{
b.Property<int>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("int");
SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property<int>("Id"));
b.Property<string>("Name")
.IsRequired()
.HasMaxLength(100)
.HasColumnType("nvarchar(100)");
b.HasKey("Id");
b.HasIndex("Name")
.IsUnique();
b.ToTable("Merchants");
});
modelBuilder.Entity("MoneyMap.Models.Receipt", b =>
{
b.Property<long>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("bigint");
SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property<long>("Id"));
b.Property<string>("ContentType")
.IsRequired()
.ValueGeneratedOnAdd()
.HasMaxLength(100)
.HasColumnType("nvarchar(100)")
.HasDefaultValue("application/octet-stream");
b.Property<string>("Currency")
.HasMaxLength(8)
.HasColumnType("nvarchar(8)");
b.Property<DateTime?>("DueDate")
.HasColumnType("datetime2");
b.Property<string>("FileHashSha256")
.IsRequired()
.HasMaxLength(64)
.HasColumnType("nvarchar(64)");
b.Property<string>("FileName")
.IsRequired()
.HasMaxLength(260)
.HasColumnType("nvarchar(260)");
b.Property<long>("FileSizeBytes")
.HasColumnType("bigint");
b.Property<string>("Merchant")
.HasMaxLength(200)
.HasColumnType("nvarchar(200)");
b.Property<string>("ParsingNotes")
.HasMaxLength(2000)
.HasColumnType("nvarchar(2000)");
b.Property<DateTime?>("ReceiptDate")
.HasColumnType("datetime2");
b.Property<string>("StoragePath")
.IsRequired()
.HasMaxLength(1024)
.HasColumnType("nvarchar(1024)");
b.Property<decimal?>("Subtotal")
.HasColumnType("decimal(18,2)");
b.Property<decimal?>("Tax")
.HasColumnType("decimal(18,2)");
b.Property<decimal?>("Total")
.HasColumnType("decimal(18,2)");
b.Property<long?>("TransactionId")
.HasColumnType("bigint");
b.Property<DateTime>("UploadedAtUtc")
.HasColumnType("datetime2");
b.HasKey("Id");
b.HasIndex("FileHashSha256");
b.HasIndex("TransactionId", "FileHashSha256")
.IsUnique()
.HasFilter("[TransactionId] IS NOT NULL");
b.HasIndex("TransactionId", "ReceiptDate");
b.ToTable("Receipts");
});
modelBuilder.Entity("MoneyMap.Models.ReceiptLineItem", b =>
{
b.Property<long>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("bigint");
SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property<long>("Id"));
b.Property<string>("Category")
.HasMaxLength(100)
.HasColumnType("nvarchar(100)");
b.Property<string>("Description")
.IsRequired()
.HasMaxLength(300)
.HasColumnType("nvarchar(300)");
b.Property<int>("LineNumber")
.HasColumnType("int");
b.Property<decimal?>("LineTotal")
.HasColumnType("decimal(18,2)");
b.Property<decimal?>("Quantity")
.HasColumnType("decimal(18,4)");
b.Property<long>("ReceiptId")
.HasColumnType("bigint");
b.Property<string>("Sku")
.HasMaxLength(64)
.HasColumnType("nvarchar(64)");
b.Property<string>("Unit")
.HasMaxLength(16)
.HasColumnType("nvarchar(16)");
b.Property<decimal?>("UnitPrice")
.HasColumnType("decimal(18,4)");
b.Property<bool>("Voided")
.HasColumnType("bit");
b.HasKey("Id");
b.HasIndex("ReceiptId", "LineNumber");
b.ToTable("ReceiptLineItems");
});
modelBuilder.Entity("MoneyMap.Models.ReceiptParseLog", b =>
{
b.Property<long>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("bigint");
SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property<long>("Id"));
b.Property<DateTime?>("CompletedAtUtc")
.HasColumnType("datetime2");
b.Property<decimal?>("Confidence")
.HasColumnType("decimal(5,4)");
b.Property<string>("Error")
.HasColumnType("nvarchar(max)");
b.Property<string>("ExtractedTextPath")
.HasMaxLength(1024)
.HasColumnType("nvarchar(1024)");
b.Property<string>("Model")
.IsRequired()
.HasMaxLength(100)
.HasColumnType("nvarchar(100)");
b.Property<string>("Provider")
.IsRequired()
.HasMaxLength(50)
.HasColumnType("nvarchar(50)");
b.Property<string>("ProviderJobId")
.HasMaxLength(100)
.HasColumnType("nvarchar(100)");
b.Property<string>("RawProviderPayloadJson")
.IsRequired()
.HasColumnType("nvarchar(max)");
b.Property<long>("ReceiptId")
.HasColumnType("bigint");
b.Property<DateTime>("StartedAtUtc")
.HasColumnType("datetime2");
b.Property<bool>("Success")
.HasColumnType("bit");
b.HasKey("Id");
b.HasIndex("ReceiptId", "StartedAtUtc");
b.ToTable("ReceiptParseLogs");
});
modelBuilder.Entity("MoneyMap.Models.Transaction", b =>
{
b.Property<long>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("bigint");
SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property<long>("Id"));
b.Property<int>("AccountId")
.HasColumnType("int");
b.Property<decimal>("Amount")
.HasColumnType("decimal(18,2)");
b.Property<int?>("CardId")
.HasColumnType("int");
b.Property<string>("Category")
.IsRequired()
.HasMaxLength(100)
.HasColumnType("nvarchar(100)");
b.Property<DateTime>("Date")
.HasColumnType("datetime2");
b.Property<string>("Last4")
.HasMaxLength(4)
.HasColumnType("nvarchar(4)");
b.Property<string>("Memo")
.IsRequired()
.ValueGeneratedOnAdd()
.HasMaxLength(500)
.HasColumnType("nvarchar(500)")
.HasDefaultValue("");
b.Property<int?>("MerchantId")
.HasColumnType("int");
b.Property<string>("Name")
.IsRequired()
.HasMaxLength(200)
.HasColumnType("nvarchar(200)");
b.Property<string>("Notes")
.IsRequired()
.HasColumnType("nvarchar(max)");
b.Property<string>("TransactionType")
.IsRequired()
.HasMaxLength(20)
.HasColumnType("nvarchar(20)");
b.Property<int?>("TransferToAccountId")
.HasColumnType("int");
b.HasKey("Id");
b.HasIndex("Amount");
b.HasIndex("Category");
b.HasIndex("Date");
b.HasIndex("MerchantId");
b.HasIndex("TransferToAccountId");
b.HasIndex("AccountId", "Category");
b.HasIndex("AccountId", "Date");
b.HasIndex("CardId", "Date");
b.HasIndex("MerchantId", "Date");
b.HasIndex("Date", "Amount", "Name", "Memo", "AccountId", "CardId")
.IsUnique()
.HasFilter("[CardId] IS NOT NULL");
b.ToTable("Transactions");
});
modelBuilder.Entity("MoneyMap.Models.Transfer", b =>
{
b.Property<long>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("bigint");
SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property<long>("Id"));
b.Property<decimal>("Amount")
.HasColumnType("decimal(18,2)");
b.Property<DateTime>("CreatedAt")
.HasColumnType("datetime2");
b.Property<DateTime>("Date")
.HasColumnType("datetime2");
b.Property<string>("Description")
.IsRequired()
.HasMaxLength(500)
.HasColumnType("nvarchar(500)");
b.Property<int?>("DestinationAccountId")
.HasColumnType("int");
b.Property<string>("Notes")
.IsRequired()
.HasColumnType("nvarchar(max)");
b.Property<long?>("OriginalTransactionId")
.HasColumnType("bigint");
b.Property<int?>("SourceAccountId")
.HasColumnType("int");
b.HasKey("Id");
b.HasIndex("Date");
b.HasIndex("DestinationAccountId");
b.HasIndex("OriginalTransactionId");
b.HasIndex("SourceAccountId");
b.ToTable("Transfers");
});
modelBuilder.Entity("MoneyMap.Models.Card", b =>
{
b.HasOne("MoneyMap.Models.Account", "Account")
.WithMany("Cards")
.HasForeignKey("AccountId")
.OnDelete(DeleteBehavior.Restrict);
b.Navigation("Account");
});
modelBuilder.Entity("MoneyMap.Models.CategoryMapping", b =>
{
b.HasOne("MoneyMap.Models.Merchant", "Merchant")
.WithMany("CategoryMappings")
.HasForeignKey("MerchantId")
.OnDelete(DeleteBehavior.SetNull);
b.Navigation("Merchant");
});
modelBuilder.Entity("MoneyMap.Models.Receipt", b =>
{
b.HasOne("MoneyMap.Models.Transaction", "Transaction")
.WithMany("Receipts")
.HasForeignKey("TransactionId")
.OnDelete(DeleteBehavior.Cascade);
b.Navigation("Transaction");
});
modelBuilder.Entity("MoneyMap.Models.ReceiptLineItem", b =>
{
b.HasOne("MoneyMap.Models.Receipt", "Receipt")
.WithMany("LineItems")
.HasForeignKey("ReceiptId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.Navigation("Receipt");
});
modelBuilder.Entity("MoneyMap.Models.ReceiptParseLog", b =>
{
b.HasOne("MoneyMap.Models.Receipt", "Receipt")
.WithMany("ParseLogs")
.HasForeignKey("ReceiptId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.Navigation("Receipt");
});
modelBuilder.Entity("MoneyMap.Models.Transaction", b =>
{
b.HasOne("MoneyMap.Models.Account", "Account")
.WithMany("Transactions")
.HasForeignKey("AccountId")
.OnDelete(DeleteBehavior.Restrict);
b.HasOne("MoneyMap.Models.Card", "Card")
.WithMany("Transactions")
.HasForeignKey("CardId")
.OnDelete(DeleteBehavior.Restrict);
b.HasOne("MoneyMap.Models.Merchant", "Merchant")
.WithMany("Transactions")
.HasForeignKey("MerchantId")
.OnDelete(DeleteBehavior.SetNull);
b.HasOne("MoneyMap.Models.Account", "TransferToAccount")
.WithMany()
.HasForeignKey("TransferToAccountId");
b.Navigation("Account");
b.Navigation("Card");
b.Navigation("Merchant");
b.Navigation("TransferToAccount");
});
modelBuilder.Entity("MoneyMap.Models.Transfer", b =>
{
b.HasOne("MoneyMap.Models.Account", "DestinationAccount")
.WithMany("DestinationTransfers")
.HasForeignKey("DestinationAccountId")
.OnDelete(DeleteBehavior.Restrict);
b.HasOne("MoneyMap.Models.Transaction", "OriginalTransaction")
.WithMany()
.HasForeignKey("OriginalTransactionId")
.OnDelete(DeleteBehavior.SetNull);
b.HasOne("MoneyMap.Models.Account", "SourceAccount")
.WithMany("SourceTransfers")
.HasForeignKey("SourceAccountId")
.OnDelete(DeleteBehavior.Restrict);
b.Navigation("DestinationAccount");
b.Navigation("OriginalTransaction");
b.Navigation("SourceAccount");
});
modelBuilder.Entity("MoneyMap.Models.Account", b =>
{
b.Navigation("Cards");
b.Navigation("DestinationTransfers");
b.Navigation("SourceTransfers");
b.Navigation("Transactions");
});
modelBuilder.Entity("MoneyMap.Models.Card", b =>
{
b.Navigation("Transactions");
});
modelBuilder.Entity("MoneyMap.Models.Merchant", b =>
{
b.Navigation("CategoryMappings");
b.Navigation("Transactions");
});
modelBuilder.Entity("MoneyMap.Models.Receipt", b =>
{
b.Navigation("LineItems");
b.Navigation("ParseLogs");
});
modelBuilder.Entity("MoneyMap.Models.Transaction", b =>
{
b.Navigation("Receipts");
});
#pragma warning restore 612, 618
}
}
}

View File

@@ -0,0 +1,29 @@
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace MoneyMap.Migrations
{
/// <inheritdoc />
public partial class AddReceiptParsingNotes : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.AddColumn<string>(
name: "ParsingNotes",
table: "Receipts",
type: "nvarchar(2000)",
maxLength: 2000,
nullable: true);
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropColumn(
name: "ParsingNotes",
table: "Receipts");
}
}
}

View File

@@ -236,6 +236,10 @@ namespace MoneyMap.Migrations
.HasMaxLength(200)
.HasColumnType("nvarchar(200)");
b.Property<string>("ParsingNotes")
.HasMaxLength(2000)
.HasColumnType("nvarchar(2000)");
b.Property<DateTime?>("ReceiptDate")
.HasColumnType("datetime2");

View File

@@ -51,6 +51,10 @@ public class Receipt
[MaxLength(8)]
public string? Currency { get; set; }
// User notes provided to AI parser
[MaxLength(2000)]
public string? ParsingNotes { get; set; }
// One receipt -> many parse attempts + many line items
public ICollection<ReceiptParseLog> ParseLogs { get; set; } = new List<ReceiptParseLog>();
public ICollection<ReceiptLineItem> LineItems { get; set; } = new List<ReceiptLineItem>();

View File

@@ -140,6 +140,12 @@
<dt class="col-sm-4">Uploaded</dt>
<dd class="col-sm-8">@Model.Receipt.UploadedAtUtc.ToLocalTime().ToString("MMM d, yyyy h:mm tt")</dd>
@if (!string.IsNullOrWhiteSpace(Model.Receipt.ParsingNotes))
{
<dt class="col-sm-4">AI Notes</dt>
<dd class="col-sm-8">@Model.Receipt.ParsingNotes</dd>
}
</dl>
</div>
</div>
@@ -158,6 +164,11 @@
Using: <strong>@Model.SelectedModel</strong>
<a href="/Settings" class="ms-2 small">Change</a>
</p>
<div class="mb-2">
<label for="ParsingNotes" class="form-label small text-muted mb-1">Notes for AI</label>
<textarea asp-for="ParsingNotes" class="form-control form-control-sm" rows="3"
placeholder="Optional hints for parsing (e.g., 'This is a restaurant receipt', 'Ignore the voided items')"></textarea>
</div>
<button type="submit" class="btn btn-primary btn-sm w-100">
Parse Receipt
</button>

View File

@@ -39,6 +39,9 @@ namespace MoneyMap.Pages
[TempData]
public string? ErrorMessage { get; set; }
[BindProperty]
public string? ParsingNotes { get; set; }
public async Task<IActionResult> OnGetAsync(long id)
{
Receipt = await _db.Receipts
@@ -53,6 +56,9 @@ namespace MoneyMap.Pages
LineItems = Receipt.LineItems?.OrderBy(li => li.LineNumber).ToList() ?? new();
ParseLogs = Receipt.ParseLogs?.OrderByDescending(pl => pl.StartedAtUtc).ToList() ?? new();
// Load saved parsing notes
ParsingNotes = Receipt.ParsingNotes;
// Get receipt URL for display - use handler parameter
ReceiptUrl = $"/ViewReceipt/{id}?handler=file";
@@ -98,8 +104,8 @@ namespace MoneyMap.Pages
return RedirectToPage(new { id });
}
// Use the configured model from settings
var result = await selectedParser.ParseReceiptAsync(id, SelectedModel);
// Use the configured model from settings, pass user notes
var result = await selectedParser.ParseReceiptAsync(id, SelectedModel, ParsingNotes);
if (result.IsSuccess)
{

View File

@@ -7,7 +7,7 @@ namespace MoneyMap.Services
{
public interface IReceiptParser
{
Task<ReceiptParseResult> ParseReceiptAsync(long receiptId, string? model = null);
Task<ReceiptParseResult> ParseReceiptAsync(long receiptId, string? model = null, string? notes = null);
}
public class AIReceiptParser : IReceiptParser
@@ -15,10 +15,7 @@ namespace MoneyMap.Services
private readonly MoneyMapContext _db;
private readonly IReceiptManager _receiptManager;
private readonly IPdfToImageConverter _pdfConverter;
private readonly OpenAIVisionClient _openAIClient;
private readonly ClaudeVisionClient _claudeClient;
private readonly OllamaVisionClient _ollamaClient;
private readonly LlamaCppVisionClient _llamaCppClient;
private readonly IAIVisionClientResolver _clientResolver;
private readonly IMerchantService _merchantService;
private readonly IServiceProvider _serviceProvider;
private readonly IConfiguration _configuration;
@@ -29,10 +26,7 @@ namespace MoneyMap.Services
MoneyMapContext db,
IReceiptManager receiptManager,
IPdfToImageConverter pdfConverter,
OpenAIVisionClient openAIClient,
ClaudeVisionClient claudeClient,
OllamaVisionClient ollamaClient,
LlamaCppVisionClient llamaCppClient,
IAIVisionClientResolver clientResolver,
IMerchantService merchantService,
IServiceProvider serviceProvider,
IConfiguration configuration,
@@ -41,17 +35,14 @@ namespace MoneyMap.Services
_db = db;
_receiptManager = receiptManager;
_pdfConverter = pdfConverter;
_openAIClient = openAIClient;
_claudeClient = claudeClient;
_ollamaClient = ollamaClient;
_llamaCppClient = llamaCppClient;
_clientResolver = clientResolver;
_merchantService = merchantService;
_serviceProvider = serviceProvider;
_configuration = configuration;
_logger = logger;
}
public async Task<ReceiptParseResult> ParseReceiptAsync(long receiptId, string? model = null)
public async Task<ReceiptParseResult> ParseReceiptAsync(long receiptId, string? model = null, string? notes = null)
{
var receipt = await _db.Receipts
.Include(r => r.Transaction)
@@ -60,17 +51,13 @@ namespace MoneyMap.Services
if (receipt == null)
return ReceiptParseResult.Failure("Receipt not found.");
var selectedModel = model ?? _configuration["AI:ReceiptParsingModel"] ?? "gpt-4o-mini";
var isLlamaCpp = selectedModel.StartsWith("llamacpp:");
var isOllama = selectedModel.StartsWith("ollama:");
var isClaude = selectedModel.StartsWith("claude-");
var provider = isLlamaCpp ? "LlamaCpp" : (isOllama ? "Ollama" : (isClaude ? "Anthropic" : "OpenAI"));
var filePath = _receiptManager.GetReceiptPhysicalPath(receipt);
if (!File.Exists(filePath))
return ReceiptParseResult.Failure("Receipt file not found on disk.");
var selectedModel = model ?? _configuration["AI:ReceiptParsingModel"] ?? "gpt-4o-mini";
var (client, provider) = _clientResolver.Resolve(selectedModel);
var parseLog = new ReceiptParseLog
{
ReceiptId = receiptId,
@@ -82,130 +69,153 @@ namespace MoneyMap.Services
try
{
string base64Data;
string mediaType;
if (receipt.ContentType == "application/pdf")
{
base64Data = await _pdfConverter.ConvertFirstPageToBase64Async(filePath);
mediaType = "image/png";
}
else
{
var fileBytes = await File.ReadAllBytesAsync(filePath);
base64Data = Convert.ToBase64String(fileBytes);
mediaType = receipt.ContentType;
}
// Build prompt
var promptText = await LoadPromptTemplateAsync();
var transactionName = receipt.Transaction?.Name;
if (!string.IsNullOrWhiteSpace(transactionName))
{
promptText += $"\n\nNote: This transaction was recorded as \"{transactionName}\" in the bank statement, which may help identify the merchant if the receipt is unclear.";
}
promptText += "\n\nRespond ONLY with valid JSON, no other text.";
// Call appropriate vision API
IAIVisionClient client = isLlamaCpp ? _llamaCppClient
: isOllama ? _ollamaClient
: isClaude ? _claudeClient
: _openAIClient;
var (base64Data, mediaType) = await PrepareImageDataAsync(receipt, filePath);
var promptText = await BuildPromptAsync(receipt, notes);
var visionResult = await client.AnalyzeImageAsync(base64Data, mediaType, promptText, selectedModel);
if (!visionResult.IsSuccess)
{
parseLog.Error = visionResult.ErrorMessage;
parseLog.CompletedAtUtc = DateTime.UtcNow;
_db.ReceiptParseLogs.Add(parseLog);
await _db.SaveChangesAsync();
await SaveParseLogAsync(parseLog, visionResult.ErrorMessage);
return ReceiptParseResult.Failure(visionResult.ErrorMessage!);
}
// Parse the JSON response
var parseData = string.IsNullOrWhiteSpace(visionResult.Content)
? new ParsedReceiptData()
: JsonSerializer.Deserialize<ParsedReceiptData>(visionResult.Content, new JsonSerializerOptions
{
PropertyNameCaseInsensitive = true
}) ?? new ParsedReceiptData();
// Update receipt with parsed data
receipt.Merchant = parseData.Merchant;
receipt.Total = parseData.Total;
receipt.Subtotal = parseData.Subtotal;
receipt.Tax = parseData.Tax;
receipt.ReceiptDate = parseData.ReceiptDate;
receipt.DueDate = parseData.DueDate;
// Update transaction merchant if extracted and transaction doesn't have one
if (receipt.Transaction != null &&
!string.IsNullOrWhiteSpace(parseData.Merchant) &&
receipt.Transaction.MerchantId == null)
{
var merchantId = await _merchantService.GetOrCreateIdAsync(parseData.Merchant);
receipt.Transaction.MerchantId = merchantId;
}
// Remove existing line items
var existingItems = await _db.ReceiptLineItems
.Where(li => li.ReceiptId == receiptId)
.ToListAsync();
_db.ReceiptLineItems.RemoveRange(existingItems);
// Add new line items
var lineItems = parseData.LineItems.Select((item, index) => new ReceiptLineItem
{
ReceiptId = receiptId,
LineNumber = index + 1,
Description = item.Description,
Sku = item.Upc,
Quantity = item.Quantity,
UnitPrice = item.UnitPrice,
LineTotal = item.LineTotal,
Voided = item.Voided
}).ToList();
_db.ReceiptLineItems.AddRange(lineItems);
var parseData = ParseResponse(visionResult.Content);
await ApplyParseResultAsync(receipt, receiptId, parseData, notes);
parseLog.Success = true;
parseLog.CompletedAtUtc = DateTime.UtcNow;
parseLog.Confidence = parseData.Confidence;
parseLog.RawProviderPayloadJson = JsonSerializer.Serialize(parseData);
await SaveParseLogAsync(parseLog);
_db.ReceiptParseLogs.Add(parseLog);
await _db.SaveChangesAsync();
await TryAutoMapReceiptAsync(receipt, receiptId);
// Attempt auto-mapping after successful parse
if (!receipt.TransactionId.HasValue)
{
try
{
using var scope = _serviceProvider.CreateScope();
var autoMapper = scope.ServiceProvider.GetRequiredService<IReceiptAutoMapper>();
await autoMapper.AutoMapReceiptAsync(receiptId);
_logger.LogInformation("Auto-mapping completed for receipt {ReceiptId}", receiptId);
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Auto-mapping failed for receipt {ReceiptId}: {Message}", receiptId, ex.Message);
}
}
return ReceiptParseResult.Success($"Parsed {lineItems.Count} line items from receipt.");
var lineCount = parseData.LineItems.Count;
return ReceiptParseResult.Success($"Parsed {lineCount} line items from receipt.");
}
catch (Exception ex)
{
parseLog.Error = ex.Message;
parseLog.CompletedAtUtc = DateTime.UtcNow;
_db.ReceiptParseLogs.Add(parseLog);
await _db.SaveChangesAsync();
await SaveParseLogAsync(parseLog, ex.Message);
_logger.LogError(ex, "Error parsing receipt {ReceiptId}: {Message}", receiptId, ex.Message);
return ReceiptParseResult.Failure($"Error parsing receipt: {ex.Message}");
}
}
private async Task<(string Base64Data, string MediaType)> PrepareImageDataAsync(Receipt receipt, string filePath)
{
if (receipt.ContentType == "application/pdf")
{
var base64 = await _pdfConverter.ConvertFirstPageToBase64Async(filePath);
return (base64, "image/png");
}
var fileBytes = await File.ReadAllBytesAsync(filePath);
return (Convert.ToBase64String(fileBytes), receipt.ContentType);
}
private async Task<string> BuildPromptAsync(Receipt receipt, string? userNotes = null)
{
var promptText = await LoadPromptTemplateAsync();
var transactionName = receipt.Transaction?.Name;
if (!string.IsNullOrWhiteSpace(transactionName))
{
promptText += $"\n\nNote: This transaction was recorded as \"{transactionName}\" in the bank statement, which may help identify the merchant if the receipt is unclear.";
}
var parsingNotes = _configuration["AI:ReceiptParsingNotes"];
if (!string.IsNullOrWhiteSpace(parsingNotes))
{
promptText += $"\n\nAdditional notes: {parsingNotes}";
}
if (!string.IsNullOrWhiteSpace(userNotes))
{
promptText += $"\n\nUser notes for this receipt: {userNotes}";
}
promptText += "\n\nRespond ONLY with valid JSON, no other text.";
return promptText;
}
private static ParsedReceiptData ParseResponse(string? content)
{
if (string.IsNullOrWhiteSpace(content))
return new ParsedReceiptData();
return JsonSerializer.Deserialize<ParsedReceiptData>(content, new JsonSerializerOptions
{
PropertyNameCaseInsensitive = true
}) ?? new ParsedReceiptData();
}
private async Task ApplyParseResultAsync(Receipt receipt, long receiptId, ParsedReceiptData parseData, string? notes)
{
// Update receipt fields
receipt.ParsingNotes = notes;
receipt.Merchant = parseData.Merchant;
receipt.Total = parseData.Total;
receipt.Subtotal = parseData.Subtotal;
receipt.Tax = parseData.Tax;
receipt.ReceiptDate = parseData.ReceiptDate;
receipt.DueDate = parseData.DueDate;
// Update transaction merchant if needed
if (receipt.Transaction != null &&
!string.IsNullOrWhiteSpace(parseData.Merchant) &&
receipt.Transaction.MerchantId == null)
{
var merchantId = await _merchantService.GetOrCreateIdAsync(parseData.Merchant);
receipt.Transaction.MerchantId = merchantId;
}
// Replace line items
var existingItems = await _db.ReceiptLineItems
.Where(li => li.ReceiptId == receiptId)
.ToListAsync();
_db.ReceiptLineItems.RemoveRange(existingItems);
var lineItems = parseData.LineItems.Select((item, index) => new ReceiptLineItem
{
ReceiptId = receiptId,
LineNumber = index + 1,
Description = item.Description,
Sku = item.Upc,
Quantity = item.Quantity,
UnitPrice = item.UnitPrice,
LineTotal = item.LineTotal,
Voided = item.Voided
}).ToList();
_db.ReceiptLineItems.AddRange(lineItems);
await _db.SaveChangesAsync();
}
private async Task SaveParseLogAsync(ReceiptParseLog parseLog, string? error = null)
{
parseLog.Error = error;
parseLog.CompletedAtUtc = DateTime.UtcNow;
_db.ReceiptParseLogs.Add(parseLog);
await _db.SaveChangesAsync();
}
private async Task TryAutoMapReceiptAsync(Receipt receipt, long receiptId)
{
if (receipt.TransactionId.HasValue)
return;
try
{
using var scope = _serviceProvider.CreateScope();
var autoMapper = scope.ServiceProvider.GetRequiredService<IReceiptAutoMapper>();
await autoMapper.AutoMapReceiptAsync(receiptId);
_logger.LogInformation("Auto-mapping completed for receipt {ReceiptId}", receiptId);
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Auto-mapping failed for receipt {ReceiptId}: {Message}", receiptId, ex.Message);
}
}
private async Task<string> LoadPromptTemplateAsync()
{
if (_promptTemplate != null)
@@ -221,6 +231,48 @@ namespace MoneyMap.Services
}
}
/// <summary>
/// Resolves the appropriate AI vision client based on model name.
/// </summary>
public interface IAIVisionClientResolver
{
(IAIVisionClient Client, string Provider) Resolve(string model);
}
public class AIVisionClientResolver : IAIVisionClientResolver
{
private readonly OpenAIVisionClient _openAIClient;
private readonly ClaudeVisionClient _claudeClient;
private readonly OllamaVisionClient _ollamaClient;
private readonly LlamaCppVisionClient _llamaCppClient;
public AIVisionClientResolver(
OpenAIVisionClient openAIClient,
ClaudeVisionClient claudeClient,
OllamaVisionClient ollamaClient,
LlamaCppVisionClient llamaCppClient)
{
_openAIClient = openAIClient;
_claudeClient = claudeClient;
_ollamaClient = ollamaClient;
_llamaCppClient = llamaCppClient;
}
public (IAIVisionClient Client, string Provider) Resolve(string model)
{
if (model.StartsWith("llamacpp:"))
return (_llamaCppClient, "LlamaCpp");
if (model.StartsWith("ollama:"))
return (_ollamaClient, "Ollama");
if (model.StartsWith("claude-"))
return (_claudeClient, "Anthropic");
return (_openAIClient, "OpenAI");
}
}
public class ParsedReceiptData
{
public string? Merchant { get; set; }