diff --git a/MoneyMap/Prompts/ReceiptParserPrompt.txt b/MoneyMap/Prompts/ReceiptParserPrompt.txt index 15c1866..be0b829 100644 --- a/MoneyMap/Prompts/ReceiptParserPrompt.txt +++ b/MoneyMap/Prompts/ReceiptParserPrompt.txt @@ -1,47 +1,62 @@ -Analyze this receipt image and extract the following information as JSON: +Analyze this receipt image and extract structured data. Respond with a single JSON object matching this exact schema. Use JSON null (not the string "null") for missing values. Do not include comments in the JSON. + { "merchant": "store name", - "receiptDate": "YYYY-MM-DD" (or null if not found), - "dueDate": "YYYY-MM-DD" (or null if not found - for bills only), - "subtotal": 0.00 (or null if not found), - "tax": 0.00 (or null if not found), + "receiptDate": "YYYY-MM-DD", + "dueDate": null, + "subtotal": 0.00, + "tax": 0.00, "total": 0.00, "confidence": 0.95, + "suggestedCategory": null, + "suggestedTransactionId": null, "lineItems": [ { "description": "item name", - "upc": "1234567890123" (or null if not found), + "upc": null, "quantity": 1.0, - "unitPrice": 0.00 (or null), + "unitPrice": 0.00, "lineTotal": 0.00, + "category": null, "voided": false } ] } -Extract all line items you can see on the receipt. For each item: -- description: The item or service name (include any count/size info in the description itself, like "4CT" or "12 OZ") -- upc: The UPC/barcode number if visible (usually a 12-13 digit number near the item). This helps track price changes over time. Set to null if not found. -- quantity: ALWAYS set to 1.0 for ALL retail products (groceries, goods, merchandise, etc.) - this is the default. ONLY use null for utility bills, service fees, or taxes (non-product items). If it's a physical item on a retail receipt, use 1.0. -- unitPrice: Calculate as lineTotal divided by quantity (so usually equals lineTotal for retail items). Set to null only if quantity is null. -- lineTotal: The total amount for this line (the price shown on the receipt, or 0.00 if voided) -- voided: Set to true if this item appears immediately after a "** VOIDED ENTRY **" marker or similar void indicator. Set to false for all other items. +FIELD TYPES (you must follow these exactly): +- merchant: string +- receiptDate: string "YYYY-MM-DD" or null +- dueDate: string "YYYY-MM-DD" or null (only for bills with a payment deadline) +- subtotal: number or null +- tax: number or null +- total: number +- confidence: number between 0 and 1 +- suggestedCategory: string or null +- suggestedTransactionId: integer or null (MUST be a JSON number like 123, NEVER a string like "123") +- lineItems: array of objects -CRITICAL - HANDLING VOIDED ITEMS: -- NEVER skip or ignore ANY line items on the receipt -- When you see "** VOIDED ENTRY **" or similar void markers, the item immediately after it is voided +LINE ITEM FIELDS: +- description: string (the item or service name, include count/size info like "4CT" or "12 OZ") +- upc: string or null (UPC/barcode number if visible, usually 12-13 digits) +- quantity: number (default 1.0 for all retail products; null only for service fees or taxes) +- unitPrice: number or null (lineTotal divided by quantity; null only if quantity is null) +- lineTotal: number (the price shown on the receipt; 0.00 if voided) +- category: string or null +- voided: boolean + +RULES FOR LINE ITEMS: +- Extract ALL line items from top to bottom - never stop early +- quantity is 1.0 for ALL physical retail items unless you see "2 @" or "QTY 3" etc. +- Do not confuse product descriptions (like "4CT BLUE MUF" = 4-count muffin package) with quantity +- UPC/barcode numbers are long numeric codes (12-13 digits) near the item + +VOIDED ITEMS: +- When you see "** VOIDED ENTRY **" or similar, the item immediately after it is voided - For voided items: set "voided": true and "lineTotal": 0.00 - For all other items: set "voided": false -- CONTINUE reading and extracting ALL items that appear after void markers - do NOT stop parsing -- The receipt may have many items listed after a void marker - you MUST include every single one -- Include EVERY line item you can see, whether voided or not +- NEVER skip voided items - include them in the lineItems array +- CONTINUE reading ALL items after void markers -OTHER IMPORTANT RULES: -- Quantity MUST be 1.0 for ALL physical retail items (groceries, food, household goods, etc.) - do NOT leave it null -- Every item on a grocery/retail receipt gets quantity: 1.0 unless you see explicit indicators like "2 @" or "QTY 3" -- Only utility bills, service charges, fees, or taxes (non-product line items) should have null quantity -- Don't confuse product descriptions (like "4CT BLUE MUF" meaning 4-count muffin package) with quantity fields (like "2 @ $3.99") -- Extract UPC/barcode numbers when visible - they're usually long numeric codes (12-13 digits) -- Read through the ENTIRE receipt from top to bottom - don't stop early - -If this is a bill (utility, credit card, etc.), look for a due date, payment due date, or deadline and extract it as dueDate. For regular receipts, dueDate should be null. \ No newline at end of file +DUE DATE: +- Only for bills (utility, credit card, etc.) - extract the payment due date +- For regular store receipts, dueDate must be null \ No newline at end of file diff --git a/MoneyMap/Services/AIReceiptParser.cs b/MoneyMap/Services/AIReceiptParser.cs index 3519025..165b441 100644 --- a/MoneyMap/Services/AIReceiptParser.cs +++ b/MoneyMap/Services/AIReceiptParser.cs @@ -1,7 +1,9 @@ using Microsoft.EntityFrameworkCore; using MoneyMap.Data; using MoneyMap.Models; +using MoneyMap.Services.AITools; using System.Text.Json; +using System.Text.Json.Serialization; namespace MoneyMap.Services { @@ -17,6 +19,7 @@ namespace MoneyMap.Services private readonly IPdfToImageConverter _pdfConverter; private readonly IAIVisionClientResolver _clientResolver; private readonly IMerchantService _merchantService; + private readonly IAIToolExecutor _toolExecutor; private readonly IServiceProvider _serviceProvider; private readonly IConfiguration _configuration; private readonly ILogger _logger; @@ -28,6 +31,7 @@ namespace MoneyMap.Services IPdfToImageConverter pdfConverter, IAIVisionClientResolver clientResolver, IMerchantService merchantService, + IAIToolExecutor toolExecutor, IServiceProvider serviceProvider, IConfiguration configuration, ILogger logger) @@ -37,6 +41,7 @@ namespace MoneyMap.Services _pdfConverter = pdfConverter; _clientResolver = clientResolver; _merchantService = merchantService; + _toolExecutor = toolExecutor; _serviceProvider = serviceProvider; _configuration = configuration; _logger = logger; @@ -58,6 +63,10 @@ namespace MoneyMap.Services var selectedModel = model ?? _configuration["AI:ReceiptParsingModel"] ?? "gpt-4o-mini"; var (client, provider) = _clientResolver.Resolve(selectedModel); + // Let model-aware clients evaluate tool support for the specific model + if (client is LlamaCppVisionClient llamaCpp) + llamaCpp.SetCurrentModel(selectedModel); + var parseLog = new ReceiptParseLog { ReceiptId = receiptId, @@ -70,8 +79,8 @@ namespace MoneyMap.Services try { var (base64Data, mediaType) = await PrepareImageDataAsync(receipt, filePath); - var promptText = await BuildPromptAsync(receipt, notes); - var visionResult = await client.AnalyzeImageAsync(base64Data, mediaType, promptText, selectedModel); + var promptText = await BuildPromptAsync(receipt, notes, client); + var visionResult = await CallVisionClientAsync(client, base64Data, mediaType, promptText, selectedModel); if (!visionResult.IsSuccess) { @@ -87,7 +96,7 @@ namespace MoneyMap.Services parseLog.RawProviderPayloadJson = JsonSerializer.Serialize(parseData); await SaveParseLogAsync(parseLog); - await TryAutoMapReceiptAsync(receipt, receiptId); + await TryAutoMapReceiptAsync(receipt, receiptId, parseData.SuggestedTransactionId); var lineCount = parseData.LineItems.Count; return ReceiptParseResult.Success($"Parsed {lineCount} line items from receipt."); @@ -100,6 +109,29 @@ namespace MoneyMap.Services } } + /// + /// Call the vision client, using tool-use if the client supports it, or enriched prompt fallback for Ollama. + /// + private async Task CallVisionClientAsync( + IAIVisionClient client, string base64Data, string mediaType, string prompt, string model) + { + if (client is IAIToolAwareVisionClient toolAwareClient && toolAwareClient.SupportsToolUse) + { + _logger.LogInformation("Using tool-aware vision client for model {Model}", model); + var tools = AIToolRegistry.GetAllTools(); + + return await toolAwareClient.AnalyzeImageWithToolsAsync( + base64Data, mediaType, prompt, model, + tools, + toolCall => _toolExecutor.ExecuteAsync(toolCall), + maxToolRounds: 5); + } + + // Fallback: standard call (Ollama gets enriched prompt via BuildPromptAsync) + _logger.LogInformation("Using standard vision client for model {Model} (no tool use)", model); + return await client.AnalyzeImageAsync(base64Data, mediaType, prompt, model); + } + private async Task<(string Base64Data, string MediaType)> PrepareImageDataAsync(Receipt receipt, string filePath) { if (receipt.ContentType == "application/pdf") @@ -112,7 +144,7 @@ namespace MoneyMap.Services return (Convert.ToBase64String(fileBytes), receipt.ContentType); } - private async Task BuildPromptAsync(Receipt receipt, string? userNotes = null) + private async Task BuildPromptAsync(Receipt receipt, string? userNotes, IAIVisionClient client) { var promptText = await LoadPromptTemplateAsync(); @@ -133,6 +165,43 @@ namespace MoneyMap.Services promptText += $"\n\nUser notes for this receipt: {userNotes}"; } + // Add tool-use or enriched context instructions based on client capability + if (client is IAIToolAwareVisionClient toolAwareClient && toolAwareClient.SupportsToolUse) + { + // Tool-aware client: instruct to use tools for lookups + promptText += @" + +TOOL USE INSTRUCTIONS: +You have access to tools that can query the application's database. You MUST call them before generating your JSON response: +1. Call search_categories to find existing category names. Use ONLY categories returned by this tool for suggestedCategory and line item category fields. Do not invent new category names. +2. Call search_transactions to find a matching bank transaction for this receipt (search by date, amount, merchant name). Set suggestedTransactionId to the numeric ID of the best match, or null if no good match. Remember: suggestedTransactionId must be a JSON integer or null, never a string. +3. Call search_merchants to look up the correct merchant name."; + } + else + { + // Non-tool client (Ollama): inject pre-fetched database context + try + { + var merchantHint = receipt.Transaction?.Name ?? receipt.Merchant; + var enrichedContext = await _toolExecutor.GetEnrichedContextAsync( + receipt.ReceiptDate, + receipt.Total, + merchantHint); + + promptText += $"\n\n{enrichedContext}"; + promptText += @" + +Using the database context above, populate these fields in your JSON response: +- suggestedCategory: Use the best matching category name from the EXISTING CATEGORIES list. Do not invent new categories. +- suggestedTransactionId: Use the numeric transaction ID from CANDIDATE TRANSACTIONS that best matches this receipt, or null if none match. Must be a JSON integer or null, never a string. +- For each line item, set category to the best matching category from the EXISTING CATEGORIES list."; + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to get enriched context for Ollama, proceeding without it"); + } + } + promptText += "\n\nRespond ONLY with valid JSON, no other text."; return promptText; } @@ -168,6 +237,16 @@ namespace MoneyMap.Services receipt.Transaction.MerchantId = merchantId; } + // Update transaction category if AI suggested one and the transaction has no category + if (receipt.Transaction != null && + !string.IsNullOrWhiteSpace(parseData.SuggestedCategory) && + string.IsNullOrWhiteSpace(receipt.Transaction.Category)) + { + receipt.Transaction.Category = parseData.SuggestedCategory; + _logger.LogInformation("Set transaction {TransactionId} category to '{Category}' from AI suggestion", + receipt.Transaction.Id, parseData.SuggestedCategory); + } + // Replace line items var existingItems = await _db.ReceiptLineItems .Where(li => li.ReceiptId == receiptId) @@ -183,6 +262,7 @@ namespace MoneyMap.Services Quantity = item.Quantity, UnitPrice = item.UnitPrice, LineTotal = item.LineTotal, + Category = item.Category, Voided = item.Voided }).ToList(); @@ -198,8 +278,41 @@ namespace MoneyMap.Services await _db.SaveChangesAsync(); } - private async Task TryAutoMapReceiptAsync(Receipt receipt, long receiptId) + private async Task TryAutoMapReceiptAsync(Receipt receipt, long receiptId, long? suggestedTransactionId) { + // If AI suggested a specific transaction, try mapping directly + if (!receipt.TransactionId.HasValue && suggestedTransactionId.HasValue) + { + try + { + var transaction = await _db.Transactions.FindAsync(suggestedTransactionId.Value); + if (transaction != null) + { + // Verify the transaction isn't already mapped to another receipt + var alreadyMapped = await _db.Receipts + .AnyAsync(r => r.TransactionId == suggestedTransactionId.Value && r.Id != receiptId); + + if (!alreadyMapped) + { + var success = await _receiptManager.MapReceiptToTransactionAsync(receiptId, suggestedTransactionId.Value); + if (success) + { + _logger.LogInformation( + "AI-suggested mapping: receipt {ReceiptId} → transaction {TransactionId}", + receiptId, suggestedTransactionId.Value); + return; + } + } + } + } + catch (Exception ex) + { + _logger.LogWarning(ex, "AI-suggested mapping failed for receipt {ReceiptId} → transaction {TransactionId}", + receiptId, suggestedTransactionId.Value); + } + } + + // Fall back to the existing auto-mapper if (receipt.TransactionId.HasValue) return; @@ -282,6 +395,9 @@ namespace MoneyMap.Services public decimal? Tax { get; set; } public decimal? Total { get; set; } public decimal Confidence { get; set; } = 0.5m; + public string? SuggestedCategory { get; set; } + [JsonConverter(typeof(NullableLongConverter))] + public long? SuggestedTransactionId { get; set; } public List LineItems { get; set; } = new(); } @@ -292,6 +408,7 @@ namespace MoneyMap.Services public decimal? Quantity { get; set; } public decimal? UnitPrice { get; set; } public decimal LineTotal { get; set; } + public string? Category { get; set; } public bool Voided { get; set; } } @@ -306,4 +423,41 @@ namespace MoneyMap.Services public static ReceiptParseResult Failure(string message) => new() { IsSuccess = false, Message = message }; } + + /// + /// Handles AI responses that return suggestedTransactionId as a string ("null", "N/A", "123") + /// instead of as a JSON number or null. + /// + public class NullableLongConverter : JsonConverter + { + public override long? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + switch (reader.TokenType) + { + case JsonTokenType.Number: + return reader.GetInt64(); + case JsonTokenType.String: + var str = reader.GetString(); + if (string.IsNullOrWhiteSpace(str) || + str.Equals("null", StringComparison.OrdinalIgnoreCase) || + str.Equals("N/A", StringComparison.OrdinalIgnoreCase) || + str.Equals("none", StringComparison.OrdinalIgnoreCase)) + return null; + return long.TryParse(str, out var val) ? val : null; + case JsonTokenType.Null: + return null; + default: + reader.Skip(); + return null; + } + } + + public override void Write(Utf8JsonWriter writer, long? value, JsonSerializerOptions options) + { + if (value.HasValue) + writer.WriteNumberValue(value.Value); + else + writer.WriteNullValue(); + } + } }