Feature: Database-aware receipt parsing with tool-use and enriched prompts

AIReceiptParser now routes to tool-aware or standard vision clients.
Tool-capable models (OpenAI, Claude, LlamaCpp) call search_categories,
search_transactions, and search_merchants during parsing. Ollama gets
pre-fetched DB context injected into the prompt. Adds suggestedCategory
and suggestedTransactionId fields with AI-driven transaction mapping.
Includes NullableLongConverter for resilient JSON deserialization and
restructured receipt prompt with strict field types.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

MoneyMap/Prompts/ReceiptParserPrompt.txt

@@ -1,47 +1,62 @@
-Analyze this receipt image and extract the following information as JSON:
+Analyze this receipt image and extract structured data. Respond with a single JSON object matching this exact schema. Use JSON null (not the string "null") for missing values. Do not include comments in the JSON.
+
 {
   "merchant": "store name",
-  "receiptDate": "YYYY-MM-DD" (or null if not found),
+  "receiptDate": "YYYY-MM-DD",
-  "dueDate": "YYYY-MM-DD" (or null if not found - for bills only),
+  "dueDate": null,
-  "subtotal": 0.00 (or null if not found),
+  "subtotal": 0.00,
-  "tax": 0.00 (or null if not found),
+  "tax": 0.00,
   "total": 0.00,
   "confidence": 0.95,
+  "suggestedCategory": null,
+  "suggestedTransactionId": null,
   "lineItems": [
     {
       "description": "item name",
-      "upc": "1234567890123" (or null if not found),
+      "upc": null,
       "quantity": 1.0,
-      "unitPrice": 0.00 (or null),
+      "unitPrice": 0.00,
       "lineTotal": 0.00,
+      "category": null,
       "voided": false
     }
   ]
 }
 
-Extract all line items you can see on the receipt. For each item:
+FIELD TYPES (you must follow these exactly):
-- description: The item or service name (include any count/size info in the description itself, like "4CT" or "12 OZ")
+- merchant: string
-- upc: The UPC/barcode number if visible (usually a 12-13 digit number near the item). This helps track price changes over time. Set to null if not found.
+- receiptDate: string "YYYY-MM-DD" or null
-- quantity: ALWAYS set to 1.0 for ALL retail products (groceries, goods, merchandise, etc.) - this is the default. ONLY use null for utility bills, service fees, or taxes (non-product items). If it's a physical item on a retail receipt, use 1.0.
+- dueDate: string "YYYY-MM-DD" or null (only for bills with a payment deadline)
-- unitPrice: Calculate as lineTotal divided by quantity (so usually equals lineTotal for retail items). Set to null only if quantity is null.
+- subtotal: number or null
-- lineTotal: The total amount for this line (the price shown on the receipt, or 0.00 if voided)
+- tax: number or null
-- voided: Set to true if this item appears immediately after a "** VOIDED ENTRY **" marker or similar void indicator. Set to false for all other items.
+- total: number
+- confidence: number between 0 and 1
+- suggestedCategory: string or null
+- suggestedTransactionId: integer or null (MUST be a JSON number like 123, NEVER a string like "123")
+- lineItems: array of objects
 
-CRITICAL - HANDLING VOIDED ITEMS:
+LINE ITEM FIELDS:
-- NEVER skip or ignore ANY line items on the receipt
+- description: string (the item or service name, include count/size info like "4CT" or "12 OZ")
-- When you see "** VOIDED ENTRY **" or similar void markers, the item immediately after it is voided
+- upc: string or null (UPC/barcode number if visible, usually 12-13 digits)
+- quantity: number (default 1.0 for all retail products; null only for service fees or taxes)
+- unitPrice: number or null (lineTotal divided by quantity; null only if quantity is null)
+- lineTotal: number (the price shown on the receipt; 0.00 if voided)
+- category: string or null
+- voided: boolean
+
+RULES FOR LINE ITEMS:
+- Extract ALL line items from top to bottom - never stop early
+- quantity is 1.0 for ALL physical retail items unless you see "2 @" or "QTY 3" etc.
+- Do not confuse product descriptions (like "4CT BLUE MUF" = 4-count muffin package) with quantity
+- UPC/barcode numbers are long numeric codes (12-13 digits) near the item
+
+VOIDED ITEMS:
+- When you see "** VOIDED ENTRY **" or similar, the item immediately after it is voided
 - For voided items: set "voided": true and "lineTotal": 0.00
 - For all other items: set "voided": false
-- CONTINUE reading and extracting ALL items that appear after void markers - do NOT stop parsing
+- NEVER skip voided items - include them in the lineItems array
-- The receipt may have many items listed after a void marker - you MUST include every single one
+- CONTINUE reading ALL items after void markers
-- Include EVERY line item you can see, whether voided or not
 
-OTHER IMPORTANT RULES:
+DUE DATE:
-- Quantity MUST be 1.0 for ALL physical retail items (groceries, food, household goods, etc.) - do NOT leave it null
+- Only for bills (utility, credit card, etc.) - extract the payment due date
-- Every item on a grocery/retail receipt gets quantity: 1.0 unless you see explicit indicators like "2 @" or "QTY 3"
+- For regular store receipts, dueDate must be null
-- Only utility bills, service charges, fees, or taxes (non-product line items) should have null quantity
-- Don't confuse product descriptions (like "4CT BLUE MUF" meaning 4-count muffin package) with quantity fields (like "2 @ $3.99")
-- Extract UPC/barcode numbers when visible - they're usually long numeric codes (12-13 digits)
-- Read through the ENTIRE receipt from top to bottom - don't stop early
-
-If this is a bill (utility, credit card, etc.), look for a due date, payment due date, or deadline and extract it as dueDate. For regular receipts, dueDate should be null.

MoneyMap/Services/AIReceiptParser.cs

@@ -1,7 +1,9 @@
 using Microsoft.EntityFrameworkCore;
 using MoneyMap.Data;
 using MoneyMap.Models;
+using MoneyMap.Services.AITools;
 using System.Text.Json;
+using System.Text.Json.Serialization;
 
 namespace MoneyMap.Services
 {
@@ -17,6 +19,7 @@ namespace MoneyMap.Services
         private readonly IPdfToImageConverter _pdfConverter;
         private readonly IAIVisionClientResolver _clientResolver;
         private readonly IMerchantService _merchantService;
+        private readonly IAIToolExecutor _toolExecutor;
         private readonly IServiceProvider _serviceProvider;
         private readonly IConfiguration _configuration;
         private readonly ILogger<AIReceiptParser> _logger;
@@ -28,6 +31,7 @@ namespace MoneyMap.Services
             IPdfToImageConverter pdfConverter,
             IAIVisionClientResolver clientResolver,
             IMerchantService merchantService,
+            IAIToolExecutor toolExecutor,
             IServiceProvider serviceProvider,
             IConfiguration configuration,
             ILogger<AIReceiptParser> logger)
@@ -37,6 +41,7 @@ namespace MoneyMap.Services
             _pdfConverter = pdfConverter;
             _clientResolver = clientResolver;
             _merchantService = merchantService;
+            _toolExecutor = toolExecutor;
             _serviceProvider = serviceProvider;
             _configuration = configuration;
             _logger = logger;
@@ -58,6 +63,10 @@ namespace MoneyMap.Services
             var selectedModel = model ?? _configuration["AI:ReceiptParsingModel"] ?? "gpt-4o-mini";
             var (client, provider) = _clientResolver.Resolve(selectedModel);
 
+            // Let model-aware clients evaluate tool support for the specific model
+            if (client is LlamaCppVisionClient llamaCpp)
+                llamaCpp.SetCurrentModel(selectedModel);
+
             var parseLog = new ReceiptParseLog
             {
                 ReceiptId = receiptId,
@@ -70,8 +79,8 @@ namespace MoneyMap.Services
             try
             {
                 var (base64Data, mediaType) = await PrepareImageDataAsync(receipt, filePath);
-                var promptText = await BuildPromptAsync(receipt, notes);
+                var promptText = await BuildPromptAsync(receipt, notes, client);
-                var visionResult = await client.AnalyzeImageAsync(base64Data, mediaType, promptText, selectedModel);
+                var visionResult = await CallVisionClientAsync(client, base64Data, mediaType, promptText, selectedModel);
 
                 if (!visionResult.IsSuccess)
                 {
@@ -87,7 +96,7 @@ namespace MoneyMap.Services
                 parseLog.RawProviderPayloadJson = JsonSerializer.Serialize(parseData);
                 await SaveParseLogAsync(parseLog);
 
-                await TryAutoMapReceiptAsync(receipt, receiptId);
+                await TryAutoMapReceiptAsync(receipt, receiptId, parseData.SuggestedTransactionId);
 
                 var lineCount = parseData.LineItems.Count;
                 return ReceiptParseResult.Success($"Parsed {lineCount} line items from receipt.");
@@ -100,6 +109,29 @@ namespace MoneyMap.Services
             }
         }
 
+        /// <summary>
+        /// Call the vision client, using tool-use if the client supports it, or enriched prompt fallback for Ollama.
+        /// </summary>
+        private async Task<VisionApiResult> CallVisionClientAsync(
+            IAIVisionClient client, string base64Data, string mediaType, string prompt, string model)
+        {
+            if (client is IAIToolAwareVisionClient toolAwareClient && toolAwareClient.SupportsToolUse)
+            {
+                _logger.LogInformation("Using tool-aware vision client for model {Model}", model);
+                var tools = AIToolRegistry.GetAllTools();
+
+                return await toolAwareClient.AnalyzeImageWithToolsAsync(
+                    base64Data, mediaType, prompt, model,
+                    tools,
+                    toolCall => _toolExecutor.ExecuteAsync(toolCall),
+                    maxToolRounds: 5);
+            }
+
+            // Fallback: standard call (Ollama gets enriched prompt via BuildPromptAsync)
+            _logger.LogInformation("Using standard vision client for model {Model} (no tool use)", model);
+            return await client.AnalyzeImageAsync(base64Data, mediaType, prompt, model);
+        }
+
         private async Task<(string Base64Data, string MediaType)> PrepareImageDataAsync(Receipt receipt, string filePath)
         {
             if (receipt.ContentType == "application/pdf")
@@ -112,7 +144,7 @@ namespace MoneyMap.Services
             return (Convert.ToBase64String(fileBytes), receipt.ContentType);
         }
 
-        private async Task<string> BuildPromptAsync(Receipt receipt, string? userNotes = null)
+        private async Task<string> BuildPromptAsync(Receipt receipt, string? userNotes, IAIVisionClient client)
         {
             var promptText = await LoadPromptTemplateAsync();
 
@@ -133,6 +165,43 @@ namespace MoneyMap.Services
                 promptText += $"\n\nUser notes for this receipt: {userNotes}";
             }
 
+            // Add tool-use or enriched context instructions based on client capability
+            if (client is IAIToolAwareVisionClient toolAwareClient && toolAwareClient.SupportsToolUse)
+            {
+                // Tool-aware client: instruct to use tools for lookups
+                promptText += @"
+
+TOOL USE INSTRUCTIONS:
+You have access to tools that can query the application's database. You MUST call them before generating your JSON response:
+1. Call search_categories to find existing category names. Use ONLY categories returned by this tool for suggestedCategory and line item category fields. Do not invent new category names.
+2. Call search_transactions to find a matching bank transaction for this receipt (search by date, amount, merchant name). Set suggestedTransactionId to the numeric ID of the best match, or null if no good match. Remember: suggestedTransactionId must be a JSON integer or null, never a string.
+3. Call search_merchants to look up the correct merchant name.";
+            }
+            else
+            {
+                // Non-tool client (Ollama): inject pre-fetched database context
+                try
+                {
+                    var merchantHint = receipt.Transaction?.Name ?? receipt.Merchant;
+                    var enrichedContext = await _toolExecutor.GetEnrichedContextAsync(
+                        receipt.ReceiptDate,
+                        receipt.Total,
+                        merchantHint);
+
+                    promptText += $"\n\n{enrichedContext}";
+                    promptText += @"
+
+Using the database context above, populate these fields in your JSON response:
+- suggestedCategory: Use the best matching category name from the EXISTING CATEGORIES list. Do not invent new categories.
+- suggestedTransactionId: Use the numeric transaction ID from CANDIDATE TRANSACTIONS that best matches this receipt, or null if none match. Must be a JSON integer or null, never a string.
+- For each line item, set category to the best matching category from the EXISTING CATEGORIES list.";
+                }
+                catch (Exception ex)
+                {
+                    _logger.LogWarning(ex, "Failed to get enriched context for Ollama, proceeding without it");
+                }
+            }
+
             promptText += "\n\nRespond ONLY with valid JSON, no other text.";
             return promptText;
         }
@@ -168,6 +237,16 @@ namespace MoneyMap.Services
                 receipt.Transaction.MerchantId = merchantId;
             }
 
+            // Update transaction category if AI suggested one and the transaction has no category
+            if (receipt.Transaction != null &&
+                !string.IsNullOrWhiteSpace(parseData.SuggestedCategory) &&
+                string.IsNullOrWhiteSpace(receipt.Transaction.Category))
+            {
+                receipt.Transaction.Category = parseData.SuggestedCategory;
+                _logger.LogInformation("Set transaction {TransactionId} category to '{Category}' from AI suggestion",
+                    receipt.Transaction.Id, parseData.SuggestedCategory);
+            }
+
             // Replace line items
             var existingItems = await _db.ReceiptLineItems
                 .Where(li => li.ReceiptId == receiptId)
@@ -183,6 +262,7 @@ namespace MoneyMap.Services
                 Quantity = item.Quantity,
                 UnitPrice = item.UnitPrice,
                 LineTotal = item.LineTotal,
+                Category = item.Category,
                 Voided = item.Voided
             }).ToList();
 
@@ -198,8 +278,41 @@ namespace MoneyMap.Services
             await _db.SaveChangesAsync();
         }
 
-        private async Task TryAutoMapReceiptAsync(Receipt receipt, long receiptId)
+        private async Task TryAutoMapReceiptAsync(Receipt receipt, long receiptId, long? suggestedTransactionId)
         {
+            // If AI suggested a specific transaction, try mapping directly
+            if (!receipt.TransactionId.HasValue && suggestedTransactionId.HasValue)
+            {
+                try
+                {
+                    var transaction = await _db.Transactions.FindAsync(suggestedTransactionId.Value);
+                    if (transaction != null)
+                    {
+                        // Verify the transaction isn't already mapped to another receipt
+                        var alreadyMapped = await _db.Receipts
+                            .AnyAsync(r => r.TransactionId == suggestedTransactionId.Value && r.Id != receiptId);
+
+                        if (!alreadyMapped)
+                        {
+                            var success = await _receiptManager.MapReceiptToTransactionAsync(receiptId, suggestedTransactionId.Value);
+                            if (success)
+                            {
+                                _logger.LogInformation(
+                                    "AI-suggested mapping: receipt {ReceiptId} → transaction {TransactionId}",
+                                    receiptId, suggestedTransactionId.Value);
+                                return;
+                            }
+                        }
+                    }
+                }
+                catch (Exception ex)
+                {
+                    _logger.LogWarning(ex, "AI-suggested mapping failed for receipt {ReceiptId} → transaction {TransactionId}",
+                        receiptId, suggestedTransactionId.Value);
+                }
+            }
+
+            // Fall back to the existing auto-mapper
             if (receipt.TransactionId.HasValue)
                 return;
 
@@ -282,6 +395,9 @@ namespace MoneyMap.Services
         public decimal? Tax { get; set; }
         public decimal? Total { get; set; }
         public decimal Confidence { get; set; } = 0.5m;
+        public string? SuggestedCategory { get; set; }
+        [JsonConverter(typeof(NullableLongConverter))]
+        public long? SuggestedTransactionId { get; set; }
         public List<ParsedLineItem> LineItems { get; set; } = new();
     }
 
@@ -292,6 +408,7 @@ namespace MoneyMap.Services
         public decimal? Quantity { get; set; }
         public decimal? UnitPrice { get; set; }
         public decimal LineTotal { get; set; }
+        public string? Category { get; set; }
         public bool Voided { get; set; }
     }
 
@@ -306,4 +423,41 @@ namespace MoneyMap.Services
         public static ReceiptParseResult Failure(string message) =>
             new() { IsSuccess = false, Message = message };
     }
+
+    /// <summary>
+    /// Handles AI responses that return suggestedTransactionId as a string ("null", "N/A", "123")
+    /// instead of as a JSON number or null.
+    /// </summary>
+    public class NullableLongConverter : JsonConverter<long?>
+    {
+        public override long? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
+        {
+            switch (reader.TokenType)
+            {
+                case JsonTokenType.Number:
+                    return reader.GetInt64();
+                case JsonTokenType.String:
+                    var str = reader.GetString();
+                    if (string.IsNullOrWhiteSpace(str) ||
+                        str.Equals("null", StringComparison.OrdinalIgnoreCase) ||
+                        str.Equals("N/A", StringComparison.OrdinalIgnoreCase) ||
+                        str.Equals("none", StringComparison.OrdinalIgnoreCase))
+                        return null;
+                    return long.TryParse(str, out var val) ? val : null;
+                case JsonTokenType.Null:
+                    return null;
+                default:
+                    reader.Skip();
+                    return null;
+            }
+        }
+
+        public override void Write(Utf8JsonWriter writer, long? value, JsonSerializerOptions options)
+        {
+            if (value.HasValue)
+                writer.WriteNumberValue(value.Value);
+            else
+                writer.WriteNullValue();
+        }
+    }
 }