refactor: move services and AITools to MoneyMap.Core
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,466 @@
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using MoneyMap.Data;
|
||||
using MoneyMap.Models;
|
||||
using MoneyMap.Services.AITools;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace MoneyMap.Services
|
||||
{
|
||||
public interface IReceiptParser
|
||||
{
|
||||
Task<ReceiptParseResult> ParseReceiptAsync(long receiptId, string? model = null, string? notes = null);
|
||||
}
|
||||
|
||||
public class AIReceiptParser : IReceiptParser
|
||||
{
|
||||
private readonly MoneyMapContext _db;
|
||||
private readonly IReceiptManager _receiptManager;
|
||||
private readonly IPdfToImageConverter _pdfConverter;
|
||||
private readonly IAIVisionClientResolver _clientResolver;
|
||||
private readonly IMerchantService _merchantService;
|
||||
private readonly IAIToolExecutor _toolExecutor;
|
||||
private readonly IServiceProvider _serviceProvider;
|
||||
private readonly IConfiguration _configuration;
|
||||
private readonly ILogger<AIReceiptParser> _logger;
|
||||
private string? _promptTemplate;
|
||||
|
||||
public AIReceiptParser(
|
||||
MoneyMapContext db,
|
||||
IReceiptManager receiptManager,
|
||||
IPdfToImageConverter pdfConverter,
|
||||
IAIVisionClientResolver clientResolver,
|
||||
IMerchantService merchantService,
|
||||
IAIToolExecutor toolExecutor,
|
||||
IServiceProvider serviceProvider,
|
||||
IConfiguration configuration,
|
||||
ILogger<AIReceiptParser> logger)
|
||||
{
|
||||
_db = db;
|
||||
_receiptManager = receiptManager;
|
||||
_pdfConverter = pdfConverter;
|
||||
_clientResolver = clientResolver;
|
||||
_merchantService = merchantService;
|
||||
_toolExecutor = toolExecutor;
|
||||
_serviceProvider = serviceProvider;
|
||||
_configuration = configuration;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
public async Task<ReceiptParseResult> ParseReceiptAsync(long receiptId, string? model = null, string? notes = null)
|
||||
{
|
||||
var receipt = await _db.Receipts
|
||||
.Include(r => r.Transaction)
|
||||
.FirstOrDefaultAsync(r => r.Id == receiptId);
|
||||
|
||||
if (receipt == null)
|
||||
return ReceiptParseResult.Failure("Receipt not found.");
|
||||
|
||||
var filePath = _receiptManager.GetReceiptPhysicalPath(receipt);
|
||||
if (!File.Exists(filePath))
|
||||
return ReceiptParseResult.Failure("Receipt file not found on disk.");
|
||||
|
||||
// Fall back to receipt.ParsingNotes if notes parameter is null
|
||||
var effectiveNotes = notes ?? receipt.ParsingNotes;
|
||||
|
||||
var selectedModel = model ?? _configuration["AI:ReceiptParsingModel"] ?? "gpt-4o-mini";
|
||||
var (client, provider) = _clientResolver.Resolve(selectedModel);
|
||||
|
||||
// Let model-aware clients evaluate tool support for the specific model
|
||||
if (client is LlamaCppVisionClient llamaCpp)
|
||||
llamaCpp.SetCurrentModel(selectedModel);
|
||||
|
||||
var parseLog = new ReceiptParseLog
|
||||
{
|
||||
ReceiptId = receiptId,
|
||||
Provider = provider,
|
||||
Model = selectedModel,
|
||||
StartedAtUtc = DateTime.UtcNow,
|
||||
Success = false
|
||||
};
|
||||
|
||||
try
|
||||
{
|
||||
var (base64Data, mediaType) = await PrepareImageDataAsync(receipt, filePath);
|
||||
var promptText = await BuildPromptAsync(receipt, effectiveNotes, client);
|
||||
var visionResult = await CallVisionClientAsync(client, base64Data, mediaType, promptText, selectedModel);
|
||||
|
||||
if (!visionResult.IsSuccess)
|
||||
{
|
||||
await SaveParseLogAsync(parseLog, visionResult.ErrorMessage);
|
||||
return ReceiptParseResult.Failure(visionResult.ErrorMessage!);
|
||||
}
|
||||
|
||||
var parseData = ParseResponse(visionResult.Content);
|
||||
await ApplyParseResultAsync(receipt, receiptId, parseData, effectiveNotes);
|
||||
|
||||
parseLog.Success = true;
|
||||
parseLog.Confidence = parseData.Confidence;
|
||||
parseLog.RawProviderPayloadJson = JsonSerializer.Serialize(parseData);
|
||||
await SaveParseLogAsync(parseLog);
|
||||
|
||||
await TryAutoMapReceiptAsync(receipt, receiptId, parseData.SuggestedTransactionId);
|
||||
|
||||
var lineCount = parseData.LineItems.Count;
|
||||
return ReceiptParseResult.Success($"Parsed {lineCount} line items from receipt.");
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
await SaveParseLogAsync(parseLog, ex.Message);
|
||||
_logger.LogError(ex, "Error parsing receipt {ReceiptId}: {Message}", receiptId, ex.Message);
|
||||
return ReceiptParseResult.Failure($"Error parsing receipt: {ex.Message}");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Call the vision client, using tool-use if the client supports it, or enriched prompt fallback for Ollama.
|
||||
/// </summary>
|
||||
private async Task<VisionApiResult> CallVisionClientAsync(
|
||||
IAIVisionClient client, string base64Data, string mediaType, string prompt, string model)
|
||||
{
|
||||
if (client is IAIToolAwareVisionClient toolAwareClient && toolAwareClient.SupportsToolUse)
|
||||
{
|
||||
_logger.LogInformation("Using tool-aware vision client for model {Model}", model);
|
||||
var tools = AIToolRegistry.GetAllTools();
|
||||
|
||||
return await toolAwareClient.AnalyzeImageWithToolsAsync(
|
||||
base64Data, mediaType, prompt, model,
|
||||
tools,
|
||||
toolCall => _toolExecutor.ExecuteAsync(toolCall),
|
||||
maxToolRounds: 5);
|
||||
}
|
||||
|
||||
// Fallback: standard call (Ollama gets enriched prompt via BuildPromptAsync)
|
||||
_logger.LogInformation("Using standard vision client for model {Model} (no tool use)", model);
|
||||
return await client.AnalyzeImageAsync(base64Data, mediaType, prompt, model);
|
||||
}
|
||||
|
||||
private async Task<(string Base64Data, string MediaType)> PrepareImageDataAsync(Receipt receipt, string filePath)
|
||||
{
|
||||
if (receipt.ContentType == "application/pdf")
|
||||
{
|
||||
var base64 = await _pdfConverter.ConvertFirstPageToBase64Async(filePath);
|
||||
return (base64, "image/png");
|
||||
}
|
||||
|
||||
var fileBytes = await File.ReadAllBytesAsync(filePath);
|
||||
return (Convert.ToBase64String(fileBytes), receipt.ContentType);
|
||||
}
|
||||
|
||||
private async Task<string> BuildPromptAsync(Receipt receipt, string? userNotes, IAIVisionClient client)
|
||||
{
|
||||
var promptText = await LoadPromptTemplateAsync();
|
||||
|
||||
var transactionName = receipt.Transaction?.Name;
|
||||
if (!string.IsNullOrWhiteSpace(transactionName))
|
||||
{
|
||||
promptText += $"\n\nNote: This transaction was recorded as \"{transactionName}\" in the bank statement, which may help identify the merchant if the receipt is unclear.";
|
||||
}
|
||||
|
||||
var parsingNotes = _configuration["AI:ReceiptParsingNotes"];
|
||||
if (!string.IsNullOrWhiteSpace(parsingNotes))
|
||||
{
|
||||
promptText += $"\n\nAdditional notes: {parsingNotes}";
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(userNotes))
|
||||
{
|
||||
promptText += $"\n\nUser notes for this receipt: {userNotes}";
|
||||
}
|
||||
|
||||
// Add tool-use or enriched context instructions based on client capability
|
||||
if (client is IAIToolAwareVisionClient toolAwareClient && toolAwareClient.SupportsToolUse)
|
||||
{
|
||||
// Tool-aware client: instruct to use tools for lookups
|
||||
promptText += @"
|
||||
|
||||
TOOL USE INSTRUCTIONS:
|
||||
You have access to tools that can query the application's database. You MUST call them before generating your JSON response:
|
||||
1. Call search_categories to find existing category names. Use ONLY categories returned by this tool for suggestedCategory and line item category fields. Do not invent new category names.
|
||||
2. Call search_transactions to find a matching bank transaction for this receipt (search by date, amount, merchant name). Set suggestedTransactionId to the numeric ID of the best match, or null if no good match. Remember: suggestedTransactionId must be a JSON integer or null, never a string.
|
||||
3. Call search_merchants to look up the correct merchant name.";
|
||||
}
|
||||
else
|
||||
{
|
||||
// Non-tool client (Ollama): inject pre-fetched database context
|
||||
try
|
||||
{
|
||||
var merchantHint = receipt.Transaction?.Name ?? receipt.Merchant;
|
||||
var enrichedContext = await _toolExecutor.GetEnrichedContextAsync(
|
||||
receipt.ReceiptDate,
|
||||
receipt.Total,
|
||||
merchantHint);
|
||||
|
||||
promptText += $"\n\n{enrichedContext}";
|
||||
promptText += @"
|
||||
|
||||
Using the database context above, populate these fields in your JSON response:
|
||||
- suggestedCategory: Use the best matching category name from the EXISTING CATEGORIES list. Do not invent new categories.
|
||||
- suggestedTransactionId: Use the numeric transaction ID from CANDIDATE TRANSACTIONS that best matches this receipt, or null if none match. Must be a JSON integer or null, never a string.
|
||||
- For each line item, set category to the best matching category from the EXISTING CATEGORIES list.";
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Failed to get enriched context for Ollama, proceeding without it");
|
||||
}
|
||||
}
|
||||
|
||||
promptText += "\n\nRespond ONLY with valid JSON, no other text.";
|
||||
return promptText;
|
||||
}
|
||||
|
||||
private static ParsedReceiptData ParseResponse(string? content)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(content))
|
||||
return new ParsedReceiptData();
|
||||
|
||||
return JsonSerializer.Deserialize<ParsedReceiptData>(content, new JsonSerializerOptions
|
||||
{
|
||||
PropertyNameCaseInsensitive = true
|
||||
}) ?? new ParsedReceiptData();
|
||||
}
|
||||
|
||||
private async Task ApplyParseResultAsync(Receipt receipt, long receiptId, ParsedReceiptData parseData, string? notes)
|
||||
{
|
||||
// Update receipt fields
|
||||
receipt.ParsingNotes = notes;
|
||||
receipt.Merchant = parseData.Merchant;
|
||||
receipt.Total = parseData.Total;
|
||||
receipt.Subtotal = parseData.Subtotal;
|
||||
receipt.Tax = parseData.Tax;
|
||||
receipt.ReceiptDate = parseData.ReceiptDate;
|
||||
receipt.DueDate = parseData.DueDate;
|
||||
|
||||
// Update transaction merchant if needed
|
||||
if (receipt.Transaction != null &&
|
||||
!string.IsNullOrWhiteSpace(parseData.Merchant) &&
|
||||
receipt.Transaction.MerchantId == null)
|
||||
{
|
||||
var merchantId = await _merchantService.GetOrCreateIdAsync(parseData.Merchant);
|
||||
receipt.Transaction.MerchantId = merchantId;
|
||||
}
|
||||
|
||||
// Update transaction category if AI suggested one and the transaction has no category
|
||||
if (receipt.Transaction != null &&
|
||||
!string.IsNullOrWhiteSpace(parseData.SuggestedCategory) &&
|
||||
string.IsNullOrWhiteSpace(receipt.Transaction.Category))
|
||||
{
|
||||
receipt.Transaction.Category = parseData.SuggestedCategory;
|
||||
_logger.LogInformation("Set transaction {TransactionId} category to '{Category}' from AI suggestion",
|
||||
receipt.Transaction.Id, parseData.SuggestedCategory);
|
||||
}
|
||||
|
||||
// Replace line items
|
||||
var existingItems = await _db.ReceiptLineItems
|
||||
.Where(li => li.ReceiptId == receiptId)
|
||||
.ToListAsync();
|
||||
_db.ReceiptLineItems.RemoveRange(existingItems);
|
||||
|
||||
var lineItems = parseData.LineItems.Select((item, index) => new ReceiptLineItem
|
||||
{
|
||||
ReceiptId = receiptId,
|
||||
LineNumber = index + 1,
|
||||
Description = item.Description,
|
||||
Sku = item.Upc,
|
||||
Quantity = item.Quantity,
|
||||
UnitPrice = item.UnitPrice,
|
||||
LineTotal = item.LineTotal,
|
||||
Category = item.Category,
|
||||
Voided = item.Voided
|
||||
}).ToList();
|
||||
|
||||
_db.ReceiptLineItems.AddRange(lineItems);
|
||||
await _db.SaveChangesAsync();
|
||||
}
|
||||
|
||||
private async Task SaveParseLogAsync(ReceiptParseLog parseLog, string? error = null)
|
||||
{
|
||||
parseLog.Error = error;
|
||||
parseLog.CompletedAtUtc = DateTime.UtcNow;
|
||||
_db.ReceiptParseLogs.Add(parseLog);
|
||||
await _db.SaveChangesAsync();
|
||||
}
|
||||
|
||||
private async Task TryAutoMapReceiptAsync(Receipt receipt, long receiptId, long? suggestedTransactionId)
|
||||
{
|
||||
// If AI suggested a specific transaction, try mapping directly
|
||||
if (!receipt.TransactionId.HasValue && suggestedTransactionId.HasValue)
|
||||
{
|
||||
try
|
||||
{
|
||||
var transaction = await _db.Transactions.FindAsync(suggestedTransactionId.Value);
|
||||
if (transaction != null)
|
||||
{
|
||||
// Verify the transaction isn't already mapped to another receipt
|
||||
var alreadyMapped = await _db.Receipts
|
||||
.AnyAsync(r => r.TransactionId == suggestedTransactionId.Value && r.Id != receiptId);
|
||||
|
||||
if (!alreadyMapped)
|
||||
{
|
||||
var success = await _receiptManager.MapReceiptToTransactionAsync(receiptId, suggestedTransactionId.Value);
|
||||
if (success)
|
||||
{
|
||||
_logger.LogInformation(
|
||||
"AI-suggested mapping: receipt {ReceiptId} → transaction {TransactionId}",
|
||||
receiptId, suggestedTransactionId.Value);
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "AI-suggested mapping failed for receipt {ReceiptId} → transaction {TransactionId}",
|
||||
receiptId, suggestedTransactionId.Value);
|
||||
}
|
||||
}
|
||||
|
||||
// Fall back to the existing auto-mapper
|
||||
if (receipt.TransactionId.HasValue)
|
||||
return;
|
||||
|
||||
try
|
||||
{
|
||||
using var scope = _serviceProvider.CreateScope();
|
||||
var autoMapper = scope.ServiceProvider.GetRequiredService<IReceiptAutoMapper>();
|
||||
await autoMapper.AutoMapReceiptAsync(receiptId);
|
||||
_logger.LogInformation("Auto-mapping completed for receipt {ReceiptId}", receiptId);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Auto-mapping failed for receipt {ReceiptId}: {Message}", receiptId, ex.Message);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<string> LoadPromptTemplateAsync()
|
||||
{
|
||||
if (_promptTemplate != null)
|
||||
return _promptTemplate;
|
||||
|
||||
var promptPath = Path.Combine(AppContext.BaseDirectory, "Prompts", "ReceiptParserPrompt.txt");
|
||||
|
||||
if (!File.Exists(promptPath))
|
||||
throw new FileNotFoundException($"Receipt parser prompt template not found at: {promptPath}");
|
||||
|
||||
_promptTemplate = await File.ReadAllTextAsync(promptPath);
|
||||
return _promptTemplate;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Resolves the appropriate AI vision client based on model name.
|
||||
/// </summary>
|
||||
public interface IAIVisionClientResolver
|
||||
{
|
||||
(IAIVisionClient Client, string Provider) Resolve(string model);
|
||||
}
|
||||
|
||||
public class AIVisionClientResolver : IAIVisionClientResolver
|
||||
{
|
||||
private readonly OpenAIVisionClient _openAIClient;
|
||||
private readonly ClaudeVisionClient _claudeClient;
|
||||
private readonly OllamaVisionClient _ollamaClient;
|
||||
private readonly LlamaCppVisionClient _llamaCppClient;
|
||||
|
||||
public AIVisionClientResolver(
|
||||
OpenAIVisionClient openAIClient,
|
||||
ClaudeVisionClient claudeClient,
|
||||
OllamaVisionClient ollamaClient,
|
||||
LlamaCppVisionClient llamaCppClient)
|
||||
{
|
||||
_openAIClient = openAIClient;
|
||||
_claudeClient = claudeClient;
|
||||
_ollamaClient = ollamaClient;
|
||||
_llamaCppClient = llamaCppClient;
|
||||
}
|
||||
|
||||
public (IAIVisionClient Client, string Provider) Resolve(string model)
|
||||
{
|
||||
if (model.StartsWith("llamacpp:"))
|
||||
return (_llamaCppClient, "LlamaCpp");
|
||||
|
||||
if (model.StartsWith("ollama:"))
|
||||
return (_ollamaClient, "Ollama");
|
||||
|
||||
if (model.StartsWith("claude-"))
|
||||
return (_claudeClient, "Anthropic");
|
||||
|
||||
return (_openAIClient, "OpenAI");
|
||||
}
|
||||
}
|
||||
|
||||
public class ParsedReceiptData
|
||||
{
|
||||
public string? Merchant { get; set; }
|
||||
public DateTime? ReceiptDate { get; set; }
|
||||
public DateTime? DueDate { get; set; }
|
||||
public decimal? Subtotal { get; set; }
|
||||
public decimal? Tax { get; set; }
|
||||
public decimal? Total { get; set; }
|
||||
public decimal Confidence { get; set; } = 0.5m;
|
||||
public string? SuggestedCategory { get; set; }
|
||||
[JsonConverter(typeof(NullableLongConverter))]
|
||||
public long? SuggestedTransactionId { get; set; }
|
||||
public List<ParsedLineItem> LineItems { get; set; } = new();
|
||||
}
|
||||
|
||||
public class ParsedLineItem
|
||||
{
|
||||
public string Description { get; set; } = "";
|
||||
public string? Upc { get; set; }
|
||||
public decimal? Quantity { get; set; }
|
||||
public decimal? UnitPrice { get; set; }
|
||||
public decimal LineTotal { get; set; }
|
||||
public string? Category { get; set; }
|
||||
public bool Voided { get; set; }
|
||||
}
|
||||
|
||||
public class ReceiptParseResult
|
||||
{
|
||||
public bool IsSuccess { get; init; }
|
||||
public string? Message { get; init; }
|
||||
|
||||
public static ReceiptParseResult Success(string message) =>
|
||||
new() { IsSuccess = true, Message = message };
|
||||
|
||||
public static ReceiptParseResult Failure(string message) =>
|
||||
new() { IsSuccess = false, Message = message };
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Handles AI responses that return suggestedTransactionId as a string ("null", "N/A", "123")
|
||||
/// instead of as a JSON number or null.
|
||||
/// </summary>
|
||||
public class NullableLongConverter : JsonConverter<long?>
|
||||
{
|
||||
public override long? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
|
||||
{
|
||||
switch (reader.TokenType)
|
||||
{
|
||||
case JsonTokenType.Number:
|
||||
return reader.GetInt64();
|
||||
case JsonTokenType.String:
|
||||
var str = reader.GetString();
|
||||
if (string.IsNullOrWhiteSpace(str) ||
|
||||
str.Equals("null", StringComparison.OrdinalIgnoreCase) ||
|
||||
str.Equals("N/A", StringComparison.OrdinalIgnoreCase) ||
|
||||
str.Equals("none", StringComparison.OrdinalIgnoreCase))
|
||||
return null;
|
||||
return long.TryParse(str, out var val) ? val : null;
|
||||
case JsonTokenType.Null:
|
||||
return null;
|
||||
default:
|
||||
reader.Skip();
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
public override void Write(Utf8JsonWriter writer, long? value, JsonSerializerOptions options)
|
||||
{
|
||||
if (value.HasValue)
|
||||
writer.WriteNumberValue(value.Value);
|
||||
else
|
||||
writer.WriteNullValue();
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user