From f7c6b2691b239126e60ea51704a8d1404cad8ee3 Mon Sep 17 00:00:00 2001
From: AJ <ajisaacs27@gmail.com>
Date: Sun, 19 Oct 2025 16:15:25 -0400
Subject: [PATCH] Feature: add Anthropic Claude model support for receipt
 parsing
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Add Claude 3.5 Haiku and Sonnet as parsing options:
- Add Claude models to AI Model dropdown (Haiku for fast, Sonnet for best quality)
- Update OpenAIReceiptParser to detect provider based on model name (claude-* prefix)
- Add CallClaudeVisionAsync method using Anthropic Messages API
- Support ANTHROPIC_API_KEY environment variable or Anthropic:ApiKey config
- Parse logs now correctly show "Anthropic" or "OpenAI" as provider
- Both providers use the same prompt template and return structure

Users can now choose from 4 models:
- GPT-4o Mini (fast & cheap)
- GPT-4o (smarter)
- Claude 3.5 Haiku (fast)
- Claude 3.5 Sonnet (best quality)

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>
---
 MoneyMap/Pages/ViewReceipt.cshtml        |  10 +-
 MoneyMap/Services/OpenAIReceiptParser.cs | 120 ++++++++++++++++++++---
 2 files changed, 117 insertions(+), 13 deletions(-)

diff --git a/MoneyMap/Pages/ViewReceipt.cshtml b/MoneyMap/Pages/ViewReceipt.cshtml
index 938191a..0a4535e 100644
--- a/MoneyMap/Pages/ViewReceipt.cshtml
+++ b/MoneyMap/Pages/ViewReceipt.cshtml
@@ -165,8 +165,14 @@
                         <div class="mb-2">
                             <label for="model" class="form-label small">AI Model</label>
                             <select name="model" id="model" class="form-select form-select-sm">
-                                <option value="gpt-4o-mini" selected>GPT-4o Mini (Fast & Cheap)</option>
-                                <option value="gpt-4o">GPT-4o (Smarter)</option>
+                                <optgroup label="OpenAI">
+                                    <option value="gpt-4o-mini" selected>GPT-4o Mini (Fast & Cheap)</option>
+                                    <option value="gpt-4o">GPT-4o (Smarter)</option>
+                                </optgroup>
+                                <optgroup label="Anthropic">
+                                    <option value="claude-3-5-haiku-20241022">Claude 3.5 Haiku (Fast)</option>
+                                    <option value="claude-3-5-sonnet-20241022">Claude 3.5 Sonnet (Best)</option>
+                                </optgroup>
                             </select>
                         </div>
                         <button type="submit" class="btn btn-primary btn-sm w-100">
diff --git a/MoneyMap/Services/OpenAIReceiptParser.cs b/MoneyMap/Services/OpenAIReceiptParser.cs
index 3bfb7d6..0dadb9b 100644
--- a/MoneyMap/Services/OpenAIReceiptParser.cs
+++ b/MoneyMap/Services/OpenAIReceiptParser.cs
@@ -47,25 +47,39 @@ namespace MoneyMap.Services
             if (receipt == null)
                 return ReceiptParseResult.Failure("Receipt not found.");
 
-            // Try environment variable first, then configuration
-            var apiKey = Environment.GetEnvironmentVariable("OPENAI_API_KEY")
-                ?? _configuration["OpenAI:ApiKey"];
+            // Default to gpt-4o-mini if no model specified
+            var selectedModel = model ?? "gpt-4o-mini";
 
-            if (string.IsNullOrWhiteSpace(apiKey))
-                return ReceiptParseResult.Failure("OpenAI API key not configured. Set OPENAI_API_KEY environment variable or OpenAI:ApiKey in appsettings.json");
+            // Determine provider based on model name
+            var isClaude = selectedModel.StartsWith("claude-");
+            var provider = isClaude ? "Anthropic" : "OpenAI";
+
+            // Get appropriate API key
+            string? apiKey;
+            if (isClaude)
+            {
+                apiKey = Environment.GetEnvironmentVariable("ANTHROPIC_API_KEY")
+                    ?? _configuration["Anthropic:ApiKey"];
+                if (string.IsNullOrWhiteSpace(apiKey))
+                    return ReceiptParseResult.Failure("Anthropic API key not configured. Set ANTHROPIC_API_KEY environment variable or Anthropic:ApiKey in appsettings.json");
+            }
+            else
+            {
+                apiKey = Environment.GetEnvironmentVariable("OPENAI_API_KEY")
+                    ?? _configuration["OpenAI:ApiKey"];
+                if (string.IsNullOrWhiteSpace(apiKey))
+                    return ReceiptParseResult.Failure("OpenAI API key not configured. Set OPENAI_API_KEY environment variable or OpenAI:ApiKey in appsettings.json");
+            }
 
             var filePath = Path.Combine(_environment.WebRootPath, receipt.StoragePath.Replace("/", Path.DirectorySeparatorChar.ToString()));
 
             if (!File.Exists(filePath))
                 return ReceiptParseResult.Failure("Receipt file not found on disk.");
 
-            // Default to gpt-4o-mini if no model specified
-            var selectedModel = model ?? "gpt-4o-mini";
-
             var parseLog = new ReceiptParseLog
             {
                 ReceiptId = receiptId,
-                Provider = "OpenAI",
+                Provider = provider,
                 Model = selectedModel,
                 StartedAtUtc = DateTime.UtcNow,
                 Success = false
@@ -90,9 +104,11 @@ namespace MoneyMap.Services
                     mediaType = receipt.ContentType;
                 }
 
-                // Call OpenAI Vision API with transaction name context
+                // Call Vision API with transaction name context
                 var transactionName = receipt.Transaction?.Name;
-                var parseData = await CallOpenAIVisionAsync(apiKey, base64Data, mediaType, selectedModel, transactionName);
+                var parseData = isClaude
+                    ? await CallClaudeVisionAsync(apiKey, base64Data, mediaType, selectedModel, transactionName)
+                    : await CallOpenAIVisionAsync(apiKey, base64Data, mediaType, selectedModel, transactionName);
 
                 // Update receipt with parsed data
                 receipt.Merchant = parseData.Merchant;
@@ -297,6 +313,88 @@ namespace MoneyMap.Services
 
             return parsedData ?? new ParsedReceiptData();
         }
+
+        private async Task<ParsedReceiptData> CallClaudeVisionAsync(string apiKey, string base64Image, string mediaType, string model, string? transactionName = null)
+        {
+            // Load the prompt template from file
+            var promptText = await LoadPromptTemplateAsync();
+
+            // Add transaction context if available
+            if (!string.IsNullOrWhiteSpace(transactionName))
+            {
+                promptText += $"\n\nNote: This transaction was recorded as \"{transactionName}\" in the bank statement, which may help identify the merchant if the receipt is unclear.";
+            }
+
+            promptText += "\n\nRespond ONLY with valid JSON, no other text.";
+
+            var requestBody = new
+            {
+                model = model,
+                max_tokens = 2000,
+                messages = new[]
+                {
+                    new
+                    {
+                        role = "user",
+                        content = new object[]
+                        {
+                            new
+                            {
+                                type = "image",
+                                source = new
+                                {
+                                    type = "base64",
+                                    media_type = mediaType,
+                                    data = base64Image
+                                }
+                            },
+                            new
+                            {
+                                type = "text",
+                                text = promptText
+                            }
+                        }
+                    }
+                }
+            };
+
+            _httpClient.DefaultRequestHeaders.Clear();
+            _httpClient.DefaultRequestHeaders.Add("x-api-key", apiKey);
+            _httpClient.DefaultRequestHeaders.Add("anthropic-version", "2023-06-01");
+
+            var json = JsonSerializer.Serialize(requestBody);
+            var content = new StringContent(json, Encoding.UTF8, "application/json");
+
+            var response = await _httpClient.PostAsync("https://api.anthropic.com/v1/messages", content);
+
+            if (!response.IsSuccessStatusCode)
+            {
+                var errorContent = await response.Content.ReadAsStringAsync();
+                throw new Exception($"Anthropic API error ({response.StatusCode}): {errorContent}");
+            }
+
+            var responseJson = await response.Content.ReadAsStringAsync();
+            var responseObj = JsonSerializer.Deserialize<JsonElement>(responseJson);
+
+            var messageContent = responseObj
+                .GetProperty("content")[0]
+                .GetProperty("text")
+                .GetString();
+
+            // Clean up the response - remove markdown code blocks if present
+            messageContent = messageContent?.Trim();
+            if (messageContent?.StartsWith("```json") == true)
+            {
+                messageContent = messageContent.Replace("```json", "").Replace("```", "").Trim();
+            }
+
+            var parsedData = JsonSerializer.Deserialize<ParsedReceiptData>(messageContent, new JsonSerializerOptions
+            {
+                PropertyNameCaseInsensitive = true
+            });
+
+            return parsedData ?? new ParsedReceiptData();
+        }
     }
 
     public class ParsedReceiptData