使用Microsoft.Extensions.AI.Evaluation测试你的AI应用

1
2

SET EVAL_SAMPLE_AZURE_OPENAI_ENDPOINT=https://<your azure openai resource name>.openai.azure.com/
SET EVAL_SAMPLE_AZURE_OPENAI_MODEL=<your model deployment name (e.g., gpt-4o)>

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131

using Azure.AI.OpenAI; 
using Azure.Identity; 
using Microsoft.Extensions.AI; 
using Microsoft.Extensions.AI.Evaluation; 
using Microsoft.Extensions.AI.Evaluation.Quality; 
using Microsoft.Extensions.AI.Evaluation.Reporting; 
using Microsoft.Extensions.AI.Evaluation.Reporting.Storage; 
 
namespace EvaluationTests; 
 
[TestClass] 
public class Test1 
{ 
    // Create a ReportingConfiguration. The ReportingConfiguration includes the IChatClient used to interact with the 
    // model, the set of evaluators to be used to assess the quality of the model's responses, and other configuration 
    // related to persisting the evaluation results and generating reports based on these results. 
    private static readonly ReportingConfiguration s_reportingConfig = CreateReportingConfiguration(); 
 
    [TestMethod] 
    public async Task Test1_DistanceBetweenEarthAndVenus() 
    { 
        // Create a ScenarioRun using the ReportingConfiguration created above. 
        await using ScenarioRun scenarioRun = await s_reportingConfig.CreateScenarioRunAsync("Scenarios.Venus"); 
 
        // Get a conversation that includes a query related to astronomy and the model's response to this query. 
        (IList<ChatMessage> messages, ChatResponse response) = 
            await GetAstronomyConversationAsync( 
                chatClient: scenarioRun.ChatConfiguration!.ChatClient, 
                query: "How far is the planet Venus from the Earth at its closest and furthest points?"); 
 
        // CoherenceEvaluator and FluencyEvaluator do not require any additional context beyond the messages and 
        // response to perform their assessments. GroundednessEvaluator on the other hand requires additional context 
        // to perform its assessment - it assesses how well the model's response is grounded in the grounding context 
        // provided below. 
        List<EvaluationContext> additionalContext = 
            [ 
                new GroundednessEvaluatorContext( 
                    """ 
                    Distance between Venus and Earth at inferior conjunction: Between 23 and 25 million miles approximately. 
                    Distance between Venus and Earth at superior conjunction: Between 160 and 164 million miles approximately. 
                    The exact distances can vary due to the specific orbital positions of the planets at any given time. 
                    """) 
            ]; 
 
        // Run the evaluators included in the ReportingConfiguration to assess the quality of the above response. 
        EvaluationResult result = await scenarioRun.EvaluateAsync(messages, response, additionalContext); 
 
        // Retrieve one of the metrics in the EvaluationResult (example: Groundedness). 
        NumericMetric groundedness = result.Get<NumericMetric>(GroundednessEvaluator.GroundednessMetricName); 
        Assert.IsFalse(groundedness.Interpretation!.Failed); 
 
        // Results are persisted to disk under the storageRootPath specified below once the scenarioRun is disposed. 
    } 
 
    [TestMethod] 
    public async Task Test2_DistanceBetweenEarthAndMars() 
    { 
        // Create another ScenarioRun using the same ReportingConfiguration created above. 
        await using ScenarioRun scenarioRun = await s_reportingConfig.CreateScenarioRunAsync("Scenarios.Mars"); 
 
        // Get another conversation for a different astronomy-related query. 
        (IList<ChatMessage> messages, ChatResponse response) = 
            await GetAstronomyConversationAsync( 
                chatClient: scenarioRun.ChatConfiguration!.ChatClient, 
                query: "How far is the planet Mars from the Earth at its closest and furthest points?"); 
 
        List<EvaluationContext> additionalContext = 
            [ 
                new GroundednessEvaluatorContext( 
                    """ 
                    Distance between Mars and Earth at inferior conjunction: Between 33.9 and 62.1 million miles approximately. 
                    Distance between Mars and Earth at superior conjunction: Between 249 and 250 million miles approximately. 
                    The exact distances can vary due to the specific orbital positions of the planets at any given time. 
                    """) 
            ]; 
 
        EvaluationResult result = await scenarioRun.EvaluateAsync(messages, response, additionalContext); 
 
        // Retrieve one of the metrics in the EvaluationResult (example: Coherence). 
        NumericMetric coherence = result.Get<NumericMetric>(CoherenceEvaluator.CoherenceMetricName); 
        Assert.IsFalse(coherence.Interpretation!.Failed); 
    } 
 
    private static ReportingConfiguration CreateReportingConfiguration() 
    { 
        // Create an IChatClient to interact with a model deployed on Azure OpenAI. 
        string endpoint = Environment.GetEnvironmentVariable("EVAL_SAMPLE_AZURE_OPENAI_ENDPOINT")!; 
        string model = Environment.GetEnvironmentVariable("EVAL_SAMPLE_AZURE_OPENAI_MODEL")!; 
        var client = new AzureOpenAIClient(new Uri(endpoint), new DefaultAzureCredential()); 
        IChatClient chatClient = client.GetChatClient(deploymentName: model).AsIChatClient(); 
 
        // Create a ReportingConfiguration for evaluating the quality of supplied responses. 
        return DiskBasedReportingConfiguration.Create( 
            storageRootPath: "./eval-results", // The evaluation results will be persisted to disk under this folder. 
            evaluators: [new CoherenceEvaluator(), new FluencyEvaluator(), new GroundednessEvaluator()], 
            chatConfiguration: new ChatConfiguration(chatClient), 
            enableResponseCaching: true); 
 
        // Since response caching is enabled above, all LLM responses produced via the chatClient above will also be 
        // cached under the storageRootPath so long as the inputs being evaluated stay unchanged, and so long as the 
        // cache entries do not expire (cache expiry is set at 14 days by default). 
    } 
 
    private static async Task<(IList<ChatMessage> Messages, ChatResponse ModelResponse)> GetAstronomyConversationAsync( 
        IChatClient chatClient, 
        string query) 
    { 
        const string SystemPrompt = 
            """ 
            You are an AI assistant that can answer questions related to astronomy. 
            Keep your responses concise staying under 100 words as much as possible. 
            Use the imperial measurement system for all measurements in your response. 
            """; 
 
        List<ChatMessage> messages = 
            [ 
                new ChatMessage(ChatRole.System, SystemPrompt), 
                new ChatMessage(ChatRole.User, query) 
            ]; 
 
        var chatOptions = 
            new ChatOptions 
            { 
                Temperature = 0.0f, 
                ResponseFormat = ChatResponseFormat.Text 
            }; 
 
        ChatResponse response = await chatClient.GetResponseAsync(messages, chatOptions); 
        return (messages, response); 
    } 
}

1

dotnet tool install Microsoft.Extensions.AI.Evaluation.Console --create-manifest-if-needed

1

dotnet aieval report -p <path to 'eval-results' folder under the build output directory for the above project> -o .\report.html --open