Add video generation support to Microsoft.Extensions.AI

samples/VideoGenerationPOC/Program.cs

@@ -0,0 +1,218 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+// Video Generation POC - Microsoft.Extensions.AI
+// Usage: set OPENAI_API_KEY environment variable, then run:
+//   dotnet run -- "A cat playing piano"
+//   dotnet run -- "She turns and smiles" --input reference.jpg
+//   dotnet run -- "Change the sky to sunset" --edit video_abc123
+//   dotnet run -- "Continue the scene" --extend video_abc123
+//   dotnet run -- "A tracking shot of Mossy" --character char_abc123
+
+using System.CommandLine;
+using System.Drawing;
+using System.Text.Json.Nodes;
+using Microsoft.Extensions.AI;
+using Microsoft.Extensions.Logging;
+using OpenAI;
+
+var promptArg = new Argument<string>("prompt", () => "A serene lake at sunset with gentle ripples", "Text prompt describing the video to generate.");
+var modelOption = new Option<string>("--model", () => "sora-2", "Model ID to use for video generation.");
+var outputOption = new Option<string>("--output", () => $"video_{DateTime.Now:yyyyMMdd_HHmmss}.mp4", "Output file path for the generated video.");
+var inputOption = new Option<string[]>("--input", "Input file(s) — images for image-to-video, or a video for editing.") { AllowMultipleArgumentsPerToken = true };
+var editOption = new Option<string?>("--edit", "Video ID of an existing generation to edit (POST /videos/edits).");
+var extendOption = new Option<string?>("--extend", "Video ID of a completed video to extend (POST /videos/extensions).");
+var characterOption = new Option<string[]>("--character", "Character ID(s) to include in the generation.") { AllowMultipleArgumentsPerToken = true };
+
+var rootCommand = new RootCommand("Video Generation POC — demonstrates Microsoft.Extensions.AI video generation with OpenAI.")
+{
+    promptArg,
+    modelOption,
+    outputOption,
+    inputOption,
+    editOption,
+    extendOption,
+    characterOption,
+};
+
+rootCommand.SetHandler(async (context) =>
+{
+    string prompt = context.ParseResult.GetValueForArgument(promptArg);
+    string model = context.ParseResult.GetValueForOption(modelOption)!;
+    string outputPath = context.ParseResult.GetValueForOption(outputOption)!;
+    string[] inputPaths = context.ParseResult.GetValueForOption(inputOption) ?? [];
+    string? editVideoId = context.ParseResult.GetValueForOption(editOption);
+    string? extendVideoId = context.ParseResult.GetValueForOption(extendOption);
+    string[] characterIds = context.ParseResult.GetValueForOption(characterOption) ?? [];
+
+    // --- API key ---
+    string? apiKey = Environment.GetEnvironmentVariable("OPENAI_API_KEY");
+    if (string.IsNullOrEmpty(apiKey))
+    {
+        Console.Error.WriteLine("Error: Set the OPENAI_API_KEY environment variable.");
+        context.ExitCode = 1;
+        return;
+    }
+
+    Console.WriteLine($"Prompt:  {prompt}");
+    Console.WriteLine($"Model:   {model}");
+    Console.WriteLine($"Output:  {outputPath}");
+    if (inputPaths.Length > 0)
+    {
+        Console.WriteLine($"Inputs:  {string.Join(", ", inputPaths)}");
+    }
+
+    if (editVideoId is not null)
+    {
+        Console.WriteLine($"Edit:    {editVideoId}");
+    }
+
+    if (extendVideoId is not null)
+    {
+        Console.WriteLine($"Extend:  {extendVideoId}");
+    }
+
+    if (characterIds.Length > 0)
+    {
+        Console.WriteLine($"Characters: {string.Join(", ", characterIds)}");
+    }
+
+    Console.WriteLine();
+
+    // --- Create the video generator with middleware pipeline ---
+    using var loggerFactory = LoggerFactory.Create(builder => builder.AddConsole().SetMinimumLevel(LogLevel.Debug));
+
+    var openAIClient = new OpenAIClient(apiKey);
+    using IVideoGenerator generator = openAIClient
+        .GetVideoClient()
+        .AsIVideoGenerator(model)
+        .AsBuilder()
+        .UseLogging(loggerFactory)
+        .UseOpenTelemetry(loggerFactory)
+        .ConfigureOptions(options =>
+        {
+            options.Count ??= 1;
+            options.Duration ??= TimeSpan.FromSeconds(12);
+            options.VideoSize ??= new Size(1280, 720);
+        })
+        .Build();
+
+    // --- Show metadata ---
+    var metadata = generator.GetService<VideoGeneratorMetadata>();
+    if (metadata is not null)
+    {
+        Console.WriteLine($"Provider:      {metadata.ProviderName}");
+        Console.WriteLine($"Endpoint:      {metadata.ProviderUri}");
+        Console.WriteLine($"Default Model: {metadata.DefaultModelId}");
+        Console.WriteLine();
+    }
+
+    // --- Build request ---
+    List<AIContent>? originalMedia = null;
+    if (inputPaths.Length > 0)
+    {
+        originalMedia = [];
+        foreach (string inputPath in inputPaths)
+        {
+            if (!File.Exists(inputPath))
+            {
+                Console.Error.WriteLine($"Error: Input file not found: {inputPath}");
+                context.ExitCode = 1;
+                return;
+            }
+
+            DataContent loaded = await DataContent.LoadFromAsync(inputPath);
+            originalMedia.Add(loaded);
+            Console.WriteLine($"  Loaded input: {inputPath} ({loaded.MediaType}, {loaded.Data.Length} bytes)");
+        }
+
+        Console.WriteLine();
+    }
+
+    // --- Generate video ---
+    string mode =
+        extendVideoId is not null ? "Extending" :
+        editVideoId is not null ? "Editing (by video ID)" :
+        originalMedia?.Exists(c => c is DataContent dc && dc.HasTopLevelMediaType("video")) == true ? "Editing (uploaded video)" :
+        originalMedia is not null ? "Generating (image-to-video)" :
+        "Generating (text-to-video)";
+    Console.WriteLine($"{mode}...");
+    var stopwatch = System.Diagnostics.Stopwatch.StartNew();
+
+    var generateOptions = new VideoGenerationOptions
+    {
+        ResponseFormat = VideoGenerationResponseFormat.Data,
+    };
+
+    if (editVideoId is not null)
+    {
+        generateOptions.AdditionalProperties ??= [];
+        generateOptions.AdditionalProperties["edit_video_id"] = editVideoId;
+    }
+
+    if (extendVideoId is not null)
+    {
+        generateOptions.AdditionalProperties ??= [];
+        generateOptions.AdditionalProperties["extend_video_id"] = extendVideoId;
+    }
+
+    if (characterIds.Length > 0)
+    {
+        var chars = new JsonArray();
+        foreach (string charId in characterIds)
+        {
+            chars.Add(new JsonObject { ["id"] = charId });
+        }
+
+        generateOptions.AdditionalProperties ??= [];
+        generateOptions.AdditionalProperties["characters"] = chars;
+    }
+
+    var response = await generator.GenerateAsync(
+        new VideoGenerationRequest(prompt, originalMedia),
+        generateOptions,
+        new Progress<VideoGenerationProgress>(p =>
+            Console.WriteLine($"  Status: {p.Status}{(p.PercentComplete.HasValue ? $" ({p.PercentComplete}%)" : string.Empty)}")));
+
+    stopwatch.Stop();
+    Console.WriteLine($"Completed in {stopwatch.Elapsed.TotalSeconds:F1}s");
+    Console.WriteLine();
+
+    // --- Process response ---
+    if (response.Usage is { } usage)
+    {
+        Console.WriteLine($"Token Usage: input={usage.InputTokenCount}, output={usage.OutputTokenCount}, total={usage.TotalTokenCount}");
+    }
+
+    Console.WriteLine($"Generated {response.Contents.Count} content item(s):");
+    for (int i = 0; i < response.Contents.Count; i++)
+    {
+        var content = response.Contents[i];
+        switch (content)
+        {
+            case DataContent dc:
+                string filePath = response.Contents.Count == 1
+                    ? outputPath
+                    : Path.Combine(
+                        Path.GetDirectoryName(outputPath) ?? ".",
+                        $"{Path.GetFileNameWithoutExtension(outputPath)}_{i}{Path.GetExtension(outputPath)}");
+
+                await dc.SaveToAsync(filePath);
+                Console.WriteLine($"  [{i}] Saved {dc.Data.Length} bytes ({dc.MediaType}) -> {filePath}");
+                break;
+
+            case UriContent uc:
+                Console.WriteLine($"  [{i}] URI: {uc.Uri} ({uc.MediaType})");
+                break;
+
+            default:
+                Console.WriteLine($"  [{i}] {content.GetType().Name}: {content}");
+                break;
+        }
+    }
+
+    Console.WriteLine();
+    Console.WriteLine("Done!");
+});
+
+return await rootCommand.InvokeAsync(args);

samples/VideoGenerationPOC/VideoGenerationPOC.csproj

@@ -0,0 +1,18 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net10.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+    <NoWarn>$(NoWarn);MEAI001;OPENAI001</NoWarn>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.Extensions.Logging.Console" />
+    <PackageReference Include="System.CommandLine" />
+    <ProjectReference Include="..\..\src\Libraries\Microsoft.Extensions.AI\Microsoft.Extensions.AI.csproj" />
+    <ProjectReference Include="..\..\src\Libraries\Microsoft.Extensions.AI.OpenAI\Microsoft.Extensions.AI.OpenAI.csproj" />
+  </ItemGroup>
+
+</Project>

src/Libraries/Microsoft.Extensions.AI.Abstractions/Video/DelegatingVideoGenerator.cs

@@ -0,0 +1,70 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.Diagnostics.CodeAnalysis;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Shared.DiagnosticIds;
+using Microsoft.Shared.Diagnostics;
+
+namespace Microsoft.Extensions.AI;
+
+/// <summary>
+/// Provides an optional base class for an <see cref="IVideoGenerator"/> that passes through calls to another instance.
+/// </summary>
+/// <remarks>
+/// This is recommended as a base type when building generators that can be chained in any order around an underlying <see cref="IVideoGenerator"/>.
+/// The default implementation simply passes each call to the inner generator instance.
+/// </remarks>
+[Experimental(DiagnosticIds.Experiments.AIVideoGeneration, UrlFormat = DiagnosticIds.UrlFormat)]
+public class DelegatingVideoGenerator : IVideoGenerator
+{
+    /// <summary>
+    /// Initializes a new instance of the <see cref="DelegatingVideoGenerator"/> class.
+    /// </summary>
+    /// <param name="innerGenerator">The wrapped generator instance.</param>
+    /// <exception cref="ArgumentNullException"><paramref name="innerGenerator"/> is <see langword="null"/>.</exception>
+    protected DelegatingVideoGenerator(IVideoGenerator innerGenerator)
+    {
+        InnerGenerator = Throw.IfNull(innerGenerator);
+    }
+
+    /// <inheritdoc />
+    public void Dispose()
+    {
+        Dispose(disposing: true);
+        GC.SuppressFinalize(this);
+    }
+
+    /// <summary>Gets the inner <see cref="IVideoGenerator" />.</summary>
+    protected IVideoGenerator InnerGenerator { get; }
+
+    /// <inheritdoc />
+    public virtual Task<VideoGenerationResponse> GenerateAsync(
+        VideoGenerationRequest request, VideoGenerationOptions? options = null, IProgress<VideoGenerationProgress>? progress = null, CancellationToken cancellationToken = default)
+    {
+        return InnerGenerator.GenerateAsync(request, options, progress, cancellationToken);
+    }
+
+    /// <inheritdoc />
+    public virtual object? GetService(Type serviceType, object? serviceKey = null)
+    {
+        _ = Throw.IfNull(serviceType);
+
+        // If the key is non-null, we don't know what it means so pass through to the inner service.
+        return
+            serviceKey is null && serviceType.IsInstanceOfType(this) ? this :
+            InnerGenerator.GetService(serviceType, serviceKey);
+    }
+
+    /// <summary>Provides a mechanism for releasing unmanaged resources.</summary>
+    /// <param name="disposing"><see langword="true"/> if being called from <see cref="Dispose()"/>; otherwise, <see langword="false"/>.</param>
+    protected virtual void Dispose(bool disposing)
+    {
+        if (disposing)
+        {
+            InnerGenerator.Dispose();
+        }
+    }
+}

src/Libraries/Microsoft.Extensions.AI.Abstractions/Video/HostedVideoGenerationTool.cs

@@ -0,0 +1,45 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
+using Microsoft.Shared.DiagnosticIds;
+
+namespace Microsoft.Extensions.AI;
+
+/// <summary>Represents a hosted tool that can be specified to an AI service to enable it to perform video generation.</summary>
+/// <remarks>
+/// This tool does not itself implement video generation. It is a marker that can be used to inform a service
+/// that the service is allowed to perform video generation if the service is capable of doing so.
+/// </remarks>
+[Experimental(DiagnosticIds.Experiments.AIVideoGeneration, UrlFormat = DiagnosticIds.UrlFormat)]
+public class HostedVideoGenerationTool : AITool
+{
+    /// <summary>Any additional properties associated with the tool.</summary>
+    private IReadOnlyDictionary<string, object?>? _additionalProperties;
+
+    /// <summary>
+    /// Initializes a new instance of the <see cref="HostedVideoGenerationTool"/> class with the specified options.
+    /// </summary>
+    public HostedVideoGenerationTool()
+    {
+    }
+
+    /// <summary>Initializes a new instance of the <see cref="HostedVideoGenerationTool"/> class.</summary>
+    /// <param name="additionalProperties">Any additional properties associated with the tool.</param>
+    public HostedVideoGenerationTool(IReadOnlyDictionary<string, object?>? additionalProperties)
+    {
+        _additionalProperties = additionalProperties;
+    }
+
+    /// <inheritdoc />
+    public override string Name => "video_generation";
+
+    /// <inheritdoc />
+    public override IReadOnlyDictionary<string, object?> AdditionalProperties => _additionalProperties ?? base.AdditionalProperties;
+
+    /// <summary>
+    /// Gets or sets the options used to configure video generation.
+    /// </summary>
+    public VideoGenerationOptions? Options { get; set; }
+}

src/Libraries/Microsoft.Extensions.AI.Abstractions/Video/IVideoGenerator.cs

@@ -0,0 +1,39 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.Diagnostics.CodeAnalysis;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Shared.DiagnosticIds;
+
+namespace Microsoft.Extensions.AI;
+
+/// <summary>
+/// Represents a generator of videos.
+/// </summary>
+[Experimental(DiagnosticIds.Experiments.AIVideoGeneration, UrlFormat = DiagnosticIds.UrlFormat)]
+public interface IVideoGenerator : IDisposable
+{
+    /// <summary>
+    /// Sends a video generation request and returns the generated video as a <see cref="VideoGenerationResponse"/>.
+    /// </summary>
+    /// <param name="request">The video generation request containing the prompt and optional original videos for editing.</param>
+    /// <param name="options">The video generation options to configure the request.</param>
+    /// <param name="progress">An optional <see cref="IProgress{T}"/> to receive progress updates during the generation process.</param>
+    /// <param name="cancellationToken">The <see cref="CancellationToken"/> to monitor for cancellation requests. The default is <see cref="CancellationToken.None"/>.</param>
+    /// <exception cref="ArgumentNullException"><paramref name="request"/> is <see langword="null"/>.</exception>
+    /// <returns>The videos generated by the <see cref="VideoGenerationRequest"/>.</returns>
+    Task<VideoGenerationResponse> GenerateAsync(VideoGenerationRequest request, VideoGenerationOptions? options = null, IProgress<VideoGenerationProgress>? progress = null, CancellationToken cancellationToken = default);
+
+    /// <summary>Asks the <see cref="IVideoGenerator"/> for an object of the specified type <paramref name="serviceType"/>.</summary>
+    /// <param name="serviceType">The type of object being requested.</param>
+    /// <param name="serviceKey">An optional key that can be used to help identify the target service.</param>
+    /// <returns>The found object, otherwise <see langword="null"/>.</returns>
+    /// <exception cref="ArgumentNullException"><paramref name="serviceType"/> is <see langword="null"/>.</exception>
+    /// <remarks>
+    /// The purpose of this method is to allow for the retrieval of strongly typed services that might be provided by the <see cref="IVideoGenerator"/>,
+    /// including itself or any services it might be wrapping.
+    /// </remarks>
+    object? GetService(Type serviceType, object? serviceKey = null);
+}