StableDiffusionCpp Backend Host

Apps/Amuse.App/Backend/StableDiffusionCppClient.cs

@@ -0,0 +1,45 @@
+using Amuse.App.Services;
+using Amuse.Common;
+using Amuse.Common.Config;
+using Microsoft.Extensions.Logging;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace Amuse.App.Runtime
+{
+    /// <summary>
+    /// PipelineClient implemntation for Amuse.Host.StableDiffusionCpp
+    /// Implements the <see cref="Amuse.App.Runtime.BackendClient" />
+    /// </summary>
+    /// <seealso cref="Amuse.App.Runtime.BackendClient" />
+    public sealed class StableDiffusionCppClient : BackendClient
+    {
+        /// <summary>
+        /// Initializes a new instance of the <see cref="StableDiffusionCppClient"/> class.
+        /// </summary>
+        /// <param name="settings">The settings.</param>
+        /// <param name="mediaService">The media service.</param>
+        /// <param name="logger">The logger.</param>
+        public StableDiffusionCppClient(Settings settings, IMediaService mediaService, ILogger logger)
+            : base(settings, mediaService, logger) { }
+
+
+        /// <summary>
+        /// Create PipelineClient targeting Amuse.Host.StableDiffusionCpp.
+        /// </summary>
+        /// <param name="cancellationToken">The cancellation token.</param>
+        /// <exception cref="System.OperationCanceledException"></exception>
+        protected override async Task<PipelineClient> CreatePipelineClientAsync(CancellationToken cancellationToken = default)
+        {
+            var createOptions = new PipelineCreateOptions();
+            var clientConfig = new ClientConfig
+            {
+                ServerPath = App.DirectoryServer,
+                ServerType = ServerType.StableDiffusionCpp,
+                IsDebugMode = Settings.IsServerDebugEnabled,
+            };
+            return await CreatePipelineClientAsync(clientConfig, createOptions, cancellationToken);
+        }
+
+    }
+}

Apps/Amuse.App/Services/GenerateService.cs

@@ -112,6 +112,7 @@ public async Task LoadAsync(PipelineModel pipeline, IProgress<PipelineProgress>
                 {
                     BackendType.PyTorch => new PyTorchBackendClient(_settings, _mediaService, _environmentService, _logger),
                     BackendType.OnnxRuntime => new OnnxBackendClient(_settings, _mediaService, _logger),
+                    BackendType.StableDiffusionCpp => new StableDiffusionCppClient(_settings, _mediaService, _logger),
                     _ => throw new NotImplementedException()
                 };
 

Apps/Amuse.Common/Config/ServerConfig.cs

@@ -45,6 +45,17 @@ public static ServerConfig GetConfig(ServerType serverType, string directoryBase
                     ChannelPipeName = "AmusePyTorch.PipeName",
                     ChannelProgress = "AmusePyTorch.Progress"
                 }
+            },
+            {
+                ServerType.StableDiffusionCpp,  new ServerConfig
+                {
+                    Name = "AmuseStableDiffusionCpp",
+                    Arguments = [nameof(ServerType.StableDiffusionCpp)],
+                    Executable = "AmuseHost.StableDiffusionCpp.exe",
+                    ChannelCommand = "AmuseStableDiffusionCpp.Command",
+                    ChannelPipeName = "AmuseStableDiffusionCpp.PipeName",
+                    ChannelProgress = "AmuseStableDiffusionCpp.Progress"
+                }
             }
         };
     }

Apps/Amuse.Common/Enums.cs

@@ -5,7 +5,8 @@ namespace Amuse.Common
     public enum ServerType
     {
         OnnxRuntime = 0,
-        PyTorch = 10
+        PyTorch = 10,
+        StableDiffusionCpp = 20
     }
 
     public enum ProcessType

Apps/Amuse.Host.StableDiffusionCpp/Amuse.Host.StableDiffusionCpp.csproj

@@ -0,0 +1,45 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <AssemblyName>AmuseHost.StableDiffusionCpp</AssemblyName>
+    <TargetFramework>net10.0-windows10.0.17763.0</TargetFramework>
+    <PlatformTarget>x64</PlatformTarget>
+    <ApplicationIcon>Icon.ico</ApplicationIcon>
+    <Configurations>Debug;Release;Release_Installer</Configurations>
+    <ErrorOnDuplicatePublishOutputFiles>false</ErrorOnDuplicatePublishOutputFiles>
+    <BaseOutputPath>..\Amuse.App\bin</BaseOutputPath>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <Compile Remove="bin\**" />
+    <EmbeddedResource Remove="bin\**" />
+    <None Remove="bin\**" />
+  </ItemGroup>
+
+
+  <!--Projects-->
+  <ItemGroup>
+    <ProjectReference Include="..\Amuse.Common\Amuse.Common.csproj" />
+  </ItemGroup>
+
+  <!--Common-->
+  <ItemGroup>
+    <PackageReference Include="Serilog.Extensions.Logging" Version="10.0.0" />
+    <PackageReference Include="Serilog.Sinks.File" Version="7.0.0" />
+  </ItemGroup>
+
+  <!--Debug-->
+  <ItemGroup Condition="'$(Configuration)' == 'Debug'">
+  </ItemGroup>
+
+  <!--Release-->
+  <ItemGroup Condition="'$(Configuration)' == 'Release' OR '$(Configuration)' == 'Release_Installer'">
+  </ItemGroup>
+
+  <!--Images-->
+  <ItemGroup>
+    <Resource Include="Icon.ico" />
+  </ItemGroup>
+
+</Project>

Apps/Amuse.Host.StableDiffusionCpp/Common/BackendType.cs

@@ -0,0 +1,23 @@
+using System.ComponentModel.DataAnnotations;
+
+namespace Amuse.Host.StableDiffusionCpp.Common
+{
+    public enum BackendType
+    {
+        [Display(Name = "", ShortName = "cpu")]
+        CPU = 0,
+
+        [Display(Name = "", ShortName = "cuda")]
+        CUDA = 1,
+
+        [Display(Name = "", ShortName = "vulkan")]
+        Vulkan = 2,
+
+        [Display(Name = "", ShortName = "metal")]
+        Metal = 3,
+
+        [Display(Name = "", ShortName = "rocm")]
+        ROCM = 4
+    }
+
+}

Apps/Amuse.Host.StableDiffusionCpp/Common/CapabilitiesModel.cs

@@ -0,0 +1,19 @@
+using System.Text.Json.Serialization;
+
+namespace Amuse.Host.StableDiffusionCpp.Common
+{
+    public record CapabilitiesModel
+    {
+        [JsonPropertyName("samplers")]
+        public string[] Samplers { get; set; }
+
+
+        [JsonPropertyName("schedulers")]
+        public string[] Schedulers { get; set; }
+
+
+        [JsonPropertyName("defaults_by_mode")]
+        public DefaultParams DefaultParams { get; set; }
+    }
+
+}

Apps/Amuse.Host.StableDiffusionCpp/Common/DefaultParams.cs

@@ -0,0 +1,14 @@
+using System.Text.Json.Serialization;
+
+namespace Amuse.Host.StableDiffusionCpp.Common
+{
+    public record DefaultParams
+    {
+        [JsonPropertyName("img_gen")]
+        public ImageParams ImageParams { get; set; }
+
+        [JsonPropertyName("vid_gen")]
+        public VideoParams VideoParams { get; set; }
+    }
+
+}

Apps/Amuse.Host.StableDiffusionCpp/Common/GuidanceParams.cs

@@ -0,0 +1,15 @@
+using System.Text.Json.Serialization;
+namespace Amuse.Host.StableDiffusionCpp.Common
+{
+    public record GuidanceParams
+    {
+        [JsonPropertyName("txt_cfg")]
+        public float? TxtCfg { get; set; }
+
+        [JsonPropertyName("img_cfg")]
+        public float? ImgCfg { get; set; }
+
+        [JsonPropertyName("distilled_guidance")]
+        public float? DistilledGuidance { get; set; }
+    }
+}

Apps/Amuse.Host.StableDiffusionCpp/Common/ImageParams.cs

@@ -0,0 +1,71 @@
+using System.Collections.Generic;
+using System.Text.Json.Serialization;
+
+namespace Amuse.Host.StableDiffusionCpp.Common
+{
+    public record ImageParams
+    {
+        [JsonPropertyName("prompt")]
+        public string Prompt { get; set; }
+
+        [JsonPropertyName("negative_prompt")]
+        public string NegativePrompt { get; set; }
+
+        [JsonPropertyName("width")]
+        public int Width { get; set; }
+
+        [JsonPropertyName("height")]
+        public int Height { get; set; }
+
+        [JsonPropertyName("seed")]
+        public int Seed { get; set; }
+
+        [JsonPropertyName("strength")]
+        public double Strength { get; set; }
+
+        [JsonPropertyName("batch_count")]
+        public int BatchCount { get;} = 1;
+
+        [JsonPropertyName("clip_skip")]
+        public int ClipSkip { get; set; } = -1;
+
+        [JsonPropertyName("control_strength")]
+        public double ControlStrength { get; set; }
+
+        [JsonPropertyName("embed_image_metadata")]
+        public bool EmbedImageMetadata { get; set; }
+
+        [JsonPropertyName("init_image")]
+        public string InitImage { get; set; }
+
+        [JsonPropertyName("ref_images")]
+        public List<string> RefImages { get; set; } = [];
+
+        [JsonPropertyName("mask_image")]
+        public string MaskImage { get; set; }
+
+        [JsonPropertyName("control_image")]
+        public string ControlImage { get; set; }
+
+        [JsonPropertyName("sample_params")]
+        public SampleParams SampleParams { get; set; }
+
+        [JsonPropertyName("lora")]
+        public List<LoraParams> Lora { get; set; } = [];
+
+        [JsonPropertyName("vae_tiling_params")]
+        public VaeTilingParams VaeTilingParams { get; set; }
+
+        [JsonPropertyName("output_format")]
+        public string OutputFormat { get; set; } = "png";
+
+        [JsonPropertyName("output_compression")]
+        public int OutputCompression { get; set; } = 100;
+
+        [JsonPropertyName("auto_resize_ref_image")]
+        public bool AutoResizeRefImage { get; set; }
+
+        [JsonPropertyName("increase_ref_index")]
+        public bool IncreaseRefIndex { get; set; }
+    }
+}

Apps/Amuse.Host.StableDiffusionCpp/Common/ImageResult.cs

@@ -0,0 +1,14 @@
+using System.Text.Json.Serialization;
+
+namespace Amuse.Host.StableDiffusionCpp.Common
+{
+    public sealed class ImageResult
+    {
+        [JsonPropertyName("index")]
+        public int Index { get; set; }
+
+        [JsonPropertyName("b64_json")]
+        public string B64Json { get; set; }
+    }
+
+}

Apps/Amuse.Host.StableDiffusionCpp/Common/JobModel.cs

@@ -0,0 +1,32 @@
+using System.Text.Json.Serialization;
+
+namespace Amuse.Host.StableDiffusionCpp.Common
+{
+    public sealed class JobModel
+    {
+        [JsonPropertyName("id")]
+        public string Id { get; set; }
+
+        [JsonPropertyName("kind")]
+        public string Kind { get; set; }
+
+        [JsonPropertyName("status")]
+        public JobStatus Status { get; set; }
+
+        [JsonPropertyName("created")]
+        public long Created { get; set; }
+
+        [JsonPropertyName("started")]
+        public long Started { get; set; }
+
+        [JsonPropertyName("completed")]
+        public long? Completed { get; set; }
+
+        [JsonPropertyName("queue_position")]
+        public int QueuePosition { get; set; }
+
+        [JsonPropertyName("result")]
+        public JobResult Result { get; set; }
+    }
+
+}

Apps/Amuse.Host.StableDiffusionCpp/Common/JobResult.cs

@@ -0,0 +1,47 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text.Json.Serialization;
+
+namespace Amuse.Host.StableDiffusionCpp.Common
+{
+    public sealed class JobResult
+    {
+        [JsonPropertyName("output_format")]
+        public string OutputFormat { get; set; }
+
+        [JsonPropertyName("mime_type")]
+        public string MimeType { get; set; }
+
+        [JsonPropertyName("fps")]
+        public int FrameRate { get; set; }
+
+        [JsonPropertyName("frame_count")]
+        public int FrameCount { get; set; }
+
+        [JsonPropertyName("images")]
+        public List<ImageResult> Images { get; set; } = [];
+
+        [JsonPropertyName("b64_json")]
+        public string Video { get; set; }
+
+        public byte[] GetImageBytes(int index = 0)
+        {
+            var image = Images.ElementAtOrDefault(index);
+            if (image == null)
+                return null;
+
+            return Convert.FromBase64String(image.B64Json);
+        }
+
+
+        public byte[] GetVideoBytes()
+        {
+            if (string.IsNullOrEmpty(Video))
+                return null;
+
+            return Convert.FromBase64String(Video);
+        }
+    }
+
+}

Apps/Amuse.Host.StableDiffusionCpp/Common/JobStatus.cs

@@ -0,0 +1,23 @@
+using System.Text.Json.Serialization;
+
+namespace Amuse.Host.StableDiffusionCpp.Common
+{
+    public enum JobStatus
+    {
+        [JsonStringEnumMemberName("queued")]
+        Queued = 0,
+
+        [JsonStringEnumMemberName("generating")]
+        Generating = 1,
+
+        [JsonStringEnumMemberName("completed")]
+        Completed = 2,
+
+        [JsonStringEnumMemberName("failed")]
+        Failed = 3,
+
+        [JsonStringEnumMemberName("cancelled")]
+        Cancelled = 4
+    }
+
+}