- [Core] Added Shader decompilation/disassembly for DXCB program.

RazTools · Feb 6, 2024 · 6b31f63 · 6b31f63
1 parent 61ff13f
commit 6b31f63
Show file tree

Hide file tree

Showing 7 changed files with 97 additions and 12 deletions.
diff --git a/AssetStudio.GUI/AssetStudio.GUI.csproj b/AssetStudio.GUI/AssetStudio.GUI.csproj
@@ -89,6 +89,25 @@
 		</ContentWithTargetPath>
 	</ItemGroup>
 
+	<ItemGroup>
+		<ContentWithTargetPath Include="Libraries\x86\HLSLDecompiler.dll">
+			<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+			<TargetPath>x86\HLSLDecompiler.dll</TargetPath>
+		</ContentWithTargetPath>
+		<ContentWithTargetPath Include="Libraries\x64\HLSLDecompiler.dll">
+			<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+			<TargetPath>x64\HLSLDecompiler.dll</TargetPath>
+		</ContentWithTargetPath>
+		<ContentWithTargetPath Include="Libraries\x86\BinaryDecompiler.lib">
+			<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+			<TargetPath>x86\BinaryDecompiler.lib</TargetPath>
+		</ContentWithTargetPath>
+		<ContentWithTargetPath Include="Libraries\x64\BinaryDecompiler.lib">
+			<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+			<TargetPath>x64\BinaryDecompiler.lib</TargetPath>
+		</ContentWithTargetPath>
+	</ItemGroup>
+
 	<ItemGroup>
 		<PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
 		<PackageReference Include="OpenTK" Version="4.8.0" />

diff --git a/AssetStudio.GUI/Libraries/x64/BinaryDecompiler.lib b/AssetStudio.GUI/Libraries/x64/BinaryDecompiler.lib
diff --git a/AssetStudio.GUI/Libraries/x64/HLSLDecompiler.dll b/AssetStudio.GUI/Libraries/x64/HLSLDecompiler.dll
diff --git a/AssetStudio.GUI/Libraries/x86/BinaryDecompiler.lib b/AssetStudio.GUI/Libraries/x86/BinaryDecompiler.lib
diff --git a/AssetStudio.GUI/Libraries/x86/HLSLDecompiler.dll b/AssetStudio.GUI/Libraries/x86/HLSLDecompiler.dll
diff --git a/AssetStudio.Utility/AssetStudio.Utility.csproj b/AssetStudio.Utility/AssetStudio.Utility.csproj
@@ -15,6 +15,7 @@
     <PackageReference Include="Kyaru.Texture2DDecoder.Windows" Version="0.1.0" />
     <PackageReference Include="Mono.Cecil" Version="0.11.5" />
     <PackageReference Include="SixLabors.ImageSharp.Drawing" Version="1.0.0-beta15" />
+    <PackageReference Include="Vortice.D3DCompiler" Version="3.4.3-beta" />
   </ItemGroup>
 
   <ItemGroup>

diff --git a/AssetStudio.Utility/ShaderConverter.cs b/AssetStudio.Utility/ShaderConverter.cs
@@ -1,11 +1,15 @@
-using SpirV;
+using AssetStudio.PInvoke;
+using SharpGen.Runtime;
+using SpirV;
 using System;
 using System.Collections.Generic;
 using System.Globalization;
 using System.IO;
 using System.Linq;
+using System.Runtime.InteropServices;
 using System.Text;
 using System.Text.RegularExpressions;
+using Vortice.D3DCompiler;
 
 namespace AssetStudio
 {
@@ -1038,9 +1042,17 @@ public string Export()
                     case ShaderGpuProgramType.DX9PixelSM20:
                     case ShaderGpuProgramType.DX9PixelSM30:
                         {
-                            /*var shaderBytecode = new ShaderBytecode(m_ProgramCode);
-                            sb.Append(shaderBytecode.Disassemble());*/
-                            sb.Append("// shader disassembly not supported on DXBC");
+                            try
+                            {
+                                var programCodeSpan = m_ProgramCode.AsSpan();
+                                var g = Compiler.Disassemble(programCodeSpan.GetPinnableReference(), programCodeSpan.Length, DisasmFlags.None, "");
+                                sb.Append(g.AsString());
+                            }
+                            catch (Exception e)
+                            {
+                                sb.Append($"// disassembly error {e.Message}\n");
+                            }
+
                             break;
                         }
                     case ShaderGpuProgramType.DX10Level9Vertex:
@@ -1054,16 +1066,42 @@ public string Export()
                     case ShaderGpuProgramType.DX11HullSM50:
                     case ShaderGpuProgramType.DX11DomainSM50:
                         {
-                            /*int start = 6;
-                            if (m_Version == 201509030) // 5.3
+                            int type = m_ProgramCode[0];
+                            int start = 1;
+                            if (type > 0)
                             {
-                                start = 5;
+                                if (type == 1)
+                                {
+                                    start = 6;
+                                }
+                                else if (type == 2)
+                                {
+                                    start = 38;
+                                }
+                            }
+
+                            var buffSpan = m_ProgramCode.AsSpan(start);
+
+                            try
+                            {
+                                HLSLDecompiler.DecompileShader(buffSpan.ToArray(), buffSpan.Length, out var hlslText);
+                                sb.Append(hlslText);
+                            }
+                            catch (Exception e)
+                            {
+                                Logger.Verbose($"Decompile error {e.Message}");
+                                Logger.Verbose($"Attempting to disassemble...");
+
+                                try
+                                {
+                                    var g = Compiler.Disassemble(buffSpan.GetPinnableReference(), buffSpan.Length, DisasmFlags.None, "");
+                                    sb.Append(g.AsString());
+                                }
+                                catch (Exception ex)
+                                {
+                                    sb.Append($"// decompile/disassembly error {ex.Message}\n");
+                                }
                             }
-                            var buff = new byte[m_ProgramCode.Length - start];
-                            Buffer.BlockCopy(m_ProgramCode, start, buff, 0, buff.Length);
-                            var shaderBytecode = new ShaderBytecode(buff);
-                            sb.Append(shaderBytecode.Disassemble());*/
-                            sb.Append("// shader disassembly not supported on DXBC");
                             break;
                         }
                     case ShaderGpuProgramType.MetalVS:
@@ -1107,4 +1145,31 @@ public string Export()
             return sb.ToString();
         }
     }
+
+    public static class HLSLDecompiler
+    {
+        private const string DLL_NAME = "HLSLDecompiler";
+        static HLSLDecompiler()
+        {
+            DllLoader.PreloadDll(DLL_NAME);
+        }
+        public static void DecompileShader(byte[] shaderByteCode, int shaderByteCodeSize, out string hlslText)
+        {
+            var code = Decompile(shaderByteCode, shaderByteCodeSize, out var shaderText, out var shaderTextSize);
+            if (code != 0)
+            {
+                throw new Exception($"Unable to decompile shader, Error code: {code}");
+            }
+
+            hlslText = Marshal.PtrToStringAnsi(shaderText, shaderTextSize);
+            Marshal.FreeHGlobal(shaderText);
+        }
+
+        #region importfunctions
+
+        [DllImport(DLL_NAME, CallingConvention = CallingConvention.Cdecl)]
+        private static extern int Decompile(byte[] shaderByteCode, int shaderByteCodeSize, out IntPtr shaderText, out int shaderTextSize);
+
+        #endregion
+    }
 }