video_core: Properly handle non RGBA8 shadow textures (#2047)

src/video_core/rasterizer_cache/rasterizer_cache.h

@@ -460,7 +460,8 @@ void RasterizerCache<T>::CopySurface(Surface& src_surface, Surface& dst_surface,
 
 template <class T>
 SurfaceId RasterizerCache<T>::GetSurface(const SurfaceParams& params, ScaleMatch match_res_scale,
-                                         bool load_if_create) {
+                                         bool load_if_create,
+                                         const SurfaceFlagBits& create_initial_flags) {
     if (params.addr == 0 || params.height * params.width == 0) {
         return {};
     }
@@ -472,7 +473,7 @@ SurfaceId RasterizerCache<T>::GetSurface(const SurfaceParams& params, ScaleMatch
     SurfaceId surface_id = FindMatch<MatchFlags::Exact>(params, match_res_scale);
 
     if (!surface_id) {
-        surface_id = CreateSurface(params);
+        surface_id = CreateSurface(params, create_initial_flags);
         RegisterSurface(surface_id);
     }
 
@@ -485,7 +486,8 @@ SurfaceId RasterizerCache<T>::GetSurface(const SurfaceParams& params, ScaleMatch
 
 template <class T>
 typename RasterizerCache<T>::SurfaceRect_Tuple RasterizerCache<T>::GetSurfaceSubRect(
-    const SurfaceParams& params, ScaleMatch match_res_scale, bool load_if_create) {
+    const SurfaceParams& params, ScaleMatch match_res_scale, bool load_if_create,
+    const SurfaceFlagBits& create_initial_flags) {
     if (params.addr == 0 || params.height * params.width == 0) {
         return std::make_pair(SurfaceId{}, Common::Rectangle<u32>{});
     }
@@ -501,7 +503,7 @@ typename RasterizerCache<T>::SurfaceRect_Tuple RasterizerCache<T>::GetSurfaceSub
             SurfaceParams new_params = slot_surfaces[surface_id];
             new_params.res_scale = params.res_scale;
 
-            surface_id = CreateSurface(new_params);
+            surface_id = CreateSurface(new_params, create_initial_flags);
             RegisterSurface(surface_id);
         }
     }
@@ -521,7 +523,7 @@ typename RasterizerCache<T>::SurfaceRect_Tuple RasterizerCache<T>::GetSurfaceSub
         new_params.width = aligned_params.stride;
         new_params.UpdateParams();
         // GetSurface will create the new surface and possibly adjust res_scale if necessary
-        surface_id = GetSurface(new_params, match_res_scale, load_if_create);
+        surface_id = GetSurface(new_params, match_res_scale, load_if_create, create_initial_flags);
     } else if (load_if_create) {
         ValidateSurface(surface_id, aligned_params.addr, aligned_params.size);
     }
@@ -560,6 +562,10 @@ SurfaceId RasterizerCache<T>::GetTextureSurface(const Pica::Texture::TextureInfo
     params.is_tiled = true;
     params.pixel_format = PixelFormatFromTextureFormat(info.format);
     params.res_scale = filter != Settings::TextureFilter::NoFilter ? resolution_scale_factor : 1;
+    SurfaceFlagBits initial_flags{};
+    if (info.is_shadow_source) {
+        initial_flags |= SurfaceFlagBits::ShadowSource;
+    }
     params.UpdateParams();
 
     const u32 min_width = info.width >> max_level;
@@ -570,11 +576,12 @@ SurfaceId RasterizerCache<T>::GetTextureSurface(const Pica::Texture::TextureInfo
                          min_height);
             return NULL_SURFACE_ID;
         }
-        const auto [src_surface_id, rect] = GetSurfaceSubRect(params, ScaleMatch::Ignore, true);
+        const auto [src_surface_id, rect] =
+            GetSurfaceSubRect(params, ScaleMatch::Ignore, true, initial_flags);
         Surface& src_surface = slot_surfaces[src_surface_id];
 
         params.res_scale = src_surface.res_scale;
-        SurfaceId tmp_surface_id = CreateSurface(params);
+        SurfaceId tmp_surface_id = CreateSurface(params, initial_flags);
         Surface& tmp_surface = slot_surfaces[tmp_surface_id];
         sentenced.emplace_back(tmp_surface_id, frame_tick);
 
@@ -593,7 +600,7 @@ SurfaceId RasterizerCache<T>::GetTextureSurface(const Pica::Texture::TextureInfo
         return NULL_SURFACE_ID;
     }
 
-    SurfaceId surface_id = GetSurface(params, ScaleMatch::Ignore, true);
+    SurfaceId surface_id = GetSurface(params, ScaleMatch::Ignore, true, initial_flags);
     return surface_id ? surface_id : NULL_SURFACE_ID;
 }
 
@@ -1026,7 +1033,7 @@ void RasterizerCache<T>::UploadSurface(Surface& surface, SurfaceInterval interva
 
     const auto upload_data = source_ptr.GetWriteBytes(load_info.end - load_info.addr);
     DecodeTexture(load_info, load_info.addr, load_info.end, upload_data, staging.mapped,
-                  runtime.NeedsConversion(surface.pixel_format));
+                  runtime.NeedsConversion(surface));
 
     const bool should_dump = False(surface.flags & SurfaceFlagBits::Custom) &&
                              False(surface.flags & SurfaceFlagBits::RenderTarget);
@@ -1135,7 +1142,7 @@ void RasterizerCache<T>::DownloadSurface(Surface& surface, SurfaceInterval inter
 
     const auto download_dest = dest_ptr.GetWriteBytes(flush_end - flush_start);
     EncodeTexture(flush_info, flush_start, flush_end, staging.mapped, download_dest,
-                  runtime.NeedsConversion(surface.pixel_format));
+                  runtime.NeedsConversion(surface));
 }
 
 template <class T>
@@ -1336,13 +1343,14 @@ void RasterizerCache<T>::InvalidateRegion(PAddr addr, u32 size, SurfaceId region
 }
 
 template <class T>
-SurfaceId RasterizerCache<T>::CreateSurface(const SurfaceParams& params) {
+SurfaceId RasterizerCache<T>::CreateSurface(const SurfaceParams& params,
+                                            const SurfaceFlagBits& initial_flags) {
     const SurfaceId surface_id = [&] {
         const auto it = std::find_if(sentenced.begin(), sentenced.end(), [&](const auto& pair) {
             return slot_surfaces[pair.first] == params;
         });
         if (it == sentenced.end()) {
-            return slot_surfaces.insert(runtime, params);
+            return slot_surfaces.insert(runtime, params, initial_flags);
         }
         const SurfaceId surface_id = it->first;
         sentenced.erase(it);

src/video_core/rasterizer_cache/rasterizer_cache_base.h

@@ -1,4 +1,4 @@
-// Copyright 2023 Citra Emulator Project
+// Copyright Citra Emulator Project / Azahar Emulator Project
 // Licensed under GPLv2 or any later version
 // Refer to the license.txt file included.
 
@@ -56,6 +56,7 @@ DECLARE_ENUM_FLAG_OPERATORS(MatchFlags);
 
 class CustomTexManager;
 class RendererBase;
+enum class SurfaceFlagBits : u32;
 
 template <class T>
 class RasterizerCache {
@@ -104,12 +105,13 @@ public:
 
     /// Load a texture from 3DS memory to OpenGL and cache it (if not already cached)
     SurfaceId GetSurface(const SurfaceParams& params, ScaleMatch match_res_scale,
-                         bool load_if_create);
+                         bool load_if_create, const SurfaceFlagBits& create_initial_flags = {});
 
     /// Attempt to find a subrect (resolution scaled) of a surface, otherwise loads a texture from
     /// 3DS memory to OpenGL and caches it (if not already cached)
     SurfaceRect_Tuple GetSurfaceSubRect(const SurfaceParams& params, ScaleMatch match_res_scale,
-                                        bool load_if_create);
+                                        bool load_if_create,
+                                        const SurfaceFlagBits& create_initial_flags = {});
 
     /// Get a surface based on the texture configuration
     Surface& GetTextureSurface(const Pica::TexturingRegs::FullTextureConfig& config);
@@ -194,7 +196,7 @@ private:
                                     const SurfaceInterval& interval);
 
     /// Create a new surface
-    SurfaceId CreateSurface(const SurfaceParams& params);
+    SurfaceId CreateSurface(const SurfaceParams& params, const SurfaceFlagBits& initial_flags = {});
 
     /// Register surface into the cache
     void RegisterSurface(SurfaceId surface);

src/video_core/rasterizer_cache/surface_base.cpp

@@ -1,4 +1,4 @@
-// Copyright 2023 Citra Emulator Project
+// Copyright Citra Emulator Project / Azahar Emulator Project
 // Licensed under GPLv2 or any later version
 // Refer to the license.txt file included.
 
@@ -9,7 +9,8 @@
 
 namespace VideoCore {
 
-SurfaceBase::SurfaceBase(const SurfaceParams& params) : SurfaceParams{params} {}
+SurfaceBase::SurfaceBase(const SurfaceParams& params, const SurfaceFlagBits& initial_flag_bits)
+    : SurfaceParams{params}, flags(initial_flag_bits) {}
 
 SurfaceBase::~SurfaceBase() = default;
 

src/video_core/rasterizer_cache/surface_base.h

@@ -1,4 +1,4 @@
-// Copyright 2023 Citra Emulator Project
+// Copyright Citra Emulator Project / Azahar Emulator Project
 // Licensed under GPLv2 or any later version
 // Refer to the license.txt file included.
 
@@ -19,14 +19,14 @@ enum class SurfaceFlagBits : u32 {
     Picked = 1 << 1,       ///< Surface has been picked when searching for a match.
     Tracked = 1 << 2,      ///< Surface is part of a texture cube and should be tracked.
     Custom = 1 << 3,       ///< Surface texture has been replaced with a custom texture.
-    ShadowMap = 1 << 4,    ///< Surface is used during shadow rendering.
+    ShadowSource = 1 << 4, ///< Surface is used as a shadow source.
     RenderTarget = 1 << 5, ///< Surface was a render target.
 };
 DECLARE_ENUM_FLAG_OPERATORS(SurfaceFlagBits);
 
 class SurfaceBase : public SurfaceParams {
 public:
-    SurfaceBase(const SurfaceParams& params);
+    SurfaceBase(const SurfaceParams& params, const SurfaceFlagBits& initial_flag_bits);
     ~SurfaceBase();
 
     /// Returns true when this surface can be used to fill the fill_interval of dest_surface
@@ -88,7 +88,7 @@ public:
     const Material* material = nullptr;
     SurfaceRegions invalid_regions;
     u32 fill_size = 0;
-    std::array<u8, 4> fill_data;
+    std::array<u8, 4> fill_data{};
     u64 modification_tick = 1;
 };
 

src/video_core/renderer_opengl/gl_rasterizer.cpp

@@ -676,7 +676,7 @@ void RasterizerOpenGL::SyncTextureUnits(const Framebuffer* framebuffer) {
             switch (texture.config.type.Value()) {
             case TextureType::Shadow2D: {
                 Surface& surface = res_cache.GetTextureSurface(texture);
-                surface.flags |= VideoCore::SurfaceFlagBits::ShadowMap;
+                surface.flags |= VideoCore::SurfaceFlagBits::ShadowSource;
                 state.image_shadow_texture_px = surface.Handle();
                 continue;
             }
@@ -724,7 +724,7 @@ void RasterizerOpenGL::BindShadowCube(const Pica::TexturingRegs::FullTextureConf
 
         VideoCore::SurfaceId surface_id = res_cache.GetTextureSurface(info);
         Surface& surface = res_cache.GetSurface(surface_id);
-        surface.flags |= VideoCore::SurfaceFlagBits::ShadowMap;
+        surface.flags |= VideoCore::SurfaceFlagBits::ShadowSource;
         state.image_shadow_texture[binding] = surface.Handle();
     }
 }

src/video_core/renderer_opengl/gl_texture_runtime.cpp

@@ -1,4 +1,4 @@
-// Copyright 2023 Citra Emulator Project
+// Copyright Citra Emulator Project / Azahar Emulator Project
 // Licensed under GPLv2 or any later version
 // Refer to the license.txt file included.
 
@@ -127,7 +127,8 @@ u32 TextureRuntime::RemoveThreshold() {
     return SWAP_CHAIN_SIZE;
 }
 
-bool TextureRuntime::NeedsConversion(VideoCore::PixelFormat pixel_format) const {
+bool TextureRuntime::NeedsConversion(const Surface& surface) const {
+    const auto& pixel_format = surface.pixel_format;
     const bool should_convert = pixel_format == PixelFormat::RGBA8 || // Needs byteswap
                                 pixel_format == PixelFormat::RGB8;    // Is converted to RGBA8
     return driver.IsOpenGLES() && should_convert;
@@ -290,7 +291,7 @@ bool TextureRuntime::BlitTextures(Surface& source, Surface& dest,
     // Note: shadow map is treated as RGBA8 format in PICA, as well as in the rasterizer cache, but
     // doing linear intepolation componentwise would cause incorrect value.
     const GLbitfield buffer_mask = MakeBufferMask(source.type);
-    const bool is_shadow_map = True(source.flags & SurfaceFlagBits::ShadowMap);
+    const bool is_shadow_map = True(source.flags & SurfaceFlagBits::ShadowSource);
     const GLenum filter =
         buffer_mask == GL_COLOR_BUFFER_BIT && !is_shadow_map ? GL_LINEAR : GL_NEAREST;
     glBlitFramebuffer(blit.src_rect.left, blit.src_rect.bottom, blit.src_rect.right,
@@ -316,8 +317,9 @@ void TextureRuntime::GenerateMipmaps(Surface& surface) {
     }
 }
 
-Surface::Surface(TextureRuntime& runtime_, const VideoCore::SurfaceParams& params)
+Surface::Surface(TextureRuntime& runtime_, const VideoCore::SurfaceParams& params,
-    : SurfaceBase{params}, driver{&runtime_.GetDriver()}, runtime{&runtime_},
+                 const VideoCore::SurfaceFlagBits& initial_flag_bits)
+    : SurfaceBase{params, initial_flag_bits}, driver{&runtime_.GetDriver()}, runtime{&runtime_},
       tuple{runtime->GetFormatTuple(pixel_format)} {
     if (pixel_format == PixelFormat::Invalid) {
         return;
@@ -334,9 +336,10 @@ Surface::Surface(TextureRuntime& runtime_, const VideoCore::SurfaceParams& param
     }
 }
 
-Surface::Surface(TextureRuntime& runtime, const VideoCore::SurfaceBase& surface,
+Surface::Surface(TextureRuntime& runtime_, const VideoCore::SurfaceBase& surface,
                  const VideoCore::Material* mat)
-    : SurfaceBase{surface}, tuple{runtime.GetFormatTuple(mat->format)} {
+    : SurfaceBase{surface, {}}, driver{&runtime_.GetDriver()}, runtime{&runtime_},
+      tuple{runtime_.GetFormatTuple(mat->format)} {
     if (mat && !driver->IsCustomFormatSupported(mat->format)) {
         return;
     }

src/video_core/renderer_opengl/gl_texture_runtime.h

@@ -1,4 +1,4 @@
-// Copyright 2023 Citra Emulator Project
+// Copyright Citra Emulator Project / Azahar Emulator Project
 // Licensed under GPLv2 or any later version
 // Refer to the license.txt file included.
 
@@ -49,7 +49,7 @@ public:
     void Finish() {}
 
     /// Returns true if the provided pixel format cannot be used natively by the runtime.
-    bool NeedsConversion(VideoCore::PixelFormat pixel_format) const;
+    bool NeedsConversion(const Surface& surface) const;
 
     /// Maps an internal staging buffer of the provided size of pixel uploads/downloads
     VideoCore::StagingData FindStaging(u32 size, bool upload);
@@ -97,7 +97,8 @@ private:
 
 class Surface : public VideoCore::SurfaceBase {
 public:
-    explicit Surface(TextureRuntime& runtime, const VideoCore::SurfaceParams& params);
+    explicit Surface(TextureRuntime& runtime, const VideoCore::SurfaceParams& params,
+                     const VideoCore::SurfaceFlagBits& initial_flag_bits = {});
     explicit Surface(TextureRuntime& runtime, const VideoCore::SurfaceBase& surface,
                      const VideoCore::Material* material);
     ~Surface();

src/video_core/renderer_vulkan/vk_rasterizer.cpp

@@ -648,7 +648,7 @@ void RasterizerVulkan::SyncTextureUnits(const Framebuffer* framebuffer) {
             case TextureType::Shadow2D: {
                 Surface& surface = res_cache.GetTextureSurface(texture);
                 Sampler& sampler = res_cache.GetSampler(texture.config);
-                surface.flags |= VideoCore::SurfaceFlagBits::ShadowMap;
+                surface.flags |= VideoCore::SurfaceFlagBits::ShadowSource;
                 update_queue.AddImageSampler(texture_set, texture_index, 0, surface.StorageView(),
                                              sampler.Handle());
                 continue;
@@ -704,7 +704,7 @@ void RasterizerVulkan::BindShadowCube(const Pica::TexturingRegs::FullTextureConf
 
         const VideoCore::SurfaceId surface_id = res_cache.GetTextureSurface(info);
         Surface& surface = res_cache.GetSurface(surface_id);
-        surface.flags |= VideoCore::SurfaceFlagBits::ShadowMap;
+        surface.flags |= VideoCore::SurfaceFlagBits::ShadowSource;
         update_queue.AddImageSampler(texture_set, 0, binding, surface.StorageView(),
                                      sampler.Handle());
     }

src/video_core/renderer_vulkan/vk_texture_runtime.cpp

@@ -720,15 +720,16 @@ void TextureRuntime::GenerateMipmaps(Surface& surface) {
     }
 }
 
-bool TextureRuntime::NeedsConversion(VideoCore::PixelFormat format) const {
+bool TextureRuntime::NeedsConversion(const Surface& surface) const {
-    const FormatTraits traits = instance.GetTraits(format);
+    const FormatTraits& traits = surface.traits;
     return traits.needs_conversion &&
            // DepthStencil formats are handled elsewhere due to de-interleaving.
            traits.aspect != (vk::ImageAspectFlagBits::eDepth | vk::ImageAspectFlagBits::eStencil);
 }
 
-Surface::Surface(TextureRuntime& runtime_, const VideoCore::SurfaceParams& params)
+Surface::Surface(TextureRuntime& runtime_, const VideoCore::SurfaceParams& params,
-    : SurfaceBase{params}, runtime{runtime_}, instance{runtime_.GetInstance()},
+                 const VideoCore::SurfaceFlagBits& initial_flag_bits)
+    : SurfaceBase{params, initial_flag_bits}, runtime{runtime_}, instance{runtime_.GetInstance()},
       scheduler{runtime_.GetScheduler()}, traits{instance.GetTraits(pixel_format)},
       handles{Handle(instance), Handle(instance), Handle(instance), Handle(instance)} {
 
@@ -736,7 +737,17 @@ Surface::Surface(TextureRuntime& runtime_, const VideoCore::SurfaceParams& param
         return;
     }
 
-    const bool is_mutable = pixel_format == VideoCore::PixelFormat::RGBA8;
+    bool is_mutable = traits.native == vk::Format::eR8G8B8A8Unorm;
+
+    if (True(flags & VideoCore::SurfaceFlagBits::ShadowSource) &&
+        traits.native != vk::Format::eR8G8B8A8Unorm) {
+        // If the surface is a shadow source, it needs conversion
+        // to be forced as it always has to be RGBA8
+        traits = instance.GetTraits(VideoCore::PixelFormat::RGBA8);
+        traits.needs_conversion = true;
+        is_mutable = true;
+    }
+
     const vk::Format format = traits.native;
 
     ASSERT_MSG(format != vk::Format::eUndefined && levels >= 1,
@@ -1278,7 +1289,7 @@ vk::ImageView Surface::ImageView(ViewType view_type, Type type) noexcept {
     auto aspect = traits.aspect;
 
     if (view_type == ViewType::Storage) {
-        ASSERT(pixel_format == PixelFormat::RGBA8);
+        ASSERT(traits.native == vk::Format::eR8G8B8A8Unorm);
         is_storage = true;
     }
     if (view_type == ViewType::Depth || view_type == ViewType::Stencil) {

src/video_core/renderer_vulkan/vk_texture_runtime.h

@@ -155,7 +155,7 @@ public:
     void GenerateMipmaps(Surface& surface);
 
     /// Returns true if the provided pixel format needs convertion
-    bool NeedsConversion(VideoCore::PixelFormat format) const;
+    bool NeedsConversion(const Surface& surface) const;
 
 private:
     /// Clears a partial texture rect using a clear rectangle
@@ -175,7 +175,8 @@ class Surface : public VideoCore::SurfaceBase {
     friend class TextureRuntime;
 
 public:
-    explicit Surface(TextureRuntime& runtime, const VideoCore::SurfaceParams& params);
+    explicit Surface(TextureRuntime& runtime, const VideoCore::SurfaceParams& params,
+                     const VideoCore::SurfaceFlagBits& initial_flag_bits = {});
     explicit Surface(TextureRuntime& runtime, const VideoCore::SurfaceBase& surface,
                      const VideoCore::Material* materal);
 

src/video_core/texture/texture_decode.cpp

@@ -1,4 +1,4 @@
-// Copyright 2017 Citra Emulator Project
+// Copyright Citra Emulator Project / Azahar Emulator Project
 // Licensed under GPLv2 or any later version
 // Refer to the license.txt file included.
 
@@ -219,6 +219,8 @@ TextureInfo TextureInfo::FromPicaRegister(const TexturingRegs::TextureConfig& co
     info.height = config.height;
     info.format = format;
     info.SetDefaultStride();
+    info.is_shadow_source = config.type == TexturingRegs::TextureConfig::TextureType::Shadow2D ||
+                            config.type == TexturingRegs::TextureConfig::TextureType::ShadowCube;
     return info;
 }
 

src/video_core/texture/texture_decode.h

@@ -1,4 +1,4 @@
-// Copyright 2017 Citra Emulator Project
+// Copyright Citra Emulator Project / Azahar Emulator Project
 // Licensed under GPLv2 or any later version
 // Refer to the license.txt file included.
 
@@ -19,6 +19,7 @@ struct TextureInfo {
     u32 height;
     ptrdiff_t stride;
     TexturingRegs::TextureFormat format;
+    bool is_shadow_source;
 
     static TextureInfo FromPicaRegister(const TexturingRegs::TextureConfig& config,
                                         const TexturingRegs::TextureFormat& format);