MirrorRepos/eden
1
0
Fork 0
mirror of https://git.eden-emu.dev/eden-emu/eden.git synced 2026-06-06 01:13:45 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

[video_core] use bool params for read/writes and cascade them thru the calltree

Browse source 
Signed-off-by: lizzie <lizzie@eden-emu.dev>
This commit is contained in:
lizzie 2026-05-22 20:03:50 +00:00
parent 9b18d0b111
commit 99efe2e727
2 changed files with 103 additions and 167 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

211
src/video_core/memory_manager.cpp
View file

@ -58,9 +58,8 @@ MemoryManager::MemoryManager(Core::System& system_, u64 address_space_bits_, GPU
MemoryManager::~MemoryManager() = default; MemoryManager::~MemoryManager() = default;
template <bool is_big_page> MemoryManager::EntryType MemoryManager::GetEntry(size_t position, bool is_big_page) const {
MemoryManager::EntryType MemoryManager::GetEntry(size_t position) const { if (is_big_page) {
if constexpr (is_big_page) {
position = position >> big_page_bits; position = position >> big_page_bits;
const u64 entry_mask = big_entries[position / 32]; const u64 entry_mask = big_entries[position / 32];
const size_t sub_index = position % 32; const size_t sub_index = position % 32;
@ -73,9 +72,8 @@ MemoryManager::EntryType MemoryManager::GetEntry(size_t position) const {
} }
} }
template <bool is_big_page> void MemoryManager::SetEntry(size_t position, MemoryManager::EntryType entry, bool is_big_page) {
void MemoryManager::SetEntry(size_t position, MemoryManager::EntryType entry) { if (is_big_page) {
if constexpr (is_big_page) {
position = position >> big_page_bits; position = position >> big_page_bits;
const u64 entry_mask = big_entries[position / 32]; const u64 entry_mask = big_entries[position / 32];
const size_t sub_index = position % 32; const size_t sub_index = position % 32;
@ -108,23 +106,21 @@ inline void MemoryManager::SetBigPageContinuous(size_t big_page_index, bool valu
(~(1ULL << sub_index) & continuous_mask) | (value ? 1ULL << sub_index : 0); (~(1ULL << sub_index) & continuous_mask) | (value ? 1ULL << sub_index : 0);
} }
template <MemoryManager::EntryType entry_type> GPUVAddr MemoryManager::PageTableOp(GPUVAddr gpu_addr, [[maybe_unused]] DAddr dev_addr, size_t size, PTEKind kind, MemoryManager::EntryType entry_type) {
GPUVAddr MemoryManager::PageTableOp(GPUVAddr gpu_addr, [[maybe_unused]] DAddr dev_addr, size_t size,
PTEKind kind) {
[[maybe_unused]] u64 remaining_size{size}; [[maybe_unused]] u64 remaining_size{size};
if constexpr (entry_type == EntryType::Mapped) { if (entry_type == EntryType::Mapped) {
page_table.ReserveRange(gpu_addr, size); page_table.ReserveRange(gpu_addr, size);
} }
for (u64 offset{}; offset < size; offset += page_size) { for (u64 offset{}; offset < size; offset += page_size) {
const GPUVAddr current_gpu_addr = gpu_addr + offset; const GPUVAddr current_gpu_addr = gpu_addr + offset;
[[maybe_unused]] const auto current_entry_type = GetEntry<false>(current_gpu_addr); [[maybe_unused]] const auto current_entry_type = GetEntry(current_gpu_addr, false);
SetEntry<false>(current_gpu_addr, entry_type); SetEntry(current_gpu_addr, entry_type, false);
if (current_entry_type != entry_type) { if (current_entry_type != entry_type) {
rasterizer->ModifyGPUMemory(unique_identifier, current_gpu_addr, page_size); rasterizer->ModifyGPUMemory(unique_identifier, current_gpu_addr, page_size);
} }
if constexpr (entry_type == EntryType::Mapped) { if (entry_type == EntryType::Mapped) {
const DAddr current_dev_addr = dev_addr + offset; const DAddr current_dev_addr = dev_addr + offset;
const auto index = PageEntryIndex<false>(current_gpu_addr); const auto index = PageEntryIndex(current_gpu_addr, false);
const u32 sub_value = static_cast<u32>(current_dev_addr >> cpu_page_bits); const u32 sub_value = static_cast<u32>(current_dev_addr >> cpu_page_bits);
page_table[index] = sub_value; page_table[index] = sub_value;
} }
@ -134,20 +130,18 @@ GPUVAddr MemoryManager::PageTableOp(GPUVAddr gpu_addr, [[maybe_unused]] DAddr de
return gpu_addr; return gpu_addr;
} }
template <MemoryManager::EntryType entry_type> GPUVAddr MemoryManager::BigPageTableOp(GPUVAddr gpu_addr, [[maybe_unused]] DAddr dev_addr, size_t size, PTEKind kind, MemoryManager::EntryType entry_type) {
GPUVAddr MemoryManager::BigPageTableOp(GPUVAddr gpu_addr, [[maybe_unused]] DAddr dev_addr,
size_t size, PTEKind kind) {
[[maybe_unused]] u64 remaining_size{size}; [[maybe_unused]] u64 remaining_size{size};
for (u64 offset{}; offset < size; offset += big_page_size) { for (u64 offset{}; offset < size; offset += big_page_size) {
const GPUVAddr current_gpu_addr = gpu_addr + offset; const GPUVAddr current_gpu_addr = gpu_addr + offset;
[[maybe_unused]] const auto current_entry_type = GetEntry<true>(current_gpu_addr); [[maybe_unused]] const auto current_entry_type = GetEntry(current_gpu_addr, true);
SetEntry<true>(current_gpu_addr, entry_type); SetEntry(current_gpu_addr, entry_type, true);
if (current_entry_type != entry_type) { if (current_entry_type != entry_type) {
rasterizer->ModifyGPUMemory(unique_identifier, current_gpu_addr, big_page_size); rasterizer->ModifyGPUMemory(unique_identifier, current_gpu_addr, big_page_size);
} }
if constexpr (entry_type == EntryType::Mapped) { if (entry_type == EntryType::Mapped) {
const DAddr current_dev_addr = dev_addr + offset; const DAddr current_dev_addr = dev_addr + offset;
const auto index = PageEntryIndex<true>(current_gpu_addr); const auto index = PageEntryIndex(current_gpu_addr, true);
const u32 sub_value = static_cast<u32>(current_dev_addr >> cpu_page_bits); const u32 sub_value = static_cast<u32>(current_dev_addr >> cpu_page_bits);
big_page_table_dev[index] = sub_value; big_page_table_dev[index] = sub_value;
const bool is_continuous = ([&] { const bool is_continuous = ([&] {
@ -181,19 +175,16 @@ void MemoryManager::BindRasterizer(VideoCore::RasterizerInterface* rasterizer_)
rasterizer = rasterizer_; rasterizer = rasterizer_;
} }
GPUVAddr MemoryManager::Map(GPUVAddr gpu_addr, DAddr dev_addr, std::size_t size, PTEKind kind, GPUVAddr MemoryManager::Map(GPUVAddr gpu_addr, DAddr dev_addr, std::size_t size, PTEKind kind, bool is_big_pages) {
bool is_big_pages) { if (is_big_pages)
if (is_big_pages) [[likely]] { return BigPageTableOp(gpu_addr, dev_addr, size, kind, EntryType::Mapped);
return BigPageTableOp<EntryType::Mapped>(gpu_addr, dev_addr, size, kind); return PageTableOp(gpu_addr, dev_addr, size, kind, EntryType::Mapped);
}
return PageTableOp<EntryType::Mapped>(gpu_addr, dev_addr, size, kind);
} }
GPUVAddr MemoryManager::MapSparse(GPUVAddr gpu_addr, std::size_t size, bool is_big_pages) { GPUVAddr MemoryManager::MapSparse(GPUVAddr gpu_addr, std::size_t size, bool is_big_pages) {
if (is_big_pages) [[likely]] { if (is_big_pages)
return BigPageTableOp<EntryType::Reserved>(gpu_addr, 0, size, PTEKind::INVALID); return BigPageTableOp(gpu_addr, 0, size, PTEKind::INVALID, EntryType::Reserved);
} return PageTableOp(gpu_addr, 0, size, PTEKind::INVALID, EntryType::Reserved);
return PageTableOp<EntryType::Reserved>(gpu_addr, 0, size, PTEKind::INVALID);
} }
void MemoryManager::Unmap(GPUVAddr gpu_addr, std::size_t size) { void MemoryManager::Unmap(GPUVAddr gpu_addr, std::size_t size) {
@ -207,26 +198,21 @@ void MemoryManager::Unmap(GPUVAddr gpu_addr, std::size_t size) {
} }
page_stash.clear(); page_stash.clear();
BigPageTableOp<EntryType::Free>(gpu_addr, 0, size, PTEKind::INVALID); BigPageTableOp(gpu_addr, 0, size, PTEKind::INVALID, EntryType::Free);
PageTableOp<EntryType::Free>(gpu_addr, 0, size, PTEKind::INVALID); PageTableOp(gpu_addr, 0, size, PTEKind::INVALID, EntryType::Free);
} }
std::optional<DAddr> MemoryManager::GpuToCpuAddress(GPUVAddr gpu_addr) const { std::optional<DAddr> MemoryManager::GpuToCpuAddress(GPUVAddr gpu_addr) const {
if (!IsWithinGPUAddressRange(gpu_addr)) [[unlikely]] { if (!IsWithinGPUAddressRange(gpu_addr)) [[unlikely]] {
return std::nullopt; return std::nullopt;
} }
if (GetEntry<true>(gpu_addr) != EntryType::Mapped) [[unlikely]] { if (GetEntry(gpu_addr, true) != EntryType::Mapped) [[unlikely]] {
if (GetEntry<false>(gpu_addr) != EntryType::Mapped) { if (GetEntry(gpu_addr, false) != EntryType::Mapped)
return std::nullopt; return std::nullopt;
} const DAddr dev_addr_base = DAddr(page_table[PageEntryIndex(gpu_addr, false)]) << cpu_page_bits;
const DAddr dev_addr_base = static_cast<DAddr>(page_table[PageEntryIndex<false>(gpu_addr)])
<< cpu_page_bits;
return dev_addr_base + (gpu_addr & page_mask); return dev_addr_base + (gpu_addr & page_mask);
} }
const DAddr dev_addr_base = DAddr(big_page_table_dev[PageEntryIndex(gpu_addr, true)]) << cpu_page_bits;
const DAddr dev_addr_base =
static_cast<DAddr>(big_page_table_dev[PageEntryIndex<true>(gpu_addr)]) << cpu_page_bits;
return dev_addr_base + (gpu_addr & big_page_mask); return dev_addr_base + (gpu_addr & big_page_mask);
} }
@ -299,10 +285,8 @@ const u8* MemoryManager::GetPointer(GPUVAddr gpu_addr) const {
#pragma inline_recursion(on) #pragma inline_recursion(on)
#endif #endif
template <bool is_big_pages, typename FuncMapped, typename FuncReserved, typename FuncUnmapped> template <typename FuncMapped, typename FuncReserved, typename FuncUnmapped>
inline void MemoryManager::MemoryOperation(GPUVAddr gpu_src_addr, std::size_t size, inline void MemoryManager::MemoryOperation(GPUVAddr gpu_src_addr, std::size_t size, bool is_big_page, FuncMapped&& func_mapped, FuncReserved&& func_reserved, FuncUnmapped&& func_unmapped) const {
FuncMapped&& func_mapped, FuncReserved&& func_reserved,
FuncUnmapped&& func_unmapped) const {
using FuncMappedReturn = using FuncMappedReturn =
typename std::invoke_result<FuncMapped, std::size_t, std::size_t, std::size_t>::type; typename std::invoke_result<FuncMapped, std::size_t, std::size_t, std::size_t>::type;
using FuncReservedReturn = using FuncReservedReturn =
@ -315,7 +299,7 @@ inline void MemoryManager::MemoryOperation(GPUVAddr gpu_src_addr, std::size_t si
u64 used_page_size; u64 used_page_size;
u64 used_page_mask; u64 used_page_mask;
u64 used_page_bits; u64 used_page_bits;
if constexpr (is_big_pages) { if (is_big_page) {
used_page_size = big_page_size; used_page_size = big_page_size;
used_page_mask = big_page_mask; used_page_mask = big_page_mask;
used_page_bits = big_page_bits; used_page_bits = big_page_bits;
@ -332,7 +316,7 @@ inline void MemoryManager::MemoryOperation(GPUVAddr gpu_src_addr, std::size_t si
while (remaining_size > 0) { while (remaining_size > 0) {
const std::size_t copy_amount{ const std::size_t copy_amount{
(std::min)(static_cast<std::size_t>(used_page_size) - page_offset, remaining_size)}; (std::min)(static_cast<std::size_t>(used_page_size) - page_offset, remaining_size)};
auto entry = GetEntry<is_big_pages>(current_address); auto entry = GetEntry(current_address, is_big_page);
if (entry == EntryType::Mapped) [[likely]] { if (entry == EntryType::Mapped) [[likely]] {
if constexpr (BOOL_BREAK_MAPPED) { if constexpr (BOOL_BREAK_MAPPED) {
if (func_mapped(page_index, page_offset, copy_amount)) { if (func_mapped(page_index, page_offset, copy_amount)) {
@ -367,18 +351,14 @@ inline void MemoryManager::MemoryOperation(GPUVAddr gpu_src_addr, std::size_t si
} }
} }
template <bool is_safe> void MemoryManager::ReadBlockImpl(GPUVAddr gpu_src_addr, void* dest_buffer, std::size_t size, [[maybe_unused]] VideoCommon::CacheType which, bool unsafe) const {
void MemoryManager::ReadBlockImpl(GPUVAddr gpu_src_addr, void* dest_buffer, std::size_t size, auto set_to_zero = [&]([[maybe_unused]] std::size_t page_index, [[maybe_unused]] std::size_t offset, std::size_t copy_amount) {
[[maybe_unused]] VideoCommon::CacheType which) const {
auto set_to_zero = [&]([[maybe_unused]] std::size_t page_index,
[[maybe_unused]] std::size_t offset, std::size_t copy_amount) {
std::memset(dest_buffer, 0, copy_amount); std::memset(dest_buffer, 0, copy_amount);
dest_buffer = static_cast<u8*>(dest_buffer) + copy_amount; dest_buffer = static_cast<u8*>(dest_buffer) + copy_amount;
}; };
auto mapped_normal = [&](std::size_t page_index, std::size_t offset, std::size_t copy_amount) { auto mapped_normal = [&](std::size_t page_index, std::size_t offset, std::size_t copy_amount) {
const DAddr dev_addr_base = const DAddr dev_addr_base = (DAddr(page_table[page_index]) << cpu_page_bits) + offset;
(static_cast<DAddr>(page_table[page_index]) << cpu_page_bits) + offset; if (!unsafe) {
if constexpr (is_safe) {
rasterizer->FlushRegion(dev_addr_base, copy_amount, which); rasterizer->FlushRegion(dev_addr_base, copy_amount, which);
} }
u8* physical = memory.GetPointer<u8>(dev_addr_base); u8* physical = memory.GetPointer<u8>(dev_addr_base);
@ -386,9 +366,8 @@ void MemoryManager::ReadBlockImpl(GPUVAddr gpu_src_addr, void* dest_buffer, std:
dest_buffer = static_cast<u8*>(dest_buffer) + copy_amount; dest_buffer = static_cast<u8*>(dest_buffer) + copy_amount;
}; };
auto mapped_big = [&](std::size_t page_index, std::size_t offset, std::size_t copy_amount) { auto mapped_big = [&](std::size_t page_index, std::size_t offset, std::size_t copy_amount) {
const DAddr dev_addr_base = const DAddr dev_addr_base = (DAddr(big_page_table_dev[page_index]) << cpu_page_bits) + offset;
(static_cast<DAddr>(big_page_table_dev[page_index]) << cpu_page_bits) + offset; if (!unsafe) {
if constexpr (is_safe) {
rasterizer->FlushRegion(dev_addr_base, copy_amount, which); rasterizer->FlushRegion(dev_addr_base, copy_amount, which);
} }
if (!IsBigPageContinuous(page_index)) [[unlikely]] { if (!IsBigPageContinuous(page_index)) [[unlikely]] {
@ -399,35 +378,28 @@ void MemoryManager::ReadBlockImpl(GPUVAddr gpu_src_addr, void* dest_buffer, std:
} }
dest_buffer = static_cast<u8*>(dest_buffer) + copy_amount; dest_buffer = static_cast<u8*>(dest_buffer) + copy_amount;
}; };
auto read_short_pages = [&](std::size_t page_index, std::size_t offset, auto read_short_pages = [&](std::size_t page_index, std::size_t offset, std::size_t copy_amount) {
std::size_t copy_amount) {
GPUVAddr base = (page_index << big_page_bits) + offset; GPUVAddr base = (page_index << big_page_bits) + offset;
MemoryOperation<false>(base, copy_amount, mapped_normal, set_to_zero, set_to_zero); MemoryOperation(base, copy_amount, false, mapped_normal, set_to_zero, set_to_zero);
}; };
MemoryOperation<true>(gpu_src_addr, size, mapped_big, set_to_zero, read_short_pages); MemoryOperation(gpu_src_addr, size, true, mapped_big, set_to_zero, read_short_pages);
} }
void MemoryManager::ReadBlock(GPUVAddr gpu_src_addr, void* dest_buffer, std::size_t size, void MemoryManager::ReadBlock(GPUVAddr gpu_src_addr, void* dest_buffer, std::size_t size, VideoCommon::CacheType which) const {
VideoCommon::CacheType which) const { ReadBlockImpl(gpu_src_addr, dest_buffer, size, which, false);
ReadBlockImpl<true>(gpu_src_addr, dest_buffer, size, which);
} }
void MemoryManager::ReadBlockUnsafe(GPUVAddr gpu_src_addr, void* dest_buffer, void MemoryManager::ReadBlockUnsafe(GPUVAddr gpu_src_addr, void* dest_buffer, const std::size_t size) const {
const std::size_t size) const { ReadBlockImpl(gpu_src_addr, dest_buffer, size, VideoCommon::CacheType::None, true);
ReadBlockImpl<false>(gpu_src_addr, dest_buffer, size, VideoCommon::CacheType::None);
} }
template <bool is_safe> void MemoryManager::WriteBlockImpl(GPUVAddr gpu_dest_addr, const void* src_buffer, std::size_t size, [[maybe_unused]] VideoCommon::CacheType which, bool unsafe) {
void MemoryManager::WriteBlockImpl(GPUVAddr gpu_dest_addr, const void* src_buffer, std::size_t size, auto just_advance = [&]([[maybe_unused]] std::size_t page_index, [[maybe_unused]] std::size_t offset, std::size_t copy_amount) {
[[maybe_unused]] VideoCommon::CacheType which) {
auto just_advance = [&]([[maybe_unused]] std::size_t page_index,
[[maybe_unused]] std::size_t offset, std::size_t copy_amount) {
src_buffer = static_cast<const u8*>(src_buffer) + copy_amount; src_buffer = static_cast<const u8*>(src_buffer) + copy_amount;
}; };
auto mapped_normal = [&](std::size_t page_index, std::size_t offset, std::size_t copy_amount) { auto mapped_normal = [&](std::size_t page_index, std::size_t offset, std::size_t copy_amount) {
const DAddr dev_addr_base = const DAddr dev_addr_base = (DAddr(page_table[page_index]) << cpu_page_bits) + offset;
(static_cast<DAddr>(page_table[page_index]) << cpu_page_bits) + offset; if (!unsafe) {
if constexpr (is_safe) {
rasterizer->InvalidateRegion(dev_addr_base, copy_amount, which); rasterizer->InvalidateRegion(dev_addr_base, copy_amount, which);
} }
u8* physical = memory.GetPointer<u8>(dev_addr_base); u8* physical = memory.GetPointer<u8>(dev_addr_base);
@ -435,9 +407,8 @@ void MemoryManager::WriteBlockImpl(GPUVAddr gpu_dest_addr, const void* src_buffe
src_buffer = static_cast<const u8*>(src_buffer) + copy_amount; src_buffer = static_cast<const u8*>(src_buffer) + copy_amount;
}; };
auto mapped_big = [&](std::size_t page_index, std::size_t offset, std::size_t copy_amount) { auto mapped_big = [&](std::size_t page_index, std::size_t offset, std::size_t copy_amount) {
const DAddr dev_addr_base = const DAddr dev_addr_base = (DAddr(big_page_table_dev[page_index]) << cpu_page_bits) + offset;
(static_cast<DAddr>(big_page_table_dev[page_index]) << cpu_page_bits) + offset; if (!unsafe) {
if constexpr (is_safe) {
rasterizer->InvalidateRegion(dev_addr_base, copy_amount, which); rasterizer->InvalidateRegion(dev_addr_base, copy_amount, which);
} }
if (!IsBigPageContinuous(page_index)) [[unlikely]] { if (!IsBigPageContinuous(page_index)) [[unlikely]] {
@ -448,26 +419,23 @@ void MemoryManager::WriteBlockImpl(GPUVAddr gpu_dest_addr, const void* src_buffe
} }
src_buffer = static_cast<const u8*>(src_buffer) + copy_amount; src_buffer = static_cast<const u8*>(src_buffer) + copy_amount;
}; };
auto write_short_pages = [&](std::size_t page_index, std::size_t offset, auto write_short_pages = [&](std::size_t page_index, std::size_t offset, std::size_t copy_amount) {
std::size_t copy_amount) {
GPUVAddr base = (page_index << big_page_bits) + offset; GPUVAddr base = (page_index << big_page_bits) + offset;
MemoryOperation<false>(base, copy_amount, mapped_normal, just_advance, just_advance); MemoryOperation(base, copy_amount, false, mapped_normal, just_advance, just_advance);
}; };
MemoryOperation<true>(gpu_dest_addr, size, mapped_big, just_advance, write_short_pages); MemoryOperation(gpu_dest_addr, size, true, mapped_big, just_advance, write_short_pages);
} }
void MemoryManager::WriteBlock(GPUVAddr gpu_dest_addr, const void* src_buffer, std::size_t size, void MemoryManager::WriteBlock(GPUVAddr gpu_dest_addr, const void* src_buffer, std::size_t size, VideoCommon::CacheType which) {
VideoCommon::CacheType which) { WriteBlockImpl(gpu_dest_addr, src_buffer, size, which, false);
WriteBlockImpl<true>(gpu_dest_addr, src_buffer, size, which);
} }
void MemoryManager::WriteBlockUnsafe(GPUVAddr gpu_dest_addr, const void* src_buffer, void MemoryManager::WriteBlockUnsafe(GPUVAddr gpu_dest_addr, const void* src_buffer, std::size_t size) {
std::size_t size) { WriteBlockImpl(gpu_dest_addr, src_buffer, size, VideoCommon::CacheType::None, true);
WriteBlockImpl<false>(gpu_dest_addr, src_buffer, size, VideoCommon::CacheType::None);
} }
void MemoryManager::WriteBlockCached(GPUVAddr gpu_dest_addr, const void* src_buffer, std::size_t size) { void MemoryManager::WriteBlockCached(GPUVAddr gpu_dest_addr, const void* src_buffer, std::size_t size) {
WriteBlockImpl<false>(gpu_dest_addr, src_buffer, size, VideoCommon::CacheType::None); WriteBlockImpl(gpu_dest_addr, src_buffer, size, VideoCommon::CacheType::None, true);
accumulator.Add(gpu_dest_addr, size); accumulator.Add(gpu_dest_addr, size);
} }
@ -478,21 +446,18 @@ void MemoryManager::FlushRegion(GPUVAddr gpu_addr, size_t size,
[[maybe_unused]] std::size_t copy_amount) {}; [[maybe_unused]] std::size_t copy_amount) {};
auto mapped_normal = [&](std::size_t page_index, std::size_t offset, std::size_t copy_amount) { auto mapped_normal = [&](std::size_t page_index, std::size_t offset, std::size_t copy_amount) {
const DAddr dev_addr_base = const DAddr dev_addr_base = (DAddr(page_table[page_index]) << cpu_page_bits) + offset;
(static_cast<DAddr>(page_table[page_index]) << cpu_page_bits) + offset;
rasterizer->FlushRegion(dev_addr_base, copy_amount, which); rasterizer->FlushRegion(dev_addr_base, copy_amount, which);
}; };
auto mapped_big = [&](std::size_t page_index, std::size_t offset, std::size_t copy_amount) { auto mapped_big = [&](std::size_t page_index, std::size_t offset, std::size_t copy_amount) {
const DAddr dev_addr_base = const DAddr dev_addr_base = (DAddr(big_page_table_dev[page_index]) << cpu_page_bits) + offset;
(static_cast<DAddr>(big_page_table_dev[page_index]) << cpu_page_bits) + offset;
rasterizer->FlushRegion(dev_addr_base, copy_amount, which); rasterizer->FlushRegion(dev_addr_base, copy_amount, which);
}; };
auto flush_short_pages = [&](std::size_t page_index, std::size_t offset, auto flush_short_pages = [&](std::size_t page_index, std::size_t offset, std::size_t copy_amount) {
std::size_t copy_amount) {
GPUVAddr base = (page_index << big_page_bits) + offset; GPUVAddr base = (page_index << big_page_bits) + offset;
MemoryOperation<false>(base, copy_amount, mapped_normal, do_nothing, do_nothing); MemoryOperation(base, copy_amount, false, mapped_normal, do_nothing, do_nothing);
}; };
MemoryOperation<true>(gpu_addr, size, mapped_big, do_nothing, flush_short_pages); MemoryOperation(gpu_addr, size, true, mapped_big, do_nothing, flush_short_pages);
} }
bool MemoryManager::IsMemoryDirty(GPUVAddr gpu_addr, size_t size, bool MemoryManager::IsMemoryDirty(GPUVAddr gpu_addr, size_t size,
@ -517,10 +482,10 @@ bool MemoryManager::IsMemoryDirty(GPUVAddr gpu_addr, size_t size,
auto check_short_pages = [&](std::size_t page_index, std::size_t offset, auto check_short_pages = [&](std::size_t page_index, std::size_t offset,
std::size_t copy_amount) { std::size_t copy_amount) {
GPUVAddr base = (page_index << big_page_bits) + offset; GPUVAddr base = (page_index << big_page_bits) + offset;
MemoryOperation<false>(base, copy_amount, mapped_normal, do_nothing, do_nothing); MemoryOperation(base, copy_amount, false, mapped_normal, do_nothing, do_nothing);
return result; return result;
}; };
MemoryOperation<true>(gpu_addr, size, mapped_big, do_nothing, check_short_pages); MemoryOperation(gpu_addr, size, true, mapped_big, do_nothing, check_short_pages);
return result; return result;
} }
@ -557,10 +522,10 @@ size_t MemoryManager::MaxContinuousRange(GPUVAddr gpu_addr, size_t size) const {
auto check_short_pages = [&](std::size_t page_index, std::size_t offset, auto check_short_pages = [&](std::size_t page_index, std::size_t offset,
std::size_t copy_amount) { std::size_t copy_amount) {
GPUVAddr base = (page_index << big_page_bits) + offset; GPUVAddr base = (page_index << big_page_bits) + offset;
MemoryOperation<false>(base, copy_amount, short_check, fail, fail); MemoryOperation(base, copy_amount, false, short_check, fail, fail);
return result; return result;
}; };
MemoryOperation<true>(gpu_addr, size, big_check, fail, check_short_pages); MemoryOperation(gpu_addr, size, true, big_check, fail, check_short_pages);
return range_so_far; return range_so_far;
} }
@ -576,21 +541,18 @@ void MemoryManager::InvalidateRegion(GPUVAddr gpu_addr, size_t size,
[[maybe_unused]] std::size_t copy_amount) {}; [[maybe_unused]] std::size_t copy_amount) {};
auto mapped_normal = [&](std::size_t page_index, std::size_t offset, std::size_t copy_amount) { auto mapped_normal = [&](std::size_t page_index, std::size_t offset, std::size_t copy_amount) {
const DAddr dev_addr_base = const DAddr dev_addr_base = (DAddr(page_table[page_index]) << cpu_page_bits) + offset;
(static_cast<DAddr>(page_table[page_index]) << cpu_page_bits) + offset;
rasterizer->InvalidateRegion(dev_addr_base, copy_amount, which); rasterizer->InvalidateRegion(dev_addr_base, copy_amount, which);
}; };
auto mapped_big = [&](std::size_t page_index, std::size_t offset, std::size_t copy_amount) { auto mapped_big = [&](std::size_t page_index, std::size_t offset, std::size_t copy_amount) {
const DAddr dev_addr_base = const DAddr dev_addr_base = (DAddr(big_page_table_dev[page_index]) << cpu_page_bits) + offset;
(static_cast<DAddr>(big_page_table_dev[page_index]) << cpu_page_bits) + offset;
rasterizer->InvalidateRegion(dev_addr_base, copy_amount, which); rasterizer->InvalidateRegion(dev_addr_base, copy_amount, which);
}; };
auto invalidate_short_pages = [&](std::size_t page_index, std::size_t offset, auto invalidate_short_pages = [&](std::size_t page_index, std::size_t offset, std::size_t copy_amount) {
std::size_t copy_amount) {
GPUVAddr base = (page_index << big_page_bits) + offset; GPUVAddr base = (page_index << big_page_bits) + offset;
MemoryOperation<false>(base, copy_amount, mapped_normal, do_nothing, do_nothing); MemoryOperation(base, copy_amount, false, mapped_normal, do_nothing, do_nothing);
}; };
MemoryOperation<true>(gpu_addr, size, mapped_big, do_nothing, invalidate_short_pages); MemoryOperation(gpu_addr, size, true, mapped_big, do_nothing, invalidate_short_pages);
} }
void MemoryManager::CopyBlock(GPUVAddr gpu_dest_addr, GPUVAddr gpu_src_addr, std::size_t size, void MemoryManager::CopyBlock(GPUVAddr gpu_dest_addr, GPUVAddr gpu_src_addr, std::size_t size,
@ -602,7 +564,7 @@ void MemoryManager::CopyBlock(GPUVAddr gpu_dest_addr, GPUVAddr gpu_src_addr, std
} }
bool MemoryManager::IsGranularRange(GPUVAddr gpu_addr, std::size_t size) const { bool MemoryManager::IsGranularRange(GPUVAddr gpu_addr, std::size_t size) const {
if (GetEntry<true>(gpu_addr) == EntryType::Mapped) [[likely]] { if (GetEntry(gpu_addr, true) == EntryType::Mapped) [[likely]] {
size_t page_index = gpu_addr >> big_page_bits; size_t page_index = gpu_addr >> big_page_bits;
if (IsBigPageContinuous(page_index)) [[likely]] { if (IsBigPageContinuous(page_index)) [[likely]] {
const std::size_t page{(page_index & big_page_mask) + size}; const std::size_t page{(page_index & big_page_mask) + size};
@ -611,7 +573,7 @@ bool MemoryManager::IsGranularRange(GPUVAddr gpu_addr, std::size_t size) const {
const std::size_t page{(gpu_addr & Core::DEVICE_PAGEMASK) + size}; const std::size_t page{(gpu_addr & Core::DEVICE_PAGEMASK) + size};
return page <= Core::DEVICE_PAGESIZE; return page <= Core::DEVICE_PAGESIZE;
} }
if (GetEntry<false>(gpu_addr) != EntryType::Mapped) { if (GetEntry(gpu_addr, false) != EntryType::Mapped) {
return false; return false;
} }
const std::size_t page{(gpu_addr & Core::DEVICE_PAGEMASK) + size}; const std::size_t page{(gpu_addr & Core::DEVICE_PAGEMASK) + size};
@ -649,10 +611,10 @@ bool MemoryManager::IsContinuousRange(GPUVAddr gpu_addr, std::size_t size) const
auto check_short_pages = [&](std::size_t page_index, std::size_t offset, auto check_short_pages = [&](std::size_t page_index, std::size_t offset,
std::size_t copy_amount) { std::size_t copy_amount) {
GPUVAddr base = (page_index << big_page_bits) + offset; GPUVAddr base = (page_index << big_page_bits) + offset;
MemoryOperation<false>(base, copy_amount, short_check, fail, fail); MemoryOperation(base, copy_amount, false, short_check, fail, fail);
return !result; return !result;
}; };
MemoryOperation<true>(gpu_addr, size, big_check, fail, check_short_pages); MemoryOperation(gpu_addr, size, true, big_check, fail, check_short_pages);
return result; return result;
} }
@ -665,13 +627,12 @@ bool MemoryManager::IsFullyMappedRange(GPUVAddr gpu_addr, std::size_t size) cons
}; };
auto pass = [&]([[maybe_unused]] std::size_t page_index, [[maybe_unused]] std::size_t offset, auto pass = [&]([[maybe_unused]] std::size_t page_index, [[maybe_unused]] std::size_t offset,
[[maybe_unused]] std::size_t copy_amount) { return false; }; [[maybe_unused]] std::size_t copy_amount) { return false; };
auto check_short_pages = [&](std::size_t page_index, std::size_t offset, auto check_short_pages = [&](std::size_t page_index, std::size_t offset, std::size_t copy_amount) {
std::size_t copy_amount) {
GPUVAddr base = (page_index << big_page_bits) + offset; GPUVAddr base = (page_index << big_page_bits) + offset;
MemoryOperation<false>(base, copy_amount, pass, pass, fail); MemoryOperation(base, copy_amount, false, pass, pass, fail);
return !result; return !result;
}; };
MemoryOperation<true>(gpu_addr, size, pass, fail, check_short_pages); MemoryOperation(gpu_addr, size, true, pass, fail, check_short_pages);
return result; return result;
} }
@ -683,13 +644,9 @@ MemoryManager::GetSubmappedRange(GPUVAddr gpu_addr, std::size_t size) const {
} }
template <bool is_gpu_address> template <bool is_gpu_address>
void MemoryManager::GetSubmappedRangeImpl( void MemoryManager::GetSubmappedRangeImpl(GPUVAddr gpu_addr, std::size_t size, boost::container::small_vector<std::pair<std::conditional_t<is_gpu_address, GPUVAddr, DAddr>, std::size_t>, 32>& result)
GPUVAddr gpu_addr, std::size_t size,
boost::container::small_vector<
std::pair<std::conditional_t<is_gpu_address, GPUVAddr, DAddr>, std::size_t>, 32>& result)
const { const {
std::optional<std::pair<std::conditional_t<is_gpu_address, GPUVAddr, DAddr>, std::size_t>> std::optional<std::pair<std::conditional_t<is_gpu_address, GPUVAddr, DAddr>, std::size_t>> last_segment{};
last_segment{};
std::optional<DAddr> old_page_addr{}; std::optional<DAddr> old_page_addr{};
const auto split = [&last_segment, &result]([[maybe_unused]] std::size_t page_index, const auto split = [&last_segment, &result]([[maybe_unused]] std::size_t page_index,
[[maybe_unused]] std::size_t offset, [[maybe_unused]] std::size_t offset,
@ -745,9 +702,9 @@ void MemoryManager::GetSubmappedRangeImpl(
}; };
auto do_short_pages = [&](std::size_t page_index, std::size_t offset, std::size_t copy_amount) { auto do_short_pages = [&](std::size_t page_index, std::size_t offset, std::size_t copy_amount) {
GPUVAddr base = (page_index << big_page_bits) + offset; GPUVAddr base = (page_index << big_page_bits) + offset;
MemoryOperation<false>(base, copy_amount, extend_size_short, split, split); MemoryOperation(base, copy_amount, false, extend_size_short, split, split);
}; };
MemoryOperation<true>(gpu_addr, size, extend_size_big, split, do_short_pages); MemoryOperation(gpu_addr, size, true, extend_size_big, split, do_short_pages);
split(0, 0, 0); split(0, 0, 0);
} }

59
src/video_core/memory_manager.h
View file

@ -45,7 +45,7 @@ public:
static constexpr bool HAS_FLUSH_INVALIDATION = true; static constexpr bool HAS_FLUSH_INVALIDATION = true;
size_t GetID() const { inline size_t GetID() const noexcept {
return unique_identifier; return unique_identifier;
} }
@ -66,16 +66,15 @@ public:
[[nodiscard]] const u8* GetPointer(GPUVAddr addr) const; [[nodiscard]] const u8* GetPointer(GPUVAddr addr) const;
template <typename T> template <typename T>
[[nodiscard]] T* GetPointer(GPUVAddr addr) { [[nodiscard]] inline T* GetPointer(GPUVAddr addr) noexcept {
const auto address{GpuToCpuAddress(addr)}; const auto address = GpuToCpuAddress(addr);
if (!address) { if (!address)
return {}; return {};
}
return memory.GetPointer<T>(*address); return memory.GetPointer<T>(*address);
} }
template <typename T> template <typename T>
[[nodiscard]] const T* GetPointer(GPUVAddr addr) const { [[nodiscard]] inline const T* GetPointer(GPUVAddr addr) const noexcept {
return GetPointer<T*>(addr); return GetPointer<T*>(addr);
} }
@ -85,12 +84,9 @@ public:
* in the Host Memory counterpart. Note: This functions cause Host GPU Memory * in the Host Memory counterpart. Note: This functions cause Host GPU Memory
* Flushes and Invalidations, respectively to each operation. * Flushes and Invalidations, respectively to each operation.
*/ */
void ReadBlock(GPUVAddr gpu_src_addr, void* dest_buffer, std::size_t size, void ReadBlock(GPUVAddr gpu_src_addr, void* dest_buffer, std::size_t size, VideoCommon::CacheType which = VideoCommon::CacheType::All) const;
VideoCommon::CacheType which = VideoCommon::CacheType::All) const; void WriteBlock(GPUVAddr gpu_dest_addr, const void* src_buffer, std::size_t size, VideoCommon::CacheType which = VideoCommon::CacheType::All);
void WriteBlock(GPUVAddr gpu_dest_addr, const void* src_buffer, std::size_t size, void CopyBlock(GPUVAddr gpu_dest_addr, GPUVAddr gpu_src_addr, std::size_t size, VideoCommon::CacheType which = VideoCommon::CacheType::All);
VideoCommon::CacheType which = VideoCommon::CacheType::All);
void CopyBlock(GPUVAddr gpu_dest_addr, GPUVAddr gpu_src_addr, std::size_t size,
VideoCommon::CacheType which = VideoCommon::CacheType::All);
/** /**
* ReadBlockUnsafe and WriteBlockUnsafe are special versions of ReadBlock and * ReadBlockUnsafe and WriteBlockUnsafe are special versions of ReadBlock and
@ -160,21 +156,14 @@ public:
u8* GetSpan(const GPUVAddr src_addr, const std::size_t size); u8* GetSpan(const GPUVAddr src_addr, const std::size_t size);
private: private:
template <bool is_big_pages, typename FuncMapped, typename FuncReserved, typename FuncUnmapped> template <typename FuncMapped, typename FuncReserved, typename FuncUnmapped>
inline void MemoryOperation(GPUVAddr gpu_src_addr, std::size_t size, FuncMapped&& func_mapped, inline void MemoryOperation(GPUVAddr gpu_src_addr, std::size_t size, bool is_big_page, FuncMapped&& func_mapped, FuncReserved&& func_reserved, FuncUnmapped&& func_unmapped) const;
FuncReserved&& func_reserved, FuncUnmapped&& func_unmapped) const;
template <bool is_safe> void ReadBlockImpl(GPUVAddr gpu_src_addr, void* dest_buffer, std::size_t size, VideoCommon::CacheType which, bool unsafe) const;
void ReadBlockImpl(GPUVAddr gpu_src_addr, void* dest_buffer, std::size_t size, void WriteBlockImpl(GPUVAddr gpu_dest_addr, const void* src_buffer, std::size_t size, VideoCommon::CacheType which, bool unsafe);
VideoCommon::CacheType which) const;
template <bool is_safe> [[nodiscard]] std::size_t PageEntryIndex(GPUVAddr gpu_addr, bool is_big_page) const {
void WriteBlockImpl(GPUVAddr gpu_dest_addr, const void* src_buffer, std::size_t size, if (is_big_page) {
VideoCommon::CacheType which);
template <bool is_big_page>
[[nodiscard]] std::size_t PageEntryIndex(GPUVAddr gpu_addr) const {
if constexpr (is_big_page) {
return (gpu_addr >> big_page_bits) & big_page_table_mask; return (gpu_addr >> big_page_bits) & big_page_table_mask;
} else { } else {
return (gpu_addr >> page_bits) & page_table_mask; return (gpu_addr >> page_bits) & page_table_mask;
@ -187,9 +176,7 @@ private:
template <bool is_gpu_address> template <bool is_gpu_address>
void GetSubmappedRangeImpl( void GetSubmappedRangeImpl(
GPUVAddr gpu_addr, std::size_t size, GPUVAddr gpu_addr, std::size_t size,
boost::container::small_vector< boost::container::small_vector<std::pair<std::conditional_t<is_gpu_address, GPUVAddr, DAddr>, std::size_t>, 32>& result) const;
std::pair<std::conditional_t<is_gpu_address, GPUVAddr, DAddr>, std::size_t>, 32>&
result) const;
Core::System& system; Core::System& system;
MaxwellDeviceMemoryManager& memory; MaxwellDeviceMemoryManager& memory;
@ -219,19 +206,11 @@ private:
std::vector<u64> entries; std::vector<u64> entries;
std::vector<u64> big_entries; std::vector<u64> big_entries;
template <EntryType entry_type> GPUVAddr PageTableOp(GPUVAddr gpu_addr, [[maybe_unused]] DAddr dev_addr, size_t size, PTEKind kind, EntryType entry_type);
GPUVAddr PageTableOp(GPUVAddr gpu_addr, [[maybe_unused]] DAddr dev_addr, size_t size, GPUVAddr BigPageTableOp(GPUVAddr gpu_addr, [[maybe_unused]] DAddr dev_addr, size_t size, PTEKind kind, EntryType entry_type);
PTEKind kind);
template <EntryType entry_type> inline EntryType GetEntry(size_t position, bool is_big_page) const;
GPUVAddr BigPageTableOp(GPUVAddr gpu_addr, [[maybe_unused]] DAddr dev_addr, size_t size, inline void SetEntry(size_t position, EntryType entry, bool is_big_page);
PTEKind kind);
template <bool is_big_page>
inline EntryType GetEntry(size_t position) const;
template <bool is_big_page>
inline void SetEntry(size_t position, EntryType entry);
Common::MultiLevelPageTable<u32> page_table; Common::MultiLevelPageTable<u32> page_table;
Common::RangeMap<GPUVAddr, PTEKind> kind_map; Common::RangeMap<GPUVAddr, PTEKind> kind_map;