perf(agc): move storage texels between the image and its tiled bytes in one pass
A storage write-back copied the image into a linear buffer and retiled that buffer; an upload detiled into a linear buffer and copied it into the image. With storage image access without a format, the retile now reads each texel from the image through an unsigned integer view of the element size, and the detile writes it there, so the linear buffer, its copy and one barrier per move go away. The texels move bit for bit as before, through the same tiling equations and windows. This covers the block-unit windows of write-backs and uploads and the whole-surface direct upload, for 2D, array and cube images. agc_driver_recorder_tests singlePassTests: a first upload, a write-back of new texels in every mip and a second upload over the stored memory leave the images and guest memory byte for byte equal to the copies through a linear buffer, in plain and write-watched memory.
This commit is contained in:
Dean Galvin
parent
a29ef59219
commit
25738ff67f
|
|
@ -45,6 +45,17 @@ add_custom_command(
|
|||
DEPENDS "${agcTextureDetileShaderSource}" glslang-standalone
|
||||
VERBATIM
|
||||
)
|
||||
set(agcTextureDetileImageShaderSpv ${agcTextureDetileGeneratedDir}/TextureDetileImage.spv)
|
||||
set(agcTextureDetileImageShaderHeader ${agcTextureDetileGeneratedDir}/TextureDetileImage_spv.h)
|
||||
add_custom_command(
|
||||
OUTPUT "${agcTextureDetileImageShaderHeader}"
|
||||
COMMAND ${CMAKE_COMMAND} -E make_directory "${agcTextureDetileGeneratedDir}"
|
||||
COMMAND "$<TARGET_FILE:glslang-standalone>" -V --target-env vulkan1.1 -DLINEAR_IMAGE=1 -o "${agcTextureDetileImageShaderSpv}" "${agcTextureDetileShaderSource}"
|
||||
COMMAND ${CMAKE_COMMAND} -DINPUT=${agcTextureDetileImageShaderSpv} -DOUTPUT=${agcTextureDetileImageShaderHeader}
|
||||
-DSYMBOL=TEXTURE_DETILE_IMAGE_SPV -P "${CMAKE_CURRENT_SOURCE_DIR}/embed_spirv.cmake"
|
||||
DEPENDS "${agcTextureDetileShaderSource}" glslang-standalone
|
||||
VERBATIM
|
||||
)
|
||||
|
||||
set(agcMeshArgumentsShaderSource ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/shaders/MeshArguments.comp)
|
||||
set(agcMeshArgumentsShaderSpv ${agcTextureDetileGeneratedDir}/MeshArguments.spv)
|
||||
|
|
@ -188,6 +199,7 @@ set(agcDriverSources
|
|||
${agcTextureDetileShaderHeader}
|
||||
${agcCmaskClearShaderHeader}
|
||||
${agcMeshArgumentsShaderHeader}
|
||||
${agcTextureDetileImageShaderHeader}
|
||||
${agcSampleCounterShaderHeader}
|
||||
${agcSampleDumpsShaderHeader}
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/Eq/src/Event.cpp
|
||||
|
|
@ -756,6 +768,7 @@ add_test_executable(agc_driver_graphics_tests EXCLUDE_FROM_ALL
|
|||
${agcTextureDetileShaderHeader}
|
||||
${agcCmaskClearShaderHeader}
|
||||
${agcMeshArgumentsShaderHeader}
|
||||
${agcTextureDetileImageShaderHeader}
|
||||
${agcSampleCounterShaderHeader}
|
||||
${agcSampleDumpsShaderHeader}
|
||||
Execution/src/GuestMemory.cpp
|
||||
|
|
|
|||
|
|
@ -227,6 +227,7 @@ struct VulkanDevice::State {
|
|||
bool pipelineExecutableInfo = false;
|
||||
bool maintenance8 = false;
|
||||
std::uint32_t srgbDecodeFormats = 0;
|
||||
bool storageImageWithoutFormat = false;
|
||||
bool depthClamp = false;
|
||||
bool depthBounds = false;
|
||||
bool depthBiasClamp = false;
|
||||
|
|
@ -1117,6 +1118,7 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window) : state(std::make_u
|
|||
enabled.shaderClipDistance = available.shaderClipDistance;
|
||||
if (enabled.shaderStorageImageWriteWithoutFormat) state->capabilities.push_back(spv::CapabilityStorageImageWriteWithoutFormat);
|
||||
if (enabled.shaderStorageImageReadWithoutFormat) state->capabilities.push_back(spv::CapabilityStorageImageReadWithoutFormat);
|
||||
state->storageImageWithoutFormat = enabled.shaderStorageImageWriteWithoutFormat == VK_TRUE && enabled.shaderStorageImageReadWithoutFormat == VK_TRUE;
|
||||
// Bindless image tables index an image array with a wave-uniform runtime slot.
|
||||
enabled.shaderSampledImageArrayDynamicIndexing = available.shaderSampledImageArrayDynamicIndexing;
|
||||
enabled.shaderStorageImageArrayDynamicIndexing = available.shaderStorageImageArrayDynamicIndexing;
|
||||
|
|
@ -2729,6 +2731,7 @@ Graphics::Context VulkanDevice::buildContext() const {
|
|||
context.pipelineExecutableInfo = state->pipelineExecutableInfo;
|
||||
context.srgbDecodeFormats = state->srgbDecodeFormats;
|
||||
context.memoryProperties2 = state->memoryBudget ? state->InstanceFunction<PFN_vkGetPhysicalDeviceMemoryProperties2>("vkGetPhysicalDeviceMemoryProperties2") : nullptr;
|
||||
context.singlePassStorage = state->storageImageWithoutFormat;
|
||||
return context;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -165,6 +165,7 @@ struct Context {
|
|||
std::uint32_t srgbDecodeFormats = 0;
|
||||
bool provokingVertexLast = false;
|
||||
bool provokingVertexModePerPipeline = false;
|
||||
bool singlePassStorage = false;
|
||||
|
||||
template<typename TFunction>
|
||||
TFunction Function(const char* name) const {
|
||||
|
|
|
|||
|
|
@ -295,6 +295,7 @@ public:
|
|||
// was still current (nothing uploaded).
|
||||
// Keeps the image current with guest memory (see GuestMemory::CollectWrites).
|
||||
bool Refresh();
|
||||
static std::uint64_t SinglePassMoves();
|
||||
std::uint64_t GuestBytes() const;
|
||||
VkDeviceSize AllocationBytes() const { return memoryBytes; }
|
||||
|
||||
|
|
@ -430,6 +431,10 @@ private:
|
|||
bool keysFillMatches(std::uint64_t address, std::size_t bytes, bool overlapped) const;
|
||||
bool pendingUnitInside(std::uint64_t address, std::size_t bytes) const;
|
||||
VkImageView createView(std::uint32_t mip, bool firstLayer, VkFormat format) const;
|
||||
bool singlePass();
|
||||
VkImageView elementLayerView(std::uint32_t level, std::uint32_t layer);
|
||||
void recordDirectUploadBarrier(VkCommandBuffer commands, bool discard);
|
||||
void recordDirectUploadDone(VkCommandBuffer commands);
|
||||
void release() noexcept;
|
||||
|
||||
Context context;
|
||||
|
|
@ -483,6 +488,8 @@ private:
|
|||
std::map<std::pair<std::uint32_t, bool>, VkImageView> atomicViews;
|
||||
std::map<std::pair<std::uint32_t, bool>, VkImageView> uintViews;
|
||||
VkImageView elementView = VK_NULL_HANDLE;
|
||||
std::map<std::uint32_t, VkImageView> elementLayerViews;
|
||||
std::int8_t singlePassState = -1;
|
||||
bool attachable = false;
|
||||
std::map<std::tuple<VkFormat, std::uint32_t, std::uint32_t>, VkImageView> attachmentViews;
|
||||
VkImage proxyImage = VK_NULL_HANDLE;
|
||||
|
|
|
|||
|
|
@ -48,15 +48,18 @@ namespace AgcDriver::Graphics {
|
|||
// restricts the move to part of the mip (a partial upload or write-back, StorageTexture),
|
||||
// leaving the other elements of both buffers alone.
|
||||
void Dispatch(VkCommandBuffer commands, TextureTileMode tileMode, std::uint32_t elementBytes, VkBuffer source, std::uint64_t sourceOffset, VkBuffer destination, std::uint64_t destinationOffset, const TileMipLayout& layout, bool retile = false, std::uint32_t slice = 0, bool thick = false, const DetileWindow& window = {});
|
||||
void DispatchImage(VkCommandBuffer commands, TextureTileMode tileMode, std::uint32_t elementBytes, VkBuffer tiled, std::uint64_t tiledOffset, VkImageView view, const TileMipLayout& layout, bool retile = false, std::uint32_t slice = 0, const DetileWindow& window = {});
|
||||
static VkFormat ImageElementFormat(std::uint32_t elementBytes);
|
||||
// Recycles the descriptor sets of the previous batch; call before recording a new command batch.
|
||||
void BeginBatch();
|
||||
void DispatchCmaskClear(VkCommandBuffer commands, VkBuffer cmask, std::uint64_t cmaskOffset, std::size_t cmaskBytes, VkImageView view, std::uint32_t width, std::uint32_t height, std::uint32_t elementBytes, const std::array<std::uint32_t, 2>& clearWords, bool write);
|
||||
std::uint32_t CmaskErrors();
|
||||
|
||||
private:
|
||||
VkPipeline pipeline(TextureTileMode tileMode, std::uint32_t elementBytes, bool retile, bool thick);
|
||||
VkPipeline pipeline(TextureTileMode tileMode, std::uint32_t elementBytes, bool retile, bool thick, bool image = false);
|
||||
void release() noexcept;
|
||||
VkDescriptorSet allocateSet();
|
||||
VkDescriptorSet allocateSet(bool image = false);
|
||||
void checkWindow(const TileMipLayout& layout, const DetileWindow& window, std::uint32_t& columnEnd, std::uint32_t& rowEnd) const;
|
||||
|
||||
const Context context;
|
||||
VkDescriptorSetLayout descriptorLayout = VK_NULL_HANDLE;
|
||||
|
|
@ -71,6 +74,11 @@ namespace AgcDriver::Graphics {
|
|||
std::vector<VkDescriptorPool> cmaskDescriptorPools;
|
||||
std::size_t allocatedCmaskSets = 0;
|
||||
std::unique_ptr<Buffer> cmaskErrors;
|
||||
VkDescriptorSetLayout imageDescriptorLayout = VK_NULL_HANDLE;
|
||||
VkPipelineLayout imagePipelineLayout = VK_NULL_HANDLE;
|
||||
VkShaderModule imageModule = VK_NULL_HANDLE;
|
||||
std::vector<VkDescriptorPool> imageDescriptorPools;
|
||||
std::size_t allocatedImageSets = 0;
|
||||
};
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,4 +1,7 @@
|
|||
#version 450
|
||||
#ifdef LINEAR_IMAGE
|
||||
#extension GL_EXT_shader_image_load_formatted : require
|
||||
#endif
|
||||
|
||||
layout(constant_id = 0) const uint ELEMENT_BYTES = 4u;
|
||||
layout(constant_id = 1) const uint BLOCK_BYTES = 256u;
|
||||
|
|
@ -29,8 +32,13 @@ layout(constant_id = 21) const uint BLOCK_HEIGHT = 0u;
|
|||
|
||||
layout(local_size_x = 8, local_size_y = 8) in;
|
||||
|
||||
#ifdef LINEAR_IMAGE
|
||||
layout(set = 0, binding = 0, std430) buffer Tiled { uint data[]; } tiledBuffer;
|
||||
layout(set = 0, binding = 1) uniform uimage2D linearImage;
|
||||
#else
|
||||
layout(set = 0, binding = 0, std430) readonly buffer Input { uint data[]; } inputBuffer;
|
||||
layout(set = 0, binding = 1, std430) buffer Output { uint data[]; } outputBuffer;
|
||||
#endif
|
||||
|
||||
layout(push_constant) uniform Push {
|
||||
uint srcBase;
|
||||
|
|
@ -142,6 +150,43 @@ uvec2 blockExtent() {
|
|||
return uvec2(width, BLOCK_BYTES / (width * ELEMENT_BYTES));
|
||||
}
|
||||
|
||||
#ifdef LINEAR_IMAGE
|
||||
void storeElement(uint src, uvec2 p) {
|
||||
uvec4 value = uvec4(0u);
|
||||
uint word = src >> 2;
|
||||
if (ELEMENT_BYTES == 1u) {
|
||||
value.x = (tiledBuffer.data[word] >> ((src & 3u) * 8u)) & 0xffu;
|
||||
} else if (ELEMENT_BYTES == 2u) {
|
||||
value.x = (tiledBuffer.data[word] >> ((src & 3u) * 8u)) & 0xffffu;
|
||||
} else {
|
||||
value.x = tiledBuffer.data[word];
|
||||
if (ELEMENT_BYTES >= 8u) value.y = tiledBuffer.data[word + 1u];
|
||||
if (ELEMENT_BYTES >= 16u) {
|
||||
value.z = tiledBuffer.data[word + 2u];
|
||||
value.w = tiledBuffer.data[word + 3u];
|
||||
}
|
||||
}
|
||||
imageStore(linearImage, ivec2(p), value);
|
||||
}
|
||||
|
||||
void loadElement(uvec2 p, uint dst) {
|
||||
uvec4 value = imageLoad(linearImage, ivec2(p));
|
||||
uint word = dst >> 2;
|
||||
if (ELEMENT_BYTES >= 4u) {
|
||||
tiledBuffer.data[word] = value.x;
|
||||
if (ELEMENT_BYTES >= 8u) tiledBuffer.data[word + 1u] = value.y;
|
||||
if (ELEMENT_BYTES >= 16u) {
|
||||
tiledBuffer.data[word + 2u] = value.z;
|
||||
tiledBuffer.data[word + 3u] = value.w;
|
||||
}
|
||||
} else {
|
||||
uint mask = ELEMENT_BYTES == 1u ? 0xffu : 0xffffu;
|
||||
uint shift = (dst & 3u) * 8u;
|
||||
atomicAnd(tiledBuffer.data[word], ~(mask << shift));
|
||||
atomicOr(tiledBuffer.data[word], (value.x & mask) << shift);
|
||||
}
|
||||
}
|
||||
#else
|
||||
void copyElement(uint src, uint dst) {
|
||||
if (ELEMENT_BYTES >= 4u) {
|
||||
for (uint i = 0u; i < ELEMENT_BYTES; i += 4u) {
|
||||
|
|
@ -155,6 +200,7 @@ void copyElement(uint src, uint dst) {
|
|||
atomicOr(outputBuffer.data[dst >> 2], value << shift);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
void main() {
|
||||
uvec2 p = gl_GlobalInvocationID.xy + uvec2(params.columnBegin, params.rowBegin);
|
||||
|
|
@ -170,11 +216,19 @@ void main() {
|
|||
if (linearOffset < params.rangeBegin || linearOffset >= params.rangeEnd) {
|
||||
return;
|
||||
}
|
||||
#ifdef LINEAR_IMAGE
|
||||
if (RETILE != 0u) {
|
||||
loadElement(p, params.dstBase + linearOffset - params.tiledBase);
|
||||
} else {
|
||||
storeElement(params.srcBase + linearOffset - params.tiledBase, p);
|
||||
}
|
||||
#else
|
||||
if (RETILE != 0u) {
|
||||
copyElement(params.srcBase + linearOffset - params.linearBase, params.dstBase + linearOffset - params.tiledBase);
|
||||
} else {
|
||||
copyElement(params.srcBase + linearOffset - params.tiledBase, params.dstBase + linearOffset - params.linearBase);
|
||||
}
|
||||
#endif
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
@ -194,10 +248,18 @@ void main() {
|
|||
tiledOffset -= params.tiledBase;
|
||||
linearOffset -= params.linearBase;
|
||||
|
||||
#ifdef LINEAR_IMAGE
|
||||
if (RETILE != 0u) {
|
||||
loadElement(p, params.dstBase + tiledOffset);
|
||||
} else {
|
||||
storeElement(params.srcBase + tiledOffset, p);
|
||||
}
|
||||
#else
|
||||
if (RETILE != 0u) {
|
||||
copyElement(params.srcBase + linearOffset, params.dstBase + tiledOffset);
|
||||
} else {
|
||||
copyElement(params.srcBase + tiledOffset, params.dstBase + linearOffset);
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -77,6 +77,7 @@ thread_local const char* uploadReason = "first";
|
|||
// selections widened to every pending unit (the hysteresis in writeBack), units dropped as dead
|
||||
// (DiscardPendingInside) and units an alias's own results superseded (markLayersPending); relaxed,
|
||||
// reported only.
|
||||
std::atomic<std::uint64_t> singlePassMoves{0};
|
||||
std::atomic<std::uint64_t> partialUploads{0}, partialUploadBytes{0}, partialWriteBacks{0}, partialWriteBackBytes{0}, emptyWriteBacks{0}, coalescedWriteBacks{0}, unitsDropped{0}, unitsSuperseded{0};
|
||||
// FlushPending listings skipped because no pending unit of the image lay inside the range, images
|
||||
// whose selected pending units were dropped because their memory is no longer registered (see
|
||||
|
|
@ -1107,6 +1108,39 @@ VkImageView StorageTexture::createView(std::uint32_t mip, bool firstLayer, VkFor
|
|||
return created;
|
||||
}
|
||||
|
||||
bool StorageTexture::singlePass() {
|
||||
if (singlePassState < 0) {
|
||||
const auto elementBytes = BytesPerElement(descriptor.format);
|
||||
const auto format = TextureDetiler::ImageElementFormat(elementBytes);
|
||||
bool eligible = context.singlePassStorage && format != VK_FORMAT_UNDEFINED && BlockWidth(descriptor.format) == 1 && !geometry.thick && geometry.imageDepth == 1 && (descriptor.dimension == TextureDimension::k2D || descriptor.dimension == TextureDimension::k2DArray || descriptor.dimension == TextureDimension::kCube);
|
||||
if (eligible) {
|
||||
VkFormatProperties properties{};
|
||||
context.formatProperties(context.physical, format, &properties);
|
||||
eligible = (properties.optimalTilingFeatures & VK_FORMAT_FEATURE_STORAGE_IMAGE_BIT) != 0;
|
||||
}
|
||||
singlePassState = eligible ? 1 : 0;
|
||||
}
|
||||
return singlePassState == 1;
|
||||
}
|
||||
|
||||
VkImageView StorageTexture::elementLayerView(std::uint32_t level, std::uint32_t layer) {
|
||||
const auto key = (level << 16) | layer;
|
||||
if (const auto found = elementLayerViews.find(key); found != elementLayerViews.end()) return found->second;
|
||||
VkImageViewUsageCreateInfo usage{VK_STRUCTURE_TYPE_IMAGE_VIEW_USAGE_CREATE_INFO};
|
||||
usage.usage = VK_IMAGE_USAGE_STORAGE_BIT;
|
||||
VkImageViewCreateInfo viewInfo{VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO};
|
||||
viewInfo.pNext = &usage;
|
||||
viewInfo.image = image;
|
||||
viewInfo.viewType = VK_IMAGE_VIEW_TYPE_2D;
|
||||
viewInfo.format = TextureDetiler::ImageElementFormat(BytesPerElement(descriptor.format));
|
||||
viewInfo.components = {VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY};
|
||||
viewInfo.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, level, 1u, geometry.CopyLayer(layer), 1u};
|
||||
VkImageView created = VK_NULL_HANDLE;
|
||||
Check(context.Function<PFN_vkCreateImageView>("vkCreateImageView")(context.device, &viewInfo, nullptr, &created), "vkCreateImageView storage element");
|
||||
elementLayerViews.emplace(key, created);
|
||||
return created;
|
||||
}
|
||||
|
||||
VkImageView StorageTexture::AttachmentView(VkFormat format, std::uint32_t mip, std::uint32_t depthSlice) {
|
||||
Require(attachable, "storage image cannot be a color attachment");
|
||||
Require(mip < descriptor.mipCount, "attachment mip exceeds the storage image");
|
||||
|
|
@ -1566,6 +1600,10 @@ DccKeys StorageTexture::ProvedKeys() const {
|
|||
return *keys;
|
||||
}
|
||||
|
||||
std::uint64_t StorageTexture::SinglePassMoves() {
|
||||
return singlePassMoves.load(std::memory_order_relaxed);
|
||||
}
|
||||
|
||||
bool StorageTexture::ServesKeysAt(std::uint64_t dccAddress) const {
|
||||
const bool locked = GuestMemory::GpuMutex().HeldByThisThread();
|
||||
const auto readFollowed = [&] {
|
||||
|
|
@ -1771,7 +1809,8 @@ void StorageTexture::upload(const std::vector<bool>* layers) {
|
|||
if ((*layers)[layer]) uploadedBytes += trackedLayerBytes;
|
||||
}
|
||||
}
|
||||
auto linear = std::make_shared<DeviceBuffer>(context, static_cast<std::size_t>(linearBytes), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT);
|
||||
const bool direct = singlePass();
|
||||
auto linear = direct ? nullptr : std::make_shared<DeviceBuffer>(context, static_cast<std::size_t>(linearBytes), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT);
|
||||
detiler.BeginBatch();
|
||||
auto* recorder = Recorder::Active();
|
||||
std::unique_ptr<CommandBatch> batch;
|
||||
|
|
@ -1782,12 +1821,12 @@ void StorageTexture::upload(const std::vector<bool>* layers) {
|
|||
recorder->FlushStoresOverlapping(descriptor.baseAddress, static_cast<std::size_t>(guestBytes));
|
||||
commands = recorder->Commands();
|
||||
timing = recorder->BeginGpuTiming(Recorder::CommandClass::StorageUpload);
|
||||
recorder->Keep(linear, linear->Size());
|
||||
if (linear) recorder->Keep(linear, linear->Size());
|
||||
// The image itself must outlive the recorded copy: the cache may evict it right after.
|
||||
if (auto self = weak_from_this().lock()) recorder->Keep(std::move(self));
|
||||
// The detile reads the tiled bytes from the import when the batch runs.
|
||||
recorder->NotePendingRead(descriptor.baseAddress, static_cast<std::size_t>(guestBytes), Recorder::ReadKind::StorageUpload);
|
||||
Recorder::CountBarriers(Recorder::CommandClass::StorageUpload, 3);
|
||||
Recorder::CountBarriers(Recorder::CommandClass::StorageUpload, direct ? 2 : 3);
|
||||
if (Recorder::BarrierValidate()) {
|
||||
const std::pair<std::uint64_t, std::uint64_t> read{descriptor.baseAddress, descriptor.baseAddress + guestBytes};
|
||||
const std::pair<VkImage, bool> written{image, true};
|
||||
|
|
@ -1798,6 +1837,26 @@ void StorageTexture::upload(const std::vector<bool>* layers) {
|
|||
commands = batch->Handle();
|
||||
}
|
||||
const auto importOffset = descriptor.baseAddress - import->base;
|
||||
if (direct) {
|
||||
recordDirectUploadBarrier(commands, layers == nullptr);
|
||||
for (std::uint32_t layer = 0; layer < arrayLayers; ++layer) {
|
||||
if (layers != nullptr && !(*layers)[layer]) continue;
|
||||
for (std::uint32_t level = 0; level < mips.size(); ++level) {
|
||||
if (!geometry.HasLayer(level, layer)) continue;
|
||||
const auto& mip = mips[level];
|
||||
detiler.DispatchImage(commands, descriptor.tileMode, elementBytes, import->buffer, importOffset + geometry.GuestLayerOffset(layer) + mip.tiledOffset, elementLayerView(level, layer), mip, false, layer, {.pipeBankXor = descriptor.pipeBankXor});
|
||||
}
|
||||
}
|
||||
recordDirectUploadDone(commands);
|
||||
singlePassMoves.fetch_add(1, std::memory_order_relaxed);
|
||||
if (batch) batch->SubmitAndWait();
|
||||
else recorder->EndGpuTiming(timing, uploadedBytes);
|
||||
countStorageUpload(1, uploadedBytes);
|
||||
++Profile().storageDirectUploads;
|
||||
++version;
|
||||
if (profile) LookupOutcomes::Add(LookupOutcomes::UploadDirect, start);
|
||||
return;
|
||||
}
|
||||
RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_HOST_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT);
|
||||
for (std::uint32_t layer = 0; layer < arrayLayers; ++layer) {
|
||||
if (layers != nullptr && !(*layers)[layer]) continue;
|
||||
|
|
@ -2055,6 +2114,33 @@ StorageTexture::PaddingSeeds StorageTexture::paddingSeeds(const HostImport& impo
|
|||
return seeds;
|
||||
}
|
||||
|
||||
void StorageTexture::recordDirectUploadBarrier(VkCommandBuffer commands, bool discard) {
|
||||
const VkMemoryBarrier sourceReady{VK_STRUCTURE_TYPE_MEMORY_BARRIER, nullptr, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_HOST_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT};
|
||||
VkImageMemoryBarrier toWrite{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER};
|
||||
toWrite.srcAccessMask = discard ? 0u : VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT;
|
||||
toWrite.dstAccessMask = VK_ACCESS_SHADER_WRITE_BIT;
|
||||
toWrite.oldLayout = discard ? VK_IMAGE_LAYOUT_UNDEFINED : VK_IMAGE_LAYOUT_GENERAL;
|
||||
toWrite.newLayout = VK_IMAGE_LAYOUT_GENERAL;
|
||||
toWrite.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
|
||||
toWrite.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
|
||||
toWrite.image = image;
|
||||
toWrite.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, descriptor.mipCount, 0, geometry.imageLayers};
|
||||
context.Function<PFN_vkCmdPipelineBarrier>("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, 0, 1, &sourceReady, 0, nullptr, 1, &toWrite);
|
||||
}
|
||||
|
||||
void StorageTexture::recordDirectUploadDone(VkCommandBuffer commands) {
|
||||
VkImageMemoryBarrier written{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER};
|
||||
written.srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT;
|
||||
written.dstAccessMask = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_COLOR_ATTACHMENT_READ_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT;
|
||||
written.oldLayout = VK_IMAGE_LAYOUT_GENERAL;
|
||||
written.newLayout = VK_IMAGE_LAYOUT_GENERAL;
|
||||
written.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
|
||||
written.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
|
||||
written.image = image;
|
||||
written.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, descriptor.mipCount, 0, geometry.imageLayers};
|
||||
context.Function<PFN_vkCmdPipelineBarrier>("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, 0, 0, nullptr, 0, nullptr, 1, &written);
|
||||
}
|
||||
|
||||
std::uint64_t StorageTexture::uploadWindows(const HostImport& import, std::span<const std::pair<std::uint64_t, std::uint64_t>> runs, bool discard) {
|
||||
const auto elementBytes = BytesPerElement(descriptor.format);
|
||||
// Each run's pieces by source (a unit shadow's slab while fresh, else the import), the tail
|
||||
|
|
@ -2094,7 +2180,8 @@ std::uint64_t StorageTexture::uploadWindows(const HostImport& import, std::span<
|
|||
linearTotal += (window.linearBytes + 255) & ~std::uint64_t{255};
|
||||
}
|
||||
for (const auto& [begin, end] : runs) uploadedBytes += end - begin;
|
||||
auto linear = std::make_shared<DeviceBuffer>(context, static_cast<std::size_t>(linearTotal), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT);
|
||||
const bool direct = singlePass();
|
||||
auto linear = direct ? nullptr : std::make_shared<DeviceBuffer>(context, static_cast<std::size_t>(linearTotal), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT);
|
||||
detiler.BeginBatch();
|
||||
auto* recorder = Recorder::Active();
|
||||
std::unique_ptr<CommandBatch> batch;
|
||||
|
|
@ -2111,13 +2198,13 @@ std::uint64_t StorageTexture::uploadWindows(const HostImport& import, std::span<
|
|||
for (const auto& [begin, end] : reads) recorder->FlushStoresOverlapping(begin, static_cast<std::size_t>(end - begin));
|
||||
commands = recorder->Commands();
|
||||
timing = recorder->BeginGpuTiming(Recorder::CommandClass::StorageUpload);
|
||||
recorder->Keep(linear, linear->Size());
|
||||
if (linear) recorder->Keep(linear, linear->Size());
|
||||
for (const auto& source : sources) {
|
||||
if (source.slab != nullptr) recorder->Keep(source.slab);
|
||||
}
|
||||
if (auto self = weak_from_this().lock()) recorder->Keep(std::move(self));
|
||||
if (!reads.empty()) recorder->NotePendingReads(reads, Recorder::ReadKind::StorageUpload);
|
||||
Recorder::CountBarriers(Recorder::CommandClass::StorageUpload, 3);
|
||||
Recorder::CountBarriers(Recorder::CommandClass::StorageUpload, direct ? 2 : 3);
|
||||
if (Recorder::BarrierValidate()) {
|
||||
const std::pair<VkImage, bool> written{image, true};
|
||||
recorder->NoteAccess(Recorder::CommandClass::StorageUpload, Recorder::Access{reads, {}, std::span(&written, 1), VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_TRANSFER_BIT});
|
||||
|
|
@ -2127,6 +2214,27 @@ std::uint64_t StorageTexture::uploadWindows(const HostImport& import, std::span<
|
|||
commands = batch->Handle();
|
||||
}
|
||||
const auto importOffset = descriptor.baseAddress - import.base;
|
||||
if (direct) {
|
||||
recordDirectUploadBarrier(commands, discard);
|
||||
for (const auto& window : windows) {
|
||||
const auto& mip = mips[window.level];
|
||||
const auto& source = sourceOf(window);
|
||||
auto detile = window.window;
|
||||
detile.pipeBankXor = descriptor.pipeBankXor;
|
||||
const auto view = elementLayerView(window.level, window.layer);
|
||||
if (source.shadow) {
|
||||
detiler.DispatchImage(commands, descriptor.tileMode, elementBytes, source.buffer, source.offset + (window.tiledBegin - source.begin), view, mip, false, window.layer, detile);
|
||||
} else {
|
||||
detile.tiledBase = 0;
|
||||
detiler.DispatchImage(commands, descriptor.tileMode, elementBytes, import.buffer, importOffset + geometry.GuestLayerOffset(window.layer) + mip.tiledOffset, view, mip, false, window.layer, detile);
|
||||
}
|
||||
}
|
||||
recordDirectUploadDone(commands);
|
||||
singlePassMoves.fetch_add(1, std::memory_order_relaxed);
|
||||
if (batch) batch->SubmitAndWait();
|
||||
else recorder->EndGpuTiming(timing, uploadedBytes);
|
||||
return uploadedBytes;
|
||||
}
|
||||
RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_HOST_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT);
|
||||
std::vector<VkBufferImageCopy> regions;
|
||||
for (std::size_t i = 0; i < windows.size(); ++i) {
|
||||
|
|
@ -2265,7 +2373,8 @@ std::uint64_t StorageTexture::writeBackWindows(const HostImport& import, std::sp
|
|||
flushBegin = std::min(flushBegin, seed.begin);
|
||||
flushEnd = std::max(flushEnd, seed.end);
|
||||
}
|
||||
auto linear = std::make_shared<DeviceBuffer>(context, static_cast<std::size_t>(linearTotal), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT);
|
||||
const bool direct = singlePass();
|
||||
auto linear = direct ? nullptr : std::make_shared<DeviceBuffer>(context, static_cast<std::size_t>(linearTotal), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT);
|
||||
auto tiledScratch = std::make_shared<DeviceBuffer>(context, static_cast<std::size_t>(scratchTotal), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT);
|
||||
detiler.BeginBatch();
|
||||
auto* recorder = Recorder::Active();
|
||||
|
|
@ -2278,12 +2387,12 @@ std::uint64_t StorageTexture::writeBackWindows(const HostImport& import, std::sp
|
|||
recorder->FlushStoresOverlapping(flushBegin, static_cast<std::size_t>(flushEnd - flushBegin));
|
||||
commands = recorder->Commands(&covered);
|
||||
timing = recorder->BeginGpuTiming(Recorder::CommandClass::StorageWriteBack);
|
||||
recorder->Keep(linear, linear->Size());
|
||||
if (linear) recorder->Keep(linear, linear->Size());
|
||||
recorder->Keep(tiledScratch, tiledScratch->Size());
|
||||
for (const auto& pieces : slabPieces) recorder->Keep(pieces.slab);
|
||||
for (const auto& slab : padding.slabs) recorder->Keep(slab);
|
||||
if (auto self = weak_from_this().lock()) recorder->Keep(std::move(self));
|
||||
Recorder::CountBarriers(Recorder::CommandClass::StorageWriteBack, 4);
|
||||
Recorder::CountBarriers(Recorder::CommandClass::StorageWriteBack, direct ? 3 : 4);
|
||||
if (Recorder::BarrierValidate()) {
|
||||
const std::pair<VkImage, bool> read{image, false};
|
||||
std::vector<std::pair<std::uint64_t, std::uint64_t>> seedReads;
|
||||
|
|
@ -2304,7 +2413,7 @@ std::uint64_t StorageTexture::writeBackWindows(const HostImport& import, std::sp
|
|||
toSource.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
|
||||
toSource.image = image;
|
||||
toSource.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, descriptor.mipCount, 0, geometry.imageLayers};
|
||||
context.Function<PFN_vkCmdPipelineBarrier>("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 1, &toSource);
|
||||
if (!direct) context.Function<PFN_vkCmdPipelineBarrier>("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 1, &toSource);
|
||||
if (!seeds.empty() || !padding.copies.empty()) {
|
||||
// The seeded units' import bytes (every earlier writer of them, host stores included)
|
||||
// precede the seed copies; the previous command's trailing barrier may have covered that.
|
||||
|
|
@ -2325,22 +2434,37 @@ std::uint64_t StorageTexture::writeBackWindows(const HostImport& import, std::sp
|
|||
// orders the scratch writes).
|
||||
for (const auto& [source, copies] : padding.copies) context.Resolved(&DeviceFunctions::cmdCopyBuffer, "vkCmdCopyBuffer")(commands, source, tiledScratch->Handle(), static_cast<std::uint32_t>(copies.size()), copies.data());
|
||||
}
|
||||
std::vector<VkBufferImageCopy> regions;
|
||||
for (std::size_t i = 0; i < windows.size(); ++i) {
|
||||
for (auto region : windows[i].regions) {
|
||||
region.bufferOffset += linearPositions[i];
|
||||
regions.push_back(region);
|
||||
}
|
||||
}
|
||||
context.Function<PFN_vkCmdCopyImageToBuffer>("vkCmdCopyImageToBuffer")(commands, image, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, linear->Handle(), static_cast<std::uint32_t>(regions.size()), regions.data());
|
||||
const auto linearRead = WholeBufferBarrier(linear->Handle(), VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT);
|
||||
const VkMemoryBarrier importReady{VK_STRUCTURE_TYPE_MEMORY_BARRIER, nullptr, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_HOST_WRITE_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_SHADER_READ_BIT};
|
||||
context.Function<PFN_vkCmdPipelineBarrier>("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, 0, 1, &importReady, 1, &linearRead, 0, nullptr);
|
||||
for (std::size_t i = 0; i < windows.size(); ++i) {
|
||||
const auto& window = windows[i];
|
||||
auto retile = window.window;
|
||||
retile.pipeBankXor = descriptor.pipeBankXor;
|
||||
detiler.Dispatch(commands, descriptor.tileMode, elementBytes, linear->Handle(), linearPositions[i], tiledScratch->Handle(), scratchPositions[i], mips[window.level], true, window.layer, false, retile);
|
||||
if (direct) {
|
||||
VkImageMemoryBarrier toRead = toSource;
|
||||
toRead.srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT;
|
||||
toRead.dstAccessMask = VK_ACCESS_SHADER_READ_BIT;
|
||||
toRead.newLayout = VK_IMAGE_LAYOUT_GENERAL;
|
||||
context.Function<PFN_vkCmdPipelineBarrier>("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, 0, 1, &importReady, 0, nullptr, 1, &toRead);
|
||||
for (std::size_t i = 0; i < windows.size(); ++i) {
|
||||
const auto& window = windows[i];
|
||||
auto retile = window.window;
|
||||
retile.pipeBankXor = descriptor.pipeBankXor;
|
||||
detiler.DispatchImage(commands, descriptor.tileMode, elementBytes, tiledScratch->Handle(), scratchPositions[i], elementLayerView(window.level, window.layer), mips[window.level], true, window.layer, retile);
|
||||
}
|
||||
singlePassMoves.fetch_add(1, std::memory_order_relaxed);
|
||||
} else {
|
||||
std::vector<VkBufferImageCopy> regions;
|
||||
for (std::size_t i = 0; i < windows.size(); ++i) {
|
||||
for (auto region : windows[i].regions) {
|
||||
region.bufferOffset += linearPositions[i];
|
||||
regions.push_back(region);
|
||||
}
|
||||
}
|
||||
context.Function<PFN_vkCmdCopyImageToBuffer>("vkCmdCopyImageToBuffer")(commands, image, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, linear->Handle(), static_cast<std::uint32_t>(regions.size()), regions.data());
|
||||
const auto linearRead = WholeBufferBarrier(linear->Handle(), VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT);
|
||||
context.Function<PFN_vkCmdPipelineBarrier>("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, 0, 1, &importReady, 1, &linearRead, 0, nullptr);
|
||||
for (std::size_t i = 0; i < windows.size(); ++i) {
|
||||
const auto& window = windows[i];
|
||||
auto retile = window.window;
|
||||
retile.pipeBankXor = descriptor.pipeBankXor;
|
||||
detiler.Dispatch(commands, descriptor.tileMode, elementBytes, linear->Handle(), linearPositions[i], tiledScratch->Handle(), scratchPositions[i], mips[window.level], true, window.layer, false, retile);
|
||||
}
|
||||
}
|
||||
{
|
||||
// The retiled scratch (and a seed's slab bytes, which the scratch copies overwrite in
|
||||
|
|
@ -2360,7 +2484,7 @@ std::uint64_t StorageTexture::writeBackWindows(const HostImport& import, std::sp
|
|||
// merges its leading barrier.
|
||||
constexpr VkAccessFlags storedAccess = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_HOST_READ_BIT;
|
||||
const VkMemoryBarrier stored{VK_STRUCTURE_TYPE_MEMORY_BARRIER, nullptr, VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_SHADER_WRITE_BIT, storedAccess};
|
||||
context.Function<PFN_vkCmdPipelineBarrier>("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, 0, 1, &stored, 0, nullptr, 1, &backToGeneral);
|
||||
context.Function<PFN_vkCmdPipelineBarrier>("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, 0, 1, &stored, 0, nullptr, direct ? 0u : 1u, &backToGeneral);
|
||||
if (batch) {
|
||||
batch->SubmitAndWait();
|
||||
} else {
|
||||
|
|
@ -3733,7 +3857,7 @@ void StorageTexture::writeBackLayers(const std::vector<bool>& layers) {
|
|||
recorder->FlushStoresOverlapping(firstStored, static_cast<std::size_t>(lastStored - firstStored));
|
||||
commands = recorder->Commands();
|
||||
timing = recorder->BeginGpuTiming(Recorder::CommandClass::StorageWriteBack);
|
||||
recorder->Keep(linear, linear->Size());
|
||||
if (linear) recorder->Keep(linear, linear->Size());
|
||||
recorder->Keep(tiledScratch, tiledScratch->Size());
|
||||
for (const auto& slab : padding.slabs) recorder->Keep(slab);
|
||||
// The image itself must outlive the recorded retile: the cache may evict it right after.
|
||||
|
|
@ -3965,6 +4089,8 @@ void StorageTexture::release() noexcept {
|
|||
uintViews.clear();
|
||||
if (elementView) context.Function<PFN_vkDestroyImageView>("vkDestroyImageView")(context.device, elementView, nullptr);
|
||||
elementView = VK_NULL_HANDLE;
|
||||
for (const auto& [key, element] : elementLayerViews) context.Function<PFN_vkDestroyImageView>("vkDestroyImageView")(context.device, element, nullptr);
|
||||
elementLayerViews.clear();
|
||||
for (const auto& [format, attachment] : attachmentViews) context.Function<PFN_vkDestroyImageView>("vkDestroyImageView")(context.device, attachment, nullptr);
|
||||
attachmentViews.clear();
|
||||
if (proxyView) context.Function<PFN_vkDestroyImageView>("vkDestroyImageView")(context.device, proxyView, nullptr);
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@
|
|||
#include "prx/libSceAgcDriver/Graphics/include/TextureTiling.hpp"
|
||||
#include "prx/libSceAgcDriver/Graphics/shaders/TextureDetile_spv.h"
|
||||
#include "prx/libSceAgcDriver/Graphics/shaders/CmaskClear_spv.h"
|
||||
#include "prx/libSceAgcDriver/Graphics/shaders/TextureDetileImage_spv.h"
|
||||
#include <algorithm>
|
||||
#include <array>
|
||||
#include <bit>
|
||||
|
|
@ -58,8 +59,8 @@ std::uint32_t BlockBytesFor(TextureTileMode tileMode) {
|
|||
throw std::runtime_error("AGC graphics: TextureDetiler encountered an unknown tile mode");
|
||||
}
|
||||
|
||||
std::uint32_t PipelineKey(TextureTileMode tileMode, std::uint32_t elementBytes, bool retile, bool thick) {
|
||||
return (thick ? 1u << 17 : 0u) | (retile ? 1u << 16 : 0u) | (static_cast<std::uint32_t>(tileMode) << 8) | elementBytes;
|
||||
std::uint32_t PipelineKey(TextureTileMode tileMode, std::uint32_t elementBytes, bool retile, bool thick, bool image) {
|
||||
return (image ? 1u << 18 : 0u) | (thick ? 1u << 17 : 0u) | (retile ? 1u << 16 : 0u) | (static_cast<std::uint32_t>(tileMode) << 8) | elementBytes;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -91,6 +92,10 @@ TextureDetiler::TextureDetiler(const Context& context) : context(context) {
|
|||
moduleInfo.codeSize = sizeof(TEXTURE_DETILE_SPV);
|
||||
moduleInfo.pCode = TEXTURE_DETILE_SPV;
|
||||
Check(context.Function<PFN_vkCreateShaderModule>("vkCreateShaderModule")(context.device, &moduleInfo, nullptr, &module), "vkCreateShaderModule");
|
||||
bindings[1].descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE;
|
||||
Check(context.Function<PFN_vkCreateDescriptorSetLayout>("vkCreateDescriptorSetLayout")(context.device, &layoutInfo, nullptr, &imageDescriptorLayout), "vkCreateDescriptorSetLayout");
|
||||
pipelineLayoutInfo.pSetLayouts = &imageDescriptorLayout;
|
||||
Check(context.Function<PFN_vkCreatePipelineLayout>("vkCreatePipelineLayout")(context.device, &pipelineLayoutInfo, nullptr, &imagePipelineLayout), "vkCreatePipelineLayout");
|
||||
} catch (...) {
|
||||
release();
|
||||
throw;
|
||||
|
|
@ -108,15 +113,30 @@ void TextureDetiler::release() noexcept {
|
|||
if (cmaskDescriptorLayout) context.Function<PFN_vkDestroyDescriptorSetLayout>("vkDestroyDescriptorSetLayout")(context.device, cmaskDescriptorLayout, nullptr);
|
||||
cmaskErrors.reset();
|
||||
for (const auto pool : descriptorPools) context.Function<PFN_vkDestroyDescriptorPool>("vkDestroyDescriptorPool")(context.device, pool, nullptr);
|
||||
for (const auto pool : imageDescriptorPools) context.Function<PFN_vkDestroyDescriptorPool>("vkDestroyDescriptorPool")(context.device, pool, nullptr);
|
||||
if (imageModule) context.Function<PFN_vkDestroyShaderModule>("vkDestroyShaderModule")(context.device, imageModule, nullptr);
|
||||
if (imagePipelineLayout) context.Function<PFN_vkDestroyPipelineLayout>("vkDestroyPipelineLayout")(context.device, imagePipelineLayout, nullptr);
|
||||
if (imageDescriptorLayout) context.Function<PFN_vkDestroyDescriptorSetLayout>("vkDestroyDescriptorSetLayout")(context.device, imageDescriptorLayout, nullptr);
|
||||
for (const auto& entry : pipelines) context.Function<PFN_vkDestroyPipeline>("vkDestroyPipeline")(context.device, entry.second, nullptr);
|
||||
if (module) context.Function<PFN_vkDestroyShaderModule>("vkDestroyShaderModule")(context.device, module, nullptr);
|
||||
if (pipelineLayout) context.Function<PFN_vkDestroyPipelineLayout>("vkDestroyPipelineLayout")(context.device, pipelineLayout, nullptr);
|
||||
if (descriptorLayout) context.Function<PFN_vkDestroyDescriptorSetLayout>("vkDestroyDescriptorSetLayout")(context.device, descriptorLayout, nullptr);
|
||||
}
|
||||
|
||||
VkPipeline TextureDetiler::pipeline(TextureTileMode tileMode, std::uint32_t elementBytes, bool retile, bool thick) {
|
||||
VkFormat TextureDetiler::ImageElementFormat(std::uint32_t elementBytes) {
|
||||
switch (elementBytes) {
|
||||
case 1: return VK_FORMAT_R8_UINT;
|
||||
case 2: return VK_FORMAT_R16_UINT;
|
||||
case 4: return VK_FORMAT_R32_UINT;
|
||||
case 8: return VK_FORMAT_R32G32_UINT;
|
||||
case 16: return VK_FORMAT_R32G32B32A32_UINT;
|
||||
default: return VK_FORMAT_UNDEFINED;
|
||||
}
|
||||
}
|
||||
|
||||
VkPipeline TextureDetiler::pipeline(TextureTileMode tileMode, std::uint32_t elementBytes, bool retile, bool thick, bool image) {
|
||||
Require(std::has_single_bit(elementBytes) && elementBytes <= 16u, "unsupported element size for texture detiling");
|
||||
const auto key = PipelineKey(tileMode, elementBytes, retile, thick);
|
||||
const auto key = PipelineKey(tileMode, elementBytes, retile, thick, image);
|
||||
for (const auto& entry : pipelines) {
|
||||
if (entry.first == key) return entry.second;
|
||||
}
|
||||
|
|
@ -139,6 +159,12 @@ VkPipeline TextureDetiler::pipeline(TextureTileMode tileMode, std::uint32_t elem
|
|||
values[2] = 2u;
|
||||
std::copy(equation->bits.begin(), equation->bits.end(), values.begin() + 4);
|
||||
}
|
||||
if (image && imageModule == VK_NULL_HANDLE) {
|
||||
VkShaderModuleCreateInfo moduleInfo{VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO};
|
||||
moduleInfo.codeSize = sizeof(TEXTURE_DETILE_IMAGE_SPV);
|
||||
moduleInfo.pCode = TEXTURE_DETILE_IMAGE_SPV;
|
||||
Check(context.Function<PFN_vkCreateShaderModule>("vkCreateShaderModule")(context.device, &moduleInfo, nullptr, &imageModule), "vkCreateShaderModule");
|
||||
}
|
||||
std::array<VkSpecializationMapEntry, 22> entries{};
|
||||
for (std::uint32_t index = 0; index < entries.size(); ++index) entries[index] = {index, index * 4u, 4};
|
||||
VkSpecializationInfo specialization{};
|
||||
|
|
@ -148,27 +174,32 @@ VkPipeline TextureDetiler::pipeline(TextureTileMode tileMode, std::uint32_t elem
|
|||
specialization.pData = values.data();
|
||||
VkPipelineShaderStageCreateInfo stage{VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO};
|
||||
stage.stage = VK_SHADER_STAGE_COMPUTE_BIT;
|
||||
stage.module = module;
|
||||
stage.module = image ? imageModule : module;
|
||||
stage.pName = "main";
|
||||
stage.pSpecializationInfo = &specialization;
|
||||
VkComputePipelineCreateInfo createInfo{VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO};
|
||||
createInfo.stage = stage;
|
||||
createInfo.layout = pipelineLayout;
|
||||
createInfo.layout = image ? imagePipelineLayout : pipelineLayout;
|
||||
VkPipeline result = VK_NULL_HANDLE;
|
||||
Check(context.Function<PFN_vkCreateComputePipelines>("vkCreateComputePipelines")(context.device, context.pipelineCache, 1, &createInfo, nullptr, &result), "vkCreateComputePipelines");
|
||||
pipelines.emplace_back(key, result);
|
||||
return result;
|
||||
}
|
||||
|
||||
void TextureDetiler::checkWindow(const TileMipLayout& layout, const DetileWindow& window, std::uint32_t& columnEnd, std::uint32_t& rowEnd) const {
|
||||
Require(layout.width != 0 && layout.height != 0, "texture detiling requires a non-empty mip layout");
|
||||
Require(layout.tiledSize != 0 && layout.linearSize != 0, "texture detiling requires a non-empty mip layout");
|
||||
columnEnd = window.columnEnd != 0 ? std::min(window.columnEnd, layout.width) : layout.width;
|
||||
rowEnd = window.rowEnd != 0 ? std::min(window.rowEnd, layout.height) : layout.height;
|
||||
if (!(window.columnBegin < columnEnd && window.rowBegin < rowEnd)) Require(false, "texture detiling window lies outside the mip: columns " + std::to_string(window.columnBegin) + ".." + std::to_string(columnEnd) + ", rows " + std::to_string(window.rowBegin) + ".." + std::to_string(rowEnd) + " of " + std::to_string(layout.width) + "x" + std::to_string(layout.height));
|
||||
}
|
||||
|
||||
void TextureDetiler::Dispatch(VkCommandBuffer commands, TextureTileMode tileMode, std::uint32_t elementBytes, VkBuffer source, std::uint64_t sourceOffset, VkBuffer destination, std::uint64_t destinationOffset, const TileMipLayout& layout, bool retile, std::uint32_t slice, bool thick, const DetileWindow& window) {
|
||||
Require(commands != VK_NULL_HANDLE, "texture detiling requires an active command buffer");
|
||||
Require(window.rangeBegin < window.rangeEnd && window.tiledBase <= window.rangeBegin, "texture detiling window is empty");
|
||||
Require(source != VK_NULL_HANDLE && destination != VK_NULL_HANDLE, "texture detiling requires source and destination buffers");
|
||||
Require(layout.width != 0 && layout.height != 0, "texture detiling requires a non-empty mip layout");
|
||||
Require(layout.tiledSize != 0 && layout.linearSize != 0, "texture detiling requires a non-empty mip layout");
|
||||
const auto columnEnd = window.columnEnd != 0 ? std::min(window.columnEnd, layout.width) : layout.width;
|
||||
const auto rowEnd = window.rowEnd != 0 ? std::min(window.rowEnd, layout.height) : layout.height;
|
||||
if (!(window.columnBegin < columnEnd && window.rowBegin < rowEnd)) Require(false, "texture detiling window lies outside the mip: columns " + std::to_string(window.columnBegin) + ".." + std::to_string(columnEnd) + ", rows " + std::to_string(window.rowBegin) + ".." + std::to_string(rowEnd) + " of " + std::to_string(layout.width) + "x" + std::to_string(layout.height));
|
||||
std::uint32_t columnEnd = 0, rowEnd = 0;
|
||||
checkWindow(layout, window, columnEnd, rowEnd);
|
||||
const auto target = pipeline(tileMode, elementBytes, retile, thick);
|
||||
const auto alignment = std::max<VkDeviceSize>(context.limits.minStorageBufferOffsetAlignment, 4);
|
||||
const auto sourceDescriptorOffset = sourceOffset - sourceOffset % alignment;
|
||||
|
|
@ -332,4 +363,63 @@ void TextureDetiler::DispatchCmaskClear(VkCommandBuffer commands, VkBuffer cmask
|
|||
barrier(VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, anyAccess);
|
||||
}
|
||||
|
||||
void TextureDetiler::DispatchImage(VkCommandBuffer commands, TextureTileMode tileMode, std::uint32_t elementBytes, VkBuffer tiled, std::uint64_t tiledOffset, VkImageView view, const TileMipLayout& layout, bool retile, std::uint32_t slice, const DetileWindow& window) {
|
||||
Require(commands != VK_NULL_HANDLE, "texture detiling requires an active command buffer");
|
||||
Require(window.rangeBegin < window.rangeEnd && window.tiledBase <= window.rangeBegin, "texture detiling window is empty");
|
||||
Require(tiled != VK_NULL_HANDLE && view != VK_NULL_HANDLE, "texture detiling requires a tiled buffer and an image view");
|
||||
Require(ImageElementFormat(elementBytes) != VK_FORMAT_UNDEFINED, "unsupported element size for texture detiling through an image");
|
||||
std::uint32_t columnEnd = 0, rowEnd = 0;
|
||||
checkWindow(layout, window, columnEnd, rowEnd);
|
||||
const auto target = pipeline(tileMode, elementBytes, retile, false, true);
|
||||
const auto alignment = std::max<VkDeviceSize>(context.limits.minStorageBufferOffsetAlignment, 4);
|
||||
const auto descriptorOffset = tiledOffset - tiledOffset % alignment;
|
||||
const auto base = tiledOffset - descriptorOffset;
|
||||
Require(base <= UINT32_MAX, "texture detiling buffer offset exceeds addressable range");
|
||||
const std::uint64_t tiledBytes = window.rangeEnd == 0xffffffffu ? layout.tiledSize : std::min<std::uint64_t>(layout.tiledSize, window.rangeEnd) - window.tiledBase;
|
||||
const auto range = (base + tiledBytes + 3) / 4 * 4;
|
||||
Require(range <= context.limits.maxStorageBufferRange, "texture detiling buffer range exceeds device limits");
|
||||
const auto set = allocateSet(true);
|
||||
const VkDescriptorBufferInfo bufferInfo{tiled, descriptorOffset, range};
|
||||
const VkDescriptorImageInfo imageInfo{VK_NULL_HANDLE, view, VK_IMAGE_LAYOUT_GENERAL};
|
||||
std::array<VkWriteDescriptorSet, 2> writes{};
|
||||
writes[0].sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET;
|
||||
writes[0].dstSet = set;
|
||||
writes[0].dstBinding = 0;
|
||||
writes[0].descriptorCount = 1;
|
||||
writes[0].descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER;
|
||||
writes[0].pBufferInfo = &bufferInfo;
|
||||
writes[1].sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET;
|
||||
writes[1].dstSet = set;
|
||||
writes[1].dstBinding = 1;
|
||||
writes[1].descriptorCount = 1;
|
||||
writes[1].descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE;
|
||||
writes[1].pImageInfo = &imageInfo;
|
||||
context.Function<PFN_vkUpdateDescriptorSets>("vkUpdateDescriptorSets")(context.device, static_cast<std::uint32_t>(writes.size()), writes.data(), 0, nullptr);
|
||||
context.Function<PFN_vkCmdBindPipeline>("vkCmdBindPipeline")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, target);
|
||||
context.Function<PFN_vkCmdBindDescriptorSets>("vkCmdBindDescriptorSets")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, imagePipelineLayout, 0, 1, &set, 0, nullptr);
|
||||
Push push{};
|
||||
push.srcBase = static_cast<std::uint32_t>(retile ? 0 : base);
|
||||
push.dstBase = static_cast<std::uint32_t>(retile ? base : 0);
|
||||
push.width = layout.width;
|
||||
push.height = layout.height;
|
||||
push.pitchBytes = layout.pitchBytes;
|
||||
push.blocksPerRow = layout.blocksPerRow;
|
||||
push.tail = layout.tail ? 1u : 0u;
|
||||
push.tailX = layout.tailX;
|
||||
push.tailY = layout.tailY;
|
||||
push.elementBytes = elementBytes;
|
||||
push.slice = slice;
|
||||
push.rangeBegin = window.rangeBegin;
|
||||
push.rangeEnd = window.rangeEnd;
|
||||
push.tiledBase = window.tiledBase;
|
||||
push.linearBase = 0;
|
||||
push.columnBegin = window.columnBegin;
|
||||
push.rowBegin = window.rowBegin;
|
||||
push.pipeBankXor = window.pipeBankXor;
|
||||
context.Function<PFN_vkCmdPushConstants>("vkCmdPushConstants")(commands, imagePipelineLayout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(Push), &push);
|
||||
const auto groupsX = (columnEnd - window.columnBegin + 7u) / 8u;
|
||||
const auto groupsY = (rowEnd - window.rowBegin + 7u) / 8u;
|
||||
context.Function<PFN_vkCmdDispatch>("vkCmdDispatch")(commands, groupsX, groupsY, 1);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
#include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp"
|
||||
#include "prx/libSceAgcDriver/Graphics/include/TextureDetiler.hpp"
|
||||
#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
|
||||
#include <array>
|
||||
|
||||
namespace AgcDriver::Graphics {
|
||||
|
||||
|
|
@ -17,29 +18,37 @@ void TextureDetiler::BeginBatch() {
|
|||
Check(context.Function<PFN_vkResetDescriptorPool>("vkResetDescriptorPool")(context.device, pool, 0), "vkResetDescriptorPool CMASK clear");
|
||||
}
|
||||
allocatedCmaskSets = 0;
|
||||
for (const auto pool : imageDescriptorPools) {
|
||||
Check(context.Function<PFN_vkResetDescriptorPool>("vkResetDescriptorPool")(context.device, pool, 0), "vkResetDescriptorPool texture detiler");
|
||||
}
|
||||
allocatedImageSets = 0;
|
||||
}
|
||||
|
||||
VkDescriptorSet TextureDetiler::allocateSet() {
|
||||
VkDescriptorSet TextureDetiler::allocateSet(bool image) {
|
||||
constexpr std::uint32_t setsPerPool = 64;
|
||||
const auto poolIndex = allocatedSets / setsPerPool;
|
||||
if (poolIndex == descriptorPools.size()) {
|
||||
const VkDescriptorPoolSize size{VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, setsPerPool * 2};
|
||||
auto& pools = image ? imageDescriptorPools : descriptorPools;
|
||||
auto& allocated = image ? allocatedImageSets : allocatedSets;
|
||||
const auto poolIndex = allocated / setsPerPool;
|
||||
if (poolIndex == pools.size()) {
|
||||
std::array<VkDescriptorPoolSize, 2> sizes{};
|
||||
sizes[0] = {VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, image ? setsPerPool : setsPerPool * 2};
|
||||
sizes[1] = {VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, setsPerPool};
|
||||
VkDescriptorPoolCreateInfo info{VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO};
|
||||
info.maxSets = setsPerPool;
|
||||
info.poolSizeCount = 1;
|
||||
info.pPoolSizes = &size;
|
||||
descriptorPools.reserve(descriptorPools.size() + 1);
|
||||
info.poolSizeCount = image ? 2u : 1u;
|
||||
info.pPoolSizes = sizes.data();
|
||||
pools.reserve(pools.size() + 1);
|
||||
VkDescriptorPool pool = VK_NULL_HANDLE;
|
||||
Check(context.Function<PFN_vkCreateDescriptorPool>("vkCreateDescriptorPool")(context.device, &info, nullptr, &pool), "vkCreateDescriptorPool texture detiler");
|
||||
descriptorPools.push_back(pool);
|
||||
pools.push_back(pool);
|
||||
}
|
||||
VkDescriptorSetAllocateInfo allocation{VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO};
|
||||
allocation.descriptorPool = descriptorPools.at(poolIndex);
|
||||
allocation.descriptorPool = pools.at(poolIndex);
|
||||
allocation.descriptorSetCount = 1;
|
||||
allocation.pSetLayouts = &descriptorLayout;
|
||||
allocation.pSetLayouts = image ? &imageDescriptorLayout : &descriptorLayout;
|
||||
VkDescriptorSet set = VK_NULL_HANDLE;
|
||||
Check(context.Function<PFN_vkAllocateDescriptorSets>("vkAllocateDescriptorSets")(context.device, &allocation, &set), "vkAllocateDescriptorSets texture detiler");
|
||||
++allocatedSets;
|
||||
++allocated;
|
||||
return set;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -167,6 +167,9 @@ public:
|
|||
function<PFN_vkGetPhysicalDeviceFeatures>("vkGetPhysicalDeviceFeatures")(context.physical, &supported);
|
||||
enabled.occlusionQueryPrecise = supported.occlusionQueryPrecise;
|
||||
context.occlusionQueryPrecise = supported.occlusionQueryPrecise == VK_TRUE;
|
||||
enabled.shaderStorageImageReadWithoutFormat = supported.shaderStorageImageReadWithoutFormat;
|
||||
enabled.shaderStorageImageWriteWithoutFormat = supported.shaderStorageImageWriteWithoutFormat;
|
||||
context.singlePassStorage = supported.shaderStorageImageReadWithoutFormat == VK_TRUE && supported.shaderStorageImageWriteWithoutFormat == VK_TRUE;
|
||||
address.pNext = &bytes;
|
||||
std::vector<const char*> extensionsEnabled{VK_KHR_BUFFER_DEVICE_ADDRESS_EXTENSION_NAME, VK_KHR_8BIT_STORAGE_EXTENSION_NAME};
|
||||
VkPhysicalDeviceDriverProperties driverProperties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DRIVER_PROPERTIES};
|
||||
|
|
@ -2366,6 +2369,201 @@ void writeBackPaddingTests(const Device& device, Recorder& recorder, bool watche
|
|||
std::cout << "write-back padding" << (watched ? " (watched)" : "") << ": ok\n";
|
||||
}
|
||||
|
||||
void singlePassTests(const Device& device, Recorder& recorder, bool watched) {
|
||||
const auto& base = device.GetContext();
|
||||
if (base.hostImportAlignment == 0 || !base.singlePassStorage) {
|
||||
std::cout << "host imports or storage access without a format unavailable: single-pass storage moves not tested\n";
|
||||
return;
|
||||
}
|
||||
struct Case {
|
||||
const char* name;
|
||||
std::uint32_t format;
|
||||
TextureTileMode tileMode;
|
||||
std::uint32_t width;
|
||||
std::uint32_t height;
|
||||
std::uint32_t mipCount;
|
||||
std::uint32_t pipeBankXor = 0;
|
||||
};
|
||||
constexpr Case cases[] = {
|
||||
{"R8 64 KiB R_X 200x150", 1, TextureTileMode::kR64KBX, 200, 150, 1},
|
||||
{"R16 64 KiB R_X 200x150, 3 mips", 7, TextureTileMode::kR64KBX, 200, 150, 3},
|
||||
{"RGBA8 64 KiB R_X 520x260, 6 mips", 56, TextureTileMode::kR64KBX, 520, 260, 6},
|
||||
{"RGBA16F 64 KiB R_X 384x256", 71, TextureTileMode::kR64KBX, 384, 256, 1},
|
||||
{"RGBA16F 64 KiB R_X 256x200, 4 mips", 71, TextureTileMode::kR64KBX, 256, 200, 4},
|
||||
{"RGBA32F 64 KiB R_X 136x72, 2 mips", 77, TextureTileMode::kR64KBX, 136, 72, 2},
|
||||
{"RGBA8 64 KiB S 300x170", 56, TextureTileMode::kStandard64KB, 300, 170, 1},
|
||||
{"RGBA16F 4 KiB S 100x70, 2 mips", 71, TextureTileMode::kStandard4KB, 100, 70, 2},
|
||||
{"R8 256 B S 90x40", 1, TextureTileMode::kStandard256B, 90, 40, 1},
|
||||
{"RGBA8 64 KiB R_X 520x260, 2 mips, pipe/bank XOR 5", 56, TextureTileMode::kR64KBX, 520, 260, 2, 5},
|
||||
};
|
||||
std::string failures;
|
||||
std::size_t tested = 0;
|
||||
for (const auto& test : cases) {
|
||||
if (!StorageFormatAvailable(base, test.format)) continue;
|
||||
const std::string what = std::string(watched ? "(watched) " : "") + test.name;
|
||||
GuestTextureResource resource{};
|
||||
resource.width = test.width;
|
||||
resource.height = test.height;
|
||||
resource.mipCount = test.mipCount;
|
||||
resource.tileMode = test.tileMode;
|
||||
resource.dimension = TextureDimension::k2D;
|
||||
resource.format = test.format;
|
||||
resource.pipeBankXor = test.pipeBankXor;
|
||||
resource.dstSelX = 4;
|
||||
resource.dstSelY = 5;
|
||||
resource.dstSelZ = 6;
|
||||
resource.dstSelW = 7;
|
||||
const auto geometry = DescribeSurface(resource);
|
||||
const auto elementBytes = BytesPerElement(test.format);
|
||||
const auto bytes = (static_cast<std::size_t>(geometry.guestBytes) + 65535) / 65536 * 65536 + 65536;
|
||||
struct Result {
|
||||
std::vector<std::uint8_t> uploaded, memory, reuploaded;
|
||||
std::uint64_t moves = 0;
|
||||
};
|
||||
std::array<Result, 2> results;
|
||||
bool skipped = false;
|
||||
for (int mode = 0; mode < 2 && !skipped; ++mode) {
|
||||
auto context = base;
|
||||
context.singlePassStorage = mode == 1;
|
||||
void* block = nullptr;
|
||||
if (watched) {
|
||||
block = AllocateWatched(bytes, 65536);
|
||||
} else {
|
||||
#ifdef _WIN32
|
||||
block = VirtualAlloc(nullptr, bytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE);
|
||||
#else
|
||||
block = std::aligned_alloc(65536, bytes);
|
||||
#endif
|
||||
}
|
||||
if (block == nullptr) {
|
||||
std::cout << "no write watching: single-pass storage moves in watched memory not tested\n";
|
||||
return;
|
||||
}
|
||||
auto* memory = static_cast<std::uint8_t*>(block);
|
||||
std::uint32_t seed = 0x13579bdfu;
|
||||
const auto random = [&] {
|
||||
seed ^= seed << 13;
|
||||
seed ^= seed >> 17;
|
||||
seed ^= seed << 5;
|
||||
return static_cast<std::uint8_t>(seed >> 9);
|
||||
};
|
||||
for (std::size_t i = 0; i < bytes; ++i) memory[i] = random();
|
||||
const auto address = reinterpret_cast<std::uint64_t>(block);
|
||||
{
|
||||
GuestAllocations::Mutation mutation;
|
||||
mutation.Add(block, bytes, true, true, true);
|
||||
}
|
||||
struct Unregister {
|
||||
const Context& context;
|
||||
void* block;
|
||||
std::uint64_t address;
|
||||
std::size_t bytes;
|
||||
bool watched;
|
||||
~Unregister() {
|
||||
{
|
||||
GuestAllocations::Mutation mutation;
|
||||
mutation.Remove(block);
|
||||
}
|
||||
HostImportFor(context, address, bytes);
|
||||
if (watched) ReleaseWatched(block, bytes);
|
||||
#ifdef _WIN32
|
||||
else VirtualFree(block, 0, MEM_RELEASE);
|
||||
#else
|
||||
else std::free(block);
|
||||
#endif
|
||||
}
|
||||
} unregister{base, block, address, bytes, watched};
|
||||
if (HostImportFor(base, address, bytes) == nullptr) {
|
||||
std::cout << "host import refused: single-pass storage moves not tested\n";
|
||||
return;
|
||||
}
|
||||
TextureDetiler detiler(base);
|
||||
context.detiler = &detiler;
|
||||
resource.baseAddress = address;
|
||||
std::size_t linearBytes = 0;
|
||||
for (const auto& mip : geometry.mips) linearBytes += static_cast<std::size_t>(mip.width) * mip.height * elementBytes;
|
||||
const auto readImage = [&](const StorageTexture& image) {
|
||||
Buffer readback(context, linearBytes, VK_BUFFER_USAGE_TRANSFER_DST_BIT);
|
||||
const auto commands = recorder.Commands();
|
||||
RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT);
|
||||
std::vector<VkBufferImageCopy> copies;
|
||||
std::size_t at = 0;
|
||||
for (std::uint32_t level = 0; level < geometry.mips.size(); ++level) {
|
||||
VkBufferImageCopy copy{};
|
||||
copy.bufferOffset = at;
|
||||
copy.imageSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, level, 0, 1};
|
||||
copy.imageExtent = {geometry.mips[level].width, geometry.mips[level].height, 1};
|
||||
copies.push_back(copy);
|
||||
at += static_cast<std::size_t>(geometry.mips[level].width) * geometry.mips[level].height * elementBytes;
|
||||
}
|
||||
context.Function<PFN_vkCmdCopyImageToBuffer>("vkCmdCopyImageToBuffer")(commands, image.Image(), VK_IMAGE_LAYOUT_GENERAL, readback.Handle(), static_cast<std::uint32_t>(copies.size()), copies.data());
|
||||
RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_HOST_READ_BIT | VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT);
|
||||
recorder.Submit();
|
||||
device.WaitQueue();
|
||||
recorder.Sync();
|
||||
const auto read = readback.Bytes();
|
||||
return std::vector<std::uint8_t>(reinterpret_cast<const std::uint8_t*>(read.data()), reinterpret_cast<const std::uint8_t*>(read.data()) + linearBytes);
|
||||
};
|
||||
std::vector<std::uint8_t> texels(linearBytes);
|
||||
for (auto& value : texels) value = random();
|
||||
const auto movesBefore = StorageTexture::SinglePassMoves();
|
||||
{
|
||||
auto image = std::make_shared<StorageTexture>(context, detiler, resource, 0);
|
||||
recorder.Keep(image);
|
||||
results[mode].uploaded = readImage(*image);
|
||||
Buffer staging(context, linearBytes, VK_BUFFER_USAGE_TRANSFER_SRC_BIT);
|
||||
std::memcpy(staging.Bytes().data(), texels.data(), linearBytes);
|
||||
const auto commands = recorder.Commands();
|
||||
RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT, VK_ACCESS_TRANSFER_WRITE_BIT);
|
||||
std::vector<VkBufferImageCopy> copies;
|
||||
std::size_t at = 0;
|
||||
for (std::uint32_t level = 0; level < geometry.mips.size(); ++level) {
|
||||
VkBufferImageCopy copy{};
|
||||
copy.bufferOffset = at;
|
||||
copy.imageSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, level, 0, 1};
|
||||
copy.imageExtent = {geometry.mips[level].width, geometry.mips[level].height, 1};
|
||||
copies.push_back(copy);
|
||||
at += static_cast<std::size_t>(geometry.mips[level].width) * geometry.mips[level].height * elementBytes;
|
||||
}
|
||||
context.Function<PFN_vkCmdCopyBufferToImage>("vkCmdCopyBufferToImage")(commands, staging.Handle(), image->Image(), VK_IMAGE_LAYOUT_GENERAL, static_cast<std::uint32_t>(copies.size()), copies.data());
|
||||
RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT);
|
||||
image->MarkDirty();
|
||||
image->WriteBack();
|
||||
recorder.Submit();
|
||||
device.WaitQueue();
|
||||
recorder.Sync();
|
||||
auto again = std::make_shared<StorageTexture>(context, detiler, resource, 0);
|
||||
recorder.Keep(again);
|
||||
results[mode].reuploaded = readImage(*again);
|
||||
std::vector<std::byte> read(bytes);
|
||||
if (watched) AgcDriver::GuestMemory::Read(address, read);
|
||||
else {
|
||||
StorageTexture::FlushPending(address, bytes, nullptr, "test", PublishScope::Whole);
|
||||
recorder.Submit();
|
||||
device.WaitQueue();
|
||||
recorder.Sync();
|
||||
std::memcpy(read.data(), memory, bytes);
|
||||
}
|
||||
results[mode].memory.assign(reinterpret_cast<const std::uint8_t*>(read.data()), reinterpret_cast<const std::uint8_t*>(read.data()) + bytes);
|
||||
}
|
||||
recorder.Sync();
|
||||
results[mode].moves = StorageTexture::SinglePassMoves() - movesBefore;
|
||||
if (results[mode].reuploaded != texels) failures += what + (mode == 1 ? " (single pass)" : " (copies)") + ": the second image does not hold the texels written back\n";
|
||||
}
|
||||
if (results[0].moves != 0 || results[1].moves < 3) failures += what + ": single-pass moves " + std::to_string(results[0].moves) + " with the copies and " + std::to_string(results[1].moves) + " in single pass (want 0 and at least 3)\n";
|
||||
if (results[0].uploaded != results[1].uploaded) failures += what + ": the first upload differs between the copies and single pass\n";
|
||||
if (results[0].memory != results[1].memory) {
|
||||
std::size_t differ = 0;
|
||||
for (std::size_t i = 0; i < results[0].memory.size(); ++i) differ += results[0].memory[i] != results[1].memory[i];
|
||||
failures += what + ": guest memory after the write-back differs in " + std::to_string(differ) + " bytes between the copies and single pass\n";
|
||||
}
|
||||
++tested;
|
||||
}
|
||||
if (!failures.empty()) throw std::runtime_error("single-pass storage moves:\n" + failures);
|
||||
std::cout << "single-pass storage moves" << (watched ? " (watched)" : "") << ": ok (" << tested << " surfaces)\n";
|
||||
}
|
||||
|
||||
|
||||
void unchangedCpuStampTests(const Device& device, Recorder& recorder) {
|
||||
const auto& base = device.GetContext();
|
||||
constexpr std::uint32_t side = 256;
|
||||
|
|
@ -3906,6 +4104,8 @@ int main(int argc, char** argv) {
|
|||
storageRefreshTests(device, recorder, true);
|
||||
writeBackPaddingTests(device, recorder, false);
|
||||
writeBackPaddingTests(device, recorder, true);
|
||||
singlePassTests(device, recorder, false);
|
||||
singlePassTests(device, recorder, true);
|
||||
unchangedCpuStampTests(device, recorder);
|
||||
importWatchTests(device);
|
||||
staleGenerationTests(device, recorder);
|
||||
|
|
|
|||
Loading…
Reference in New Issue