perf(shader): hash only the recompiler's inputs into the shader cache version
SourceVersion hashed every libSceAgcDriver source, so any driver-only change sent the shader disk cache cold: 88 of the last 200 merges on main did; with the inputs below, 51 of them would have. It now covers: - core/shader/recompiler/** (sources, headers, CMake); - prx/libc/include/HostThreadLocal.hpp, the one header the recompiler includes from outside its directory (checked against the ninja deps); - the checked-out commits of SPIRV-Headers and glslang, plus SPIRV-Tools when ANYPS5_ENABLE_SPIRV_TOOLS is on (the submodule HEAD files are build dependencies, so moving a submodule regenerates the version); - the compiler, build type, architecture and C++ flags. Everything else the driver passes to the recompiler arrives in the RecompileRequest, which the per-entry key covers. Two build-time checks keep that true, failing the version step with a message naming the file: - a recompiler source that includes a project header outside the hashed set; - a driver source that calls or using-declares a ShaderRecompiler name not on the reviewed list, pulls in the whole namespace, or sets an environment variable (which could flip a recompiler APS5_* switch behind the key). APS5_* switches are collected from every hashed source. The version script sets cmake_minimum_required: without it CMake 3.x runs it with CMP0054 OLD and reads the quoted section names in its if() tests as variables.
This commit is contained in:
Dean Galvin
parent
d3ec1f15fb
commit
3886b7b31d
|
|
@ -152,7 +152,16 @@ target_include_directories(libSceAgcDriver PRIVATE
|
|||
${CMAKE_SOURCE_DIR}/3rdparty/SPIRV-Headers/include
|
||||
)
|
||||
|
||||
add_shader_recompiler(shader_recompiler_agc_driver libc)
|
||||
file(GLOB_RECURSE agcShaderCacheGuardSources CONFIGURE_DEPENDS
|
||||
"${CMAKE_CURRENT_SOURCE_DIR}/*.cpp"
|
||||
"${CMAKE_CURRENT_SOURCE_DIR}/*.hpp"
|
||||
"${CMAKE_CURRENT_SOURCE_DIR}/*.h"
|
||||
)
|
||||
list(FILTER agcShaderCacheGuardSources EXCLUDE REGEX "^${CMAKE_CURRENT_SOURCE_DIR}/(tests|tools)/")
|
||||
add_shader_recompiler(shader_recompiler_agc_driver libc
|
||||
GUARD_SOURCES
|
||||
${agcShaderCacheGuardSources}
|
||||
)
|
||||
|
||||
target_link_libraries(libSceAgcDriver PRIVATE
|
||||
shader_recompiler_agc_driver
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
function(add_shader_recompiler target runtimeTarget)
|
||||
cmake_parse_arguments(PARSE_ARGV 2 shaderCache "" "" "GUARD_SOURCES")
|
||||
if(NOT TARGET "${runtimeTarget}")
|
||||
message(FATAL_ERROR "Shader recompiler requires an existing runtime target: ${runtimeTarget}")
|
||||
endif()
|
||||
|
|
@ -155,39 +156,99 @@ function(add_shader_recompiler target runtimeTarget)
|
|||
|
||||
set(shaderCacheGeneratedDir "${CMAKE_CURRENT_BINARY_DIR}/${target}_generated")
|
||||
set(shaderCacheVersionHeader "${shaderCacheGeneratedDir}/ShaderCacheVersion.hpp")
|
||||
set(shaderCacheDriverDir "${PROJECT_SOURCE_DIR}/core/libs/prx/libSceAgcDriver")
|
||||
file(GLOB_RECURSE shaderCacheSources CONFIGURE_DEPENDS
|
||||
file(GLOB_RECURSE shaderCacheRecompilerSources CONFIGURE_DEPENDS
|
||||
"${CMAKE_CURRENT_FUNCTION_LIST_DIR}/*.cpp"
|
||||
"${CMAKE_CURRENT_FUNCTION_LIST_DIR}/*.hpp"
|
||||
"${CMAKE_CURRENT_FUNCTION_LIST_DIR}/*.h"
|
||||
"${CMAKE_CURRENT_FUNCTION_LIST_DIR}/*.cmake"
|
||||
"${CMAKE_CURRENT_FUNCTION_LIST_DIR}/CMakeLists.txt"
|
||||
"${shaderCacheDriverDir}/*.cpp"
|
||||
"${shaderCacheDriverDir}/*.hpp"
|
||||
"${shaderCacheDriverDir}/*.h"
|
||||
"${shaderCacheDriverDir}/*.comp"
|
||||
"${shaderCacheDriverDir}/*.cmake"
|
||||
"${shaderCacheDriverDir}/CMakeLists.txt"
|
||||
)
|
||||
list(FILTER shaderCacheSources EXCLUDE REGEX "^${shaderCacheDriverDir}/(tests|tools)/")
|
||||
if(ANYPS5_ENABLE_SPIRV_TOOLS AND EXISTS "${PROJECT_SOURCE_DIR}/3rdparty/SPIRV-Tools/CHANGES")
|
||||
list(APPEND shaderCacheSources "${PROJECT_SOURCE_DIR}/3rdparty/SPIRV-Tools/CHANGES")
|
||||
set(shaderCacheKeySources "${PROJECT_SOURCE_DIR}/core/libs/prx/libc/include/HostThreadLocal.hpp")
|
||||
foreach(source IN LISTS shaderCacheKeySources)
|
||||
if(NOT EXISTS "${source}")
|
||||
message(FATAL_ERROR "Shader disk cache key source does not exist: ${source}")
|
||||
endif()
|
||||
list(APPEND shaderCacheRecompilerSources "${source}")
|
||||
endforeach()
|
||||
set(shaderCacheGuardSources)
|
||||
foreach(source IN LISTS shaderCache_GUARD_SOURCES)
|
||||
get_filename_component(source "${source}" ABSOLUTE BASE_DIR "${CMAKE_CURRENT_SOURCE_DIR}")
|
||||
list(APPEND shaderCacheGuardSources "${source}")
|
||||
endforeach()
|
||||
list(REMOVE_ITEM shaderCacheGuardSources ${shaderCacheRecompilerSources})
|
||||
|
||||
find_package(Git QUIET)
|
||||
set(shaderCacheSubmodules "${PROJECT_SOURCE_DIR}/3rdparty/SPIRV-Headers" "${PROJECT_SOURCE_DIR}/3rdparty/glslang")
|
||||
if(ANYPS5_ENABLE_SPIRV_TOOLS)
|
||||
list(APPEND shaderCacheSubmodules "${PROJECT_SOURCE_DIR}/3rdparty/SPIRV-Tools")
|
||||
endif()
|
||||
list(JOIN shaderCacheSources "\n" shaderCacheSourceList)
|
||||
set(shaderCacheSubmoduleStamps)
|
||||
foreach(submodule IN LISTS shaderCacheSubmodules)
|
||||
list(APPEND shaderCacheSubmoduleStamps "${submodule}/CMakeLists.txt")
|
||||
if(NOT GIT_FOUND)
|
||||
continue()
|
||||
endif()
|
||||
foreach(ref IN ITEMS HEAD refs/heads)
|
||||
execute_process(COMMAND "${GIT_EXECUTABLE}" -C "${submodule}" rev-parse --path-format=absolute --git-path ${ref}
|
||||
OUTPUT_VARIABLE stamp OUTPUT_STRIP_TRAILING_WHITESPACE RESULT_VARIABLE status ERROR_QUIET)
|
||||
if(status EQUAL 0 AND EXISTS "${stamp}" AND NOT IS_DIRECTORY "${stamp}")
|
||||
list(APPEND shaderCacheSubmoduleStamps "${stamp}")
|
||||
endif()
|
||||
endforeach()
|
||||
endforeach()
|
||||
|
||||
get_target_property(shaderCacheIncludeDirectories ${target} INCLUDE_DIRECTORIES)
|
||||
string(TOUPPER "${CMAKE_BUILD_TYPE}" shaderCacheBuildType)
|
||||
set(shaderCacheExtra "${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION} ${CMAKE_BUILD_TYPE} spirv-tools=${ANYPS5_ENABLE_SPIRV_TOOLS} ${CMAKE_SYSTEM_PROCESSOR} flags=${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_${shaderCacheBuildType}}")
|
||||
set(shaderCacheReviewedNames
|
||||
BuildRectListShaders
|
||||
CaptureResources
|
||||
DebugProbeActive
|
||||
DepthBitsTextureWidth
|
||||
HashBytes
|
||||
PixelInput
|
||||
PixelInputBit
|
||||
ReadWholeFile
|
||||
Recompile
|
||||
RecompileCacheKey::ContextHash
|
||||
RecompileResult
|
||||
ResolveSource
|
||||
ResourceMaterializer::SpecializationNanoseconds
|
||||
SetDebugProbeActive
|
||||
ShaderCacheDirectory
|
||||
WriteFileAtomically
|
||||
)
|
||||
set(shaderCacheManifest)
|
||||
foreach(section IN ITEMS sources guard submodules includes reviewed)
|
||||
if(section STREQUAL "sources")
|
||||
set(entries ${shaderCacheRecompilerSources})
|
||||
elseif(section STREQUAL "guard")
|
||||
set(entries ${shaderCacheGuardSources})
|
||||
elseif(section STREQUAL "submodules")
|
||||
set(entries ${shaderCacheSubmodules})
|
||||
elseif(section STREQUAL "includes")
|
||||
set(entries ${shaderCacheIncludeDirectories})
|
||||
else()
|
||||
set(entries ${shaderCacheReviewedNames})
|
||||
endif()
|
||||
foreach(entry IN LISTS entries)
|
||||
string(APPEND shaderCacheManifest "${section} ${entry}\n")
|
||||
endforeach()
|
||||
endforeach()
|
||||
set(shaderCacheListFile "${shaderCacheGeneratedDir}/sources.txt")
|
||||
file(CONFIGURE OUTPUT "${shaderCacheListFile}" CONTENT "${shaderCacheSourceList}\n" @ONLY)
|
||||
set(shaderCacheExtra "${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION} ${CMAKE_BUILD_TYPE} spirv-tools=${ANYPS5_ENABLE_SPIRV_TOOLS} ${CMAKE_SYSTEM_PROCESSOR}")
|
||||
file(CONFIGURE OUTPUT "${shaderCacheListFile}" CONTENT "${shaderCacheManifest}" @ONLY)
|
||||
add_custom_command(
|
||||
OUTPUT "${shaderCacheVersionHeader}"
|
||||
COMMAND ${CMAKE_COMMAND}
|
||||
"-DROOT=${PROJECT_SOURCE_DIR}"
|
||||
"-DLIST_FILE=${shaderCacheListFile}"
|
||||
"-DRECOMPILER_DIR=${CMAKE_CURRENT_FUNCTION_LIST_DIR}"
|
||||
"-DGIT=${GIT_EXECUTABLE}"
|
||||
"-DEXTRA=${shaderCacheExtra}"
|
||||
"-DOUTPUT=${shaderCacheVersionHeader}"
|
||||
-P "${CMAKE_CURRENT_FUNCTION_LIST_DIR}/ShaderCacheVersion.cmake"
|
||||
DEPENDS ${shaderCacheSources} "${shaderCacheListFile}" "${CMAKE_CURRENT_FUNCTION_LIST_DIR}/ShaderCacheVersion.cmake"
|
||||
COMMENT "Hashing the shader recompiler and AGC driver sources for the shader disk cache"
|
||||
DEPENDS ${shaderCacheRecompilerSources} ${shaderCacheGuardSources} ${shaderCacheSubmoduleStamps} "${shaderCacheListFile}" "${CMAKE_CURRENT_FUNCTION_LIST_DIR}/ShaderCacheVersion.cmake"
|
||||
COMMENT "Hashing the shader recompiler sources for the shader disk cache"
|
||||
VERBATIM
|
||||
)
|
||||
target_sources(${target} PRIVATE "${shaderCacheVersionHeader}")
|
||||
|
|
|
|||
|
|
@ -1,5 +1,39 @@
|
|||
file(STRINGS "${LIST_FILE}" sources)
|
||||
cmake_minimum_required(VERSION 3.20)
|
||||
|
||||
file(STRINGS "${LIST_FILE}" entries)
|
||||
set(sources)
|
||||
set(guard)
|
||||
set(submodules)
|
||||
set(includeDirectories)
|
||||
set(reviewed)
|
||||
foreach(entry IN LISTS entries)
|
||||
string(FIND "${entry}" " " split)
|
||||
string(SUBSTRING "${entry}" 0 ${split} section)
|
||||
math(EXPR split "${split} + 1")
|
||||
string(SUBSTRING "${entry}" ${split} -1 value)
|
||||
if(section STREQUAL "sources")
|
||||
list(APPEND sources "${value}")
|
||||
elseif(section STREQUAL "guard")
|
||||
list(APPEND guard "${value}")
|
||||
elseif(section STREQUAL "submodules")
|
||||
list(APPEND submodules "${value}")
|
||||
elseif(section STREQUAL "includes")
|
||||
list(APPEND includeDirectories "${value}")
|
||||
elseif(section STREQUAL "reviewed")
|
||||
list(APPEND reviewed "${value}")
|
||||
endif()
|
||||
endforeach()
|
||||
list(SORT sources)
|
||||
list(SORT guard)
|
||||
get_filename_component(rootReal "${ROOT}" REALPATH)
|
||||
get_filename_component(recompilerReal "${RECOMPILER_DIR}" REALPATH)
|
||||
set(sourcesReal)
|
||||
foreach(source IN LISTS sources)
|
||||
get_filename_component(real "${source}" REALPATH)
|
||||
list(APPEND sourcesReal "${real}")
|
||||
endforeach()
|
||||
|
||||
set(failures)
|
||||
set(manifest "${EXTRA}\n")
|
||||
set(switches)
|
||||
foreach(source IN LISTS sources)
|
||||
|
|
@ -9,15 +43,97 @@ foreach(source IN LISTS sources)
|
|||
file(SHA256 "${source}" digest)
|
||||
file(RELATIVE_PATH relative "${ROOT}" "${source}")
|
||||
string(APPEND manifest "${relative} ${digest}\n")
|
||||
string(FIND "${source}" "${RECOMPILER_DIR}/" inRecompiler)
|
||||
if(inRecompiler EQUAL 0 AND source MATCHES "\\.(cpp|hpp|h)$")
|
||||
file(STRINGS "${source}" lines REGEX "\"APS5_[A-Z0-9_]+\"")
|
||||
foreach(line IN LISTS lines)
|
||||
string(REGEX MATCHALL "\"APS5_[A-Z0-9_]+\"" names "${line}")
|
||||
list(APPEND switches ${names})
|
||||
endforeach()
|
||||
if(NOT source MATCHES "\\.(cpp|hpp|h)$")
|
||||
continue()
|
||||
endif()
|
||||
file(STRINGS "${source}" lines REGEX "\"APS5_[A-Z0-9_]+\"")
|
||||
foreach(line IN LISTS lines)
|
||||
string(REGEX MATCHALL "\"APS5_[A-Z0-9_]+\"" names "${line}")
|
||||
list(APPEND switches ${names})
|
||||
endforeach()
|
||||
string(FIND "${source}" "${RECOMPILER_DIR}/" inRecompiler)
|
||||
if(NOT inRecompiler EQUAL 0)
|
||||
continue()
|
||||
endif()
|
||||
get_filename_component(sourceDirectory "${source}" DIRECTORY)
|
||||
file(STRINGS "${source}" lines REGEX "^[ \t]*#[ \t]*include[ \t]*\"[^\"]+\"")
|
||||
foreach(line IN LISTS lines)
|
||||
string(REGEX REPLACE "^[ \t]*#[ \t]*include[ \t]*\"([^\"]+)\".*$" "\\1" included "${line}")
|
||||
set(resolved)
|
||||
foreach(directory IN ITEMS "${sourceDirectory}" ${includeDirectories})
|
||||
if(EXISTS "${directory}/${included}" AND NOT IS_DIRECTORY "${directory}/${included}")
|
||||
get_filename_component(resolved "${directory}/${included}" REALPATH)
|
||||
break()
|
||||
endif()
|
||||
endforeach()
|
||||
file(RELATIVE_PATH relativeSource "${ROOT}" "${source}")
|
||||
if(NOT resolved)
|
||||
continue()
|
||||
endif()
|
||||
string(FIND "${resolved}" "${recompilerReal}/" resolvedInRecompiler)
|
||||
string(FIND "${resolved}" "${rootReal}/3rdparty/" resolvedInThirdParty)
|
||||
if(resolvedInRecompiler EQUAL 0 OR resolvedInThirdParty EQUAL 0 OR resolved IN_LIST sourcesReal)
|
||||
continue()
|
||||
endif()
|
||||
file(RELATIVE_PATH relativeResolved "${rootReal}" "${resolved}")
|
||||
list(APPEND failures "${relativeSource} includes ${relativeResolved}, which the shader disk cache key does not hash: add it to shaderCacheKeySources in core/shader/recompiler/CMakeLists.txt")
|
||||
endforeach()
|
||||
endforeach()
|
||||
|
||||
foreach(submodule IN LISTS submodules)
|
||||
file(RELATIVE_PATH relative "${ROOT}" "${submodule}")
|
||||
set(commit)
|
||||
if(GIT)
|
||||
execute_process(COMMAND "${GIT}" -C "${submodule}" rev-parse --show-toplevel HEAD
|
||||
OUTPUT_VARIABLE output OUTPUT_STRIP_TRAILING_WHITESPACE RESULT_VARIABLE status ERROR_QUIET)
|
||||
if(status EQUAL 0)
|
||||
string(REPLACE "\n" ";" output "${output}")
|
||||
list(GET output 0 topLevel)
|
||||
list(GET output 1 head)
|
||||
get_filename_component(topLevel "${topLevel}" REALPATH)
|
||||
get_filename_component(submoduleReal "${submodule}" REALPATH)
|
||||
if(topLevel STREQUAL submoduleReal)
|
||||
set(commit "${head}")
|
||||
endif()
|
||||
endif()
|
||||
endif()
|
||||
if(NOT commit AND EXISTS "${submodule}/CMakeLists.txt")
|
||||
file(SHA256 "${submodule}/CMakeLists.txt" digest)
|
||||
set(commit "unversioned ${digest}")
|
||||
endif()
|
||||
string(APPEND manifest "submodule ${relative} ${commit}\n")
|
||||
endforeach()
|
||||
|
||||
foreach(source IN LISTS guard)
|
||||
if(NOT EXISTS "${source}")
|
||||
continue()
|
||||
endif()
|
||||
file(RELATIVE_PATH relative "${ROOT}" "${source}")
|
||||
file(STRINGS "${source}" lines REGEX "ShaderRecompiler|setenv|putenv")
|
||||
foreach(line IN LISTS lines)
|
||||
if(line MATCHES "using[ \t]+namespace[ \t]+ShaderRecompiler" OR line MATCHES "namespace[ \t]+[A-Za-z_][A-Za-z0-9_]*[ \t]*=[ \t]*(::)?ShaderRecompiler[ \t]*;")
|
||||
list(APPEND failures "${relative} brings the ShaderRecompiler namespace in whole, which hides its calls from the shader disk cache key review: qualify them")
|
||||
endif()
|
||||
if(line MATCHES "(^|[^A-Za-z0-9_])(setenv|putenv|_putenv_s)[ \t]*\\(")
|
||||
list(APPEND failures "${relative} sets an environment variable, which can change the recompiler's APS5_* switches behind the shader disk cache key")
|
||||
endif()
|
||||
string(REGEX MATCHALL "ShaderRecompiler::[A-Za-z_][A-Za-z0-9_]*(::[A-Za-z_][A-Za-z0-9_]*)*[ \t]*\\(" calls "${line}")
|
||||
string(REGEX MATCHALL "using[ \t]+ShaderRecompiler::[A-Za-z_][A-Za-z0-9_]*(::[A-Za-z_][A-Za-z0-9_]*)*" declarations "${line}")
|
||||
foreach(use IN LISTS calls declarations)
|
||||
string(REGEX REPLACE "^(using[ \t]+)?ShaderRecompiler::([A-Za-z0-9_:]+).*$" "\\2" name "${use}")
|
||||
if(NOT name IN_LIST reviewed)
|
||||
list(APPEND failures "${relative} uses ShaderRecompiler::${name}, which has not been reviewed for the shader disk cache key: if it can change what a recompile produces other than through the RecompileRequest, key that input in ShaderDiskCache::BuildKey, then add the name to shaderCacheReviewedNames in core/shader/recompiler/CMakeLists.txt")
|
||||
endif()
|
||||
endforeach()
|
||||
endforeach()
|
||||
endforeach()
|
||||
|
||||
if(failures)
|
||||
list(REMOVE_DUPLICATES failures)
|
||||
list(JOIN failures "\n " failureText)
|
||||
message(FATAL_ERROR "Shader disk cache key check failed:\n ${failureText}")
|
||||
endif()
|
||||
|
||||
string(SHA256 version "${manifest}")
|
||||
string(SUBSTRING "${version}" 0 16 version)
|
||||
list(REMOVE_DUPLICATES switches)
|
||||
|
|
|
|||
Loading…
Reference in New Issue