From f4f75eac6426eb2af0347962f9ca91b86c6cc5dd Mon Sep 17 00:00:00 2001 From: Tiago Rodrigues Date: Fri, 17 Nov 2023 15:48:17 -0500 Subject: [PATCH 01/21] Add support for offline symbol resolving by setting the "TRACY_SYMBOL_OFFLINE_RESOLVE=1" env var - Add a tool "tracy-edit" that allows loading a tracy capture, patching symbols and recompress the result - Add offline symbol resolvers for linux (using addr2line) and windows (using dbghelper) --- public/client/TracyCallstack.cpp | 238 +++++++++--- server/TracyWorker.cpp | 34 +- server/TracyWorker.hpp | 7 +- tracy-edit/build/unix/Makefile | 16 + tracy-edit/build/unix/build.mk | 13 + tracy-edit/build/unix/debug.mk | 6 + tracy-edit/build/unix/release.mk | 9 + tracy-edit/build/win32/tracy-edit.sln | 25 ++ tracy-edit/build/win32/tracy-edit.vcxproj | 210 ++++++++++ .../build/win32/tracy-edit.vcxproj.filters | 366 ++++++++++++++++++ tracy-edit/src/OfflineSymbolResolver.cpp | 123 ++++++ tracy-edit/src/OfflineSymbolResolver.h | 41 ++ .../src/OfflineSymbolResolverAddr2Line.cpp | 124 ++++++ .../src/OfflineSymbolResolverDbgHelper.cpp | 142 +++++++ tracy-edit/src/tracy-edit.cpp | 181 +++++++++ 15 files changed, 1472 insertions(+), 63 deletions(-) create mode 100644 tracy-edit/build/unix/Makefile create mode 100644 tracy-edit/build/unix/build.mk create mode 100644 tracy-edit/build/unix/debug.mk create mode 100644 tracy-edit/build/unix/release.mk create mode 100644 tracy-edit/build/win32/tracy-edit.sln create mode 100644 tracy-edit/build/win32/tracy-edit.vcxproj create mode 100644 tracy-edit/build/win32/tracy-edit.vcxproj.filters create mode 100644 tracy-edit/src/OfflineSymbolResolver.cpp create mode 100644 tracy-edit/src/OfflineSymbolResolver.h create mode 100644 tracy-edit/src/OfflineSymbolResolverAddr2Line.cpp create mode 100644 tracy-edit/src/OfflineSymbolResolverDbgHelper.cpp create mode 100644 tracy-edit/src/tracy-edit.cpp diff --git a/public/client/TracyCallstack.cpp b/public/client/TracyCallstack.cpp index bef85431..8860cbe2 100644 --- a/public/client/TracyCallstack.cpp +++ b/public/client/TracyCallstack.cpp @@ -93,6 +93,14 @@ extern "C" const char* ___tracy_demangle( const char* mangled ) namespace tracy { +// when "TRACY_SYMBOL_OFFLINE_RESOLVE" is set to "1", instead of fully resolving symbols at runtime, +// simply resolve the offset and image name (which will be enough the resolving to be done offline) +bool getDoOfflineSymbolResolve() +{ + const char* symbolOfflineResolve = GetEnvVar( "TRACY_SYMBOL_OFFLINE_RESOLVE" ); + return (symbolOfflineResolve && symbolOfflineResolve[0] == '1'); +} + #if TRACY_HAS_CALLSTACK == 1 enum { MaxCbTrace = 64 }; @@ -135,19 +143,24 @@ struct KernelDriver KernelDriver* s_krnlCache = nullptr; size_t s_krnlCacheCnt; - +bool s_doOfflineSymbolResolve = false; void InitCallstackCritical() { ___tracy_RtlWalkFrameChain = (___tracy_t_RtlWalkFrameChain)GetProcAddress( GetModuleHandleA( "ntdll.dll" ), "RtlWalkFrameChain" ); } -void InitCallstack() +void dbgHelpInit() { - _SymAddrIncludeInlineTrace = (t_SymAddrIncludeInlineTrace)GetProcAddress( GetModuleHandleA( "dbghelp.dll" ), "SymAddrIncludeInlineTrace" ); - _SymQueryInlineTrace = (t_SymQueryInlineTrace)GetProcAddress( GetModuleHandleA( "dbghelp.dll" ), "SymQueryInlineTrace" ); - _SymFromInlineContext = (t_SymFromInlineContext)GetProcAddress( GetModuleHandleA( "dbghelp.dll" ), "SymFromInlineContext" ); - _SymGetLineFromInlineContext = (t_SymGetLineFromInlineContext)GetProcAddress( GetModuleHandleA( "dbghelp.dll" ), "SymGetLineFromInlineContext" ); + if( s_doOfflineSymbolResolve ) + { + return; + } + + _SymAddrIncludeInlineTrace = (t_SymAddrIncludeInlineTrace)GetProcAddress(GetModuleHandleA("dbghelp.dll"), "SymAddrIncludeInlineTrace"); + _SymQueryInlineTrace = (t_SymQueryInlineTrace)GetProcAddress(GetModuleHandleA("dbghelp.dll"), "SymQueryInlineTrace"); + _SymFromInlineContext = (t_SymFromInlineContext)GetProcAddress(GetModuleHandleA("dbghelp.dll"), "SymFromInlineContext"); + _SymGetLineFromInlineContext = (t_SymGetLineFromInlineContext)GetProcAddress(GetModuleHandleA("dbghelp.dll"), "SymGetLineFromInlineContext"); #ifdef TRACY_DBGHELP_LOCK DBGHELP_INIT; @@ -157,6 +170,65 @@ void InitCallstack() SymInitialize( GetCurrentProcess(), nullptr, true ); SymSetOptions( SYMOPT_LOAD_LINES ); +#ifdef TRACY_DBGHELP_LOCK + DBGHELP_UNLOCK; +#endif +} + +DWORD64 dbgHelpLoadSymbolsForModule(PCSTR imageName, DWORD64 baseOfDll, DWORD bllSize) +{ + if( !s_doOfflineSymbolResolve ) + { + return SymLoadModuleEx( GetCurrentProcess(), nullptr, imageName, nullptr, baseOfDll, bllSize, nullptr, 0 ); + } + return 0x0; +} + +ModuleCache* dbgHelpLoadSymbolsForModuleAndCache(PCSTR imageName, DWORD imageNameLength, DWORD64 baseOfDll, DWORD dllSize) +{ + dbgHelpLoadSymbolsForModule( imageName, baseOfDll, dllSize ); + + ModuleCache* cachedModule = s_modCache->push_next(); + cachedModule->start = baseOfDll; + cachedModule->end = baseOfDll + dllSize; + + // when doing offline symbol resolution, we must store the full path of the dll for the resolving to work + if( s_doOfflineSymbolResolve ) + { + cachedModule->name = (char*)tracy_malloc_fast(imageNameLength + 2); + memcpy(cachedModule->name, imageName, imageNameLength); + cachedModule->name[imageNameLength + 1] = '\0'; + } + else + { + auto ptr = imageName + imageNameLength; + while (ptr > imageName && *ptr != '\\' && *ptr != '/') ptr--; + if (ptr > imageName) ptr++; + const auto namelen = imageName + imageNameLength - ptr; + cachedModule->name = (char*)tracy_malloc_fast(namelen + 3); + cachedModule->name[0] = '['; + memcpy(cachedModule->name + 1, ptr, namelen); + cachedModule->name[namelen + 1] = ']'; + cachedModule->name[namelen + 2] = '\0'; + } + + return cachedModule; +} + +void InitCallstack() +{ + s_doOfflineSymbolResolve = getDoOfflineSymbolResolve(); + if( s_doOfflineSymbolResolve ) + { + TracyDebug("TRACY: enabling offline symbol resolving!\n"); + } + + dbgHelpInit(); + +#ifdef TRACY_DBGHELP_LOCK + DBGHELP_LOCK; +#endif + // use TRACY_NO_DBHELP_INIT_LOAD=1 to disable preloading of driver // and process module symbol loading at startup time - they will be loaded on demand later // Sometimes this process can take a very long time and prevent resolving callstack frames @@ -204,7 +276,7 @@ void InitCallstack() path = full; } - SymLoadModuleEx( GetCurrentProcess(), nullptr, path, nullptr, (DWORD64)dev[i], 0, nullptr, 0 ); + dbgHelpLoadSymbolsForModule( path, (DWORD64)dev[i], 0 ); const auto psz = strlen( path ); auto pptr = (char*)tracy_malloc_fast( psz+1 ); @@ -235,25 +307,12 @@ void InitCallstack() { const auto base = uint64_t( info.lpBaseOfDll ); char name[1024]; - const auto res = GetModuleFileNameA( mod[i], name, 1021 ); - if( res > 0 ) + const auto nameLength = GetModuleFileNameA( mod[i], name, 1021 ); + if( nameLength > 0 ) { // This may be a new module loaded since our call to SymInitialize. // Just in case, force DbgHelp to load its pdb ! - SymLoadModuleEx(proc, NULL, name, NULL, (DWORD64)info.lpBaseOfDll, info.SizeOfImage, NULL, 0); - - auto ptr = name + res; - while( ptr > name && *ptr != '\\' && *ptr != '/' ) ptr--; - if( ptr > name ) ptr++; - const auto namelen = name + res - ptr; - auto cache = s_modCache->push_next(); - cache->start = base; - cache->end = base + info.SizeOfImage; - cache->name = (char*)tracy_malloc_fast( namelen+3 ); - cache->name[0] = '['; - memcpy( cache->name+1, ptr, namelen ); - cache->name[namelen+1] = ']'; - cache->name[namelen+2] = '\0'; + dbgHelpLoadSymbolsForModuleAndCache( name, nameLength, (DWORD64)info.lpBaseOfDll, info.SizeOfImage ); } } } @@ -270,6 +329,11 @@ void EndCallstack() const char* DecodeCallstackPtrFast( uint64_t ptr ) { + if( s_doOfflineSymbolResolve ) + { + return "[unresolved]"; + } + static char ret[MaxNameSize]; const auto proc = GetCurrentProcess(); @@ -305,7 +369,13 @@ const char* GetKernelModulePath( uint64_t addr ) return it->path; } -static const char* GetModuleNameAndPrepareSymbols( uint64_t addr ) +struct ModuleNameAndBaseAddress +{ + const char* name; + uint64_t baseAddr; +}; + +ModuleNameAndBaseAddress GetModuleNameAndPrepareSymbols( uint64_t addr ) { if( ( addr >> 63 ) != 0 ) { @@ -314,17 +384,17 @@ static const char* GetModuleNameAndPrepareSymbols( uint64_t addr ) auto it = std::lower_bound( s_krnlCache, s_krnlCache + s_krnlCacheCnt, addr, []( const KernelDriver& lhs, const uint64_t& rhs ) { return lhs.addr > rhs; } ); if( it != s_krnlCache + s_krnlCacheCnt ) { - return it->mod; + return ModuleNameAndBaseAddress{ it->mod, it->addr }; } } - return ""; + return ModuleNameAndBaseAddress{ "", addr }; } for( auto& v : *s_modCache ) { if( addr >= v.start && addr < v.end ) { - return v.name; + return ModuleNameAndBaseAddress{ v.name, v.start }; } } @@ -345,35 +415,33 @@ static const char* GetModuleNameAndPrepareSymbols( uint64_t addr ) if( addr >= base && addr < base + info.SizeOfImage ) { char name[1024]; - const auto res = GetModuleFileNameA( mod[i], name, 1021 ); - if( res > 0 ) + const auto nameLength = GetModuleFileNameA( mod[i], name, 1021 ); + if( nameLength > 0 ) { // since this is the first time we encounter this module, load its symbols (needed for modules loaded after SymInitialize) - SymLoadModuleEx(proc, NULL, name, NULL, (DWORD64)info.lpBaseOfDll, info.SizeOfImage, NULL, 0); - auto ptr = name + res; - while( ptr > name && *ptr != '\\' && *ptr != '/' ) ptr--; - if( ptr > name ) ptr++; - const auto namelen = name + res - ptr; - auto cache = s_modCache->push_next(); - cache->start = base; - cache->end = base + info.SizeOfImage; - cache->name = (char*)tracy_malloc_fast( namelen+3 ); - cache->name[0] = '['; - memcpy( cache->name+1, ptr, namelen ); - cache->name[namelen+1] = ']'; - cache->name[namelen+2] = '\0'; - return cache->name; + ModuleCache* cachedModule = dbgHelpLoadSymbolsForModuleAndCache( name, nameLength, (DWORD64)info.lpBaseOfDll, info.SizeOfImage ); + return ModuleNameAndBaseAddress{ cachedModule->name, cachedModule->start }; } } } } } - return "[unknown]"; + + return ModuleNameAndBaseAddress{ "[unknown]", 0x0 }; } CallstackSymbolData DecodeSymbolAddress( uint64_t ptr ) { CallstackSymbolData sym; + + if( s_doOfflineSymbolResolve ) + { + sym.file = "[unknown]"; + sym.line = 0; + sym.needFree = false; + return sym; + } + IMAGEHLP_LINE64 line; DWORD displacement = 0; line.SizeOfStruct = sizeof(IMAGEHLP_LINE64); @@ -401,16 +469,29 @@ CallstackSymbolData DecodeSymbolAddress( uint64_t ptr ) CallstackEntryData DecodeCallstackPtr( uint64_t ptr ) { + InitRpmalloc(); + + const ModuleNameAndBaseAddress moduleNameAndAddress = GetModuleNameAndPrepareSymbols( ptr ); + + if( s_doOfflineSymbolResolve ) + { + cb_data[0].symAddr = ptr - moduleNameAndAddress.baseAddr; + cb_data[0].symLen = 0; + + cb_data[0].name = CopyStringFast("[unresolved]"); + cb_data[0].file = CopyStringFast("[unknown]"); + cb_data[0].line = 0; + + return { cb_data, 1, moduleNameAndAddress.name }; + } + int write; const auto proc = GetCurrentProcess(); - InitRpmalloc(); #ifdef TRACY_DBGHELP_LOCK DBGHELP_LOCK; #endif - const auto moduleName = GetModuleNameAndPrepareSymbols(ptr); - #if !defined TRACY_NO_CALLSTACK_INLINES BOOL doInline = FALSE; DWORD ctx = 0; @@ -459,7 +540,7 @@ CallstackEntryData DecodeCallstackPtr( uint64_t ptr ) cb_data[write].line = line.LineNumber; } - cb_data[write].name = symValid ? CopyStringFast( si->Name, si->NameLen ) : CopyStringFast( moduleName ); + cb_data[write].name = symValid ? CopyStringFast( si->Name, si->NameLen ) : CopyStringFast( moduleNameAndAddress.name ); cb_data[write].file = CopyStringFast( filename ); if( symValid ) { @@ -492,7 +573,7 @@ CallstackEntryData DecodeCallstackPtr( uint64_t ptr ) cb.line = line.LineNumber; } - cb.name = symInlineValid ? CopyStringFast( si->Name, si->NameLen ) : CopyStringFast( moduleName ); + cb.name = symInlineValid ? CopyStringFast( si->Name, si->NameLen ) : CopyStringFast( moduleNameAndAddress.name ); cb.file = CopyStringFast( filename ); if( symInlineValid ) { @@ -513,7 +594,7 @@ CallstackEntryData DecodeCallstackPtr( uint64_t ptr ) DBGHELP_UNLOCK; #endif - return { cb_data, uint8_t( cb_num ), moduleName }; + return { cb_data, uint8_t( cb_num ), moduleNameAndAddress.name }; } #elif TRACY_HAS_CALLSTACK == 2 || TRACY_HAS_CALLSTACK == 3 || TRACY_HAS_CALLSTACK == 4 || TRACY_HAS_CALLSTACK == 6 @@ -696,7 +777,16 @@ void InitCallstackCritical() void InitCallstack() { - cb_bts = backtrace_create_state( nullptr, 0, nullptr, nullptr ); + if( getDoOfflineSymbolResolve() ) + { + cb_bts = nullptr; // disable use of libbacktrace calls + TracyDebug("TRACY: enabling offline symbol resolving!\n"); + } + else + { + cb_bts = backtrace_create_state( nullptr, 0, nullptr, nullptr ); + } + #ifndef TRACY_DEMANGLE ___tracy_init_demangle_buffer(); #endif @@ -835,7 +925,15 @@ static void SymbolAddressErrorCb( void* data, const char* /*msg*/, int /*errnum* CallstackSymbolData DecodeSymbolAddress( uint64_t ptr ) { CallstackSymbolData sym; - backtrace_pcinfo( cb_bts, ptr, SymbolAddressDataCb, SymbolAddressErrorCb, &sym ); + if( cb_bts ) + { + backtrace_pcinfo( cb_bts, ptr, SymbolAddressDataCb, SymbolAddressErrorCb, &sym ); + } + else + { + SymbolAddressErrorCb(&sym, nullptr, 0); + } + return sym; } @@ -938,20 +1036,42 @@ void SymInfoError( void* /*data*/, const char* /*msg*/, int /*errnum*/ ) cb_data[cb_num-1].symAddr = 0; } +void getSymbolForOfflineResolve(void* address, Dl_info& dlinfo, CallstackEntry& cbEntry) +{ + // tagged with a string that we can identify as an unresolved symbol + cbEntry.name = CopyStringFast( "[unresolved]" ); + // set .so relative offset so it can be resolved offline + cbEntry.symAddr = (uint64_t)address - (uint64_t)(dlinfo.dli_fbase); + cbEntry.symLen = 0x0; + cbEntry.file = CopyStringFast( "[unknown]" ); + cbEntry.line = 0; +} + CallstackEntryData DecodeCallstackPtr( uint64_t ptr ) { InitRpmalloc(); if( ptr >> 63 == 0 ) { - cb_num = 0; - backtrace_pcinfo( cb_bts, ptr, CallstackDataCb, CallstackErrorCb, nullptr ); - assert( cb_num > 0 ); - - backtrace_syminfo( cb_bts, ptr, SymInfoCallback, SymInfoError, nullptr ); - const char* symloc = nullptr; Dl_info dlinfo; - if( dladdr( (void*)ptr, &dlinfo ) ) symloc = dlinfo.dli_fname; + if( dladdr( (void*)ptr, &dlinfo ) ) + { + symloc = dlinfo.dli_fname; + } + + if(cb_bts) + { + cb_num = 0; + backtrace_pcinfo( cb_bts, ptr, CallstackDataCb, CallstackErrorCb, nullptr ); + assert( cb_num > 0 ); + + backtrace_syminfo( cb_bts, ptr, SymInfoCallback, SymInfoError, nullptr ); + } + else + { + cb_num = 1; + getSymbolForOfflineResolve( (void*)ptr, dlinfo, cb_data[0] ); + } return { cb_data, uint8_t( cb_num ), symloc ? symloc : "[unknown]" }; } diff --git a/server/TracyWorker.cpp b/server/TracyWorker.cpp index 90dac69c..36acf354 100644 --- a/server/TracyWorker.cpp +++ b/server/TracyWorker.cpp @@ -553,11 +553,12 @@ Worker::Worker( const char* name, const char* program, const std::vector( sz+1 ); + memcpy( ptr, newString, sz ); + ptr[sz] = '\0'; + uint32_t idx = m_data.stringData.size(); + m_data.stringMap.emplace( charutil::StringKey { ptr, sz }, idx ); + m_data.stringData.push_back( ptr ); + return idx; +} + static const char* BadExternalThreadNames[] = { "ntdll.dll", "???", diff --git a/server/TracyWorker.hpp b/server/TracyWorker.hpp index 6ef503c9..081f8be9 100644 --- a/server/TracyWorker.hpp +++ b/server/TracyWorker.hpp @@ -446,7 +446,7 @@ public: Worker( const char* addr, uint16_t port ); Worker( const char* name, const char* program, const std::vector& timeline, const std::vector& messages, const std::vector& plots, const std::unordered_map& threadNames ); - Worker( FileRead& f, EventType::Type eventMask = EventType::All, bool bgTasks = true ); + Worker( FileRead& f, EventType::Type eventMask = EventType::All, bool bgTasks = true, bool allowStringModification = false); ~Worker(); const std::string& GetAddr() const { return m_addr; } @@ -549,6 +549,8 @@ public: StringIdx GetLocationForAddress( uint64_t address, uint32_t& line ) const; const uint64_t* GetInlineSymbolList( uint64_t sym, uint32_t len ); + unordered_flat_map& GetCallstackFrameMap() { return m_data.callstackFrameMap; } + #ifndef TRACY_NO_STATISTICS const VarArray& GetParentCallstack( uint32_t idx ) const { return *m_data.parentCallstackPayload[idx]; } const CallstackFrameData* GetParentCallstackFrame( const CallstackFrameId& ptr ) const; @@ -665,6 +667,8 @@ public: void CacheSourceFiles(); + uint32_t AddNewString(const char* newString); + private: void Network(); void Exec(); @@ -984,6 +988,7 @@ private: bool m_combineSamples; bool m_identifySamples = false; bool m_inconsistentSamples; + bool m_allowStringModification = false; short_ptr m_gpuCtxMap[256]; uint32_t m_pendingCallstackId = 0; diff --git a/tracy-edit/build/unix/Makefile b/tracy-edit/build/unix/Makefile new file mode 100644 index 00000000..035eda1b --- /dev/null +++ b/tracy-edit/build/unix/Makefile @@ -0,0 +1,16 @@ +all: release + +debug: + @+make -f debug.mk all + +release: + @+make -f release.mk all + +clean: + @+make -f build.mk clean + +db: clean + @bear -- $(MAKE) -f debug.mk all + @mv -f compile_commands.json ../../../ + +.PHONY: all clean debug release db diff --git a/tracy-edit/build/unix/build.mk b/tracy-edit/build/unix/build.mk new file mode 100644 index 00000000..dffcb6cc --- /dev/null +++ b/tracy-edit/build/unix/build.mk @@ -0,0 +1,13 @@ +CFLAGS += +CXXFLAGS := $(CFLAGS) -std=gnu++17 +DEFINES += -DTRACY_NO_STATISTICS +INCLUDES := -I../../../capstone/include/capstone +LIBS += -lcapstone -lpthread +LDFLAGS := -L../../../capstone +PROJECT := tracy-edit +IMAGE := $(PROJECT)-$(BUILD) + +FILTER := +include ../../../common/src-from-vcxproj.mk + +include ../../../common/unix.mk diff --git a/tracy-edit/build/unix/debug.mk b/tracy-edit/build/unix/debug.mk new file mode 100644 index 00000000..a4ec6b6a --- /dev/null +++ b/tracy-edit/build/unix/debug.mk @@ -0,0 +1,6 @@ +CFLAGS := -g3 -Wall +DEFINES := -DDEBUG +BUILD := debug + +include ../../../common/unix-debug.mk +include build.mk diff --git a/tracy-edit/build/unix/release.mk b/tracy-edit/build/unix/release.mk new file mode 100644 index 00000000..ccf07661 --- /dev/null +++ b/tracy-edit/build/unix/release.mk @@ -0,0 +1,9 @@ +CFLAGS := -O3 +ifndef TRACY_NO_LTO +CFLAGS += -flto +endif +DEFINES := -DNDEBUG +BUILD := release + +include ../../../common/unix-release.mk +include build.mk diff --git a/tracy-edit/build/win32/tracy-edit.sln b/tracy-edit/build/win32/tracy-edit.sln new file mode 100644 index 00000000..9b606fbc --- /dev/null +++ b/tracy-edit/build/win32/tracy-edit.sln @@ -0,0 +1,25 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 16 +VisualStudioVersion = 16.0.30907.101 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "tracy-edit", "tracy-edit.vcxproj", "{447D58BF-94CD-4469-BB90-549C05D03E00}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|x64 = Debug|x64 + Release|x64 = Release|x64 + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {447D58BF-94CD-4469-BB90-549C05D03E00}.Debug|x64.ActiveCfg = Debug|x64 + {447D58BF-94CD-4469-BB90-549C05D03E00}.Debug|x64.Build.0 = Debug|x64 + {447D58BF-94CD-4469-BB90-549C05D03E00}.Release|x64.ActiveCfg = Release|x64 + {447D58BF-94CD-4469-BB90-549C05D03E00}.Release|x64.Build.0 = Release|x64 + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {3E51386C-43EA-44AC-9F24-AFAFE4D63ADE} + EndGlobalSection +EndGlobal diff --git a/tracy-edit/build/win32/tracy-edit.vcxproj b/tracy-edit/build/win32/tracy-edit.vcxproj new file mode 100644 index 00000000..2fb58ced --- /dev/null +++ b/tracy-edit/build/win32/tracy-edit.vcxproj @@ -0,0 +1,210 @@ + + + + + Debug + x64 + + + Release + x64 + + + + 15.0 + {447D58BF-94CD-4469-BB90-549C05D03E00} + tracy-edit + 10.0.17763.0 + x64-windows-static + + + + Application + true + v143 + MultiByte + + + Application + false + v143 + true + MultiByte + + + + + + + + + + + + + + + + true + + + + Level3 + Disabled + true + true + true + TRACY_NO_STATISTICS;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_DEPRECATE;WIN32_LEAN_AND_MEAN;NOMINMAX;_USE_MATH_DEFINES;%(PreprocessorDefinitions) + AdvancedVectorExtensions2 + stdcpplatest + $(ProjectDir)..\..\..\vcpkg_installed\$(VcpkgTriplet)\include;$(ProjectDir)..\..\..\vcpkg_installed\$(VcpkgTriplet)\include\capstone;$(VcpkgManifestRoot)\vcpkg_installed\$(VcpkgTriplet)\$(VcpkgTriplet)\include\capstone;$(VcpkgRoot)\installed\$(VcpkgTriplet)\include\capstone + + + ws2_32.lib;capstone.lib;dbghelp.lib;%(AdditionalDependencies) + Console + $(ProjectDir)..\..\..\vcpkg_installed\$(VcpkgTriplet)\debug\lib + + + + + Level3 + MaxSpeed + true + true + true + true + true + TRACY_NO_STATISTICS;NDEBUG;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_DEPRECATE;WIN32_LEAN_AND_MEAN;NOMINMAX;_USE_MATH_DEFINES;%(PreprocessorDefinitions) + AdvancedVectorExtensions2 + stdcpplatest + $(ProjectDir)..\..\..\vcpkg_installed\$(VcpkgTriplet)\include;$(ProjectDir)..\..\..\vcpkg_installed\$(VcpkgTriplet)\include\capstone;$(VcpkgManifestRoot)\vcpkg_installed\$(VcpkgTriplet)\$(VcpkgTriplet)\include\capstone;$(VcpkgRoot)\installed\$(VcpkgTriplet)\include\capstone + + + true + true + ws2_32.lib;capstone.lib;dbghelp.lib;%(AdditionalDependencies) + Console + $(ProjectDir)..\..\..\vcpkg_installed\$(VcpkgTriplet)\lib + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/tracy-edit/build/win32/tracy-edit.vcxproj.filters b/tracy-edit/build/win32/tracy-edit.vcxproj.filters new file mode 100644 index 00000000..8b77d3ab --- /dev/null +++ b/tracy-edit/build/win32/tracy-edit.vcxproj.filters @@ -0,0 +1,366 @@ + + + + + {729c80ee-4d26-4a5e-8f1f-6c075783eb56} + + + {cf23ef7b-7694-4154-830b-00cf053350ea} + + + {e39d3623-47cd-4752-8da9-3ea324f964c1} + + + {9ec18988-3ab7-4c05-a9d0-46c0a68037de} + + + {5ee9ba63-2914-4027-997e-e743a294bba6} + + + {a166d032-7be0-4d07-9f85-a8199cc1ec7c} + + + {438fff23-197c-4b6f-91f0-74f8b3878571} + + + {e5c7021a-e0e4-45c2-b461-e806bc036d5f} + + + + + server + + + server + + + src + + + server + + + server + + + server + + + server + + + zstd\common + + + zstd\common + + + zstd\common + + + zstd\common + + + zstd\common + + + zstd\common + + + zstd\common + + + zstd\common + + + zstd\compress + + + zstd\compress + + + zstd\compress + + + zstd\compress + + + zstd\compress + + + zstd\compress + + + zstd\compress + + + zstd\compress + + + zstd\compress + + + zstd\compress + + + zstd\compress + + + zstd\compress + + + zstd\compress + + + zstd\decompress + + + zstd\decompress + + + zstd\decompress + + + zstd\decompress + + + zstd\dictBuilder + + + zstd\dictBuilder + + + zstd\dictBuilder + + + zstd\dictBuilder + + + common + + + common + + + common + + + common + + + common + + + src + + + src + + + src + + + + + + server + + + server + + + server + + + server + + + server + + + server + + + server + + + server + + + server + + + server + + + server + + + server + + + server + + + zstd + + + zstd + + + zstd\common + + + zstd\common + + + zstd\common + + + zstd\common + + + zstd\common + + + zstd\common + + + zstd\common + + + zstd\common + + + zstd\common + + + zstd\common + + + zstd\common + + + zstd\common + + + zstd\common + + + zstd\common + + + zstd\compress + + + zstd\compress + + + zstd\compress + + + zstd\compress + + + zstd\compress + + + zstd\compress + + + zstd\compress + + + zstd\compress + + + zstd\compress + + + zstd\compress + + + zstd\compress + + + zstd\compress + + + zstd\compress + + + zstd\decompress + + + zstd\decompress + + + zstd\decompress + + + zstd + + + zstd\dictBuilder + + + zstd\dictBuilder + + + zstd\common + + + zstd\compress + + + common + + + common + + + common + + + common + + + common + + + common + + + common + + + common + + + common + + + common + + + common + + + common + + + common + + + common + + + common + + + + + zstd\decompress + + + \ No newline at end of file diff --git a/tracy-edit/src/OfflineSymbolResolver.cpp b/tracy-edit/src/OfflineSymbolResolver.cpp new file mode 100644 index 00000000..5f0e6630 --- /dev/null +++ b/tracy-edit/src/OfflineSymbolResolver.cpp @@ -0,0 +1,123 @@ +#include +#include +#include +#include +#include +#include +#include + +#include "../../server/TracyWorker.hpp" +#include "../../zstd/zstd.h" + +#include "OfflineSymbolResolver.h" + +// TODO: use string hash map to reduce duplication or use some worker string internal hashing +tracy::StringIdx AddSymbolString(tracy::Worker& worker, const char* str) +{ + uint32_t newStringIdx = worker.AddNewString( str ); + return tracy::StringIdx( newStringIdx ); +} + +bool PatchSymbols(SymbolResolver* resolver, tracy::Worker& worker, bool verbose) +{ + if( !resolver ) + { + return false; + } + + uint64_t callstackFrameCount = worker.GetCallstackFrameCount(); + std::string relativeSoNameMatch = "[unresolved]"; + + std::cout << "Found '" << callstackFrameCount << "' callstack frames. Batching into image groups..." << std::endl; + + // batch the symbol queries by .so so we issue the least amount of requests + using FrameEntriesPerImageIdx = std::unordered_map; + FrameEntriesPerImageIdx entriesPerImageIdx; + + auto& callstackFrameMap = worker.GetCallstackFrameMap(); + for( auto it = callstackFrameMap.begin(); it != callstackFrameMap.end(); ++it ) + { + tracy::CallstackFrameData* frameDataPtr = it->second; + if( !frameDataPtr ) + { + continue; + } + + tracy::CallstackFrameData& frameData = *frameDataPtr; + const char* imageName = worker.GetString( frameData.imageName ); + + const uint32_t imageNameIdx = frameData.imageName.Idx(); + FrameEntryList& entries = entriesPerImageIdx[imageNameIdx]; + + for( uint8_t f = 0; f < frameData.size; f++ ) + { + tracy::CallstackFrame& frame = frameData.data[f]; + + // TODO: use a better way to identify symbols that are unresolved + const char* nameStr = worker.GetString(frame.name); + if( strncmp( nameStr, relativeSoNameMatch.c_str(), relativeSoNameMatch.length() ) == 0 ) + { + // when doing offline resolving we pass the offset from the start of the shared library in the "symAddr" + const uint64_t decodedOffset = frame.symAddr; + entries.push_back( {&frame, decodedOffset} ); + } + } + } + + std::cout << "Batched into '" << entriesPerImageIdx.size() << "' unique image groups" << std::endl; + + // FIXME: the resolving of symbols here can be slow and could be done in parallel per "image" + // - be careful with string allocation though as that would be not safe to do in parallel + for( FrameEntriesPerImageIdx::iterator imageIt = entriesPerImageIdx.begin(), + imageItEnd = entriesPerImageIdx.end(); imageIt != imageItEnd; ++imageIt ) + { + tracy::StringIdx imageIdx( imageIt->first ); + const char* imageName = worker.GetString( imageIdx ); + + FrameEntryList& entries = imageIt->second; + + std::cout << "Resolving " << entries.size() << " symbols for image: '" << imageName << "'" << std::endl; + + if(!entries.size()) + { + continue; + } + + SymbolEntryList resolvedEntries; + ResolveSymbols( resolver, imageName, entries, resolvedEntries ); + + if( resolvedEntries.size() != entries.size() ) + { + std::cerr << "ERROR: failed to resolve all entries! (got: " << resolvedEntries.size() << ")" << std::endl; + continue; + } + + // finally patch the string with the resolved symbol data + for (size_t i = 0; i < resolvedEntries.size(); ++i) + { + FrameEntry& frameEntry = entries[i]; + const SymbolEntry& symbolEntry = resolvedEntries[i]; + + tracy::CallstackFrame& frame = *frameEntry.frame; + if(!symbolEntry.name.length()) + continue; + + if(verbose) + { + const char* nameStr = worker.GetString(frame.name); + std::cout << "patching '" << nameStr << "' of '" << imageName << "' -> '" << symbolEntry.name << "'" << std::endl; + } + + frame.name = AddSymbolString(worker, symbolEntry.name.c_str()); + const char* newName = worker.GetString(frame.name); + + if(symbolEntry.file.length()) + { + frame.file = AddSymbolString(worker, symbolEntry.file.c_str()); + frame.line = symbolEntry.line; + } + } + } + + return true; +} diff --git a/tracy-edit/src/OfflineSymbolResolver.h b/tracy-edit/src/OfflineSymbolResolver.h new file mode 100644 index 00000000..584e9179 --- /dev/null +++ b/tracy-edit/src/OfflineSymbolResolver.h @@ -0,0 +1,41 @@ +#ifndef __SYMBOLRESOLVER_HPP__ +#define __SYMBOLRESOLVER_HPP__ + +#include +#include + +namespace tracy +{ + struct CallstackFrame; + class Worker; +} + +class SymbolResolver; + +SymbolResolver* CreateResolver(); +void DestroySymbolResolver(SymbolResolver* resolver); + +struct FrameEntry +{ + tracy::CallstackFrame* frame = nullptr; + uint64_t symbolOffset = 0; +}; + +using FrameEntryList = std::vector; + +struct SymbolEntry +{ + std::string name; + std::string file; + int line = 0; +}; + +using SymbolEntryList = std::vector; + +bool ResolveSymbols(SymbolResolver* resolver, const char* imageName, + const FrameEntryList& inputEntryList, + SymbolEntryList& resolvedEntries); + +bool PatchSymbols(SymbolResolver* resolver, tracy::Worker& worker, bool verbose = false); + +#endif // __SYMBOLRESOLVER_HPP__ \ No newline at end of file diff --git a/tracy-edit/src/OfflineSymbolResolverAddr2Line.cpp b/tracy-edit/src/OfflineSymbolResolverAddr2Line.cpp new file mode 100644 index 00000000..cb467f51 --- /dev/null +++ b/tracy-edit/src/OfflineSymbolResolverAddr2Line.cpp @@ -0,0 +1,124 @@ +#ifdef __linux + +#include "OfflineSymbolResolver.h" + +#include +#include +#include +#include +#include +#include +#include + +std::string ExecShellCommand(const char* cmd) +{ + std::array buffer; + std::string result; + std::unique_ptr pipe(popen(cmd, "r"), pclose); + if (!pipe) + { + return ""; + } + while (fgets(buffer.data(), buffer.size(), pipe.get()) != nullptr) + { + result += buffer.data(); + } + return result; +} + +class SymbolResolver +{ +public: + SymbolResolver(const std::string& addr2linePath) + : m_addr2LinePath(addr2linePath) + {} + + bool ResolveSymbols(const char* imageName, const FrameEntryList& inputEntryList, + SymbolEntryList& resolvedEntries) + { + // generate a single addr2line cmd line for all addresses in one invocation + std::stringstream ss; + ss << m_addr2LinePath << " -C -f -e " << imageName << " -a "; + for (const FrameEntry& entry : inputEntryList) + { + ss << " 0x" << std::hex << entry.symbolOffset; + } + + std::string resultStr = ExecShellCommand(ss.str().c_str()); + std::stringstream result(resultStr); + //printf("executing: '%s' got '%s'\n", ss.str().c_str(), result.str().c_str()); + + // The output is 2 lines per entry with the following contents: + // hex_address_of_symbol + // symbol_name + // file:line + + for (size_t i = 0; i < inputEntryList.size(); ++i) + { + const FrameEntry& inputEntry = inputEntryList[i]; + + SymbolEntry newEntry; + + std::string addr; + std::getline(result, addr); + std::getline(result, newEntry.name); + if (newEntry.name == "??") + { + newEntry.name = "[unknown] + " + std::to_string(inputEntry.symbolOffset); + } + + std::string fileLine; + std::getline(result, fileLine); + if (fileLine != "??:?") + { + size_t pos = fileLine.find_last_of(':'); + if (pos != std::string::npos) + { + newEntry.file = fileLine.substr(0, pos); + std::string lineStr = fileLine.substr(pos + 1); + char* after = nullptr; + newEntry.line = strtol(lineStr.c_str(), &after, 10); + } + } + + resolvedEntries.push_back(std::move(newEntry)); + } + + return true; + } + +private: + std::string m_addr2LinePath; +}; + +SymbolResolver* CreateResolver() +{ + std::stringstream result(ExecShellCommand("which addr2line")); + std::string addr2LinePath; + std::getline(result, addr2LinePath); + + if(!addr2LinePath.length()) + { + std::cerr << "'addr2line' was not found in the system, please installed it" << std::endl; + return nullptr; + } + std::cout << "Using 'addr2line' found at: '" << addr2LinePath.c_str() << "'" << std::endl; + return new SymbolResolver{addr2LinePath}; +} + +void DestroySymbolResolver(SymbolResolver* resolver) +{ + delete resolver; +} + +bool ResolveSymbols(SymbolResolver* resolver, const char* imageName, + const FrameEntryList& inputEntryList, SymbolEntryList& resolvedEntries) +{ + if (resolver) + { + return resolver->ResolveSymbols(imageName, inputEntryList, resolvedEntries); + } + return false; +} + +#endif // #ifdef __linux \ No newline at end of file diff --git a/tracy-edit/src/OfflineSymbolResolverDbgHelper.cpp b/tracy-edit/src/OfflineSymbolResolverDbgHelper.cpp new file mode 100644 index 00000000..40b96cd3 --- /dev/null +++ b/tracy-edit/src/OfflineSymbolResolverDbgHelper.cpp @@ -0,0 +1,142 @@ +#ifdef _WIN32 + +#ifndef WIN32_LEAN_AND_MEAN +# define WIN32_LEAN_AND_MEAN +#endif + +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "OfflineSymbolResolver.h" + +class SymbolResolver +{ +public: + SymbolResolver() + { + m_procHandle = GetCurrentProcess(); + + if (!SymInitialize(m_procHandle, NULL, FALSE)) + { + std::cerr << "SymInitialize() failed with: " << GetLastErrorString() << std::endl; + } + else + { + const DWORD symopts = SYMOPT_UNDNAME | SYMOPT_DEFERRED_LOADS | SYMOPT_DEBUG | SYMOPT_LOAD_LINES; + SymSetOptions( symopts ); + } + } + + ~SymbolResolver() + { + SymCleanup( m_procHandle ); + } + + bool ResolveSymbolsForModule(const char* fileName, const FrameEntryList& inputEntryList, + SymbolEntryList& resolvedEntries) + { + ULONG64 moduleBase = SymLoadModuleEx( m_procHandle, NULL, fileName, NULL, 0, 0, NULL, 0 ); + if (!moduleBase) + { + std::cerr << "SymLoadModuleEx() failed for module " << fileName + << ": " << GetLastErrorString() << std::endl; + return false; + } + + for (size_t i = 0; i < inputEntryList.size(); ++i) + { + uint64_t offset = inputEntryList[i].symbolOffset; + DWORD64 address = moduleBase + offset; + + SYMBOL_INFO* symbolInfo = (SYMBOL_INFO*)s_symbolResolutionBuffer; + symbolInfo->SizeOfStruct = sizeof(SYMBOL_INFO); + symbolInfo->MaxNameLen = MAX_SYM_NAME; + + SymbolEntry newEntry; + + if ( SymFromAddr( m_procHandle, address, NULL, symbolInfo ) ) + { + newEntry.name = symbolInfo->Name; + //std::cout << "Resolved symbol to: '" << newEntry.name << "'" << std::endl; + } + else + { + newEntry.name = "[unknown] + " + std::to_string(offset); + } + + IMAGEHLP_LINE lineInfo = { 0 }; + lineInfo.SizeOfStruct = sizeof(IMAGEHLP_LINE64); + DWORD displaceMent = 0; + if ( SymGetLineFromAddr64( m_procHandle, address, &displaceMent, &lineInfo ) ) + { + newEntry.file = lineInfo.FileName; + newEntry.line = int(lineInfo.LineNumber); + ///std::cout << "\tline_file: " lineInfo.FileName << ":" << int(lineInfo.LineNumber) << std::endl; + } + + resolvedEntries.push_back(std::move(newEntry)); + } + + SymUnloadModule64(m_procHandle, moduleBase); + return true; + } + +private: + static const size_t symbolResolutionBufferSize = sizeof(SYMBOL_INFO) + MAX_SYM_NAME; + static char s_symbolResolutionBuffer[symbolResolutionBufferSize]; + + std::string GetLastErrorString() + { + DWORD error = GetLastError(); + if (error == 0) + { + return ""; + } + + LPSTR messageBuffer = nullptr; + DWORD dwFlags = FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS; + size_t size = FormatMessageA( dwFlags, NULL, error, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), + (LPSTR)&messageBuffer, 0, NULL ); + + std::string message(messageBuffer, size); + LocalFree(messageBuffer); + + return message; + } + + HANDLE m_procHandle = nullptr; +}; + +char SymbolResolver::s_symbolResolutionBuffer[symbolResolutionBufferSize]; + + +SymbolResolver* CreateResolver() +{ + SymbolResolver* resolver = new SymbolResolver(); + return resolver; +} + +void DestroySymbolResolver(SymbolResolver* resolver) +{ + delete resolver; +} + +bool ResolveSymbols(SymbolResolver* resolver, const char* imageName, + const FrameEntryList& inputEntryList, + SymbolEntryList& resolvedEntries) +{ + if( resolver ) + { + return resolver->ResolveSymbolsForModule( imageName, inputEntryList, resolvedEntries ); + } + return false; +} + +#endif // #ifdef _WIN32 \ No newline at end of file diff --git a/tracy-edit/src/tracy-edit.cpp b/tracy-edit/src/tracy-edit.cpp new file mode 100644 index 00000000..dcf8a251 --- /dev/null +++ b/tracy-edit/src/tracy-edit.cpp @@ -0,0 +1,181 @@ +#ifdef _WIN32 +# include +#endif + +#include +#include +#include +#include +#include +#include + +#include "../../server/TracyFileRead.hpp" +#include "../../server/TracyFileWrite.hpp" +#include "../../server/TracyWorker.hpp" +#include "../../getopt/getopt.h" + +#include "OfflineSymbolResolver.h" + +struct Args +{ + const char* inputTracyPath = nullptr; + const char* outputTracyPath = nullptr; + bool verbose = false; + bool resolveSymbols = false; + tracy::FileWrite::Compression compressionType = tracy::FileWrite::Compression::Zstd; + int compressionLevel = 5; +}; + +void PrintUsageAndExit() +{ + std::cerr << "Modify a tracy file" << std::endl; + std::cerr << "Usage:" << std::endl; + std::cerr << " extract [OPTION...] " << std::endl; + std::cerr << std::endl; + std::cerr << " -h, --help Print usage" << std::endl; + std::cerr << " -v, --verbose Enable verbose logging" << std::endl; + std::cerr << " -r, --resolveSymbols Resolve symbols and patch callstack frames" << std::endl; + std::cerr << " -c, --compression arg Compress output with the given compression algo" << std::endl; + std::cerr << " -l, --compressesionLevel arg Level of compression" << std::endl; + exit( 1 ); +} + +static const char* compressionTypeStr[] +{ + "Fast", + "Slow", + "Extreme", + "Zstd" +}; +static_assert( uint32_t(tracy::FileWrite::Compression::Zstd)+1 == sizeof(compressionTypeStr)/sizeof(compressionTypeStr[0])); +tracy::FileWrite::Compression getCompressionFromString(const char* str) +{ + for( uint32_t i = 0; i < sizeof(compressionTypeStr)/sizeof(compressionTypeStr[0]); ++i ) + { + if( strcmp( compressionTypeStr[i], str ) == 0 ) + { + return tracy::FileWrite::Compression( i ); + } + } + return tracy::FileWrite::Compression::Zstd; +} + +Args ParseArgs( int argc, char** argv ) +{ + if ( argc < 3 ) + { + PrintUsageAndExit(); + } + + Args args; + + struct option long_opts[] = + { + { "help", no_argument, NULL, 'h' }, + { "verbose", no_argument, NULL, 'v' }, + { "resolveSymbols", no_argument, NULL, 'r' }, + { "compression", required_argument, NULL, 'c' }, + { "compressesionLevel", required_argument, NULL, 'l' }, + { NULL, 0, NULL, 0 } + }; + + int c; + while ( (c = getopt_long( argc, argv, "hvrc:l:", long_opts, NULL )) != -1 ) + { + switch (c) + { + case 'h': + PrintUsageAndExit(); + break; + case 'v': + args.verbose = true; + break; + case 'r': + args.resolveSymbols = true; + break; + case 'c': + args.compressionType = getCompressionFromString( optarg ); + break; + case 'l': + args.compressionLevel = atoi( optarg ); + break; + default: + PrintUsageAndExit(); + break; + } + } + + if (argc != optind + 2) + { + PrintUsageAndExit(); + } + + args.inputTracyPath = argv[optind + 0]; + args.outputTracyPath = argv[optind + 1]; + + return args; +} + +int main( int argc, char** argv ) +{ +#ifdef _WIN32 + if( !AttachConsole( ATTACH_PARENT_PROCESS ) ) + { + AllocConsole(); + SetConsoleMode( GetStdHandle( STD_OUTPUT_HANDLE ), 0x07 ); + } +#endif // #ifdef _WIN32 + + Args args = ParseArgs( argc, argv ); + + // load input tracy file + auto f = std::unique_ptr(tracy::FileRead::Open( args.inputTracyPath )); + if (!f) + { + std::cerr << "Could not open file: " << args.inputTracyPath; + return 1; + } + + std::cout << "Reading ..." << std::endl; + + const bool allowBgThreads = false; + bool allowStringModification = true; + tracy::Worker worker( *f, tracy::EventType::All, allowBgThreads, allowStringModification ); + + std::cout << "Loaded." << std::endl; + + // attempt to resolve symbols only if requested + if(args.resolveSymbols) + { + std::cout << "Resolving and patching symbols..." << std::endl; + + SymbolResolver* resolver = CreateResolver(); + if(!resolver) + { + std::cerr << "Failed to create symbol resolver - skipping resolving" << std::endl; + } + else + { + PatchSymbols(resolver, worker); + DestroySymbolResolver(resolver); + } + } + + // save out capture file with new compression options + std::cout << "Saving (using '" << compressionTypeStr[uint32_t(args.compressionType)] + << "', level: " << args.compressionLevel << ") ..." << std::endl; + + auto w = std::unique_ptr( + tracy::FileWrite::Open( args.outputTracyPath, args.compressionType, args.compressionLevel) ); + if( !w ) + { + std::cerr << "Cannot open output file: '" << args.outputTracyPath << "'" << std::endl; + exit( 1 ); + } + + worker.Write( *w, false ); + + std::cout << "Cleanup..." << std::endl; + + return 0; +} From a24fec91bd7475af0dd9aac9bfbb103dafb2dc0e Mon Sep 17 00:00:00 2001 From: Tiago Rodrigues Date: Fri, 17 Nov 2023 16:25:31 -0500 Subject: [PATCH 02/21] try to add tracy-edit to github workflows --- .github/workflows/macos.yml | 2 ++ .github/workflows/msvc.yml | 3 +++ 2 files changed, 5 insertions(+) diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 54697835..b8138a91 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -24,6 +24,8 @@ jobs: run: make -j`nproc` -C csvexport/build/unix debug release - name: Import-chrome utility run: make -j`nproc` -C import-chrome/build/unix debug release + - name: tracy-edit utility + run: make -j`nproc` -C tracy-edit/build/unix debug release - name: Library run: make -j`nproc` -C library/unix debug release - name: Library (meson) diff --git a/.github/workflows/msvc.yml b/.github/workflows/msvc.yml index 122b754b..3cbecd71 100644 --- a/.github/workflows/msvc.yml +++ b/.github/workflows/msvc.yml @@ -36,6 +36,8 @@ jobs: run: msbuild .\import-chrome\build\win32\import-chrome.vcxproj /property:Configuration=Debug /property:Platform=x64 - name: Import-chrome utility Release run: msbuild .\import-chrome\build\win32\import-chrome.vcxproj /property:Configuration=Release /property:Platform=x64 + - name: tracy-edit utility Release + run: msbuild .\tracy-edit\build\win32\tracy-edit.vcxproj /property:Configuration=Release /property:Platform=x64 - name: Library run: msbuild .\library\win32\TracyProfiler.vcxproj /property:Configuration=Release /property:Platform=x64 - name: Package binaries @@ -46,6 +48,7 @@ jobs: copy update\build\win32\x64\Release\update.exe bin copy capture\build\win32\x64\Release\capture.exe bin copy import-chrome\build\win32\x64\Release\import-chrome.exe bin + copy tracy-edit\build\win32\x64\Release\tracy-edit.exe bin copy csvexport\build\win32\x64\Release\csvexport.exe bin copy library\win32\x64\Release\TracyProfiler.dll bin\dev copy library\win32\x64\Release\TracyProfiler.lib bin\dev From 5c3e303f6fccead94a55ee6942bee9c7adae078b Mon Sep 17 00:00:00 2001 From: Tiago Rodrigues Date: Fri, 17 Nov 2023 16:29:06 -0500 Subject: [PATCH 03/21] add tracy-edit debug compilation workflow --- .github/workflows/msvc.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/msvc.yml b/.github/workflows/msvc.yml index 3cbecd71..fbcb441d 100644 --- a/.github/workflows/msvc.yml +++ b/.github/workflows/msvc.yml @@ -36,6 +36,8 @@ jobs: run: msbuild .\import-chrome\build\win32\import-chrome.vcxproj /property:Configuration=Debug /property:Platform=x64 - name: Import-chrome utility Release run: msbuild .\import-chrome\build\win32\import-chrome.vcxproj /property:Configuration=Release /property:Platform=x64 + - name: tracy-edit utility Debug + run: msbuild .\tracy-edit\build\win32\tracy-edit.vcxproj /property:Configuration=Debug /property:Platform=x64 - name: tracy-edit utility Release run: msbuild .\tracy-edit\build\win32\tracy-edit.vcxproj /property:Configuration=Release /property:Platform=x64 - name: Library From a887d558f1b75e148861f36745e607dc40864579 Mon Sep 17 00:00:00 2001 From: Tiago Rodrigues Date: Fri, 17 Nov 2023 16:31:09 -0500 Subject: [PATCH 04/21] enable addr2line symbol resolver in all platforms except windows --- tracy-edit/src/OfflineSymbolResolverAddr2Line.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tracy-edit/src/OfflineSymbolResolverAddr2Line.cpp b/tracy-edit/src/OfflineSymbolResolverAddr2Line.cpp index cb467f51..5e93e242 100644 --- a/tracy-edit/src/OfflineSymbolResolverAddr2Line.cpp +++ b/tracy-edit/src/OfflineSymbolResolverAddr2Line.cpp @@ -1,4 +1,4 @@ -#ifdef __linux +#ifndef _WIN32 #include "OfflineSymbolResolver.h" @@ -121,4 +121,4 @@ bool ResolveSymbols(SymbolResolver* resolver, const char* imageName, return false; } -#endif // #ifdef __linux \ No newline at end of file +#endif // #ifndef _WIN32 \ No newline at end of file From 94e7da11ec673a5234d5d47a2d12168575a01dff Mon Sep 17 00:00:00 2001 From: Tiago Rodrigues Date: Fri, 17 Nov 2023 16:52:06 -0500 Subject: [PATCH 05/21] use pkg-config to get libcapstone config --- tracy-edit/build/unix/build.mk | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tracy-edit/build/unix/build.mk b/tracy-edit/build/unix/build.mk index dffcb6cc..85fb7528 100644 --- a/tracy-edit/build/unix/build.mk +++ b/tracy-edit/build/unix/build.mk @@ -1,9 +1,8 @@ CFLAGS += CXXFLAGS := $(CFLAGS) -std=gnu++17 DEFINES += -DTRACY_NO_STATISTICS -INCLUDES := -I../../../capstone/include/capstone -LIBS += -lcapstone -lpthread -LDFLAGS := -L../../../capstone +INCLUDES := $(shell pkg-config --cflags capstone) +LIBS += $(shell pkg-config --libs capstone) -lpthread PROJECT := tracy-edit IMAGE := $(PROJECT)-$(BUILD) From fe0e5f335805019d266dd1bfa27cb8c8b86de54d Mon Sep 17 00:00:00 2001 From: Tiago Rodrigues Date: Fri, 17 Nov 2023 21:06:30 -0500 Subject: [PATCH 06/21] Add some basic documentation for offline symbols --- manual/tracy.tex | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/manual/tracy.tex b/manual/tracy.tex index 742a8368..c6da6d2e 100644 --- a/manual/tracy.tex +++ b/manual/tracy.tex @@ -1782,6 +1782,32 @@ At initilization time, tracy will attempt to preload symbols for device drivers Inline frames retrieval on Windows can be multiple orders of magnitude slower than just performing essential symbol resolution. This manifests as profiler seemingly being stuck for a long time, having hundreds of thousands of query backlog entries queued, which are slowly trickling down. If your use case requires speed of operation rather than having call stacks with inline frames included, you may define the \texttt{TRACY\_NO\_CALLSTACK\_INLINES} macro, which will make the profiler stick to the basic but fast frame resolution mode. +\paragraph{Offline symbol resolution} + +By default, tracy client resolves callstack symbols in a background thread at runtime. +This process requires that tracy client load symbols for the shared libraries +involved, which requires additial memory allocations, and potential impact runtime performance if a lot of symbol queries are involved. +As an alternative to to runtime symbol resolution, we can set the environment variable + \texttt{TRACY\_SYMBOL\_OFFLINE\_RESOLVE} to 1 and instead have tracy client only resolve + the minimal set of info required for offline resolution (a shared library path and an offset into that shared library). + +The generated tracy capture will have callstack frames symbols showing \texttt{[unresolved]}. +The \texttt{tracy-edit} tool can be used to load that capture, perforce symbol resolution offline +(by passing \texttt{-r}) and writing out a new capture with symbols resolved. +By default \texttt{tracy-edit} will use the original shared libraries paths that were recorded +in the capture (which assumes running in the same machine or a machine with identical +filesystem setup as the one used to run the tracy instrumented application). + You can do path substitution with the \texttt{-s} option to perform any number of path +substitions in order to use symbols located elsewhere. + +\begin{bclogo}[ +noborder=true, +couleur=black!5, +logo=\bcbombe +]{Important} +Beware that \texttt{tracy-edit} will use any matching symbol file to the path it resolved to (no symbol version checking is done), so if the symbol file doesn't match the code that was used when doing the callstack capturing you will get incorrect results. +\end{bclogo} + \subsection{Lua support} To profile Lua code using Tracy, include the \texttt{public/tracy/TracyLua.hpp} header file in your Lua wrapper and execute \texttt{tracy::LuaRegister(lua\_State*)} function to add instrumentation support. From 0491cad49a3eebe15a98b88553efbf20c6ad7635 Mon Sep 17 00:00:00 2001 From: Tiago Rodrigues Date: Fri, 17 Nov 2023 21:07:25 -0500 Subject: [PATCH 07/21] Add support for symbol path replacement using regex in tracy-edit --- tracy-edit/src/OfflineSymbolResolver.cpp | 59 +++++++++----- tracy-edit/src/OfflineSymbolResolver.h | 8 +- .../src/OfflineSymbolResolverAddr2Line.cpp | 34 ++++---- .../src/OfflineSymbolResolverDbgHelper.cpp | 10 +-- tracy-edit/src/tracy-edit.cpp | 79 ++++++++++++++----- 5 files changed, 128 insertions(+), 62 deletions(-) diff --git a/tracy-edit/src/OfflineSymbolResolver.cpp b/tracy-edit/src/OfflineSymbolResolver.cpp index 5f0e6630..3788e8b8 100644 --- a/tracy-edit/src/OfflineSymbolResolver.cpp +++ b/tracy-edit/src/OfflineSymbolResolver.cpp @@ -11,6 +11,19 @@ #include "OfflineSymbolResolver.h" +bool ApplyPathSubstitutions(std::string& path, const PathSubstitutionList& pathSubstituionlist) +{ + for( const auto& substituion : pathSubstituionlist ) + { + if( std::regex_match(path, substituion.first) ) + { + path = std::regex_replace( path, substituion.first, substituion.second ); + return true; + } + } + return false; +} + // TODO: use string hash map to reduce duplication or use some worker string internal hashing tracy::StringIdx AddSymbolString(tracy::Worker& worker, const char* str) { @@ -18,7 +31,8 @@ tracy::StringIdx AddSymbolString(tracy::Worker& worker, const char* str) return tracy::StringIdx( newStringIdx ); } -bool PatchSymbols(SymbolResolver* resolver, tracy::Worker& worker, bool verbose) +bool PatchSymbols(SymbolResolver* resolver, tracy::Worker& worker, + const PathSubstitutionList& pathSubstituionlist, bool verbose) { if( !resolver ) { @@ -72,48 +86,57 @@ bool PatchSymbols(SymbolResolver* resolver, tracy::Worker& worker, bool verbose) imageItEnd = entriesPerImageIdx.end(); imageIt != imageItEnd; ++imageIt ) { tracy::StringIdx imageIdx( imageIt->first ); - const char* imageName = worker.GetString( imageIdx ); + std::string imagePath = worker.GetString( imageIdx ); FrameEntryList& entries = imageIt->second; - - std::cout << "Resolving " << entries.size() << " symbols for image: '" << imageName << "'" << std::endl; - - if(!entries.size()) + if (!entries.size()) { continue; } + std::cout << "Resolving " << entries.size() << " symbols for image: '" + << imagePath << "'" << std::endl; + const bool substituted = ApplyPathSubstitutions(imagePath, pathSubstituionlist); + if (substituted) + { + std::cout << "\tPath substituted to: '" << imagePath << "'" << std::endl; + } + SymbolEntryList resolvedEntries; - ResolveSymbols( resolver, imageName, entries, resolvedEntries ); + ResolveSymbols( resolver, imagePath, entries, resolvedEntries ); if( resolvedEntries.size() != entries.size() ) { - std::cerr << "ERROR: failed to resolve all entries! (got: " << resolvedEntries.size() << ")" << std::endl; + std::cerr << " failed to resolve all entries! (got: " + << resolvedEntries.size() << ")" << std::endl; continue; } // finally patch the string with the resolved symbol data - for (size_t i = 0; i < resolvedEntries.size(); ++i) + for ( size_t i = 0; i < resolvedEntries.size(); ++i ) { FrameEntry& frameEntry = entries[i]; const SymbolEntry& symbolEntry = resolvedEntries[i]; tracy::CallstackFrame& frame = *frameEntry.frame; - if(!symbolEntry.name.length()) - continue; - - if(verbose) + if (!symbolEntry.name.length()) { - const char* nameStr = worker.GetString(frame.name); - std::cout << "patching '" << nameStr << "' of '" << imageName << "' -> '" << symbolEntry.name << "'" << std::endl; + continue; } - frame.name = AddSymbolString(worker, symbolEntry.name.c_str()); + if( verbose ) + { + const char* nameStr = worker.GetString( frame.name ); + std::cout << "patching '" << nameStr << "' of '" << imagePath + << "' -> '" << symbolEntry.name << "'" << std::endl; + } + + frame.name = AddSymbolString( worker, symbolEntry.name.c_str() ); const char* newName = worker.GetString(frame.name); - if(symbolEntry.file.length()) + if( symbolEntry.file.length() ) { - frame.file = AddSymbolString(worker, symbolEntry.file.c_str()); + frame.file = AddSymbolString( worker, symbolEntry.file.c_str() ); frame.line = symbolEntry.line; } } diff --git a/tracy-edit/src/OfflineSymbolResolver.h b/tracy-edit/src/OfflineSymbolResolver.h index 584e9179..26d483a2 100644 --- a/tracy-edit/src/OfflineSymbolResolver.h +++ b/tracy-edit/src/OfflineSymbolResolver.h @@ -3,6 +3,7 @@ #include #include +#include namespace tracy { @@ -32,10 +33,13 @@ struct SymbolEntry using SymbolEntryList = std::vector; -bool ResolveSymbols(SymbolResolver* resolver, const char* imageName, +bool ResolveSymbols(SymbolResolver* resolver, const std::string& imagePath, const FrameEntryList& inputEntryList, SymbolEntryList& resolvedEntries); -bool PatchSymbols(SymbolResolver* resolver, tracy::Worker& worker, bool verbose = false); +using PathSubstitutionList = std::vector >; + +bool PatchSymbols(SymbolResolver* resolver, tracy::Worker& worker, + const PathSubstitutionList& pathSubstituionlist, bool verbose = false); #endif // __SYMBOLRESOLVER_HPP__ \ No newline at end of file diff --git a/tracy-edit/src/OfflineSymbolResolverAddr2Line.cpp b/tracy-edit/src/OfflineSymbolResolverAddr2Line.cpp index 5e93e242..b35b65b5 100644 --- a/tracy-edit/src/OfflineSymbolResolverAddr2Line.cpp +++ b/tracy-edit/src/OfflineSymbolResolverAddr2Line.cpp @@ -33,18 +33,18 @@ public: : m_addr2LinePath(addr2linePath) {} - bool ResolveSymbols(const char* imageName, const FrameEntryList& inputEntryList, + bool ResolveSymbols(const std::string& imagePath, const FrameEntryList& inputEntryList, SymbolEntryList& resolvedEntries) { // generate a single addr2line cmd line for all addresses in one invocation std::stringstream ss; - ss << m_addr2LinePath << " -C -f -e " << imageName << " -a "; - for (const FrameEntry& entry : inputEntryList) + ss << m_addr2LinePath << " -C -f -e " << imagePath << " -a "; + for ( const FrameEntry& entry : inputEntryList ) { ss << " 0x" << std::hex << entry.symbolOffset; } - std::string resultStr = ExecShellCommand(ss.str().c_str()); + std::string resultStr = ExecShellCommand( ss.str().c_str() ); std::stringstream result(resultStr); //printf("executing: '%s' got '%s'\n", ss.str().c_str(), result.str().c_str()); @@ -53,15 +53,15 @@ public: // symbol_name // file:line - for (size_t i = 0; i < inputEntryList.size(); ++i) + for( size_t i = 0; i < inputEntryList.size(); ++i ) { const FrameEntry& inputEntry = inputEntryList[i]; SymbolEntry newEntry; std::string addr; - std::getline(result, addr); - std::getline(result, newEntry.name); + std::getline( result, addr ); + std::getline( result, newEntry.name ); if (newEntry.name == "??") { newEntry.name = "[unknown] + " + std::to_string(inputEntry.symbolOffset); @@ -69,19 +69,19 @@ public: std::string fileLine; std::getline(result, fileLine); - if (fileLine != "??:?") + if ( fileLine != "??:?" ) { size_t pos = fileLine.find_last_of(':'); - if (pos != std::string::npos) + if ( pos != std::string::npos ) { - newEntry.file = fileLine.substr(0, pos); - std::string lineStr = fileLine.substr(pos + 1); + newEntry.file = fileLine.substr( 0, pos ); + std::string lineStr = fileLine.substr( pos + 1 ); char* after = nullptr; - newEntry.line = strtol(lineStr.c_str(), &after, 10); + newEntry.line = strtol( lineStr.c_str(), &after, 10 ); } } - resolvedEntries.push_back(std::move(newEntry)); + resolvedEntries.push_back( std::move(newEntry) ); } return true; @@ -93,9 +93,9 @@ private: SymbolResolver* CreateResolver() { - std::stringstream result(ExecShellCommand("which addr2line")); + std::stringstream result( ExecShellCommand("which addr2line") ); std::string addr2LinePath; - std::getline(result, addr2LinePath); + std::getline( result, addr2LinePath ); if(!addr2LinePath.length()) { @@ -111,12 +111,12 @@ void DestroySymbolResolver(SymbolResolver* resolver) delete resolver; } -bool ResolveSymbols(SymbolResolver* resolver, const char* imageName, +bool ResolveSymbols(SymbolResolver* resolver, const std::string& imagePath, const FrameEntryList& inputEntryList, SymbolEntryList& resolvedEntries) { if (resolver) { - return resolver->ResolveSymbols(imageName, inputEntryList, resolvedEntries); + return resolver->ResolveSymbols( imagePath, inputEntryList, resolvedEntries ); } return false; } diff --git a/tracy-edit/src/OfflineSymbolResolverDbgHelper.cpp b/tracy-edit/src/OfflineSymbolResolverDbgHelper.cpp index 40b96cd3..7f5b7c04 100644 --- a/tracy-edit/src/OfflineSymbolResolverDbgHelper.cpp +++ b/tracy-edit/src/OfflineSymbolResolverDbgHelper.cpp @@ -39,13 +39,13 @@ public: SymCleanup( m_procHandle ); } - bool ResolveSymbolsForModule(const char* fileName, const FrameEntryList& inputEntryList, + bool ResolveSymbolsForModule(const std::string& imagePath, const FrameEntryList& inputEntryList, SymbolEntryList& resolvedEntries) { - ULONG64 moduleBase = SymLoadModuleEx( m_procHandle, NULL, fileName, NULL, 0, 0, NULL, 0 ); + ULONG64 moduleBase = SymLoadModuleEx( m_procHandle, NULL, imagePath.c_str(), NULL, 0, 0, NULL, 0); if (!moduleBase) { - std::cerr << "SymLoadModuleEx() failed for module " << fileName + std::cerr << "SymLoadModuleEx() failed for module " << imagePath << ": " << GetLastErrorString() << std::endl; return false; } @@ -128,13 +128,13 @@ void DestroySymbolResolver(SymbolResolver* resolver) delete resolver; } -bool ResolveSymbols(SymbolResolver* resolver, const char* imageName, +bool ResolveSymbols(SymbolResolver* resolver, const std::string& imagePath, const FrameEntryList& inputEntryList, SymbolEntryList& resolvedEntries) { if( resolver ) { - return resolver->ResolveSymbolsForModule( imageName, inputEntryList, resolvedEntries ); + return resolver->ResolveSymbolsForModule( imagePath, inputEntryList, resolvedEntries ); } return false; } diff --git a/tracy-edit/src/tracy-edit.cpp b/tracy-edit/src/tracy-edit.cpp index dcf8a251..de2e07a2 100644 --- a/tracy-edit/src/tracy-edit.cpp +++ b/tracy-edit/src/tracy-edit.cpp @@ -4,6 +4,7 @@ #include #include +#include #include #include #include @@ -22,6 +23,7 @@ struct Args const char* outputTracyPath = nullptr; bool verbose = false; bool resolveSymbols = false; + std::vector pathSubstitutions; tracy::FileWrite::Compression compressionType = tracy::FileWrite::Compression::Zstd; int compressionLevel = 5; }; @@ -30,13 +32,14 @@ void PrintUsageAndExit() { std::cerr << "Modify a tracy file" << std::endl; std::cerr << "Usage:" << std::endl; - std::cerr << " extract [OPTION...] " << std::endl; + std::cerr << " tracy-edit [OPTION...] " << std::endl; std::cerr << std::endl; - std::cerr << " -h, --help Print usage" << std::endl; - std::cerr << " -v, --verbose Enable verbose logging" << std::endl; - std::cerr << " -r, --resolveSymbols Resolve symbols and patch callstack frames" << std::endl; - std::cerr << " -c, --compression arg Compress output with the given compression algo" << std::endl; - std::cerr << " -l, --compressesionLevel arg Level of compression" << std::endl; + std::cerr << " -h, --help Print usage" << std::endl; + std::cerr << " -v, --verbose Enable verbose logging" << std::endl; + std::cerr << " -r, --resolveSymbols Resolve symbols and patch callstack frames" << std::endl; + std::cerr << " -s, --substitutePath \"REGEX_MATCH;REPLACEMENT\" Substitute symbol resolution path with an alternative" << std::endl; + std::cerr << " -c, --compression Type Set the compression algorithm used for the output [Fast,Slow,Extreme,Zstd]" << std::endl; + std::cerr << " -l, --compressesionLevel Level Level of compression" << std::endl; exit( 1 ); } @@ -74,13 +77,14 @@ Args ParseArgs( int argc, char** argv ) { "help", no_argument, NULL, 'h' }, { "verbose", no_argument, NULL, 'v' }, { "resolveSymbols", no_argument, NULL, 'r' }, + { "substitutePath", required_argument, NULL, 's' }, { "compression", required_argument, NULL, 'c' }, { "compressesionLevel", required_argument, NULL, 'l' }, { NULL, 0, NULL, 0 } }; int c; - while ( (c = getopt_long( argc, argv, "hvrc:l:", long_opts, NULL )) != -1 ) + while ( (c = getopt_long( argc, argv, "hvrc:l:s:", long_opts, NULL )) != -1 ) { switch (c) { @@ -96,6 +100,9 @@ Args ParseArgs( int argc, char** argv ) case 'c': args.compressionType = getCompressionFromString( optarg ); break; + case 's': + args.pathSubstitutions.push_back( optarg ); + break; case 'l': args.compressionLevel = atoi( optarg ); break; @@ -116,6 +123,8 @@ Args ParseArgs( int argc, char** argv ) return args; } +void PatchSymbols(tracy::Worker& worker, const Args& args); + int main( int argc, char** argv ) { #ifdef _WIN32 @@ -144,21 +153,9 @@ int main( int argc, char** argv ) std::cout << "Loaded." << std::endl; - // attempt to resolve symbols only if requested if(args.resolveSymbols) { - std::cout << "Resolving and patching symbols..." << std::endl; - - SymbolResolver* resolver = CreateResolver(); - if(!resolver) - { - std::cerr << "Failed to create symbol resolver - skipping resolving" << std::endl; - } - else - { - PatchSymbols(resolver, worker); - DestroySymbolResolver(resolver); - } + PatchSymbols(worker, args); } // save out capture file with new compression options @@ -179,3 +176,45 @@ int main( int argc, char** argv ) return 0; } + +void PatchSymbols(tracy::Worker& worker, const Args& args) +{ + std::cout << "Resolving and patching symbols..." << std::endl; + + PathSubstitutionList pathSubstitutionList; + for (const std::string& pathSubst : args.pathSubstitutions) + { + std::size_t pos = pathSubst.find(';'); + if (pos == std::string::npos) + { + std::cerr << "Ignoring invalid path substitution: '" << pathSubst + << " '(please separate the regex of the string to replace with a ';')" << std::endl; + continue; + } + + try + { + std::regex reg(pathSubst.substr(0, pos)); + std::string replacementStr(pathSubst.substr(pos + 1)); + pathSubstitutionList.push_back(std::pair(reg, replacementStr)); + } + catch (std::exception& e) + { + std::cerr << "Ignoring invalid path substitution: '" << pathSubst + << "' (" << e.what() << ")" << std::endl; + continue; + } + } + + SymbolResolver* resolver = CreateResolver(); + if (resolver) + { + PatchSymbols(resolver, worker, pathSubstitutionList, args.verbose); + + DestroySymbolResolver(resolver); + } + else + { + std::cerr << "Failed to create symbol resolver - skipping resolving" << std::endl; + } +} From 3fdd1e1863882d40964bde33bcb57ca3465a164c Mon Sep 17 00:00:00 2001 From: Tiago Rodrigues Date: Sat, 18 Nov 2023 15:02:41 -0500 Subject: [PATCH 08/21] fix typo --- manual/tracy.tex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manual/tracy.tex b/manual/tracy.tex index c6da6d2e..766f7d06 100644 --- a/manual/tracy.tex +++ b/manual/tracy.tex @@ -1792,7 +1792,7 @@ As an alternative to to runtime symbol resolution, we can set the environment va the minimal set of info required for offline resolution (a shared library path and an offset into that shared library). The generated tracy capture will have callstack frames symbols showing \texttt{[unresolved]}. -The \texttt{tracy-edit} tool can be used to load that capture, perforce symbol resolution offline +The \texttt{tracy-edit} tool can be used to load that capture, perform symbol resolution offline (by passing \texttt{-r}) and writing out a new capture with symbols resolved. By default \texttt{tracy-edit} will use the original shared libraries paths that were recorded in the capture (which assumes running in the same machine or a machine with identical From c302b509fe1e8eaff482c81a979077a8f6ebc2ca Mon Sep 17 00:00:00 2001 From: Tiago Rodrigues Date: Sat, 18 Nov 2023 15:03:12 -0500 Subject: [PATCH 09/21] fix formatting --- public/client/TracyCallstack.cpp | 46 ++++++++++++++------------------ 1 file changed, 20 insertions(+), 26 deletions(-) diff --git a/public/client/TracyCallstack.cpp b/public/client/TracyCallstack.cpp index 8860cbe2..cee781be 100644 --- a/public/client/TracyCallstack.cpp +++ b/public/client/TracyCallstack.cpp @@ -93,9 +93,11 @@ extern "C" const char* ___tracy_demangle( const char* mangled ) namespace tracy { +// TODO: use TRACY_SYMBOL_OFFLINE_RESOLVE to this at compilation time + // when "TRACY_SYMBOL_OFFLINE_RESOLVE" is set to "1", instead of fully resolving symbols at runtime, // simply resolve the offset and image name (which will be enough the resolving to be done offline) -bool getDoOfflineSymbolResolve() +bool ShouldResolveSymbolsOffline() { const char* symbolOfflineResolve = GetEnvVar( "TRACY_SYMBOL_OFFLINE_RESOLVE" ); return (symbolOfflineResolve && symbolOfflineResolve[0] == '1'); @@ -150,12 +152,9 @@ void InitCallstackCritical() ___tracy_RtlWalkFrameChain = (___tracy_t_RtlWalkFrameChain)GetProcAddress( GetModuleHandleA( "ntdll.dll" ), "RtlWalkFrameChain" ); } -void dbgHelpInit() +void DbgHelpInit() { - if( s_doOfflineSymbolResolve ) - { - return; - } + if( s_doOfflineSymbolResolve ) return; _SymAddrIncludeInlineTrace = (t_SymAddrIncludeInlineTrace)GetProcAddress(GetModuleHandleA("dbghelp.dll"), "SymAddrIncludeInlineTrace"); _SymQueryInlineTrace = (t_SymQueryInlineTrace)GetProcAddress(GetModuleHandleA("dbghelp.dll"), "SymQueryInlineTrace"); @@ -175,18 +174,16 @@ void dbgHelpInit() #endif } -DWORD64 dbgHelpLoadSymbolsForModule(PCSTR imageName, DWORD64 baseOfDll, DWORD bllSize) +DWORD64 DbgHelpLoadSymbolsForModule( const char* imageName, uint64_t baseOfDll, uint32_t bllSize ) { - if( !s_doOfflineSymbolResolve ) - { - return SymLoadModuleEx( GetCurrentProcess(), nullptr, imageName, nullptr, baseOfDll, bllSize, nullptr, 0 ); - } - return 0x0; + if( s_doOfflineSymbolResolve ) return 0; + + return SymLoadModuleEx( GetCurrentProcess(), nullptr, imageName, nullptr, baseOfDll, bllSize, nullptr, 0 ); } -ModuleCache* dbgHelpLoadSymbolsForModuleAndCache(PCSTR imageName, DWORD imageNameLength, DWORD64 baseOfDll, DWORD dllSize) +ModuleCache* DbgHelpLoadSymbolsForModuleAndCache( const char* imageName, uint32_t imageNameLength, uint64_t baseOfDll, uint32_t dllSize ) { - dbgHelpLoadSymbolsForModule( imageName, baseOfDll, dllSize ); + DbgHelpLoadSymbolsForModule( imageName, baseOfDll, dllSize ); ModuleCache* cachedModule = s_modCache->push_next(); cachedModule->start = baseOfDll; @@ -195,9 +192,9 @@ ModuleCache* dbgHelpLoadSymbolsForModuleAndCache(PCSTR imageName, DWORD imageNam // when doing offline symbol resolution, we must store the full path of the dll for the resolving to work if( s_doOfflineSymbolResolve ) { - cachedModule->name = (char*)tracy_malloc_fast(imageNameLength + 2); + cachedModule->name = (char*)tracy_malloc_fast(imageNameLength + 1); memcpy(cachedModule->name, imageName, imageNameLength); - cachedModule->name[imageNameLength + 1] = '\0'; + cachedModule->name[imageNameLength] = '\0'; } else { @@ -217,13 +214,13 @@ ModuleCache* dbgHelpLoadSymbolsForModuleAndCache(PCSTR imageName, DWORD imageNam void InitCallstack() { - s_doOfflineSymbolResolve = getDoOfflineSymbolResolve(); + s_doOfflineSymbolResolve = ShouldResolveSymbolsOffline(); if( s_doOfflineSymbolResolve ) { TracyDebug("TRACY: enabling offline symbol resolving!\n"); } - dbgHelpInit(); + DbgHelpInit(); #ifdef TRACY_DBGHELP_LOCK DBGHELP_LOCK; @@ -276,7 +273,7 @@ void InitCallstack() path = full; } - dbgHelpLoadSymbolsForModule( path, (DWORD64)dev[i], 0 ); + DbgHelpLoadSymbolsForModule( path, (DWORD64)dev[i], 0 ); const auto psz = strlen( path ); auto pptr = (char*)tracy_malloc_fast( psz+1 ); @@ -312,7 +309,7 @@ void InitCallstack() { // This may be a new module loaded since our call to SymInitialize. // Just in case, force DbgHelp to load its pdb ! - dbgHelpLoadSymbolsForModuleAndCache( name, nameLength, (DWORD64)info.lpBaseOfDll, info.SizeOfImage ); + DbgHelpLoadSymbolsForModuleAndCache( name, nameLength, (DWORD64)info.lpBaseOfDll, info.SizeOfImage ); } } } @@ -329,10 +326,7 @@ void EndCallstack() const char* DecodeCallstackPtrFast( uint64_t ptr ) { - if( s_doOfflineSymbolResolve ) - { - return "[unresolved]"; - } + if( s_doOfflineSymbolResolve ) return "[unresolved]"; static char ret[MaxNameSize]; const auto proc = GetCurrentProcess(); @@ -419,7 +413,7 @@ ModuleNameAndBaseAddress GetModuleNameAndPrepareSymbols( uint64_t addr ) if( nameLength > 0 ) { // since this is the first time we encounter this module, load its symbols (needed for modules loaded after SymInitialize) - ModuleCache* cachedModule = dbgHelpLoadSymbolsForModuleAndCache( name, nameLength, (DWORD64)info.lpBaseOfDll, info.SizeOfImage ); + ModuleCache* cachedModule = DbgHelpLoadSymbolsForModuleAndCache( name, nameLength, (DWORD64)info.lpBaseOfDll, info.SizeOfImage ); return ModuleNameAndBaseAddress{ cachedModule->name, cachedModule->start }; } } @@ -777,7 +771,7 @@ void InitCallstackCritical() void InitCallstack() { - if( getDoOfflineSymbolResolve() ) + if( ShouldResolveSymbolsOffline() ) { cb_bts = nullptr; // disable use of libbacktrace calls TracyDebug("TRACY: enabling offline symbol resolving!\n"); From 687d6817641c719ba8e52012fa578eefaa20ca61 Mon Sep 17 00:00:00 2001 From: Tiago Rodrigues Date: Sat, 18 Nov 2023 15:04:02 -0500 Subject: [PATCH 10/21] Instead of adding new method, make StoreString public --- server/TracyWorker.cpp | 13 ------------- server/TracyWorker.hpp | 3 +-- 2 files changed, 1 insertion(+), 15 deletions(-) diff --git a/server/TracyWorker.cpp b/server/TracyWorker.cpp index 36acf354..0bbf0c78 100644 --- a/server/TracyWorker.cpp +++ b/server/TracyWorker.cpp @@ -2426,19 +2426,6 @@ const char* Worker::GetString( const StringIdx& idx ) const return m_data.stringData[idx.Idx()]; } -uint32_t Worker::AddNewString(const char* newString) -{ - assert(m_allowStringModification); - uint64_t sz = strlen(newString); - auto ptr = m_slab.Alloc( sz+1 ); - memcpy( ptr, newString, sz ); - ptr[sz] = '\0'; - uint32_t idx = m_data.stringData.size(); - m_data.stringMap.emplace( charutil::StringKey { ptr, sz }, idx ); - m_data.stringData.push_back( ptr ); - return idx; -} - static const char* BadExternalThreadNames[] = { "ntdll.dll", "???", diff --git a/server/TracyWorker.hpp b/server/TracyWorker.hpp index 081f8be9..72b94d1a 100644 --- a/server/TracyWorker.hpp +++ b/server/TracyWorker.hpp @@ -667,7 +667,7 @@ public: void CacheSourceFiles(); - uint32_t AddNewString(const char* newString); + StringLocation StoreString(const char* str, size_t sz); private: void Network(); @@ -896,7 +896,6 @@ private: uint32_t GetSingleStringIdx(); uint32_t GetSecondStringIdx(); - StringLocation StoreString( const char* str, size_t sz ); const ContextSwitch* const GetContextSwitchDataImpl( uint64_t thread ); void CacheSource( const StringRef& str, const StringIdx& image = StringIdx() ); From 5c0513931a4ce8353b90b4e030d543e276a9e17b Mon Sep 17 00:00:00 2001 From: Tiago Rodrigues Date: Sat, 18 Nov 2023 15:05:43 -0500 Subject: [PATCH 11/21] - Fix formatting - delete CreateResolver/DestroySymbolResolver --- tracy-edit/src/OfflineSymbolResolver.cpp | 42 +++++------- tracy-edit/src/OfflineSymbolResolver.h | 13 +--- .../src/OfflineSymbolResolverAddr2Line.cpp | 64 ++++++++----------- .../src/OfflineSymbolResolverDbgHelper.cpp | 48 ++++++-------- tracy-edit/src/tracy-edit.cpp | 25 +++----- 5 files changed, 73 insertions(+), 119 deletions(-) diff --git a/tracy-edit/src/OfflineSymbolResolver.cpp b/tracy-edit/src/OfflineSymbolResolver.cpp index 3788e8b8..9ba7ce71 100644 --- a/tracy-edit/src/OfflineSymbolResolver.cpp +++ b/tracy-edit/src/OfflineSymbolResolver.cpp @@ -11,7 +11,7 @@ #include "OfflineSymbolResolver.h" -bool ApplyPathSubstitutions(std::string& path, const PathSubstitutionList& pathSubstituionlist) +bool ApplyPathSubstitutions( std::string& path, const PathSubstitutionList& pathSubstituionlist ) { for( const auto& substituion : pathSubstituionlist ) { @@ -24,21 +24,15 @@ bool ApplyPathSubstitutions(std::string& path, const PathSubstitutionList& pathS return false; } -// TODO: use string hash map to reduce duplication or use some worker string internal hashing -tracy::StringIdx AddSymbolString(tracy::Worker& worker, const char* str) +tracy::StringIdx AddSymbolString( tracy::Worker& worker, const std::string& str ) { - uint32_t newStringIdx = worker.AddNewString( str ); - return tracy::StringIdx( newStringIdx ); + // TODO: use string hash map to reduce potential string duplication? + tracy::StringLocation location = worker.StoreString( str.c_str(), str.length() ); + return tracy::StringIdx( location.idx ); } -bool PatchSymbols(SymbolResolver* resolver, tracy::Worker& worker, - const PathSubstitutionList& pathSubstituionlist, bool verbose) +bool PatchSymbols( tracy::Worker& worker, const PathSubstitutionList& pathSubstituionlist, bool verbose ) { - if( !resolver ) - { - return false; - } - uint64_t callstackFrameCount = worker.GetCallstackFrameCount(); std::string relativeSoNameMatch = "[unresolved]"; @@ -89,21 +83,19 @@ bool PatchSymbols(SymbolResolver* resolver, tracy::Worker& worker, std::string imagePath = worker.GetString( imageIdx ); FrameEntryList& entries = imageIt->second; - if (!entries.size()) - { - continue; - } + + if( !entries.size() ) continue; std::cout << "Resolving " << entries.size() << " symbols for image: '" << imagePath << "'" << std::endl; - const bool substituted = ApplyPathSubstitutions(imagePath, pathSubstituionlist); - if (substituted) + const bool substituted = ApplyPathSubstitutions( imagePath, pathSubstituionlist ); + if( substituted ) { std::cout << "\tPath substituted to: '" << imagePath << "'" << std::endl; } SymbolEntryList resolvedEntries; - ResolveSymbols( resolver, imagePath, entries, resolvedEntries ); + ResolveSymbols( imagePath, entries, resolvedEntries ); if( resolvedEntries.size() != entries.size() ) { @@ -119,10 +111,8 @@ bool PatchSymbols(SymbolResolver* resolver, tracy::Worker& worker, const SymbolEntry& symbolEntry = resolvedEntries[i]; tracy::CallstackFrame& frame = *frameEntry.frame; - if (!symbolEntry.name.length()) - { - continue; - } + + if( !symbolEntry.name.length() ) continue; if( verbose ) { @@ -131,12 +121,12 @@ bool PatchSymbols(SymbolResolver* resolver, tracy::Worker& worker, << "' -> '" << symbolEntry.name << "'" << std::endl; } - frame.name = AddSymbolString( worker, symbolEntry.name.c_str() ); - const char* newName = worker.GetString(frame.name); + frame.name = AddSymbolString( worker, symbolEntry.name ); + const char* newName = worker.GetString( frame.name ); if( symbolEntry.file.length() ) { - frame.file = AddSymbolString( worker, symbolEntry.file.c_str() ); + frame.file = AddSymbolString( worker, symbolEntry.file ); frame.line = symbolEntry.line; } } diff --git a/tracy-edit/src/OfflineSymbolResolver.h b/tracy-edit/src/OfflineSymbolResolver.h index 26d483a2..4ecc4db2 100644 --- a/tracy-edit/src/OfflineSymbolResolver.h +++ b/tracy-edit/src/OfflineSymbolResolver.h @@ -11,11 +11,6 @@ namespace tracy class Worker; } -class SymbolResolver; - -SymbolResolver* CreateResolver(); -void DestroySymbolResolver(SymbolResolver* resolver); - struct FrameEntry { tracy::CallstackFrame* frame = nullptr; @@ -33,13 +28,11 @@ struct SymbolEntry using SymbolEntryList = std::vector; -bool ResolveSymbols(SymbolResolver* resolver, const std::string& imagePath, - const FrameEntryList& inputEntryList, - SymbolEntryList& resolvedEntries); +bool ResolveSymbols( const std::string& imagePath, const FrameEntryList& inputEntryList, + SymbolEntryList& resolvedEntries ); using PathSubstitutionList = std::vector >; -bool PatchSymbols(SymbolResolver* resolver, tracy::Worker& worker, - const PathSubstitutionList& pathSubstituionlist, bool verbose = false); +bool PatchSymbols( tracy::Worker& worker, const PathSubstitutionList& pathSubstituionlist, bool verbose = false ); #endif // __SYMBOLRESOLVER_HPP__ \ No newline at end of file diff --git a/tracy-edit/src/OfflineSymbolResolverAddr2Line.cpp b/tracy-edit/src/OfflineSymbolResolverAddr2Line.cpp index b35b65b5..4fd0caa0 100644 --- a/tracy-edit/src/OfflineSymbolResolverAddr2Line.cpp +++ b/tracy-edit/src/OfflineSymbolResolverAddr2Line.cpp @@ -10,16 +10,16 @@ #include #include -std::string ExecShellCommand(const char* cmd) +std::string ExecShellCommand( const char* cmd ) { std::array buffer; std::string result; std::unique_ptr pipe(popen(cmd, "r"), pclose); - if (!pipe) + if( !pipe ) { return ""; } - while (fgets(buffer.data(), buffer.size(), pipe.get()) != nullptr) + while( fgets(buffer.data(), buffer.size(), pipe.get()) != nullptr ) { result += buffer.data(); } @@ -29,13 +29,26 @@ std::string ExecShellCommand(const char* cmd) class SymbolResolver { public: - SymbolResolver(const std::string& addr2linePath) - : m_addr2LinePath(addr2linePath) - {} - - bool ResolveSymbols(const std::string& imagePath, const FrameEntryList& inputEntryList, - SymbolEntryList& resolvedEntries) + SymbolResolver( const std::string& addr2linePath ) { + std::stringstream result(ExecShellCommand("which addr2line")); + std::getline(result, m_addr2LinePath); + + if( !m_addr2LinePath.length() ) + { + std::cerr << "'addr2line' was not found in the system, please installed it" << std::endl; + } + else + { + std::cout << "Using 'addr2line' found at: '" << m_addr2LinePath.c_str() << "'" << std::endl; + } + } + + bool ResolveSymbols( const std::string& imagePath, const FrameEntryList& inputEntryList, + SymbolEntryList& resolvedEntries ) + { + if (!m_addr2LinePath.length()) return false; + // generate a single addr2line cmd line for all addresses in one invocation std::stringstream ss; ss << m_addr2LinePath << " -C -f -e " << imagePath << " -a "; @@ -91,34 +104,11 @@ private: std::string m_addr2LinePath; }; -SymbolResolver* CreateResolver() +bool ResolveSymbols( const std::string& imagePath, const FrameEntryList& inputEntryList, + SymbolEntryList& resolvedEntries ) { - std::stringstream result( ExecShellCommand("which addr2line") ); - std::string addr2LinePath; - std::getline( result, addr2LinePath ); - - if(!addr2LinePath.length()) - { - std::cerr << "'addr2line' was not found in the system, please installed it" << std::endl; - return nullptr; - } - std::cout << "Using 'addr2line' found at: '" << addr2LinePath.c_str() << "'" << std::endl; - return new SymbolResolver{addr2LinePath}; + static SymbolResolver symbolResolver; + return symbolResolver.ResolveSymbols( imagePath, inputEntryList, resolvedEntries ); } -void DestroySymbolResolver(SymbolResolver* resolver) -{ - delete resolver; -} - -bool ResolveSymbols(SymbolResolver* resolver, const std::string& imagePath, - const FrameEntryList& inputEntryList, SymbolEntryList& resolvedEntries) -{ - if (resolver) - { - return resolver->ResolveSymbols( imagePath, inputEntryList, resolvedEntries ); - } - return false; -} - -#endif // #ifndef _WIN32 \ No newline at end of file +#endif // #ifndef _WIN32 diff --git a/tracy-edit/src/OfflineSymbolResolverDbgHelper.cpp b/tracy-edit/src/OfflineSymbolResolverDbgHelper.cpp index 7f5b7c04..20ed2f5e 100644 --- a/tracy-edit/src/OfflineSymbolResolverDbgHelper.cpp +++ b/tracy-edit/src/OfflineSymbolResolverDbgHelper.cpp @@ -23,14 +23,15 @@ public: { m_procHandle = GetCurrentProcess(); - if (!SymInitialize(m_procHandle, NULL, FALSE)) + if( !SymInitialize(m_procHandle, NULL, FALSE) ) { std::cerr << "SymInitialize() failed with: " << GetLastErrorString() << std::endl; } else { - const DWORD symopts = SYMOPT_UNDNAME | SYMOPT_DEFERRED_LOADS | SYMOPT_DEBUG | SYMOPT_LOAD_LINES; + const DWORD symopts = SYMOPT_UNDNAME | SYMOPT_DEFERRED_LOADS | SYMOPT_LOAD_LINES; SymSetOptions( symopts ); + m_dbgHelpInitialized = true; } } @@ -39,18 +40,20 @@ public: SymCleanup( m_procHandle ); } - bool ResolveSymbolsForModule(const std::string& imagePath, const FrameEntryList& inputEntryList, - SymbolEntryList& resolvedEntries) + bool ResolveSymbolsForModule( const std::string& imagePath, const FrameEntryList& inputEntryList, + SymbolEntryList& resolvedEntries ) { - ULONG64 moduleBase = SymLoadModuleEx( m_procHandle, NULL, imagePath.c_str(), NULL, 0, 0, NULL, 0); - if (!moduleBase) + if( !m_dbgHelpInitialized ) return false; + + ULONG64 moduleBase = SymLoadModuleEx( m_procHandle, NULL, imagePath.c_str(), NULL, 0, 0, NULL, 0 ); + if( !moduleBase ) { std::cerr << "SymLoadModuleEx() failed for module " << imagePath << ": " << GetLastErrorString() << std::endl; return false; } - for (size_t i = 0; i < inputEntryList.size(); ++i) + for( size_t i = 0; i < inputEntryList.size(); ++i ) { uint64_t offset = inputEntryList[i].symbolOffset; DWORD64 address = moduleBase + offset; @@ -61,7 +64,7 @@ public: SymbolEntry newEntry; - if ( SymFromAddr( m_procHandle, address, NULL, symbolInfo ) ) + if( SymFromAddr( m_procHandle, address, NULL, symbolInfo ) ) { newEntry.name = symbolInfo->Name; //std::cout << "Resolved symbol to: '" << newEntry.name << "'" << std::endl; @@ -84,7 +87,7 @@ public: resolvedEntries.push_back(std::move(newEntry)); } - SymUnloadModule64(m_procHandle, moduleBase); + SymUnloadModule64( m_procHandle, moduleBase ); return true; } @@ -111,32 +114,17 @@ private: return message; } + bool m_dbgHelpInitialized = false; HANDLE m_procHandle = nullptr; }; char SymbolResolver::s_symbolResolutionBuffer[symbolResolutionBufferSize]; - -SymbolResolver* CreateResolver() +bool ResolveSymbols( const std::string& imagePath, const FrameEntryList& inputEntryList, + SymbolEntryList& resolvedEntries ) { - SymbolResolver* resolver = new SymbolResolver(); - return resolver; + static SymbolResolver resolver; + return resolver.ResolveSymbolsForModule( imagePath, inputEntryList, resolvedEntries ); } -void DestroySymbolResolver(SymbolResolver* resolver) -{ - delete resolver; -} - -bool ResolveSymbols(SymbolResolver* resolver, const std::string& imagePath, - const FrameEntryList& inputEntryList, - SymbolEntryList& resolvedEntries) -{ - if( resolver ) - { - return resolver->ResolveSymbolsForModule( imagePath, inputEntryList, resolvedEntries ); - } - return false; -} - -#endif // #ifdef _WIN32 \ No newline at end of file +#endif // #ifdef _WIN32 diff --git a/tracy-edit/src/tracy-edit.cpp b/tracy-edit/src/tracy-edit.cpp index de2e07a2..f79c5dab 100644 --- a/tracy-edit/src/tracy-edit.cpp +++ b/tracy-edit/src/tracy-edit.cpp @@ -177,15 +177,15 @@ int main( int argc, char** argv ) return 0; } -void PatchSymbols(tracy::Worker& worker, const Args& args) +void PatchSymbols( tracy::Worker& worker, const Args& args ) { std::cout << "Resolving and patching symbols..." << std::endl; PathSubstitutionList pathSubstitutionList; - for (const std::string& pathSubst : args.pathSubstitutions) + for( const std::string& pathSubst : args.pathSubstitutions ) { std::size_t pos = pathSubst.find(';'); - if (pos == std::string::npos) + if( pos == std::string::npos ) { std::cerr << "Ignoring invalid path substitution: '" << pathSubst << " '(please separate the regex of the string to replace with a ';')" << std::endl; @@ -194,11 +194,11 @@ void PatchSymbols(tracy::Worker& worker, const Args& args) try { - std::regex reg(pathSubst.substr(0, pos)); - std::string replacementStr(pathSubst.substr(pos + 1)); - pathSubstitutionList.push_back(std::pair(reg, replacementStr)); + std::regex reg( pathSubst.substr(0, pos) ); + std::string replacementStr( pathSubst.substr(pos + 1) ); + pathSubstitutionList.push_back( std::pair(reg, replacementStr) ); } - catch (std::exception& e) + catch( std::exception& e ) { std::cerr << "Ignoring invalid path substitution: '" << pathSubst << "' (" << e.what() << ")" << std::endl; @@ -206,15 +206,8 @@ void PatchSymbols(tracy::Worker& worker, const Args& args) } } - SymbolResolver* resolver = CreateResolver(); - if (resolver) + if( !PatchSymbols( worker, pathSubstitutionList, args.verbose ) ) { - PatchSymbols(resolver, worker, pathSubstitutionList, args.verbose); - - DestroySymbolResolver(resolver); - } - else - { - std::cerr << "Failed to create symbol resolver - skipping resolving" << std::endl; + std::cerr << "Failed to patch symbols" << std::endl; } } From b3f68801bb7c1cc5ce07d13cf0e676596d727585 Mon Sep 17 00:00:00 2001 From: Tiago Rodrigues Date: Sat, 18 Nov 2023 15:55:09 -0500 Subject: [PATCH 12/21] fix linux compilation --- tracy-edit/src/OfflineSymbolResolverAddr2Line.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tracy-edit/src/OfflineSymbolResolverAddr2Line.cpp b/tracy-edit/src/OfflineSymbolResolverAddr2Line.cpp index 4fd0caa0..c397dcff 100644 --- a/tracy-edit/src/OfflineSymbolResolverAddr2Line.cpp +++ b/tracy-edit/src/OfflineSymbolResolverAddr2Line.cpp @@ -29,9 +29,9 @@ std::string ExecShellCommand( const char* cmd ) class SymbolResolver { public: - SymbolResolver( const std::string& addr2linePath ) + SymbolResolver() { - std::stringstream result(ExecShellCommand("which addr2line")); + std::stringstream result( ExecShellCommand("which addr2line") ); std::getline(result, m_addr2LinePath); if( !m_addr2LinePath.length() ) From 79618c5a1a537296a7f9c0fefcd70dfe9ca5ff39 Mon Sep 17 00:00:00 2001 From: Tiago Rodrigues Date: Sat, 18 Nov 2023 16:27:26 -0500 Subject: [PATCH 13/21] Add support for using defining TRACY_SYMBOL_OFFLINE_RESOLVE to enable only the offline symbol resolving codepath --- CMakeLists.txt | 1 + public/client/TracyCallstack.cpp | 110 ++++++++++++++++++++----------- 2 files changed, 74 insertions(+), 37 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7bf17c31..66b506f2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -85,6 +85,7 @@ set_option(TRACY_FIBERS "Enable fibers support" OFF) set_option(TRACY_NO_CRASH_HANDLER "Disable crash handling" OFF) set_option(TRACY_TIMER_FALLBACK "Use lower resolution timers" OFF) set_option(TRACE_CLIENT_LIBUNWIND_BACKTRACE "Use libunwind backtracing where supported" OFF) +set_option(TRACY_SYMBOL_OFFLINE_RESOLVE "Instead of runtime symbol resolving, sore offset and sharedlib path for offline symbol resolution" OFF) if(NOT TRACY_STATIC) target_compile_definitions(TracyClient PRIVATE TRACY_EXPORTS) diff --git a/public/client/TracyCallstack.cpp b/public/client/TracyCallstack.cpp index cee781be..83d29d44 100644 --- a/public/client/TracyCallstack.cpp +++ b/public/client/TracyCallstack.cpp @@ -93,15 +93,20 @@ extern "C" const char* ___tracy_demangle( const char* mangled ) namespace tracy { -// TODO: use TRACY_SYMBOL_OFFLINE_RESOLVE to this at compilation time - -// when "TRACY_SYMBOL_OFFLINE_RESOLVE" is set to "1", instead of fully resolving symbols at runtime, +// when "TRACY_SYMBOL_OFFLINE_RESOLVE" is set, instead of fully resolving symbols at runtime, // simply resolve the offset and image name (which will be enough the resolving to be done offline) +#ifdef TRACY_SYMBOL_OFFLINE_RESOLVE +static constexpr bool s_doOfflineSymbolResolve = true; +bool ShouldResolveSymbolsOffline() { return true; } + +#else +bool s_doOfflineSymbolResolve = false; bool ShouldResolveSymbolsOffline() { const char* symbolOfflineResolve = GetEnvVar( "TRACY_SYMBOL_OFFLINE_RESOLVE" ); return (symbolOfflineResolve && symbolOfflineResolve[0] == '1'); } +#endif // #ifdef TRACY_SYMBOL_OFFLINE_RESOLVE #if TRACY_HAS_CALLSTACK == 1 @@ -113,18 +118,20 @@ CallstackEntry cb_data[MaxCbTrace]; extern "C" { +#ifndef TRACY_SYMBOL_OFFLINE_RESOLVE typedef DWORD (__stdcall *t_SymAddrIncludeInlineTrace)( HANDLE hProcess, DWORD64 Address ); typedef BOOL (__stdcall *t_SymQueryInlineTrace)( HANDLE hProcess, DWORD64 StartAddress, DWORD StartContext, DWORD64 StartRetAddress, DWORD64 CurAddress, LPDWORD CurContext, LPDWORD CurFrameIndex ); typedef BOOL (__stdcall *t_SymFromInlineContext)( HANDLE hProcess, DWORD64 Address, ULONG InlineContext, PDWORD64 Displacement, PSYMBOL_INFO Symbol ); typedef BOOL (__stdcall *t_SymGetLineFromInlineContext)( HANDLE hProcess, DWORD64 qwAddr, ULONG InlineContext, DWORD64 qwModuleBaseAddress, PDWORD pdwDisplacement, PIMAGEHLP_LINE64 Line64 ); - TRACY_API ___tracy_t_RtlWalkFrameChain ___tracy_RtlWalkFrameChain = 0; t_SymAddrIncludeInlineTrace _SymAddrIncludeInlineTrace = 0; t_SymQueryInlineTrace _SymQueryInlineTrace = 0; t_SymFromInlineContext _SymFromInlineContext = 0; t_SymGetLineFromInlineContext _SymGetLineFromInlineContext = 0; -} +#endif // #ifndef TRACY_SYMBOL_OFFLINE_RESOLVE + TRACY_API ___tracy_t_RtlWalkFrameChain ___tracy_RtlWalkFrameChain = 0; +} struct ModuleCache { @@ -145,13 +152,13 @@ struct KernelDriver KernelDriver* s_krnlCache = nullptr; size_t s_krnlCacheCnt; -bool s_doOfflineSymbolResolve = false; void InitCallstackCritical() { ___tracy_RtlWalkFrameChain = (___tracy_t_RtlWalkFrameChain)GetProcAddress( GetModuleHandleA( "ntdll.dll" ), "RtlWalkFrameChain" ); } +#ifndef TRACY_SYMBOL_OFFLINE_RESOLVE void DbgHelpInit() { if( s_doOfflineSymbolResolve ) return; @@ -161,29 +168,31 @@ void DbgHelpInit() _SymFromInlineContext = (t_SymFromInlineContext)GetProcAddress(GetModuleHandleA("dbghelp.dll"), "SymFromInlineContext"); _SymGetLineFromInlineContext = (t_SymGetLineFromInlineContext)GetProcAddress(GetModuleHandleA("dbghelp.dll"), "SymGetLineFromInlineContext"); -#ifdef TRACY_DBGHELP_LOCK +# ifdef TRACY_DBGHELP_LOCK DBGHELP_INIT; DBGHELP_LOCK; -#endif +# endif SymInitialize( GetCurrentProcess(), nullptr, true ); SymSetOptions( SYMOPT_LOAD_LINES ); -#ifdef TRACY_DBGHELP_LOCK +# ifdef TRACY_DBGHELP_LOCK DBGHELP_UNLOCK; -#endif +# endif } DWORD64 DbgHelpLoadSymbolsForModule( const char* imageName, uint64_t baseOfDll, uint32_t bllSize ) { if( s_doOfflineSymbolResolve ) return 0; - return SymLoadModuleEx( GetCurrentProcess(), nullptr, imageName, nullptr, baseOfDll, bllSize, nullptr, 0 ); } +#endif //#ifndef TRACY_SYMBOL_OFFLINE_RESOLVE -ModuleCache* DbgHelpLoadSymbolsForModuleAndCache( const char* imageName, uint32_t imageNameLength, uint64_t baseOfDll, uint32_t dllSize ) +ModuleCache* LoadSymbolsForModuleAndCache( const char* imageName, uint32_t imageNameLength, uint64_t baseOfDll, uint32_t dllSize ) { +#ifndef TRACY_SYMBOL_OFFLINE_RESOLVE DbgHelpLoadSymbolsForModule( imageName, baseOfDll, dllSize ); +#endif //#ifndef TRACY_SYMBOL_OFFLINE_RESOLVE ModuleCache* cachedModule = s_modCache->push_next(); cachedModule->start = baseOfDll; @@ -196,6 +205,7 @@ ModuleCache* DbgHelpLoadSymbolsForModuleAndCache( const char* imageName, uint32_ memcpy(cachedModule->name, imageName, imageNameLength); cachedModule->name[imageNameLength] = '\0'; } +#ifndef TRACY_SYMBOL_OFFLINE_RESOLVE else { auto ptr = imageName + imageNameLength; @@ -208,12 +218,14 @@ ModuleCache* DbgHelpLoadSymbolsForModuleAndCache( const char* imageName, uint32_ cachedModule->name[namelen + 1] = ']'; cachedModule->name[namelen + 2] = '\0'; } +#endif //#ifndef TRACY_SYMBOL_OFFLINE_RESOLVE return cachedModule; } void InitCallstack() { +#ifndef TRACY_SYMBOL_OFFLINE_RESOLVE s_doOfflineSymbolResolve = ShouldResolveSymbolsOffline(); if( s_doOfflineSymbolResolve ) { @@ -222,9 +234,10 @@ void InitCallstack() DbgHelpInit(); -#ifdef TRACY_DBGHELP_LOCK +# ifdef TRACY_DBGHELP_LOCK DBGHELP_LOCK; -#endif +# endif +#endif //#ifndef TRACY_SYMBOL_OFFLINE_RESOLVE // use TRACY_NO_DBHELP_INIT_LOAD=1 to disable preloading of driver // and process module symbol loading at startup time - they will be loaded on demand later @@ -273,7 +286,9 @@ void InitCallstack() path = full; } +# ifndef TRACY_SYMBOL_OFFLINE_RESOLVE DbgHelpLoadSymbolsForModule( path, (DWORD64)dev[i], 0 ); +# endif const auto psz = strlen( path ); auto pptr = (char*)tracy_malloc_fast( psz+1 ); @@ -309,15 +324,17 @@ void InitCallstack() { // This may be a new module loaded since our call to SymInitialize. // Just in case, force DbgHelp to load its pdb ! - DbgHelpLoadSymbolsForModuleAndCache( name, nameLength, (DWORD64)info.lpBaseOfDll, info.SizeOfImage ); + LoadSymbolsForModuleAndCache( name, nameLength, (DWORD64)info.lpBaseOfDll, info.SizeOfImage ); } } } } -#ifdef TRACY_DBGHELP_LOCK +#ifndef TRACY_SYMBOL_OFFLINE_RESOLVE +# ifdef TRACY_DBGHELP_LOCK DBGHELP_UNLOCK; -#endif +# endif +#endif //#ifndef TRACY_SYMBOL_OFFLINE_RESOLVE } void EndCallstack() @@ -328,6 +345,7 @@ const char* DecodeCallstackPtrFast( uint64_t ptr ) { if( s_doOfflineSymbolResolve ) return "[unresolved]"; +#ifndef TRACY_SYMBOL_OFFLINE_RESOLVE static char ret[MaxNameSize]; const auto proc = GetCurrentProcess(); @@ -336,9 +354,9 @@ const char* DecodeCallstackPtrFast( uint64_t ptr ) si->SizeOfStruct = sizeof( SYMBOL_INFO ); si->MaxNameLen = MaxNameSize; -#ifdef TRACY_DBGHELP_LOCK +# ifdef TRACY_DBGHELP_LOCK DBGHELP_LOCK; -#endif +# endif if( SymFromAddr( proc, ptr, nullptr, si ) == 0 ) { *ret = '\0'; @@ -348,10 +366,11 @@ const char* DecodeCallstackPtrFast( uint64_t ptr ) memcpy( ret, si->Name, si->NameLen ); ret[si->NameLen] = '\0'; } -#ifdef TRACY_DBGHELP_LOCK +# ifdef TRACY_DBGHELP_LOCK DBGHELP_UNLOCK; -#endif +# endif return ret; +#endif //#ifndef TRACY_SYMBOL_OFFLINE_RESOLVE } const char* GetKernelModulePath( uint64_t addr ) @@ -413,7 +432,7 @@ ModuleNameAndBaseAddress GetModuleNameAndPrepareSymbols( uint64_t addr ) if( nameLength > 0 ) { // since this is the first time we encounter this module, load its symbols (needed for modules loaded after SymInitialize) - ModuleCache* cachedModule = DbgHelpLoadSymbolsForModuleAndCache( name, nameLength, (DWORD64)info.lpBaseOfDll, info.SizeOfImage ); + ModuleCache* cachedModule = LoadSymbolsForModuleAndCache( name, nameLength, (DWORD64)info.lpBaseOfDll, info.SizeOfImage ); return ModuleNameAndBaseAddress{ cachedModule->name, cachedModule->start }; } } @@ -436,12 +455,13 @@ CallstackSymbolData DecodeSymbolAddress( uint64_t ptr ) return sym; } +#ifndef TRACY_SYMBOL_OFFLINE_RESOLVE IMAGEHLP_LINE64 line; DWORD displacement = 0; line.SizeOfStruct = sizeof(IMAGEHLP_LINE64); -#ifdef TRACY_DBGHELP_LOCK +# ifdef TRACY_DBGHELP_LOCK DBGHELP_LOCK; -#endif +# endif const auto res = SymGetLineFromAddr64( GetCurrentProcess(), ptr, &displacement, &line ); if( res == 0 || line.LineNumber >= 0xF00000 ) { @@ -455,10 +475,11 @@ CallstackSymbolData DecodeSymbolAddress( uint64_t ptr ) sym.line = line.LineNumber; sym.needFree = true; } -#ifdef TRACY_DBGHELP_LOCK +# ifdef TRACY_DBGHELP_LOCK DBGHELP_UNLOCK; -#endif +# endif return sym; +#endif //TRACY_SYMBOL_OFFLINE_RESOLVE } CallstackEntryData DecodeCallstackPtr( uint64_t ptr ) @@ -479,14 +500,15 @@ CallstackEntryData DecodeCallstackPtr( uint64_t ptr ) return { cb_data, 1, moduleNameAndAddress.name }; } +#ifndef TRACY_SYMBOL_OFFLINE_RESOLVE int write; const auto proc = GetCurrentProcess(); -#ifdef TRACY_DBGHELP_LOCK +# ifdef TRACY_DBGHELP_LOCK DBGHELP_LOCK; -#endif +# endif -#if !defined TRACY_NO_CALLSTACK_INLINES +# if !defined TRACY_NO_CALLSTACK_INLINES BOOL doInline = FALSE; DWORD ctx = 0; DWORD inlineNum = 0; @@ -503,7 +525,7 @@ CallstackEntryData DecodeCallstackPtr( uint64_t ptr ) cb_num = 1 + inlineNum; } else -#endif +# endif { write = 0; cb_num = 1; @@ -548,7 +570,7 @@ CallstackEntryData DecodeCallstackPtr( uint64_t ptr ) } } -#if !defined TRACY_NO_CALLSTACK_INLINES +# if !defined TRACY_NO_CALLSTACK_INLINES if( doInline ) { for( DWORD i=0; i Date: Sun, 19 Nov 2023 06:31:42 -0800 Subject: [PATCH 14/21] revert #ifdef of a lot of code by TRACY_SYMBOL_OFFLINE_RESOLVE, it now only force enables the offline symbol resolving codepath. --- public/client/TracyCallstack.cpp | 105 ++++++++++++------------------- 1 file changed, 39 insertions(+), 66 deletions(-) diff --git a/public/client/TracyCallstack.cpp b/public/client/TracyCallstack.cpp index 83d29d44..b3275700 100644 --- a/public/client/TracyCallstack.cpp +++ b/public/client/TracyCallstack.cpp @@ -96,11 +96,9 @@ namespace tracy // when "TRACY_SYMBOL_OFFLINE_RESOLVE" is set, instead of fully resolving symbols at runtime, // simply resolve the offset and image name (which will be enough the resolving to be done offline) #ifdef TRACY_SYMBOL_OFFLINE_RESOLVE -static constexpr bool s_doOfflineSymbolResolve = true; -bool ShouldResolveSymbolsOffline() { return true; } - +constexpr bool s_shouldResolveSymbolsOffline = true; #else -bool s_doOfflineSymbolResolve = false; +static bool s_shouldResolveSymbolsOffline = false; bool ShouldResolveSymbolsOffline() { const char* symbolOfflineResolve = GetEnvVar( "TRACY_SYMBOL_OFFLINE_RESOLVE" ); @@ -158,54 +156,49 @@ void InitCallstackCritical() ___tracy_RtlWalkFrameChain = (___tracy_t_RtlWalkFrameChain)GetProcAddress( GetModuleHandleA( "ntdll.dll" ), "RtlWalkFrameChain" ); } -#ifndef TRACY_SYMBOL_OFFLINE_RESOLVE void DbgHelpInit() { - if( s_doOfflineSymbolResolve ) return; + if( s_shouldResolveSymbolsOffline ) return; _SymAddrIncludeInlineTrace = (t_SymAddrIncludeInlineTrace)GetProcAddress(GetModuleHandleA("dbghelp.dll"), "SymAddrIncludeInlineTrace"); _SymQueryInlineTrace = (t_SymQueryInlineTrace)GetProcAddress(GetModuleHandleA("dbghelp.dll"), "SymQueryInlineTrace"); _SymFromInlineContext = (t_SymFromInlineContext)GetProcAddress(GetModuleHandleA("dbghelp.dll"), "SymFromInlineContext"); _SymGetLineFromInlineContext = (t_SymGetLineFromInlineContext)GetProcAddress(GetModuleHandleA("dbghelp.dll"), "SymGetLineFromInlineContext"); -# ifdef TRACY_DBGHELP_LOCK +#ifdef TRACY_DBGHELP_LOCK DBGHELP_INIT; DBGHELP_LOCK; -# endif +#endif SymInitialize( GetCurrentProcess(), nullptr, true ); SymSetOptions( SYMOPT_LOAD_LINES ); -# ifdef TRACY_DBGHELP_LOCK +#ifdef TRACY_DBGHELP_LOCK DBGHELP_UNLOCK; -# endif +#endif } DWORD64 DbgHelpLoadSymbolsForModule( const char* imageName, uint64_t baseOfDll, uint32_t bllSize ) { - if( s_doOfflineSymbolResolve ) return 0; + if( s_shouldResolveSymbolsOffline ) return 0; return SymLoadModuleEx( GetCurrentProcess(), nullptr, imageName, nullptr, baseOfDll, bllSize, nullptr, 0 ); } -#endif //#ifndef TRACY_SYMBOL_OFFLINE_RESOLVE ModuleCache* LoadSymbolsForModuleAndCache( const char* imageName, uint32_t imageNameLength, uint64_t baseOfDll, uint32_t dllSize ) { -#ifndef TRACY_SYMBOL_OFFLINE_RESOLVE DbgHelpLoadSymbolsForModule( imageName, baseOfDll, dllSize ); -#endif //#ifndef TRACY_SYMBOL_OFFLINE_RESOLVE ModuleCache* cachedModule = s_modCache->push_next(); cachedModule->start = baseOfDll; cachedModule->end = baseOfDll + dllSize; // when doing offline symbol resolution, we must store the full path of the dll for the resolving to work - if( s_doOfflineSymbolResolve ) + if( s_shouldResolveSymbolsOffline ) { cachedModule->name = (char*)tracy_malloc_fast(imageNameLength + 1); memcpy(cachedModule->name, imageName, imageNameLength); cachedModule->name[imageNameLength] = '\0'; } -#ifndef TRACY_SYMBOL_OFFLINE_RESOLVE else { auto ptr = imageName + imageNameLength; @@ -218,7 +211,6 @@ ModuleCache* LoadSymbolsForModuleAndCache( const char* imageName, uint32_t image cachedModule->name[namelen + 1] = ']'; cachedModule->name[namelen + 2] = '\0'; } -#endif //#ifndef TRACY_SYMBOL_OFFLINE_RESOLVE return cachedModule; } @@ -226,18 +218,18 @@ ModuleCache* LoadSymbolsForModuleAndCache( const char* imageName, uint32_t image void InitCallstack() { #ifndef TRACY_SYMBOL_OFFLINE_RESOLVE - s_doOfflineSymbolResolve = ShouldResolveSymbolsOffline(); - if( s_doOfflineSymbolResolve ) + s_shouldResolveSymbolsOffline = ShouldResolveSymbolsOffline(); +#endif //#ifndef TRACY_SYMBOL_OFFLINE_RESOLVE + if( s_shouldResolveSymbolsOffline ) { TracyDebug("TRACY: enabling offline symbol resolving!\n"); } DbgHelpInit(); -# ifdef TRACY_DBGHELP_LOCK +#ifdef TRACY_DBGHELP_LOCK DBGHELP_LOCK; -# endif -#endif //#ifndef TRACY_SYMBOL_OFFLINE_RESOLVE +#endif // use TRACY_NO_DBHELP_INIT_LOAD=1 to disable preloading of driver // and process module symbol loading at startup time - they will be loaded on demand later @@ -286,9 +278,7 @@ void InitCallstack() path = full; } -# ifndef TRACY_SYMBOL_OFFLINE_RESOLVE DbgHelpLoadSymbolsForModule( path, (DWORD64)dev[i], 0 ); -# endif const auto psz = strlen( path ); auto pptr = (char*)tracy_malloc_fast( psz+1 ); @@ -330,11 +320,9 @@ void InitCallstack() } } -#ifndef TRACY_SYMBOL_OFFLINE_RESOLVE -# ifdef TRACY_DBGHELP_LOCK +#ifdef TRACY_DBGHELP_LOCK DBGHELP_UNLOCK; -# endif -#endif //#ifndef TRACY_SYMBOL_OFFLINE_RESOLVE +#endif } void EndCallstack() @@ -343,9 +331,8 @@ void EndCallstack() const char* DecodeCallstackPtrFast( uint64_t ptr ) { - if( s_doOfflineSymbolResolve ) return "[unresolved]"; + if( s_shouldResolveSymbolsOffline ) return "[unresolved]"; -#ifndef TRACY_SYMBOL_OFFLINE_RESOLVE static char ret[MaxNameSize]; const auto proc = GetCurrentProcess(); @@ -354,9 +341,9 @@ const char* DecodeCallstackPtrFast( uint64_t ptr ) si->SizeOfStruct = sizeof( SYMBOL_INFO ); si->MaxNameLen = MaxNameSize; -# ifdef TRACY_DBGHELP_LOCK +#ifdef TRACY_DBGHELP_LOCK DBGHELP_LOCK; -# endif +#endif if( SymFromAddr( proc, ptr, nullptr, si ) == 0 ) { *ret = '\0'; @@ -366,11 +353,10 @@ const char* DecodeCallstackPtrFast( uint64_t ptr ) memcpy( ret, si->Name, si->NameLen ); ret[si->NameLen] = '\0'; } -# ifdef TRACY_DBGHELP_LOCK +#ifdef TRACY_DBGHELP_LOCK DBGHELP_UNLOCK; -# endif +#endif return ret; -#endif //#ifndef TRACY_SYMBOL_OFFLINE_RESOLVE } const char* GetKernelModulePath( uint64_t addr ) @@ -447,7 +433,7 @@ CallstackSymbolData DecodeSymbolAddress( uint64_t ptr ) { CallstackSymbolData sym; - if( s_doOfflineSymbolResolve ) + if( s_shouldResolveSymbolsOffline ) { sym.file = "[unknown]"; sym.line = 0; @@ -455,13 +441,12 @@ CallstackSymbolData DecodeSymbolAddress( uint64_t ptr ) return sym; } -#ifndef TRACY_SYMBOL_OFFLINE_RESOLVE IMAGEHLP_LINE64 line; DWORD displacement = 0; line.SizeOfStruct = sizeof(IMAGEHLP_LINE64); -# ifdef TRACY_DBGHELP_LOCK +#ifdef TRACY_DBGHELP_LOCK DBGHELP_LOCK; -# endif +#endif const auto res = SymGetLineFromAddr64( GetCurrentProcess(), ptr, &displacement, &line ); if( res == 0 || line.LineNumber >= 0xF00000 ) { @@ -475,11 +460,10 @@ CallstackSymbolData DecodeSymbolAddress( uint64_t ptr ) sym.line = line.LineNumber; sym.needFree = true; } -# ifdef TRACY_DBGHELP_LOCK +#ifdef TRACY_DBGHELP_LOCK DBGHELP_UNLOCK; -# endif +#endif return sym; -#endif //TRACY_SYMBOL_OFFLINE_RESOLVE } CallstackEntryData DecodeCallstackPtr( uint64_t ptr ) @@ -488,7 +472,7 @@ CallstackEntryData DecodeCallstackPtr( uint64_t ptr ) const ModuleNameAndBaseAddress moduleNameAndAddress = GetModuleNameAndPrepareSymbols( ptr ); - if( s_doOfflineSymbolResolve ) + if( s_shouldResolveSymbolsOffline ) { cb_data[0].symAddr = ptr - moduleNameAndAddress.baseAddr; cb_data[0].symLen = 0; @@ -500,15 +484,14 @@ CallstackEntryData DecodeCallstackPtr( uint64_t ptr ) return { cb_data, 1, moduleNameAndAddress.name }; } -#ifndef TRACY_SYMBOL_OFFLINE_RESOLVE int write; const auto proc = GetCurrentProcess(); -# ifdef TRACY_DBGHELP_LOCK +#ifdef TRACY_DBGHELP_LOCK DBGHELP_LOCK; -# endif +#endif -# if !defined TRACY_NO_CALLSTACK_INLINES +#if !defined TRACY_NO_CALLSTACK_INLINES BOOL doInline = FALSE; DWORD ctx = 0; DWORD inlineNum = 0; @@ -525,7 +508,7 @@ CallstackEntryData DecodeCallstackPtr( uint64_t ptr ) cb_num = 1 + inlineNum; } else -# endif +#endif { write = 0; cb_num = 1; @@ -570,7 +553,7 @@ CallstackEntryData DecodeCallstackPtr( uint64_t ptr ) } } -# if !defined TRACY_NO_CALLSTACK_INLINES +#if !defined TRACY_NO_CALLSTACK_INLINES if( doInline ) { for( DWORD i=0; i Date: Sun, 19 Nov 2023 06:36:42 -0800 Subject: [PATCH 15/21] update linux github workflow after rebase --- .github/workflows/linux.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index a8a2bad5..2af87d71 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -25,6 +25,8 @@ jobs: run: make -j`nproc` -C csvexport/build/unix debug release - name: Import-chrome utility run: make -j`nproc` -C import-chrome/build/unix debug release + - name: tracy-edit utility + run: make -j`nproc` -C tracy-edit/build/unix debug release - name: Library run: make -j`nproc` -C library/unix debug release - name: Library (meson) From 6c0bb0a56a31f201cfc14aa7667dd9aa5cde3d40 Mon Sep 17 00:00:00 2001 From: trodrigues Date: Sun, 19 Nov 2023 06:50:56 -0800 Subject: [PATCH 16/21] add mising cstdint include that was saucing compilation failure in the linux github workflow (but not locally, likely due to diff compiler) --- tracy-edit/src/OfflineSymbolResolver.h | 1 + 1 file changed, 1 insertion(+) diff --git a/tracy-edit/src/OfflineSymbolResolver.h b/tracy-edit/src/OfflineSymbolResolver.h index 4ecc4db2..d61863dd 100644 --- a/tracy-edit/src/OfflineSymbolResolver.h +++ b/tracy-edit/src/OfflineSymbolResolver.h @@ -4,6 +4,7 @@ #include #include #include +#include namespace tracy { From e5e84424f68d4445e89f69d72e23fd18b28dceb0 Mon Sep 17 00:00:00 2001 From: Tiago Rodrigues Date: Fri, 24 Nov 2023 16:24:45 -0500 Subject: [PATCH 17/21] update cmake option description --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 66b506f2..bdbf33c9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -85,7 +85,7 @@ set_option(TRACY_FIBERS "Enable fibers support" OFF) set_option(TRACY_NO_CRASH_HANDLER "Disable crash handling" OFF) set_option(TRACY_TIMER_FALLBACK "Use lower resolution timers" OFF) set_option(TRACE_CLIENT_LIBUNWIND_BACKTRACE "Use libunwind backtracing where supported" OFF) -set_option(TRACY_SYMBOL_OFFLINE_RESOLVE "Instead of runtime symbol resolving, sore offset and sharedlib path for offline symbol resolution" OFF) +set_option(TRACY_SYMBOL_OFFLINE_RESOLVE "Instead of full runtime symbol resolution, only resolve the image path and offset to enable offline symbol resolution" OFF) if(NOT TRACY_STATIC) target_compile_definitions(TracyClient PRIVATE TRACY_EXPORTS) From a1ca8dd06a8d907afbf61fa094a84009b12cb95a Mon Sep 17 00:00:00 2001 From: Tiago Rodrigues Date: Fri, 24 Nov 2023 16:32:45 -0500 Subject: [PATCH 18/21] fixes according to review comments --- public/client/TracyCallstack.cpp | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/public/client/TracyCallstack.cpp b/public/client/TracyCallstack.cpp index b3275700..e346647e 100644 --- a/public/client/TracyCallstack.cpp +++ b/public/client/TracyCallstack.cpp @@ -116,7 +116,6 @@ CallstackEntry cb_data[MaxCbTrace]; extern "C" { -#ifndef TRACY_SYMBOL_OFFLINE_RESOLVE typedef DWORD (__stdcall *t_SymAddrIncludeInlineTrace)( HANDLE hProcess, DWORD64 Address ); typedef BOOL (__stdcall *t_SymQueryInlineTrace)( HANDLE hProcess, DWORD64 StartAddress, DWORD StartContext, DWORD64 StartRetAddress, DWORD64 CurAddress, LPDWORD CurContext, LPDWORD CurFrameIndex ); typedef BOOL (__stdcall *t_SymFromInlineContext)( HANDLE hProcess, DWORD64 Address, ULONG InlineContext, PDWORD64 Displacement, PSYMBOL_INFO Symbol ); @@ -126,7 +125,6 @@ extern "C" t_SymQueryInlineTrace _SymQueryInlineTrace = 0; t_SymFromInlineContext _SymFromInlineContext = 0; t_SymGetLineFromInlineContext _SymGetLineFromInlineContext = 0; -#endif // #ifndef TRACY_SYMBOL_OFFLINE_RESOLVE TRACY_API ___tracy_t_RtlWalkFrameChain ___tracy_RtlWalkFrameChain = 0; } @@ -468,12 +466,20 @@ CallstackSymbolData DecodeSymbolAddress( uint64_t ptr ) CallstackEntryData DecodeCallstackPtr( uint64_t ptr ) { +#ifdef TRACY_DBGHELP_LOCK + DBGHELP_LOCK; +#endif + InitRpmalloc(); const ModuleNameAndBaseAddress moduleNameAndAddress = GetModuleNameAndPrepareSymbols( ptr ); if( s_shouldResolveSymbolsOffline ) { +#ifdef TRACY_DBGHELP_LOCK + DBGHELP_UNLOCK; +#endif + cb_data[0].symAddr = ptr - moduleNameAndAddress.baseAddr; cb_data[0].symLen = 0; @@ -487,10 +493,6 @@ CallstackEntryData DecodeCallstackPtr( uint64_t ptr ) int write; const auto proc = GetCurrentProcess(); -#ifdef TRACY_DBGHELP_LOCK - DBGHELP_LOCK; -#endif - #if !defined TRACY_NO_CALLSTACK_INLINES BOOL doInline = FALSE; DWORD ctx = 0; @@ -1039,7 +1041,7 @@ void SymInfoError( void* /*data*/, const char* /*msg*/, int /*errnum*/ ) cb_data[cb_num-1].symAddr = 0; } -void getSymbolForOfflineResolve(void* address, Dl_info& dlinfo, CallstackEntry& cbEntry) +void GetSymbolForOfflineResolve(void* address, Dl_info& dlinfo, CallstackEntry& cbEntry) { // tagged with a string that we can identify as an unresolved symbol cbEntry.name = CopyStringFast( "[unresolved]" ); @@ -1062,7 +1064,12 @@ CallstackEntryData DecodeCallstackPtr( uint64_t ptr ) symloc = dlinfo.dli_fname; } - if(cb_bts && !s_shouldResolveSymbolsOffline) + if( s_shouldResolveSymbolsOffline ) + { + cb_num = 1; + GetSymbolForOfflineResolve( (void*)ptr, dlinfo, cb_data[0] ); + } + else { cb_num = 0; backtrace_pcinfo( cb_bts, ptr, CallstackDataCb, CallstackErrorCb, nullptr ); @@ -1070,11 +1077,6 @@ CallstackEntryData DecodeCallstackPtr( uint64_t ptr ) backtrace_syminfo( cb_bts, ptr, SymInfoCallback, SymInfoError, nullptr ); } - else - { - cb_num = 1; - getSymbolForOfflineResolve( (void*)ptr, dlinfo, cb_data[0] ); - } return { cb_data, uint8_t( cb_num ), symloc ? symloc : "[unknown]" }; } From de6e7d2fc3e67c149f9abc9c47d1f73db56f91bc Mon Sep 17 00:00:00 2001 From: Tiago Rodrigues Date: Fri, 24 Nov 2023 20:48:39 -0500 Subject: [PATCH 19/21] move the symbol resolving functionality to the update application instead of creating a new one --- manual/tracy.tex | 8 +- tracy-edit/build/unix/Makefile | 16 - tracy-edit/build/unix/build.mk | 12 - tracy-edit/build/unix/debug.mk | 6 - tracy-edit/build/unix/release.mk | 9 - tracy-edit/build/win32/tracy-edit.sln | 25 -- tracy-edit/build/win32/tracy-edit.vcxproj | 210 ---------- .../build/win32/tracy-edit.vcxproj.filters | 366 ------------------ tracy-edit/src/tracy-edit.cpp | 213 ---------- update/build/win32/update.vcxproj | 8 +- update/build/win32/update.vcxproj.filters | 12 + .../src/OfflineSymbolResolver.cpp | 51 ++- .../src/OfflineSymbolResolver.h | 5 +- .../src/OfflineSymbolResolverAddr2Line.cpp | 0 .../src/OfflineSymbolResolverDbgHelper.cpp | 0 update/src/update.cpp | 25 +- 16 files changed, 90 insertions(+), 876 deletions(-) delete mode 100644 tracy-edit/build/unix/Makefile delete mode 100644 tracy-edit/build/unix/build.mk delete mode 100644 tracy-edit/build/unix/debug.mk delete mode 100644 tracy-edit/build/unix/release.mk delete mode 100644 tracy-edit/build/win32/tracy-edit.sln delete mode 100644 tracy-edit/build/win32/tracy-edit.vcxproj delete mode 100644 tracy-edit/build/win32/tracy-edit.vcxproj.filters delete mode 100644 tracy-edit/src/tracy-edit.cpp rename {tracy-edit => update}/src/OfflineSymbolResolver.cpp (71%) rename {tracy-edit => update}/src/OfflineSymbolResolver.h (71%) rename {tracy-edit => update}/src/OfflineSymbolResolverAddr2Line.cpp (100%) rename {tracy-edit => update}/src/OfflineSymbolResolverDbgHelper.cpp (100%) diff --git a/manual/tracy.tex b/manual/tracy.tex index 766f7d06..c11cb55d 100644 --- a/manual/tracy.tex +++ b/manual/tracy.tex @@ -1792,12 +1792,12 @@ As an alternative to to runtime symbol resolution, we can set the environment va the minimal set of info required for offline resolution (a shared library path and an offset into that shared library). The generated tracy capture will have callstack frames symbols showing \texttt{[unresolved]}. -The \texttt{tracy-edit} tool can be used to load that capture, perform symbol resolution offline +The \texttt{update} tool can be used to load that capture, perform symbol resolution offline (by passing \texttt{-r}) and writing out a new capture with symbols resolved. -By default \texttt{tracy-edit} will use the original shared libraries paths that were recorded +By default \texttt{update} will use the original shared libraries paths that were recorded in the capture (which assumes running in the same machine or a machine with identical filesystem setup as the one used to run the tracy instrumented application). - You can do path substitution with the \texttt{-s} option to perform any number of path + You can do path substitution with the \texttt{-p} option to perform any number of path substitions in order to use symbols located elsewhere. \begin{bclogo}[ @@ -1805,7 +1805,7 @@ noborder=true, couleur=black!5, logo=\bcbombe ]{Important} -Beware that \texttt{tracy-edit} will use any matching symbol file to the path it resolved to (no symbol version checking is done), so if the symbol file doesn't match the code that was used when doing the callstack capturing you will get incorrect results. +Beware that \texttt{update} will use any matching symbol file to the path it resolved to (no symbol version checking is done), so if the symbol file doesn't match the code that was used when doing the callstack capturing you will get incorrect results. \end{bclogo} \subsection{Lua support} diff --git a/tracy-edit/build/unix/Makefile b/tracy-edit/build/unix/Makefile deleted file mode 100644 index 035eda1b..00000000 --- a/tracy-edit/build/unix/Makefile +++ /dev/null @@ -1,16 +0,0 @@ -all: release - -debug: - @+make -f debug.mk all - -release: - @+make -f release.mk all - -clean: - @+make -f build.mk clean - -db: clean - @bear -- $(MAKE) -f debug.mk all - @mv -f compile_commands.json ../../../ - -.PHONY: all clean debug release db diff --git a/tracy-edit/build/unix/build.mk b/tracy-edit/build/unix/build.mk deleted file mode 100644 index 85fb7528..00000000 --- a/tracy-edit/build/unix/build.mk +++ /dev/null @@ -1,12 +0,0 @@ -CFLAGS += -CXXFLAGS := $(CFLAGS) -std=gnu++17 -DEFINES += -DTRACY_NO_STATISTICS -INCLUDES := $(shell pkg-config --cflags capstone) -LIBS += $(shell pkg-config --libs capstone) -lpthread -PROJECT := tracy-edit -IMAGE := $(PROJECT)-$(BUILD) - -FILTER := -include ../../../common/src-from-vcxproj.mk - -include ../../../common/unix.mk diff --git a/tracy-edit/build/unix/debug.mk b/tracy-edit/build/unix/debug.mk deleted file mode 100644 index a4ec6b6a..00000000 --- a/tracy-edit/build/unix/debug.mk +++ /dev/null @@ -1,6 +0,0 @@ -CFLAGS := -g3 -Wall -DEFINES := -DDEBUG -BUILD := debug - -include ../../../common/unix-debug.mk -include build.mk diff --git a/tracy-edit/build/unix/release.mk b/tracy-edit/build/unix/release.mk deleted file mode 100644 index ccf07661..00000000 --- a/tracy-edit/build/unix/release.mk +++ /dev/null @@ -1,9 +0,0 @@ -CFLAGS := -O3 -ifndef TRACY_NO_LTO -CFLAGS += -flto -endif -DEFINES := -DNDEBUG -BUILD := release - -include ../../../common/unix-release.mk -include build.mk diff --git a/tracy-edit/build/win32/tracy-edit.sln b/tracy-edit/build/win32/tracy-edit.sln deleted file mode 100644 index 9b606fbc..00000000 --- a/tracy-edit/build/win32/tracy-edit.sln +++ /dev/null @@ -1,25 +0,0 @@ - -Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio Version 16 -VisualStudioVersion = 16.0.30907.101 -MinimumVisualStudioVersion = 10.0.40219.1 -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "tracy-edit", "tracy-edit.vcxproj", "{447D58BF-94CD-4469-BB90-549C05D03E00}" -EndProject -Global - GlobalSection(SolutionConfigurationPlatforms) = preSolution - Debug|x64 = Debug|x64 - Release|x64 = Release|x64 - EndGlobalSection - GlobalSection(ProjectConfigurationPlatforms) = postSolution - {447D58BF-94CD-4469-BB90-549C05D03E00}.Debug|x64.ActiveCfg = Debug|x64 - {447D58BF-94CD-4469-BB90-549C05D03E00}.Debug|x64.Build.0 = Debug|x64 - {447D58BF-94CD-4469-BB90-549C05D03E00}.Release|x64.ActiveCfg = Release|x64 - {447D58BF-94CD-4469-BB90-549C05D03E00}.Release|x64.Build.0 = Release|x64 - EndGlobalSection - GlobalSection(SolutionProperties) = preSolution - HideSolutionNode = FALSE - EndGlobalSection - GlobalSection(ExtensibilityGlobals) = postSolution - SolutionGuid = {3E51386C-43EA-44AC-9F24-AFAFE4D63ADE} - EndGlobalSection -EndGlobal diff --git a/tracy-edit/build/win32/tracy-edit.vcxproj b/tracy-edit/build/win32/tracy-edit.vcxproj deleted file mode 100644 index 2fb58ced..00000000 --- a/tracy-edit/build/win32/tracy-edit.vcxproj +++ /dev/null @@ -1,210 +0,0 @@ - - - - - Debug - x64 - - - Release - x64 - - - - 15.0 - {447D58BF-94CD-4469-BB90-549C05D03E00} - tracy-edit - 10.0.17763.0 - x64-windows-static - - - - Application - true - v143 - MultiByte - - - Application - false - v143 - true - MultiByte - - - - - - - - - - - - - - - - true - - - - Level3 - Disabled - true - true - true - TRACY_NO_STATISTICS;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_DEPRECATE;WIN32_LEAN_AND_MEAN;NOMINMAX;_USE_MATH_DEFINES;%(PreprocessorDefinitions) - AdvancedVectorExtensions2 - stdcpplatest - $(ProjectDir)..\..\..\vcpkg_installed\$(VcpkgTriplet)\include;$(ProjectDir)..\..\..\vcpkg_installed\$(VcpkgTriplet)\include\capstone;$(VcpkgManifestRoot)\vcpkg_installed\$(VcpkgTriplet)\$(VcpkgTriplet)\include\capstone;$(VcpkgRoot)\installed\$(VcpkgTriplet)\include\capstone - - - ws2_32.lib;capstone.lib;dbghelp.lib;%(AdditionalDependencies) - Console - $(ProjectDir)..\..\..\vcpkg_installed\$(VcpkgTriplet)\debug\lib - - - - - Level3 - MaxSpeed - true - true - true - true - true - TRACY_NO_STATISTICS;NDEBUG;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_DEPRECATE;WIN32_LEAN_AND_MEAN;NOMINMAX;_USE_MATH_DEFINES;%(PreprocessorDefinitions) - AdvancedVectorExtensions2 - stdcpplatest - $(ProjectDir)..\..\..\vcpkg_installed\$(VcpkgTriplet)\include;$(ProjectDir)..\..\..\vcpkg_installed\$(VcpkgTriplet)\include\capstone;$(VcpkgManifestRoot)\vcpkg_installed\$(VcpkgTriplet)\$(VcpkgTriplet)\include\capstone;$(VcpkgRoot)\installed\$(VcpkgTriplet)\include\capstone - - - true - true - ws2_32.lib;capstone.lib;dbghelp.lib;%(AdditionalDependencies) - Console - $(ProjectDir)..\..\..\vcpkg_installed\$(VcpkgTriplet)\lib - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/tracy-edit/build/win32/tracy-edit.vcxproj.filters b/tracy-edit/build/win32/tracy-edit.vcxproj.filters deleted file mode 100644 index 8b77d3ab..00000000 --- a/tracy-edit/build/win32/tracy-edit.vcxproj.filters +++ /dev/null @@ -1,366 +0,0 @@ - - - - - {729c80ee-4d26-4a5e-8f1f-6c075783eb56} - - - {cf23ef7b-7694-4154-830b-00cf053350ea} - - - {e39d3623-47cd-4752-8da9-3ea324f964c1} - - - {9ec18988-3ab7-4c05-a9d0-46c0a68037de} - - - {5ee9ba63-2914-4027-997e-e743a294bba6} - - - {a166d032-7be0-4d07-9f85-a8199cc1ec7c} - - - {438fff23-197c-4b6f-91f0-74f8b3878571} - - - {e5c7021a-e0e4-45c2-b461-e806bc036d5f} - - - - - server - - - server - - - src - - - server - - - server - - - server - - - server - - - zstd\common - - - zstd\common - - - zstd\common - - - zstd\common - - - zstd\common - - - zstd\common - - - zstd\common - - - zstd\common - - - zstd\compress - - - zstd\compress - - - zstd\compress - - - zstd\compress - - - zstd\compress - - - zstd\compress - - - zstd\compress - - - zstd\compress - - - zstd\compress - - - zstd\compress - - - zstd\compress - - - zstd\compress - - - zstd\compress - - - zstd\decompress - - - zstd\decompress - - - zstd\decompress - - - zstd\decompress - - - zstd\dictBuilder - - - zstd\dictBuilder - - - zstd\dictBuilder - - - zstd\dictBuilder - - - common - - - common - - - common - - - common - - - common - - - src - - - src - - - src - - - - - - server - - - server - - - server - - - server - - - server - - - server - - - server - - - server - - - server - - - server - - - server - - - server - - - server - - - zstd - - - zstd - - - zstd\common - - - zstd\common - - - zstd\common - - - zstd\common - - - zstd\common - - - zstd\common - - - zstd\common - - - zstd\common - - - zstd\common - - - zstd\common - - - zstd\common - - - zstd\common - - - zstd\common - - - zstd\common - - - zstd\compress - - - zstd\compress - - - zstd\compress - - - zstd\compress - - - zstd\compress - - - zstd\compress - - - zstd\compress - - - zstd\compress - - - zstd\compress - - - zstd\compress - - - zstd\compress - - - zstd\compress - - - zstd\compress - - - zstd\decompress - - - zstd\decompress - - - zstd\decompress - - - zstd - - - zstd\dictBuilder - - - zstd\dictBuilder - - - zstd\common - - - zstd\compress - - - common - - - common - - - common - - - common - - - common - - - common - - - common - - - common - - - common - - - common - - - common - - - common - - - common - - - common - - - common - - - - - zstd\decompress - - - \ No newline at end of file diff --git a/tracy-edit/src/tracy-edit.cpp b/tracy-edit/src/tracy-edit.cpp deleted file mode 100644 index f79c5dab..00000000 --- a/tracy-edit/src/tracy-edit.cpp +++ /dev/null @@ -1,213 +0,0 @@ -#ifdef _WIN32 -# include -#endif - -#include -#include -#include -#include -#include -#include -#include - -#include "../../server/TracyFileRead.hpp" -#include "../../server/TracyFileWrite.hpp" -#include "../../server/TracyWorker.hpp" -#include "../../getopt/getopt.h" - -#include "OfflineSymbolResolver.h" - -struct Args -{ - const char* inputTracyPath = nullptr; - const char* outputTracyPath = nullptr; - bool verbose = false; - bool resolveSymbols = false; - std::vector pathSubstitutions; - tracy::FileWrite::Compression compressionType = tracy::FileWrite::Compression::Zstd; - int compressionLevel = 5; -}; - -void PrintUsageAndExit() -{ - std::cerr << "Modify a tracy file" << std::endl; - std::cerr << "Usage:" << std::endl; - std::cerr << " tracy-edit [OPTION...] " << std::endl; - std::cerr << std::endl; - std::cerr << " -h, --help Print usage" << std::endl; - std::cerr << " -v, --verbose Enable verbose logging" << std::endl; - std::cerr << " -r, --resolveSymbols Resolve symbols and patch callstack frames" << std::endl; - std::cerr << " -s, --substitutePath \"REGEX_MATCH;REPLACEMENT\" Substitute symbol resolution path with an alternative" << std::endl; - std::cerr << " -c, --compression Type Set the compression algorithm used for the output [Fast,Slow,Extreme,Zstd]" << std::endl; - std::cerr << " -l, --compressesionLevel Level Level of compression" << std::endl; - exit( 1 ); -} - -static const char* compressionTypeStr[] -{ - "Fast", - "Slow", - "Extreme", - "Zstd" -}; -static_assert( uint32_t(tracy::FileWrite::Compression::Zstd)+1 == sizeof(compressionTypeStr)/sizeof(compressionTypeStr[0])); -tracy::FileWrite::Compression getCompressionFromString(const char* str) -{ - for( uint32_t i = 0; i < sizeof(compressionTypeStr)/sizeof(compressionTypeStr[0]); ++i ) - { - if( strcmp( compressionTypeStr[i], str ) == 0 ) - { - return tracy::FileWrite::Compression( i ); - } - } - return tracy::FileWrite::Compression::Zstd; -} - -Args ParseArgs( int argc, char** argv ) -{ - if ( argc < 3 ) - { - PrintUsageAndExit(); - } - - Args args; - - struct option long_opts[] = - { - { "help", no_argument, NULL, 'h' }, - { "verbose", no_argument, NULL, 'v' }, - { "resolveSymbols", no_argument, NULL, 'r' }, - { "substitutePath", required_argument, NULL, 's' }, - { "compression", required_argument, NULL, 'c' }, - { "compressesionLevel", required_argument, NULL, 'l' }, - { NULL, 0, NULL, 0 } - }; - - int c; - while ( (c = getopt_long( argc, argv, "hvrc:l:s:", long_opts, NULL )) != -1 ) - { - switch (c) - { - case 'h': - PrintUsageAndExit(); - break; - case 'v': - args.verbose = true; - break; - case 'r': - args.resolveSymbols = true; - break; - case 'c': - args.compressionType = getCompressionFromString( optarg ); - break; - case 's': - args.pathSubstitutions.push_back( optarg ); - break; - case 'l': - args.compressionLevel = atoi( optarg ); - break; - default: - PrintUsageAndExit(); - break; - } - } - - if (argc != optind + 2) - { - PrintUsageAndExit(); - } - - args.inputTracyPath = argv[optind + 0]; - args.outputTracyPath = argv[optind + 1]; - - return args; -} - -void PatchSymbols(tracy::Worker& worker, const Args& args); - -int main( int argc, char** argv ) -{ -#ifdef _WIN32 - if( !AttachConsole( ATTACH_PARENT_PROCESS ) ) - { - AllocConsole(); - SetConsoleMode( GetStdHandle( STD_OUTPUT_HANDLE ), 0x07 ); - } -#endif // #ifdef _WIN32 - - Args args = ParseArgs( argc, argv ); - - // load input tracy file - auto f = std::unique_ptr(tracy::FileRead::Open( args.inputTracyPath )); - if (!f) - { - std::cerr << "Could not open file: " << args.inputTracyPath; - return 1; - } - - std::cout << "Reading ..." << std::endl; - - const bool allowBgThreads = false; - bool allowStringModification = true; - tracy::Worker worker( *f, tracy::EventType::All, allowBgThreads, allowStringModification ); - - std::cout << "Loaded." << std::endl; - - if(args.resolveSymbols) - { - PatchSymbols(worker, args); - } - - // save out capture file with new compression options - std::cout << "Saving (using '" << compressionTypeStr[uint32_t(args.compressionType)] - << "', level: " << args.compressionLevel << ") ..." << std::endl; - - auto w = std::unique_ptr( - tracy::FileWrite::Open( args.outputTracyPath, args.compressionType, args.compressionLevel) ); - if( !w ) - { - std::cerr << "Cannot open output file: '" << args.outputTracyPath << "'" << std::endl; - exit( 1 ); - } - - worker.Write( *w, false ); - - std::cout << "Cleanup..." << std::endl; - - return 0; -} - -void PatchSymbols( tracy::Worker& worker, const Args& args ) -{ - std::cout << "Resolving and patching symbols..." << std::endl; - - PathSubstitutionList pathSubstitutionList; - for( const std::string& pathSubst : args.pathSubstitutions ) - { - std::size_t pos = pathSubst.find(';'); - if( pos == std::string::npos ) - { - std::cerr << "Ignoring invalid path substitution: '" << pathSubst - << " '(please separate the regex of the string to replace with a ';')" << std::endl; - continue; - } - - try - { - std::regex reg( pathSubst.substr(0, pos) ); - std::string replacementStr( pathSubst.substr(pos + 1) ); - pathSubstitutionList.push_back( std::pair(reg, replacementStr) ); - } - catch( std::exception& e ) - { - std::cerr << "Ignoring invalid path substitution: '" << pathSubst - << "' (" << e.what() << ")" << std::endl; - continue; - } - } - - if( !PatchSymbols( worker, pathSubstitutionList, args.verbose ) ) - { - std::cerr << "Failed to patch symbols" << std::endl; - } -} diff --git a/update/build/win32/update.vcxproj b/update/build/win32/update.vcxproj index 55f0e55e..1ed9c723 100644 --- a/update/build/win32/update.vcxproj +++ b/update/build/win32/update.vcxproj @@ -60,7 +60,7 @@ $(ProjectDir)..\..\..\vcpkg_installed\$(VcpkgTriplet)\include;$(ProjectDir)..\..\..\vcpkg_installed\$(VcpkgTriplet)\include\capstone;$(VcpkgManifestRoot)\vcpkg_installed\$(VcpkgTriplet)\$(VcpkgTriplet)\include\capstone;$(VcpkgRoot)\installed\$(VcpkgTriplet)\include\capstone - ws2_32.lib;capstone.lib;%(AdditionalDependencies) + ws2_32.lib;capstone.lib;dbghelp.lib;%(AdditionalDependencies) Console $(ProjectDir)..\..\..\vcpkg_installed\$(VcpkgTriplet)\debug\lib @@ -82,7 +82,7 @@ true true - ws2_32.lib;capstone.lib;%(AdditionalDependencies) + ws2_32.lib;capstone.lib;dbghelp.lib;%(AdditionalDependencies) Console $(ProjectDir)..\..\..\vcpkg_installed\$(VcpkgTriplet)\lib @@ -130,6 +130,9 @@ + + + @@ -200,6 +203,7 @@ + diff --git a/update/build/win32/update.vcxproj.filters b/update/build/win32/update.vcxproj.filters index c987fbf6..7c90bef3 100644 --- a/update/build/win32/update.vcxproj.filters +++ b/update/build/win32/update.vcxproj.filters @@ -159,6 +159,15 @@ common + + src + + + src + + + src + @@ -362,6 +371,9 @@ common + + src + diff --git a/tracy-edit/src/OfflineSymbolResolver.cpp b/update/src/OfflineSymbolResolver.cpp similarity index 71% rename from tracy-edit/src/OfflineSymbolResolver.cpp rename to update/src/OfflineSymbolResolver.cpp index 9ba7ce71..08be4d52 100644 --- a/tracy-edit/src/OfflineSymbolResolver.cpp +++ b/update/src/OfflineSymbolResolver.cpp @@ -11,13 +11,13 @@ #include "OfflineSymbolResolver.h" -bool ApplyPathSubstitutions( std::string& path, const PathSubstitutionList& pathSubstituionlist ) +bool ApplyPathSubstitutions( std::string& path, const PathSubstitutionList& pathSubstitutionlist ) { - for( const auto& substituion : pathSubstituionlist ) + for( const auto& substitution : pathSubstitutionlist ) { - if( std::regex_match(path, substituion.first) ) + if( std::regex_match(path, substitution.first) ) { - path = std::regex_replace( path, substituion.first, substituion.second ); + path = std::regex_replace( path, substitution.first, substitution.second ); return true; } } @@ -31,12 +31,12 @@ tracy::StringIdx AddSymbolString( tracy::Worker& worker, const std::string& str return tracy::StringIdx( location.idx ); } -bool PatchSymbols( tracy::Worker& worker, const PathSubstitutionList& pathSubstituionlist, bool verbose ) +bool PatchSymbolsWithRegex( tracy::Worker& worker, const PathSubstitutionList& pathSubstitutionlist, bool verbose ) { uint64_t callstackFrameCount = worker.GetCallstackFrameCount(); std::string relativeSoNameMatch = "[unresolved]"; - std::cout << "Found '" << callstackFrameCount << "' callstack frames. Batching into image groups..." << std::endl; + std::cout << "Found " << callstackFrameCount << " callstack frames. Batching into image groups..." << std::endl; // batch the symbol queries by .so so we issue the least amount of requests using FrameEntriesPerImageIdx = std::unordered_map; @@ -72,7 +72,7 @@ bool PatchSymbols( tracy::Worker& worker, const PathSubstitutionList& pathSubsti } } - std::cout << "Batched into '" << entriesPerImageIdx.size() << "' unique image groups" << std::endl; + std::cout << "Batched into " << entriesPerImageIdx.size() << " unique image groups" << std::endl; // FIXME: the resolving of symbols here can be slow and could be done in parallel per "image" // - be careful with string allocation though as that would be not safe to do in parallel @@ -88,7 +88,7 @@ bool PatchSymbols( tracy::Worker& worker, const PathSubstitutionList& pathSubsti std::cout << "Resolving " << entries.size() << " symbols for image: '" << imagePath << "'" << std::endl; - const bool substituted = ApplyPathSubstitutions( imagePath, pathSubstituionlist ); + const bool substituted = ApplyPathSubstitutions( imagePath, pathSubstitutionlist ); if( substituted ) { std::cout << "\tPath substituted to: '" << imagePath << "'" << std::endl; @@ -134,3 +134,38 @@ bool PatchSymbols( tracy::Worker& worker, const PathSubstitutionList& pathSubsti return true; } + +void PatchSymbols( tracy::Worker& worker, const std::vector& pathSubstitutionsStrings, bool verbose ) +{ + std::cout << "Resolving and patching symbols..." << std::endl; + + PathSubstitutionList pathSubstitutionList; + for ( const std::string& pathSubst : pathSubstitutionsStrings ) + { + std::size_t pos = pathSubst.find(';'); + if ( pos == std::string::npos ) + { + std::cerr << "Ignoring invalid path substitution: '" << pathSubst + << " '(please separate the regex of the string to replace with a ';')" << std::endl; + continue; + } + + try + { + std::regex reg(pathSubst.substr(0, pos)); + std::string replacementStr(pathSubst.substr(pos + 1)); + pathSubstitutionList.push_back(std::pair(reg, replacementStr)); + } + catch ( std::exception& e ) + { + std::cerr << "Ignoring invalid path substitution: '" << pathSubst + << "' (" << e.what() << ")" << std::endl; + continue; + } + } + + if ( !PatchSymbolsWithRegex(worker, pathSubstitutionList, verbose) ) + { + std::cerr << "Failed to patch symbols" << std::endl; + } +} \ No newline at end of file diff --git a/tracy-edit/src/OfflineSymbolResolver.h b/update/src/OfflineSymbolResolver.h similarity index 71% rename from tracy-edit/src/OfflineSymbolResolver.h rename to update/src/OfflineSymbolResolver.h index d61863dd..e3216eb5 100644 --- a/tracy-edit/src/OfflineSymbolResolver.h +++ b/update/src/OfflineSymbolResolver.h @@ -32,8 +32,9 @@ using SymbolEntryList = std::vector; bool ResolveSymbols( const std::string& imagePath, const FrameEntryList& inputEntryList, SymbolEntryList& resolvedEntries ); -using PathSubstitutionList = std::vector >; +void PatchSymbols( tracy::Worker& worker, const std::vector& pathSubstitutionsStrings, bool verbose = false ); -bool PatchSymbols( tracy::Worker& worker, const PathSubstitutionList& pathSubstituionlist, bool verbose = false ); +using PathSubstitutionList = std::vector >; +bool PatchSymbolsWithRegex( tracy::Worker& worker, const PathSubstitutionList& pathSubstituionlist, bool verbose = false ); #endif // __SYMBOLRESOLVER_HPP__ \ No newline at end of file diff --git a/tracy-edit/src/OfflineSymbolResolverAddr2Line.cpp b/update/src/OfflineSymbolResolverAddr2Line.cpp similarity index 100% rename from tracy-edit/src/OfflineSymbolResolverAddr2Line.cpp rename to update/src/OfflineSymbolResolverAddr2Line.cpp diff --git a/tracy-edit/src/OfflineSymbolResolverDbgHelper.cpp b/update/src/OfflineSymbolResolverDbgHelper.cpp similarity index 100% rename from tracy-edit/src/OfflineSymbolResolverDbgHelper.cpp rename to update/src/OfflineSymbolResolverDbgHelper.cpp diff --git a/update/src/update.cpp b/update/src/update.cpp index f3cba1b7..1643c620 100644 --- a/update/src/update.cpp +++ b/update/src/update.cpp @@ -15,6 +15,8 @@ #include "../../zstd/zstd.h" #include "../../getopt/getopt.h" +#include "OfflineSymbolResolver.h" + #ifdef __APPLE__ # define ftello64(x) ftello(x) #elif defined _WIN32 @@ -32,6 +34,9 @@ void Usage() printf( " l: locks, m: messages, p: plots, M: memory, i: frame images\n" ); printf( " c: context switches, s: sampling data, C: symbol code, S: source cache\n" ); printf( " -c: scan for source files missing in cache and add if found\n" ); + printf( " -r resolve symbols and patch callstack frames\n"); + printf( " -p: substitute symbol resolution path with an alternative: \"REGEX_MATCH;REPLACEMENT\"\n"); + exit( 1 ); } @@ -50,8 +55,11 @@ int main( int argc, char** argv ) int zstdLevel = 1; bool buildDict = false; bool cacheSource = false; + bool resolveSymbols = false; + std::vector pathSubstitutions; + int c; - while( ( c = getopt( argc, argv, "hez:ds:c" ) ) != -1 ) + while( ( c = getopt( argc, argv, "hez:ds:crp:" ) ) != -1 ) { switch( c ) { @@ -118,12 +126,19 @@ int main( int argc, char** argv ) case 'c': cacheSource = true; break; + case 'r': + resolveSymbols = true; + break; + case 'p': + pathSubstitutions.push_back(optarg); + break; default: Usage(); break; } } - if( argc - optind != 2 ) Usage(); + + if (argc != optind + 2) Usage(); const char* input = argv[optind]; const char* output = argv[optind+1]; @@ -144,7 +159,9 @@ int main( int argc, char** argv ) int inVer; { const auto t0 = std::chrono::high_resolution_clock::now(); - tracy::Worker worker( *f, (tracy::EventType::Type)events, false ); + const bool allowBgThreads = false; + const bool allowStringModification = resolveSymbols; + tracy::Worker worker( *f, (tracy::EventType::Type)events, allowBgThreads, allowStringModification); #ifndef TRACY_NO_STATISTICS while( !worker.AreSourceLocationZonesReady() ) std::this_thread::sleep_for( std::chrono::milliseconds( 10 ) ); @@ -152,6 +169,8 @@ int main( int argc, char** argv ) if( cacheSource ) worker.CacheSourceFiles(); + if ( resolveSymbols ) PatchSymbols( worker, pathSubstitutions ); + auto w = std::unique_ptr( tracy::FileWrite::Open( output, clev, zstdLevel ) ); if( !w ) { From 69b9707bc53c18b5f63ad94204d57de6abaa82d5 Mon Sep 17 00:00:00 2001 From: Tiago Rodrigues Date: Sat, 25 Nov 2023 09:19:13 -0500 Subject: [PATCH 20/21] update github workflows to remore tracy-edit --- .github/workflows/linux.yml | 2 -- .github/workflows/macos.yml | 2 -- .github/workflows/msvc.yml | 5 ----- 3 files changed, 9 deletions(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 2af87d71..a8a2bad5 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -25,8 +25,6 @@ jobs: run: make -j`nproc` -C csvexport/build/unix debug release - name: Import-chrome utility run: make -j`nproc` -C import-chrome/build/unix debug release - - name: tracy-edit utility - run: make -j`nproc` -C tracy-edit/build/unix debug release - name: Library run: make -j`nproc` -C library/unix debug release - name: Library (meson) diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index b8138a91..54697835 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -24,8 +24,6 @@ jobs: run: make -j`nproc` -C csvexport/build/unix debug release - name: Import-chrome utility run: make -j`nproc` -C import-chrome/build/unix debug release - - name: tracy-edit utility - run: make -j`nproc` -C tracy-edit/build/unix debug release - name: Library run: make -j`nproc` -C library/unix debug release - name: Library (meson) diff --git a/.github/workflows/msvc.yml b/.github/workflows/msvc.yml index fbcb441d..122b754b 100644 --- a/.github/workflows/msvc.yml +++ b/.github/workflows/msvc.yml @@ -36,10 +36,6 @@ jobs: run: msbuild .\import-chrome\build\win32\import-chrome.vcxproj /property:Configuration=Debug /property:Platform=x64 - name: Import-chrome utility Release run: msbuild .\import-chrome\build\win32\import-chrome.vcxproj /property:Configuration=Release /property:Platform=x64 - - name: tracy-edit utility Debug - run: msbuild .\tracy-edit\build\win32\tracy-edit.vcxproj /property:Configuration=Debug /property:Platform=x64 - - name: tracy-edit utility Release - run: msbuild .\tracy-edit\build\win32\tracy-edit.vcxproj /property:Configuration=Release /property:Platform=x64 - name: Library run: msbuild .\library\win32\TracyProfiler.vcxproj /property:Configuration=Release /property:Platform=x64 - name: Package binaries @@ -50,7 +46,6 @@ jobs: copy update\build\win32\x64\Release\update.exe bin copy capture\build\win32\x64\Release\capture.exe bin copy import-chrome\build\win32\x64\Release\import-chrome.exe bin - copy tracy-edit\build\win32\x64\Release\tracy-edit.exe bin copy csvexport\build\win32\x64\Release\csvexport.exe bin copy library\win32\x64\Release\TracyProfiler.dll bin\dev copy library\win32\x64\Release\TracyProfiler.lib bin\dev From 2509a4b962634630ed01aa6a440251eae77e6dfc Mon Sep 17 00:00:00 2001 From: Tiago Rodrigues Date: Mon, 27 Nov 2023 10:22:12 -0500 Subject: [PATCH 21/21] update documentation to reflect that symbol statistics are broken when using offline symbol resolving --- manual/tracy.tex | 2 ++ 1 file changed, 2 insertions(+) diff --git a/manual/tracy.tex b/manual/tracy.tex index c11cb55d..3d02e463 100644 --- a/manual/tracy.tex +++ b/manual/tracy.tex @@ -1806,6 +1806,8 @@ couleur=black!5, logo=\bcbombe ]{Important} Beware that \texttt{update} will use any matching symbol file to the path it resolved to (no symbol version checking is done), so if the symbol file doesn't match the code that was used when doing the callstack capturing you will get incorrect results. + +Also note that in the case of using offline symbol resolving, even after running the \texttt{update} tool to resolve symbols, the symbols statistics are not updated and will still report the unresolved symbols. \end{bclogo} \subsection{Lua support}