Added perfetto tracks and traces with cached data

Author: marantic-amd

projects/rocprofiler-systems/source/lib/core/config.cpp

@@ -315,6 +315,10 @@ configure_settings(bool _init)
     ROCPROFSYS_CONFIG_SETTING(bool, "ROCPROFSYS_USE_ROCPD", "Enable rocpd backend", false,
                               "backend", "rocpd");
 
+    ROCPROFSYS_CONFIG_SETTING(bool, "ROCPROFSYS_CACHING_PERFETTO",
+                              "Enable perfetto with trace cache", false, "backend",
+                              "perfetto_caching");
+
     ROCPROFSYS_CONFIG_SETTING(bool, "ROCPROFSYS_USE_ROCM",
                               "Enable ROCm API and kernel tracing", true, "backend",
                               "rocm");
@@ -2398,6 +2402,13 @@ get_use_rocpd()
     return static_cast<tim::tsettings<bool>&>(*_v).get();
 }
 
+bool&
+get_caching_perfetto()
+{
+    static auto _v = get_config()->at("ROCPROFSYS_CACHING_PERFETTO");
+    return static_cast<tim::tsettings<bool>&>(*_v).get();
+}
+
 tmp_file::tmp_file(std::string _v)
 : filename{ std::move(_v) }
 {}

projects/rocprofiler-systems/source/lib/core/config.hpp

@@ -367,6 +367,9 @@ get_database_absolute_path(std::string_view database_name, std::string_view tag)
 bool&
 get_use_rocpd() ROCPROFSYS_HOT;
 
+bool&
+get_caching_perfetto() ROCPROFSYS_HOT;
+
 struct tmp_file
 {
     tmp_file(std::string);

projects/rocprofiler-systems/source/lib/core/perfetto.cpp

@@ -208,6 +208,7 @@ post_process(tim::manager* _timemory_manager, bool& _perfetto_output_error)
             _data = char_vec_t{ tracing_session->ReadTraceBlocking() };
         }
 
+        tracing_session.reset();
         return _data;
     };
 

projects/rocprofiler-systems/source/lib/core/trace_cache/CMakeLists.txt

@@ -26,6 +26,7 @@ set(trace_cache_sources
     ${CMAKE_CURRENT_LIST_DIR}/buffer_storage.cpp
     ${CMAKE_CURRENT_LIST_DIR}/metadata_registry.cpp
     ${CMAKE_CURRENT_LIST_DIR}/rocpd_post_processing.cpp
+    ${CMAKE_CURRENT_LIST_DIR}/perfetto_post_processing.cpp
 )
 
 set(trace_cache_headers
@@ -35,6 +36,7 @@ set(trace_cache_headers
     ${CMAKE_CURRENT_LIST_DIR}/cache_utility.hpp
     ${CMAKE_CURRENT_LIST_DIR}/metadata_registry.hpp
     ${CMAKE_CURRENT_LIST_DIR}/rocpd_post_processing.hpp
+    ${CMAKE_CURRENT_LIST_DIR}/perfetto_post_processing.hpp
     ${CMAKE_CURRENT_LIST_DIR}/sample_type.hpp
 )
 

projects/rocprofiler-systems/source/lib/core/trace_cache/cache_manager.cpp

@@ -26,10 +26,12 @@
 #include "core/trace_cache/storage_parser.hpp"
 #include "debug.hpp"
 #include "library/runtime.hpp"
+#include "perfetto_post_processing.hpp"
 #include "trace_cache/cache_utility.hpp"
 #include "trace_cache/metadata_registry.hpp"
 #include "trace_cache/rocpd_post_processing.hpp"
 #include <algorithm>
+#include <iterator>
 #include <memory>
 #include <vector>
 
@@ -70,7 +72,8 @@ list_dir_files(const std::string& path)
     DIR* dir = opendir(path.c_str());
     if(dir == nullptr)
     {
-        ROCPROFSYS_THROW("Error opening directory: %s", path.c_str());
+        std::cerr << "Error opening directory: " << path << std::endl;
+        return {};
     }
 
     std::vector<std::string> result{};
@@ -153,35 +156,24 @@ cache_manager::post_process_bulk()
             shutdown();
         }
 
-        auto _cache_files = get_cache_files();
-
-        if(get_use_rocpd())
+        if(get_use_rocpd() || get_caching_perfetto())
         {
             ROCPROFSYS_PRINT(
                 "Generating rocpd with collected data. This may take a while..\n");
 
+            auto _cache_files = get_cache_files();
+            
             std::vector<std::thread> rocpd_threads;
             ROCPROFSYS_SCOPED_SAMPLING_ON_CHILD_THREADS(false);
 
-            rocpd_threads.emplace_back([this]() {
-                auto                  pid  = getpid();
-                auto                  ppid = get_root_process_id();
-                rocpd_post_processing _post_processing(
-                    m_metadata, get_agent_manager_instance(), pid, ppid);
-                storage_parser _parser(
-                    get_buffered_storage_filename(get_root_process_id(), getpid()));
-                _post_processing.register_parser_callback(_parser);
-                _post_processing.post_process_metadata();
-                _parser.consume_storage();
-            });
-
-            for(const auto& [pid, files] : _cache_files)
+            if(get_caching_perfetto())
             {
-                if(!files.buff_storage.empty() && !files.metadata.empty())
+                for(const auto& [pid, files] : _cache_files)
                 {
-                    rocpd_threads.emplace_back([pid = pid, files = files]() {
+                    if(!files.buff_storage.empty() && !files.metadata.empty())
+                    {
                         ROCPROFSYS_DEBUG(
-                            "Creating database for [%d] from buffered storage "
+                            "Creating Perfetto trace for [%d] from buffered storage "
                             "file: %s and from metadata file: %s\n",
                             pid, files.buff_storage.c_str(), files.metadata.c_str());
 
@@ -197,22 +189,82 @@ cache_manager::post_process_bulk()
                             return;
                         }
 
-                        agent_manager         _agent_manager{ _agents };
-                        auto                  ppid = get_root_process_id();
-                        rocpd_post_processing _post_processing(_metadata, _agent_manager,
-                                                               pid, ppid);
-                        storage_parser        _parser(files.buff_storage);
-                        _post_processing.register_parser_callback(_parser);
-                        _post_processing.post_process_metadata();
+                        agent_manager _agent_manager{ _agents };
+
+                        perfetto_post_processing _perfetto_post_processing(
+                            _metadata, pid, _agent_manager);
+
+                        storage_parser _parser(files.buff_storage);
+
+                        _perfetto_post_processing.register_parser_callback(_parser);
+                        _perfetto_post_processing.start_session();
+
                         _parser.consume_storage();
-                    });
+
+                        bool _perfetto_error = false;
+                        _perfetto_post_processing.post_process(_perfetto_error);
+                    }
+                }
+            }
+
+            if(get_use_rocpd())
+            {
+                rocpd_threads.emplace_back([this]() {
+                    auto                  pid  = getpid();
+                    auto                  ppid = get_root_process_id();
+                    rocpd_post_processing _post_processing(
+                        m_metadata, get_agent_manager_instance(), pid, ppid);
+                    storage_parser _parser(
+                        get_buffered_storage_filename(get_root_process_id(), getpid()));
+                    _post_processing.register_parser_callback(_parser);
+                    _post_processing.post_process_metadata();
+                    _parser.consume_storage();
+                });
+
+                for(const auto& [pid, files] : _cache_files)
+                {
+                    if(!files.buff_storage.empty() && !files.metadata.empty())
+                    {
+                        rocpd_threads.emplace_back([pid = pid, files = files]() {
+                            ROCPROFSYS_DEBUG(
+                                "Creating database for [%d] from buffered storage "
+                                "file: %s and from metadata file: %s\n",
+                                pid, files.buff_storage.c_str(), files.metadata.c_str());
+
+                            std::vector<std::shared_ptr<agent>> _agents;
+                            metadata_registry                   _metadata;
+
+                            auto res = _metadata.load_from_file(files.metadata, _agents);
+                            if(!res)
+                            {
+                                ROCPROFSYS_WARNING(
+                                    0, "Load from file for metadata failed: %s\n",
+                                    files.metadata.c_str());
+                                return;
+                            }
+
+                            agent_manager         _agent_manager{ _agents };
+                            auto                  ppid = get_root_process_id();
+                            rocpd_post_processing _post_processing(
+                                _metadata, _agent_manager, pid, ppid);
+                            storage_parser _parser(files.buff_storage);
+                            _post_processing.register_parser_callback(_parser);
+                            _post_processing.post_process_metadata();
+                            _parser.consume_storage();
+                            std::remove(files.metadata.c_str());  // Remove metadata file
+                        });
+                    }
                 }
             }
 
             for(auto& thread : rocpd_threads)
             {
                 thread.join();
             }
+            if(get_caching_perfetto())
+            {
+                clear_cache_files();
+            }
         }
 
         ROCPROFSYS_PRINT("Removing cached temporary files...\n");