diff --git a/tcmalloc/background.cc b/tcmalloc/background.cc index 777c4f39b..67c00bc64 100644 --- a/tcmalloc/background.cc +++ b/tcmalloc/background.cc @@ -99,43 +99,54 @@ void ShuffleCpuCaches() { GOOGLE_MALLOC_SECTION_END // Release memory to the system at a constant rate. -void MallocExtension_Internal_ProcessBackgroundActions() { - tcmalloc::MallocExtension::MarkThreadIdle(); +static absl::Time prev_time; +static absl::Time last_shuffle; +void MallocExtension_Internal_ProcessBackgroundActionsInit() { // Initialize storage for ReleasePerCpuMemoryToOS(). CPU_ZERO(&tcmalloc::tcmalloc_internal::prev_allowed_cpus); - absl::Time prev_time = absl::Now(); + prev_time = absl::Now(); + last_shuffle = absl::InfinitePast(); +} + +void MallocExtension_Internal_ProcessBackgroundActions() { constexpr absl::Duration kSleepTime = absl::Seconds(1); + tcmalloc::MallocExtension::MarkThreadIdle(); + MallocExtension_Internal_ProcessBackgroundActionsInit(); + + while (true) { + MallocExtension_Internal_ProcessBackgroundActionsTick(); + absl::SleepFor(kSleepTime); + } +} + +void MallocExtension_Internal_ProcessBackgroundActionsTick() { // Shuffle per-cpu caches once per kCpuCacheShufflePeriod secs. constexpr absl::Duration kCpuCacheShufflePeriod = absl::Seconds(5); - absl::Time last_shuffle = absl::InfinitePast(); - while (true) { - absl::Time now = absl::Now(); - const ssize_t bytes_to_release = - static_cast(tcmalloc::tcmalloc_internal::Parameters:: - background_release_rate()) * - absl::ToDoubleSeconds(now - prev_time); - if (bytes_to_release > 0) { // may be negative if time goes backwards - tcmalloc::MallocExtension::ReleaseMemoryToSystem(bytes_to_release); - } + absl::Time now = absl::Now(); + const ssize_t bytes_to_release = + static_cast(tcmalloc::tcmalloc_internal::Parameters:: + background_release_rate()) * + absl::ToDoubleSeconds(now - prev_time); + if (bytes_to_release > 0) { // may be negative if time goes backwards + tcmalloc::MallocExtension::ReleaseMemoryToSystem(bytes_to_release); + } - tcmalloc::tcmalloc_internal::ReleasePerCpuMemoryToOS(); + tcmalloc::tcmalloc_internal::ReleasePerCpuMemoryToOS(); - const bool shuffle_per_cpu_caches = - tcmalloc::tcmalloc_internal::Parameters::shuffle_per_cpu_caches(); + const bool shuffle_per_cpu_caches = + tcmalloc::tcmalloc_internal::Parameters::shuffle_per_cpu_caches(); - if (shuffle_per_cpu_caches) { - if (now - last_shuffle >= kCpuCacheShufflePeriod) { - tcmalloc::tcmalloc_internal::ShuffleCpuCaches(); - last_shuffle = now; - } + if (shuffle_per_cpu_caches) { + if (now - last_shuffle >= kCpuCacheShufflePeriod) { + tcmalloc::tcmalloc_internal::ShuffleCpuCaches(); + last_shuffle = now; } - - tcmalloc::tcmalloc_internal::Static().sharded_transfer_cache().Plunder(); - prev_time = now; - absl::SleepFor(kSleepTime); } + + tcmalloc::tcmalloc_internal::Static().sharded_transfer_cache().Plunder(); + prev_time = now; } diff --git a/tcmalloc/internal_malloc_extension.h b/tcmalloc/internal_malloc_extension.h index 9d12a9b1b..468ca4908 100644 --- a/tcmalloc/internal_malloc_extension.h +++ b/tcmalloc/internal_malloc_extension.h @@ -101,6 +101,8 @@ ABSL_ATTRIBUTE_WEAK void MallocExtension_Internal_SetProfileSamplingRate( int64_t); ABSL_ATTRIBUTE_WEAK void MallocExtension_Internal_ProcessBackgroundActions(); +ABSL_ATTRIBUTE_WEAK void MallocExtension_Internal_ProcessBackgroundActionsInit(); +ABSL_ATTRIBUTE_WEAK void MallocExtension_Internal_ProcessBackgroundActionsTick(); ABSL_ATTRIBUTE_WEAK tcmalloc::MallocExtension::BytesPerSecond MallocExtension_Internal_GetBackgroundReleaseRate(); diff --git a/tcmalloc/malloc_extension.cc b/tcmalloc/malloc_extension.cc index e3b5294a1..156b403af 100644 --- a/tcmalloc/malloc_extension.cc +++ b/tcmalloc/malloc_extension.cc @@ -425,6 +425,22 @@ void MallocExtension::ProcessBackgroundActions() { #endif } +void MallocExtension::ProcessBackgroundActionsInit() { +#if ABSL_INTERNAL_HAVE_WEAK_MALLOCEXTENSION_STUBS + if (NeedsProcessBackgroundActions()) { + MallocExtension_Internal_ProcessBackgroundActionsInit(); + } +#endif +} + +void MallocExtension::ProcessBackgroundActionsTick() { +#if ABSL_INTERNAL_HAVE_WEAK_MALLOCEXTENSION_STUBS + if (NeedsProcessBackgroundActions()) { + MallocExtension_Internal_ProcessBackgroundActionsTick(); + } +#endif +} + bool MallocExtension::NeedsProcessBackgroundActions() { #if ABSL_INTERNAL_HAVE_WEAK_MALLOCEXTENSION_STUBS return &MallocExtension_Internal_ProcessBackgroundActions != nullptr; diff --git a/tcmalloc/malloc_extension.h b/tcmalloc/malloc_extension.h index af3d32444..69dd8abb4 100644 --- a/tcmalloc/malloc_extension.h +++ b/tcmalloc/malloc_extension.h @@ -433,6 +433,19 @@ class MallocExtension final { // null if the implementation does not support profiling. static AllocationProfilingToken StartAllocationProfiling(); + // Initializes the state needed for this thread to call + // ProcessBackgroundActionsTick. This is the counterpart to + // ProcessBackgroundActions that allows co-operative task keeping on a thread + // that is performing other task keeping. + static void ProcessBackgroundActionsInit(); + + // This is to be called periodically to run housekeeping actions for the + // allocator off of the main allocation paths of new/delete. Note that unlike + // ProcessBackgroundActions, the thread is not automatically marked as + // idle/busy and the caller is responsible for doing that correctly. + // See ProcessBackgroundActions for details of the actions performed. + static void ProcessBackgroundActionsTick(); + // Runs housekeeping actions for the allocator off of the main allocation path // of new/delete. As of 2020, this includes: // * Inspecting the current CPU mask and releasing memory from inaccessible