diff --git a/src/snmalloc/aal/aal.h b/src/snmalloc/aal/aal.h index 3bbf63404..6f9ab81fe 100644 --- a/src/snmalloc/aal/aal.h +++ b/src/snmalloc/aal/aal.h @@ -9,21 +9,10 @@ #include "../ds_core/ds_core.h" #include "aal_concept.h" #include "aal_consts.h" - -#if __has_include() -# include -# ifdef CLOCK_MONOTONIC -# define SNMALLOC_TICK_USE_CLOCK_GETTIME -# endif -#endif #include "snmalloc/stl/utility.h" #include -#ifndef SNMALLOC_TICK_USE_CLOCK_GETTIME -# include -#endif - #if ( \ defined(__i386__) || defined(_M_IX86) || defined(_X86_) || \ defined(__amd64__) || defined(__x86_64__) || defined(_M_X64) || \ @@ -169,37 +158,17 @@ namespace snmalloc /** * Return an architecture-specific cycle counter. * - * If the compiler provides a portable prefetch builtin, use it directly, - * otherwise delegate to the architecture-specific layer. This allows new - * architectures to avoid needing to implement a custom `tick` method - * if they are used only with a compiler that provides the builtin. + * If the architecture reports that CPU cycle counters are unavailable, + * use any architecture-specific implementation that exists, otherwise + * fall back to zero. When counters are available, prefer a compiler + * builtin and then the architecture-specific implementation. */ static inline uint64_t tick() noexcept { if constexpr ( (Arch::aal_features & NoCpuCycleCounters) == NoCpuCycleCounters) { -#ifdef SNMALLOC_TICK_USE_CLOCK_GETTIME - // the buf is populated by clock_gettime - SNMALLOC_UNINITIALISED timespec buf; - // we can skip the error checking here: - // * EFAULT: for out-of-bound pointers (buf is always valid stack - // memory) - // * EINVAL: for invalid clock_id (we only use CLOCK_MONOTONIC enforced - // by POSIX.1) - // Notice that clock_gettime is a usually a vDSO call, so the overhead - // is minimal. - ::clock_gettime(CLOCK_MONOTONIC, &buf); - return static_cast(buf.tv_sec) * 1000'000'000 + - static_cast(buf.tv_nsec); -# undef SNMALLOC_TICK_USE_CLOCK_GETTIME -#else - auto tick = std::chrono::high_resolution_clock::now(); - return static_cast( - std::chrono::duration_cast( - tick.time_since_epoch()) - .count()); -#endif + return 0; } else { diff --git a/src/snmalloc/backend_helpers/backend_helpers.h b/src/snmalloc/backend_helpers/backend_helpers.h index 24e02b053..ee339337b 100644 --- a/src/snmalloc/backend_helpers/backend_helpers.h +++ b/src/snmalloc/backend_helpers/backend_helpers.h @@ -1,3 +1,5 @@ +#pragma once + #include "../mem/mem.h" #include "authmap.h" #include "buddy.h" diff --git a/src/snmalloc/ds_aal/ds_aal.h b/src/snmalloc/ds_aal/ds_aal.h index 21eeb8dd6..e0b4ac202 100644 --- a/src/snmalloc/ds_aal/ds_aal.h +++ b/src/snmalloc/ds_aal/ds_aal.h @@ -7,4 +7,5 @@ #include "../aal/aal.h" #include "flaglock.h" #include "prevent_fork.h" +#include "seqset.h" #include "singleton.h" \ No newline at end of file diff --git a/src/snmalloc/ds_core/seqset.h b/src/snmalloc/ds_aal/seqset.h similarity index 100% rename from src/snmalloc/ds_core/seqset.h rename to src/snmalloc/ds_aal/seqset.h diff --git a/src/snmalloc/ds_core/cheri.h b/src/snmalloc/ds_core/cheri.h index db809765c..f10acb4e1 100644 --- a/src/snmalloc/ds_core/cheri.h +++ b/src/snmalloc/ds_core/cheri.h @@ -1,3 +1,5 @@ +#pragma once + #include "mitigations.h" namespace snmalloc diff --git a/src/snmalloc/ds_core/ds_core.h b/src/snmalloc/ds_core/ds_core.h index 38e99dce2..cc395127b 100644 --- a/src/snmalloc/ds_core/ds_core.h +++ b/src/snmalloc/ds_core/ds_core.h @@ -15,5 +15,4 @@ #include "mitigations.h" #include "ptrwrap.h" #include "redblacktree.h" -#include "seqset.h" #include "tid.h" \ No newline at end of file diff --git a/src/snmalloc/global/global.h b/src/snmalloc/global/global.h index ffa149567..17699ca5a 100644 --- a/src/snmalloc/global/global.h +++ b/src/snmalloc/global/global.h @@ -1,3 +1,5 @@ +#pragma once + #include "bounds_checks.h" #include "globalalloc.h" #include "libc.h" diff --git a/src/snmalloc/mem/mem.h b/src/snmalloc/mem/mem.h index fc5e59965..3e9212635 100644 --- a/src/snmalloc/mem/mem.h +++ b/src/snmalloc/mem/mem.h @@ -1,3 +1,5 @@ +#pragma once + #include "backend_concept.h" #include "backend_wrappers.h" #include "check_init.h" diff --git a/src/snmalloc/pal/pal_posix.h b/src/snmalloc/pal/pal_posix.h index 8feab7ac7..3ab85bf6a 100644 --- a/src/snmalloc/pal/pal_posix.h +++ b/src/snmalloc/pal/pal_posix.h @@ -17,6 +17,7 @@ #include #include #include +#include #include #if __has_include() @@ -404,6 +405,28 @@ namespace snmalloc (static_cast(ts.tv_nsec) / 1000000); } + static uint64_t tick() + { + if constexpr ( + (Aal::aal_features & NoCpuCycleCounters) != NoCpuCycleCounters) + { + return Aal::tick(); + } + else + { + auto hold = KeepErrno(); + + struct timespec ts; + if (clock_gettime(CLOCK_MONOTONIC, &ts) == -1) + { + error("Failed to get monotonic time"); + } + + return (static_cast(ts.tv_sec) * 1'000'000'000) + + static_cast(ts.tv_nsec); + } + } + static uint64_t dev_urandom() { union diff --git a/src/snmalloc/pal/pal_windows.h b/src/snmalloc/pal/pal_windows.h index 15dfaf28e..a44079dea 100644 --- a/src/snmalloc/pal/pal_windows.h +++ b/src/snmalloc/pal/pal_windows.h @@ -7,8 +7,8 @@ #ifdef _WIN32 # ifndef _MSC_VER # include -# include # endif +# include # ifndef WIN32_LEAN_AND_MEAN # define WIN32_LEAN_AND_MEAN # endif @@ -309,12 +309,11 @@ namespace snmalloc return result; } - static uint64_t internal_time_in_ms() + static uint64_t performance_counter_frequency() { - // Performance counter is a high-precision monotonic clock. static stl::Atomic freq_cache = 0; - constexpr uint64_t ms_per_second = 1000; SNMALLOC_UNINITIALISED LARGE_INTEGER buf; + auto freq = freq_cache.load(stl::memory_order_relaxed); if (SNMALLOC_UNLIKELY(freq == 0)) { @@ -324,10 +323,36 @@ namespace snmalloc freq = static_cast(buf.QuadPart); freq_cache.store(freq, stl::memory_order_relaxed); } + + return freq; + } + + static uint64_t internal_time_in_ms() + { + constexpr uint64_t ms_per_second = 1000; + SNMALLOC_UNINITIALISED LARGE_INTEGER buf; + auto freq = performance_counter_frequency(); ::QueryPerformanceCounter(&buf); return (static_cast(buf.QuadPart) * ms_per_second) / freq; } + static uint64_t tick() + { + if constexpr ( + (Aal::aal_features & NoCpuCycleCounters) != NoCpuCycleCounters) + { + return Aal::tick(); + } + else + { + constexpr uint64_t ns_per_second = 1'000'000'000; + SNMALLOC_UNINITIALISED LARGE_INTEGER buf; + auto freq = performance_counter_frequency(); + ::QueryPerformanceCounter(&buf); + return (static_cast(buf.QuadPart) * ns_per_second) / freq; + } + } + # ifdef PLATFORM_HAS_WAITONADDRESS using WaitingWord = char; diff --git a/src/snmalloc/snmalloc_front.h b/src/snmalloc/snmalloc_front.h index 4c5aa60ca..f7cedbb4b 100644 --- a/src/snmalloc/snmalloc_front.h +++ b/src/snmalloc/snmalloc_front.h @@ -1 +1,3 @@ +#pragma once + #include "global/global.h" diff --git a/src/snmalloc/stl/cxx/utility.h b/src/snmalloc/stl/cxx/utility.h index 93ee5df18..b51d9e270 100644 --- a/src/snmalloc/stl/cxx/utility.h +++ b/src/snmalloc/stl/cxx/utility.h @@ -1,3 +1,5 @@ +#pragma once + #include namespace snmalloc diff --git a/src/test/perf/contention/contention.cc b/src/test/perf/contention/contention.cc index 06a433d43..6bed00545 100644 --- a/src/test/perf/contention/contention.cc +++ b/src/test/perf/contention/contention.cc @@ -5,6 +5,7 @@ #include #include +#include #include #include #include @@ -30,7 +31,7 @@ class ParallelTest auto prev = ready.fetch_add(1); if (prev + 1 == cores) { - start = Aal::tick(); + start = DefaultPal::tick(); flag = true; } while (!flag) @@ -41,7 +42,7 @@ class ParallelTest prev = complete.fetch_add(1); if (prev + 1 == cores) { - end = Aal::tick(); + end = DefaultPal::tick(); } } diff --git a/src/test/perf/startup/startup.cc b/src/test/perf/startup/startup.cc index bcfd455f1..dfc6cb0c2 100644 --- a/src/test/perf/startup/startup.cc +++ b/src/test/perf/startup/startup.cc @@ -5,6 +5,7 @@ #include #include +#include #include #include #include @@ -30,7 +31,7 @@ class ParallelTest auto prev = ready.fetch_add(1); if (prev + 1 == cores) { - start = Aal::tick(); + start = DefaultPal::tick(); flag = true; } while (!flag) @@ -41,7 +42,7 @@ class ParallelTest prev = complete.fetch_add(1); if (prev + 1 == cores) { - end = Aal::tick(); + end = DefaultPal::tick(); } } @@ -76,9 +77,9 @@ int main() ParallelTest test( [](size_t id) { - auto start = Aal::tick(); + auto start = DefaultPal::tick(); snmalloc::dealloc(snmalloc::alloc(1)); - auto end = Aal::tick(); + auto end = DefaultPal::tick(); counters[id] = end - start; }, nthreads); diff --git a/src/test/setup.h b/src/test/setup.h index deb903de6..e18151c5f 100644 --- a/src/test/setup.h +++ b/src/test/setup.h @@ -1,3 +1,5 @@ +#pragma once + #if defined(SNMALLOC_CI_BUILD) # include # if defined(WIN32)