diff --git a/CHANGELOG.md b/CHANGELOG.md index 6e63e3e95..d8e72cb52 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## Unreleased + +**Features**: + +- Add new offline caching options to persist envelopes locally: `sentry_options_set_cache_keep`, `sentry_options_set_cache_max_size`, and `sentry_options_set_cache_max_age`. ([#1490](https://github.com/getsentry/sentry-native/pull/1490), [#1493](https://github.com/getsentry/sentry-native/pull/1493)) + ## 0.12.5 **Features**: diff --git a/examples/example.c b/examples/example.c index 311a1822b..75bcc1612 100644 --- a/examples/example.c +++ b/examples/example.c @@ -503,6 +503,11 @@ main(int argc, char **argv) if (has_arg(argc, argv, "log-attributes")) { sentry_options_set_logs_with_attributes(options, true); } + if (has_arg(argc, argv, "cache-keep")) { + sentry_options_set_cache_keep(options, true); + sentry_options_set_cache_max_size(options, 4 * 1024 * 1024); + sentry_options_set_cache_max_age(options, 5 * 24 * 60 * 60); + } if (0 != sentry_init(options)) { return EXIT_FAILURE; diff --git a/include/sentry.h b/include/sentry.h index 242bc4e60..4052758b3 100644 --- a/include/sentry.h +++ b/include/sentry.h @@ -165,6 +165,7 @@ extern "C" { #include #include #include +#include /* context type dependencies */ #ifdef _WIN32 @@ -1375,6 +1376,37 @@ SENTRY_API void sentry_options_set_symbolize_stacktraces( SENTRY_API int sentry_options_get_symbolize_stacktraces( const sentry_options_t *opts); +/** + * Enables or disables storing envelopes in a persistent cache. + * + * When enabled, envelopes are written to a `cache/` subdirectory within the + * database directory and retained regardless of send success or failure. + * The cache is cleared on startup based on the cache_max_size and cache_max_age + * options. + */ +SENTRY_API void sentry_options_set_cache_keep( + sentry_options_t *opts, int enabled); + +/** + * Sets the maximum size (in bytes) for the cache directory. + * On startup, cached entries are removed from oldest to newest until the + * directory size is within the max size limit. + */ +SENTRY_API void sentry_options_set_cache_max_size( + sentry_options_t *opts, size_t bytes); + +/** + * Sets the maximum age (in seconds) for cache entries in the cache directory. + * On startup, cached entries exceeding the max age limit are removed. + */ +SENTRY_API void sentry_options_set_cache_max_age( + sentry_options_t *opts, time_t seconds); + +/** + * Gets the caching mode for crash reports. + */ +SENTRY_API int sentry_options_get_cache_keep(const sentry_options_t *opts); + /** * Adds a new attachment to be sent along. * diff --git a/src/backends/sentry_backend_crashpad.cpp b/src/backends/sentry_backend_crashpad.cpp index eaf44c52d..8485cc17c 100644 --- a/src/backends/sentry_backend_crashpad.cpp +++ b/src/backends/sentry_backend_crashpad.cpp @@ -18,6 +18,7 @@ extern "C" { #include "sentry_screenshot.h" #include "sentry_sync.h" #include "sentry_transport.h" +#include "sentry_value.h" #ifdef SENTRY_PLATFORM_LINUX # include "sentry_unix_pageallocator.h" #endif @@ -432,6 +433,158 @@ sentry__crashpad_handler(int signum, siginfo_t *info, ucontext_t *user_context) } #endif +static sentry_value_t +read_msgpack_file(const sentry_path_t *path) +{ + size_t size; + char *data = sentry__path_read_to_buffer(path, &size); + if (!data) { + return sentry_value_new_null(); + } + sentry_value_t value = sentry__value_from_msgpack(data, size); + sentry_free(data); + return value; +} + +static sentry_path_t * +report_minidump_path(const crashpad::CrashReportDatabase::Report &report) +{ + return +#ifdef SENTRY_PLATFORM_WINDOWS + sentry__path_from_wstr(report.file_path.value().c_str()); +#else + sentry__path_from_str(report.file_path.value().c_str()); +#endif +} + +static sentry_path_t * +report_attachments_dir(const crashpad::CrashReportDatabase::Report &report, + const sentry_options_t *options) +{ + sentry_path_t *attachments_root + = sentry__path_join_str(options->database_path, "attachments"); + if (!attachments_root) { + return nullptr; + } + + sentry_path_t *attachments_dir = sentry__path_join_str( + attachments_root, report.uuid.ToString().c_str()); + + sentry__path_free(attachments_root); + return attachments_dir; +} + +static sentry_envelope_t * +report_to_envelope(const crashpad::CrashReportDatabase::Report &report, + const sentry_options_t *options) +{ + sentry_path_t *minidump_path = report_minidump_path(report); + sentry_path_t *attachments_dir = report_attachments_dir(report, options); + + if (!minidump_path || !attachments_dir) { + sentry__path_free(minidump_path); + sentry__path_free(attachments_dir); + return nullptr; + } + + sentry_value_t event = sentry_value_new_null(); + sentry_value_t breadcrumbs1 = sentry_value_new_null(); + sentry_value_t breadcrumbs2 = sentry_value_new_null(); + sentry_attachment_t *attachments = nullptr; + + sentry_pathiter_t *iter = sentry__path_iter_directory(attachments_dir); + if (iter) { + const sentry_path_t *path; + while ((path = sentry__pathiter_next(iter)) != nullptr) { + const char *filename = sentry__path_filename(path); + if (strcmp(filename, "__sentry-event") == 0) { + event = read_msgpack_file(path); + } else if (strcmp(filename, "__sentry-breadcrumb1") == 0) { + breadcrumbs1 = read_msgpack_file(path); + } else if (strcmp(filename, "__sentry-breadcrumb2") == 0) { + breadcrumbs2 = read_msgpack_file(path); + } else { + sentry__attachments_add_path(&attachments, + sentry__path_clone(path), ATTACHMENT, nullptr); + } + } + sentry__pathiter_free(iter); + } + sentry__path_free(attachments_dir); + + sentry_envelope_t *envelope = nullptr; + if (!sentry_value_is_null(event)) { + envelope = sentry__envelope_new(); + } + if (envelope) { + sentry_value_set_by_key(event, "breadcrumbs", + sentry__value_merge_breadcrumbs( + breadcrumbs1, breadcrumbs2, options->max_breadcrumbs)); + sentry__attachments_add_path( + &attachments, minidump_path, MINIDUMP, nullptr); + + sentry__envelope_add_event(envelope, event); + sentry__envelope_add_attachments(envelope, attachments); + } else { + sentry__path_free(minidump_path); + sentry_value_decref(event); + } + + sentry_value_decref(breadcrumbs1); + sentry_value_decref(breadcrumbs2); + sentry__attachments_free(attachments); + + return envelope; +} + +static void +process_completed_reports( + crashpad_state_t *state, const sentry_options_t *options) +{ + if (!state || !state->db || !options || !options->cache_keep) { + return; + } + + std::vector reports; + if (state->db->GetCompletedReports(&reports) + != crashpad::CrashReportDatabase::kNoError + || reports.empty()) { + return; + } + + SENTRY_DEBUGF("caching %zu completed reports", reports.size()); + + sentry_path_t *cache_dir + = sentry__path_join_str(options->database_path, "cache"); + if (!cache_dir || sentry__path_create_dir_all(cache_dir) != 0) { + SENTRY_WARN("failed to create cache dir"); + sentry__path_free(cache_dir); + return; + } + + for (const auto &report : reports) { + std::string filename = report.uuid.ToString() + ".envelope"; + sentry_envelope_t *envelope = report_to_envelope(report, options); + if (!envelope) { + SENTRY_WARNF("failed to convert \"%s\"", filename.c_str()); + continue; + } + sentry_path_t *out_path + = sentry__path_join_str(cache_dir, filename.c_str()); + if (!out_path + || sentry_envelope_write_to_path(envelope, out_path) != 0) { + SENTRY_WARNF("failed to cache \"%s\"", filename.c_str()); + } else if (state->db->DeleteReport(report.uuid) + != crashpad::CrashReportDatabase::kNoError) { + SENTRY_WARNF("failed to delete \"%s\"", filename.c_str()); + } + sentry__path_free(out_path); + sentry_envelope_free(envelope); + } + + sentry__path_free(cache_dir); +} + static int crashpad_backend_startup( sentry_backend_t *backend, const sentry_options_t *options) @@ -545,6 +698,7 @@ crashpad_backend_startup( // Initialize database first, flushing the consent later on as part of // `sentry_init` will persist the upload flag. data->db = crashpad::CrashReportDatabase::Initialize(database).release(); + process_completed_reports(data, options); data->client = new crashpad::CrashpadClient; char *minidump_url = sentry__dsn_get_minidump_url(options->dsn, options->user_agent); @@ -740,6 +894,49 @@ crashpad_backend_last_crash(sentry_backend_t *backend) return crash_time; } +// seconds-based alternative to crashpad::AgePruneCondition (days) +class MaxAgePruneCondition final : public crashpad::PruneCondition { +public: + explicit MaxAgePruneCondition(time_t max_age) + : max_age_(max_age) + , oldest_report_time_(time(nullptr) - max_age) + { + } + + bool + ShouldPruneReport( + const crashpad::CrashReportDatabase::Report &report) override + { + return max_age_ > 0 && report.creation_time < oldest_report_time_; + } + +private: + const time_t max_age_; + const time_t oldest_report_time_; +}; + +// bytes-based alternative to crashpad::DatabaseSizePruneCondition (kb) +class MaxSizePruneCondition final : public crashpad::PruneCondition { +public: + explicit MaxSizePruneCondition(size_t max_size) + : max_size_(max_size) + , measured_size_(0) + { + } + + bool + ShouldPruneReport( + const crashpad::CrashReportDatabase::Report &report) override + { + measured_size_ += static_cast(report.total_size); + return max_size_ > 0 && measured_size_ > max_size_; + } + +private: + const size_t max_size_; + size_t measured_size_; +}; + static void crashpad_backend_prune_database(sentry_backend_t *backend) { @@ -749,11 +946,17 @@ crashpad_backend_prune_database(sentry_backend_t *backend) // complete database to a maximum of 8M. That might still be a lot for // an embedded use-case, but minidumps on desktop can sometimes be quite // large. - data->db->CleanDatabase(60 * 60 * 24 * 2); - crashpad::BinaryPruneCondition condition(crashpad::BinaryPruneCondition::OR, - new crashpad::DatabaseSizePruneCondition(1024 * 8), - new crashpad::AgePruneCondition(2)); - crashpad::PruneCrashReportDatabase(data->db, &condition); + SENTRY_WITH_OPTIONS (options) { + if (options->cache_max_age > 0) { + data->db->CleanDatabase(options->cache_max_age); + } + + crashpad::BinaryPruneCondition condition( + crashpad::BinaryPruneCondition::OR, + new MaxSizePruneCondition(options->cache_max_size), + new MaxAgePruneCondition(options->cache_max_age)); + crashpad::PruneCrashReportDatabase(data->db, &condition); + } } #if defined(SENTRY_PLATFORM_WINDOWS) || defined(SENTRY_PLATFORM_LINUX) diff --git a/src/path/sentry_path_unix.c b/src/path/sentry_path_unix.c index 7b7f63fa3..e831a2b22 100644 --- a/src/path/sentry_path_unix.c +++ b/src/path/sentry_path_unix.c @@ -324,6 +324,14 @@ sentry__path_remove(const sentry_path_t *path) return 1; } +int +sentry__path_rename(const sentry_path_t *src, const sentry_path_t *dst) +{ + int status; + EINTR_RETRY(rename(src->path, dst->path), &status); + return status == 0 ? 0 : 1; +} + int sentry__path_create_dir_all(const sentry_path_t *path) { diff --git a/src/path/sentry_path_windows.c b/src/path/sentry_path_windows.c index 5b76497f4..bd8269a6e 100644 --- a/src/path/sentry_path_windows.c +++ b/src/path/sentry_path_windows.c @@ -511,6 +511,18 @@ sentry__path_remove(const sentry_path_t *path) return removal_success ? 0 : !is_last_error_path_not_found(); } +int +sentry__path_rename(const sentry_path_t *src, const sentry_path_t *dst) +{ + wchar_t *src_w = src->path_w; + wchar_t *dst_w = dst->path_w; + if (!src_w || !dst_w) { + return 1; + } + // MOVEFILE_REPLACE_EXISTING allows overwriting the destination if it exists + return MoveFileExW(src_w, dst_w, MOVEFILE_REPLACE_EXISTING) ? 0 : 1; +} + int sentry__path_create_dir_all(const sentry_path_t *path) { diff --git a/src/sentry_core.c b/src/sentry_core.c index d4a2cb594..9b11af143 100644 --- a/src/sentry_core.c +++ b/src/sentry_core.c @@ -289,6 +289,10 @@ sentry_init(sentry_options_t *options) backend->prune_database_func(backend); } + if (options->cache_keep) { + sentry__cleanup_cache(options); + } + if (options->auto_session_tracking) { sentry_start_session(); } diff --git a/src/sentry_database.c b/src/sentry_database.c index 32cf8ba57..8dc6d4b23 100644 --- a/src/sentry_database.c +++ b/src/sentry_database.c @@ -6,6 +6,7 @@ #include "sentry_session.h" #include "sentry_uuid.h" #include +#include #include sentry_run_t * @@ -237,6 +238,15 @@ sentry__process_old_runs(const sentry_options_t *options, uint64_t last_crash) if (strcmp(options->run->run_path->path, run_dir->path) == 0) { continue; } + + sentry_path_t *cache_dir = NULL; + if (options->cache_keep) { + cache_dir = sentry__path_join_str(options->database_path, "cache"); + if (cache_dir) { + sentry__path_create_dir_all(cache_dir); + } + } + sentry_pathiter_t *run_iter = sentry__path_iter_directory(run_dir); const sentry_path_t *file; while (run_iter && (file = sentry__pathiter_next(run_iter)) != NULL) { @@ -281,12 +291,25 @@ sentry__process_old_runs(const sentry_options_t *options, uint64_t last_crash) } else if (sentry__path_ends_with(file, ".envelope")) { sentry_envelope_t *envelope = sentry__envelope_from_path(file); sentry__capture_envelope(options->transport, envelope); + + if (cache_dir) { + sentry_path_t *cached_file = sentry__path_join_str( + cache_dir, sentry__path_filename(file)); + if (!cached_file + || sentry__path_rename(file, cached_file) != 0) { + SENTRY_WARNF("failed to cache envelope \"%s\"", + sentry__path_filename(file)); + } + sentry__path_free(cached_file); + continue; + } } sentry__path_remove(file); } sentry__pathiter_free(run_iter); + sentry__path_free(cache_dir); sentry__path_remove_all(run_dir); sentry__filelock_free(lock); } @@ -295,6 +318,126 @@ sentry__process_old_runs(const sentry_options_t *options, uint64_t last_crash) sentry__capture_envelope(options->transport, session_envelope); } +// Cache Pruning below is based on prune_crash_reports.cc from Crashpad + +/** + * A cache entry with its metadata for sorting and pruning decisions. + */ +typedef struct { + sentry_path_t *path; + time_t mtime; + size_t size; +} cache_entry_t; + +/** + * Comparison function to sort cache entries by mtime, newest first. + */ +static int +compare_cache_entries_newest_first(const void *a, const void *b) +{ + const cache_entry_t *entry_a = (const cache_entry_t *)a; + const cache_entry_t *entry_b = (const cache_entry_t *)b; + // Newest first: if b is newer, return positive (b comes before a) + if (entry_b->mtime > entry_a->mtime) { + return 1; + } + if (entry_b->mtime < entry_a->mtime) { + return -1; + } + return 0; +} + +void +sentry__cleanup_cache(const sentry_options_t *options) +{ + if (!options->database_path) { + return; + } + + sentry_path_t *cache_dir + = sentry__path_join_str(options->database_path, "cache"); + if (!cache_dir || !sentry__path_is_dir(cache_dir)) { + sentry__path_free(cache_dir); + return; + } + + // First pass: collect all cache entries with their metadata + size_t entries_capacity = 16; + size_t entries_count = 0; + cache_entry_t *entries + = sentry_malloc(sizeof(cache_entry_t) * entries_capacity); + if (!entries) { + sentry__path_free(cache_dir); + return; + } + + sentry_pathiter_t *iter = sentry__path_iter_directory(cache_dir); + const sentry_path_t *entry; + while (iter && (entry = sentry__pathiter_next(iter)) != NULL) { + if (sentry__path_is_dir(entry)) { + continue; + } + + // Grow array if needed + if (entries_count >= entries_capacity) { + entries_capacity *= 2; + cache_entry_t *new_entries + = sentry_malloc(sizeof(cache_entry_t) * entries_capacity); + if (!new_entries) { + break; + } + memcpy(new_entries, entries, sizeof(cache_entry_t) * entries_count); + sentry_free(entries); + entries = new_entries; + } + + entries[entries_count].path = sentry__path_clone(entry); + if (!entries[entries_count].path) { + break; + } + entries[entries_count].mtime = sentry__path_get_mtime(entry); + entries[entries_count].size = sentry__path_get_size(entry); + entries_count++; + } + sentry__pathiter_free(iter); + + // Sort by mtime, newest first (like crashpad) + // This ensures we keep the newest entries when pruning by size + qsort(entries, entries_count, sizeof(cache_entry_t), + compare_cache_entries_newest_first); + + // Calculate the age threshold + time_t now = time(NULL); + time_t oldest_allowed = now - options->cache_max_age; + + // Prune entries: iterate newest-to-oldest, accumulating size + // Remove if: too old OR accumulated size exceeds limit + size_t accumulated_size = 0; + for (size_t i = 0; i < entries_count; i++) { + bool should_prune = false; + + // Age-based pruning + if (options->cache_max_age > 0 && entries[i].mtime < oldest_allowed) { + should_prune = true; + } else { + // Size-based pruning (accumulate size as we go, like crashpad) + accumulated_size += entries[i].size; + if (options->cache_max_size > 0 + && accumulated_size > options->cache_max_size) { + should_prune = true; + } + } + + if (should_prune) { + sentry__path_remove_all(entries[i].path); + } + sentry__path_free(entries[i].path); + } + + sentry_free(entries); + sentry__path_free(cache_dir); +} + static const char *g_last_crash_filename = "last_crash"; bool diff --git a/src/sentry_database.h b/src/sentry_database.h index 0cc688907..791c30d9f 100644 --- a/src/sentry_database.h +++ b/src/sentry_database.h @@ -78,6 +78,12 @@ bool sentry__run_clear_session(const sentry_run_t *run); void sentry__process_old_runs( const sentry_options_t *options, uint64_t last_crash); +/** + * Cleans up the cache based on options.max_cache_size and + * options.max_cache_age. + */ +void sentry__cleanup_cache(const sentry_options_t *options); + /** * This will write the current ISO8601 formatted timestamp into the * `/last_crash` file. diff --git a/src/sentry_options.c b/src/sentry_options.c index b9b6ea4c6..e86d7852e 100644 --- a/src/sentry_options.c +++ b/src/sentry_options.c @@ -53,6 +53,9 @@ sentry_options_new(void) opts->enable_logging_when_crashed = true; opts->propagate_traceparent = false; opts->crashpad_limit_stack_capture_to_sp = false; + opts->cache_keep = false; + opts->cache_max_age = 2 * 24 * 60 * 60; + opts->cache_max_size = 8 * 1024 * 1024; opts->symbolize_stacktraces = // AIX doesn't have reliable debug IDs for server-side symbolication, // and the diversity of Android makes it infeasible to have access to debug @@ -475,6 +478,30 @@ sentry_options_get_symbolize_stacktraces(const sentry_options_t *opts) return opts->symbolize_stacktraces; } +void +sentry_options_set_cache_keep(sentry_options_t *opts, int enabled) +{ + opts->cache_keep = !!enabled; +} + +void +sentry_options_set_cache_max_size(sentry_options_t *opts, size_t bytes) +{ + opts->cache_max_size = bytes; +} + +void +sentry_options_set_cache_max_age(sentry_options_t *opts, time_t seconds) +{ + opts->cache_max_age = seconds; +} + +int +sentry_options_get_cache_keep(const sentry_options_t *opts) +{ + return opts->cache_keep; +} + void sentry_options_set_system_crash_reporter_enabled( sentry_options_t *opts, int enabled) diff --git a/src/sentry_options.h b/src/sentry_options.h index 280703d40..e91ddfbe7 100644 --- a/src/sentry_options.h +++ b/src/sentry_options.h @@ -45,6 +45,10 @@ struct sentry_options_s { bool enable_logging_when_crashed; bool propagate_traceparent; bool crashpad_limit_stack_capture_to_sp; + bool cache_keep; + + time_t cache_max_age; + size_t cache_max_size; sentry_attachment_t *attachments; sentry_run_t *run; diff --git a/src/sentry_path.h b/src/sentry_path.h index cd5a3a917..484a6f531 100644 --- a/src/sentry_path.h +++ b/src/sentry_path.h @@ -152,6 +152,13 @@ int sentry__path_remove(const sentry_path_t *path); */ int sentry__path_remove_all(const sentry_path_t *path); +/** + * Rename/move the file or directory from `src` to `dst`. + * This will overwrite `dst` if it already exists. + * Returns 0 on success. + */ +int sentry__path_rename(const sentry_path_t *src, const sentry_path_t *dst); + /** * This will create the directory referred to by `path`, and any non-existing * parent directory. diff --git a/tests/test_integration_cache.py b/tests/test_integration_cache.py new file mode 100644 index 000000000..2eca8dfd3 --- /dev/null +++ b/tests/test_integration_cache.py @@ -0,0 +1,143 @@ +import os +import time +import pytest + +from . import run +from .conditions import has_breakpad, has_files + +pytestmark = pytest.mark.skipif(not has_files, reason="tests need local filesystem") + + +@pytest.mark.parametrize("cache_keep", [True, False]) +@pytest.mark.parametrize( + "backend", + [ + "inproc", + pytest.param( + "breakpad", + marks=pytest.mark.skipif( + not has_breakpad, reason="breakpad backend not available" + ), + ), + ], +) +def test_cache_keep(cmake, backend, cache_keep): + tmp_path = cmake( + ["sentry_example"], {"SENTRY_BACKEND": backend, "SENTRY_TRANSPORT": "none"} + ) + cache_dir = tmp_path.joinpath(".sentry-native/cache") + + # capture + run( + tmp_path, + "sentry_example", + ["log", "crash"] + (["cache-keep"] if cache_keep else []), + expect_failure=True, + ) + + assert not cache_dir.exists() or len(list(cache_dir.glob("*.envelope"))) == 0 + + # cache + run( + tmp_path, + "sentry_example", + ["log", "no-setup"] + (["cache-keep"] if cache_keep else []), + ) + + assert cache_dir.exists() or cache_keep is False + if cache_keep: + cache_files = list(cache_dir.glob("*.envelope")) + assert len(cache_files) == 1 + + +@pytest.mark.parametrize( + "backend", + [ + "inproc", + pytest.param( + "breakpad", + marks=pytest.mark.skipif( + not has_breakpad, reason="breakpad backend not available" + ), + ), + ], +) +def test_cache_max_size(cmake, backend): + tmp_path = cmake( + ["sentry_example"], {"SENTRY_BACKEND": backend, "SENTRY_TRANSPORT": "none"} + ) + cache_dir = tmp_path.joinpath(".sentry-native/cache") + + # 5 x 2mb + for i in range(5): + run( + tmp_path, + "sentry_example", + ["log", "cache-keep", "crash"], + expect_failure=True, + ) + + if cache_dir.exists(): + cache_files = list(cache_dir.glob("*.envelope")) + for f in cache_files: + with open(f, "r+b") as file: + file.truncate(2 * 1024 * 1024) + + run( + tmp_path, + "sentry_example", + ["log", "cache-keep", "no-setup"], + ) + + # max 4mb + assert cache_dir.exists() + cache_files = list(cache_dir.glob("*.envelope")) + assert len(cache_files) <= 2 + assert sum(f.stat().st_size for f in cache_files) <= 4 * 1024 * 1024 + + +@pytest.mark.parametrize( + "backend", + [ + "inproc", + pytest.param( + "breakpad", + marks=pytest.mark.skipif( + not has_breakpad, reason="breakpad backend not available" + ), + ), + ], +) +def test_cache_max_age(cmake, backend): + tmp_path = cmake( + ["sentry_example"], {"SENTRY_BACKEND": backend, "SENTRY_TRANSPORT": "none"} + ) + cache_dir = tmp_path.joinpath(".sentry-native/cache") + + for i in range(5): + run( + tmp_path, + "sentry_example", + ["log", "cache-keep", "crash"], + expect_failure=True, + ) + + # 2,4,6,8,10 days old + assert cache_dir.exists() + cache_files = list(cache_dir.glob("*.envelope")) + for i, f in enumerate(cache_files): + mtime = time.time() - ((i + 1) * 2 * 24 * 60 * 60) + os.utime(str(f), (mtime, mtime)) + + # 0 days old + run( + tmp_path, + "sentry_example", + ["log", "cache-keep", "no-setup"], + ) + + # max 5 days + cache_files = list(cache_dir.glob("*.envelope")) + assert len(cache_files) == 3 + for f in cache_files: + assert time.time() - f.stat().st_mtime <= 5 * 24 * 60 * 60 diff --git a/tests/test_integration_crashpad.py b/tests/test_integration_crashpad.py index 815cd9ea6..ad1cc5d20 100644 --- a/tests/test_integration_crashpad.py +++ b/tests/test_integration_crashpad.py @@ -765,3 +765,182 @@ def test_crashpad_external_crash_reporter(cmake, httpserver, run_args): ) def test_crashpad_external_crash_reporter_wer(cmake, httpserver, run_args): test_crashpad_external_crash_reporter(cmake, httpserver, run_args) + + +@pytest.mark.parametrize("cache_keep", [True, False]) +def test_crashpad_cache_keep(cmake, httpserver, cache_keep): + tmp_path = cmake(["sentry_example"], {"SENTRY_BACKEND": "crashpad"}) + cache_dir = tmp_path.joinpath(".sentry-native/cache") + + env = dict(os.environ, SENTRY_DSN=make_dsn(httpserver)) + httpserver.expect_oneshot_request("/api/123456/minidump/").respond_with_data("OK") + + with httpserver.wait(timeout=10) as waiting: + run( + tmp_path, + "sentry_example", + ["log", "crash"] + (["cache-keep"] if cache_keep else []), + expect_failure=True, + env=env, + ) + assert waiting.result + + assert not cache_dir.exists() or len(list(cache_dir.glob("*.envelope"))) == 0 + + # upload + run( + tmp_path, + "sentry_example", + ["log", "no-setup"] + (["cache-keep"] if cache_keep else []), + env=env, + ) + + # cache + run( + tmp_path, + "sentry_example", + ["log", "no-setup"] + (["cache-keep"] if cache_keep else []), + env=env, + ) + + assert cache_dir.exists() or cache_keep is False + if cache_keep: + cache_files = list(cache_dir.glob("*.envelope")) + assert len(cache_files) == 1 + + +def test_crashpad_cache_max_size(cmake, httpserver): + tmp_path = cmake(["sentry_example"], {"SENTRY_BACKEND": "crashpad"}) + cache_dir = tmp_path.joinpath(".sentry-native/cache") + env = dict(os.environ, SENTRY_DSN=make_dsn(httpserver)) + + # 5 x 2mb + for i in range(5): + httpserver.expect_oneshot_request("/api/123456/minidump/").respond_with_data( + "OK" + ) + + with httpserver.wait(timeout=10) as waiting: + run( + tmp_path, + "sentry_example", + ["log", "cache-keep", "crash"], + expect_failure=True, + env=env, + ) + assert waiting.result + + # upload + run( + tmp_path, + "sentry_example", + ["log", "cache-keep", "no-setup"], + env=env, + ) + + # cache + run( + tmp_path, + "sentry_example", + ["log", "cache-keep", "no-setup"], + env=env, + ) + + if cache_dir.exists(): + for f in cache_dir.glob("*.envelope"): + with open(f, "r+b") as file: + file.truncate(2 * 1024 * 1024) + + run( + tmp_path, + "sentry_example", + ["log", "cache-keep", "no-setup"], + env=env, + ) + + # max 4mb + assert cache_dir.exists() + cache_files = list(cache_dir.glob("*.envelope")) + assert len(cache_files) <= 2 + assert sum(f.stat().st_size for f in cache_files) <= 4 * 1024 * 1024 + + +def test_crashpad_cache_max_age(cmake, httpserver): + tmp_path = cmake(["sentry_example"], {"SENTRY_BACKEND": "crashpad"}) + cache_dir = tmp_path.joinpath(".sentry-native/cache") + env = dict(os.environ, SENTRY_DSN=make_dsn(httpserver)) + + # 4 crashes that get fully cached + for i in range(4): + httpserver.expect_oneshot_request("/api/123456/minidump/").respond_with_data( + "OK" + ) + + with httpserver.wait(timeout=10) as waiting: + run( + tmp_path, + "sentry_example", + ["log", "cache-keep", "crash"], + expect_failure=True, + env=env, + ) + assert waiting.result + + # upload + run( + tmp_path, + "sentry_example", + ["log", "cache-keep", "no-setup"], + env=env, + ) + + # cache + run( + tmp_path, + "sentry_example", + ["log", "cache-keep", "no-setup"], + env=env, + ) + + # 2,4,6,8 days old + assert cache_dir.exists() + cache_files = list(cache_dir.glob("*.envelope")) + assert len(cache_files) == 4 + for i, f in enumerate(cache_files): + mtime = time.time() - ((i + 1) * 2 * 24 * 60 * 60) + os.utime(str(f), (mtime, mtime)) + + # 5th crash - only upload, not cached yet + httpserver.expect_oneshot_request("/api/123456/minidump/").respond_with_data("OK") + + with httpserver.wait(timeout=10) as waiting: + run( + tmp_path, + "sentry_example", + ["log", "cache-keep", "crash"], + expect_failure=True, + env=env, + ) + assert waiting.result + + # upload + run( + tmp_path, + "sentry_example", + ["log", "cache-keep", "no-setup"], + env=env, + ) + + # 0 days old (caches 5th crash + prunes old files) + run( + tmp_path, + "sentry_example", + ["log", "cache-keep", "no-setup"], + env=env, + ) + + # max 5 days + cache_files = list(cache_dir.glob("*.envelope")) + assert len(cache_files) == 3 + for f in cache_files: + assert time.time() - f.stat().st_mtime <= 5 * 24 * 60 * 60 diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt index c889b9320..24d04ac03 100644 --- a/tests/unit/CMakeLists.txt +++ b/tests/unit/CMakeLists.txt @@ -23,6 +23,7 @@ add_executable(sentry_test_unit sentry_testsupport.h test_attachments.c test_basic.c + test_cache.c test_consent.c test_concurrency.c test_embedded_info.c diff --git a/tests/unit/test_cache.c b/tests/unit/test_cache.c new file mode 100644 index 000000000..2d474a143 --- /dev/null +++ b/tests/unit/test_cache.c @@ -0,0 +1,248 @@ +#include "sentry_core.h" +#include "sentry_database.h" +#include "sentry_envelope.h" +#include "sentry_options.h" +#include "sentry_path.h" +#include "sentry_testsupport.h" +#include "sentry_uuid.h" +#include "sentry_value.h" + +#ifdef SENTRY_PLATFORM_WINDOWS +# include +#else +# include +#endif + +static int +set_file_mtime(const sentry_path_t *path, time_t mtime) +{ +#ifdef SENTRY_PLATFORM_WINDOWS + HANDLE h = CreateFileW(path->path_w, FILE_WRITE_ATTRIBUTES, 0, NULL, + OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); + if (h == INVALID_HANDLE_VALUE) { + return -1; + } + // 100 ns intervals since January 1, 1601 (UTC) + uint64_t t = ((uint64_t)mtime * 10000000ULL) + 116444736000000000ULL; + FILETIME ft = { (DWORD)t, (DWORD)(t >> 32) }; + BOOL rv = SetFileTime(h, NULL, NULL, &ft); + CloseHandle(h); + return rv ? 0 : -1; +#else + struct utimbuf times = { .modtime = mtime, .actime = mtime }; + return utime(path->path, ×); +#endif +} + +SENTRY_TEST(cache_keep) +{ + SENTRY_TEST_OPTIONS_NEW(options); + sentry_options_set_dsn(options, "https://foo@sentry.invalid/42"); + sentry_options_set_cache_keep(options, true); + sentry_init(options); + + sentry_path_t *cache_path + = sentry__path_join_str(options->database_path, "cache"); + TEST_ASSERT(!!cache_path); + TEST_ASSERT(sentry__path_remove_all(cache_path) == 0); + + sentry_path_t *old_run_path + = sentry__path_join_str(options->database_path, "old.run"); + TEST_ASSERT(!!old_run_path); + TEST_ASSERT(sentry__path_create_dir_all(old_run_path) == 0); + + sentry_envelope_t *envelope = sentry__envelope_new(); + TEST_ASSERT(!!envelope); + sentry_uuid_t event_id = sentry_uuid_new_v4(); + sentry_value_t event = sentry__value_new_event_with_id(&event_id); + sentry__envelope_add_event(envelope, event); + + char *envelope_filename = sentry__uuid_as_filename(&event_id, ".envelope"); + TEST_ASSERT(!!envelope_filename); + sentry_path_t *old_envelope_path + = sentry__path_join_str(old_run_path, envelope_filename); + TEST_ASSERT( + sentry_envelope_write_to_path(envelope, old_envelope_path) == 0); + sentry_envelope_free(envelope); + + sentry_path_t *cached_envelope_path + = sentry__path_join_str(cache_path, envelope_filename); + TEST_ASSERT(!!cached_envelope_path); + + TEST_ASSERT(sentry__path_is_file(old_envelope_path)); + TEST_ASSERT(!sentry__path_is_file(cached_envelope_path)); + + sentry__process_old_runs(options, 0); + + TEST_ASSERT(!sentry__path_is_file(old_envelope_path)); + TEST_ASSERT(sentry__path_is_file(cached_envelope_path)); + + sentry__path_free(old_envelope_path); + sentry__path_free(cached_envelope_path); + sentry__path_free(old_run_path); + sentry__path_free(cache_path); + sentry_free(envelope_filename); + sentry_close(); +} + +SENTRY_TEST(cache_max_size) +{ + SENTRY_TEST_OPTIONS_NEW(options); + sentry_options_set_dsn(options, "https://foo@sentry.invalid/42"); + sentry_options_set_cache_keep(options, true); + sentry_options_set_cache_max_size(options, 10 * 1024); // 10 kb + sentry_init(options); + + sentry_path_t *cache_path + = sentry__path_join_str(options->database_path, "cache"); + TEST_ASSERT(!!cache_path); + TEST_ASSERT(sentry__path_remove_all(cache_path) == 0); + TEST_ASSERT(sentry__path_create_dir_all(cache_path) == 0); + + // 10 x 5 kb files + for (int i = 0; i < 10; i++) { + sentry_uuid_t event_id = sentry_uuid_new_v4(); + char *filename = sentry__uuid_as_filename(&event_id, ".envelope"); + TEST_ASSERT(!!filename); + sentry_path_t *filepath = sentry__path_join_str(cache_path, filename); + sentry_free(filename); + + sentry_filewriter_t *fw = sentry__filewriter_new(filepath); + TEST_ASSERT(!!fw); + for (int j = 0; j < 500; j++) { + sentry__filewriter_write(fw, "0123456789", 10); + } + TEST_CHECK_INT_EQUAL(sentry__filewriter_byte_count(fw), 5000); + sentry__filewriter_free(fw); + sentry__path_free(filepath); + } + + sentry__cleanup_cache(options); + + int cache_count = 0; + size_t cache_size = 0; + sentry_pathiter_t *iter = sentry__path_iter_directory(cache_path); + const sentry_path_t *entry; + while (iter && (entry = sentry__pathiter_next(iter)) != NULL) { + cache_count++; + cache_size += sentry__path_get_size(entry); + } + sentry__pathiter_free(iter); + + TEST_CHECK_INT_EQUAL(cache_count, 2); + TEST_CHECK(cache_size <= 10 * 1024); + + sentry__path_free(cache_path); + sentry_close(); +} + +SENTRY_TEST(cache_max_age) +{ + SENTRY_TEST_OPTIONS_NEW(options); + sentry_options_set_dsn(options, "https://foo@sentry.invalid/42"); + sentry_options_set_cache_keep(options, true); + sentry_options_set_cache_max_age(options, 5 * 24 * 60 * 60); // 5 days + sentry_init(options); + + sentry_path_t *cache_path + = sentry__path_join_str(options->database_path, "cache"); + TEST_ASSERT(!!cache_path); + TEST_ASSERT(sentry__path_remove_all(cache_path) == 0); + TEST_ASSERT(sentry__path_create_dir_all(cache_path) == 0); + + // 0,2,4,6,8 days ago + time_t now = time(NULL); + for (int i = 0; i < 5; i++) { + sentry_uuid_t event_id = sentry_uuid_new_v4(); + char *filename = sentry__uuid_as_filename(&event_id, ".envelope"); + TEST_ASSERT(!!filename); + sentry_path_t *filepath = sentry__path_join_str(cache_path, filename); + sentry_free(filename); + + TEST_ASSERT(sentry__path_touch(filepath) == 0); + time_t mtime = now - (i * 2 * 24 * 60 * 60); // N days ago + TEST_ASSERT(set_file_mtime(filepath, mtime) == 0); + sentry__path_free(filepath); + } + + sentry__cleanup_cache(options); + + int cache_count = 0; + sentry_pathiter_t *iter = sentry__path_iter_directory(cache_path); + const sentry_path_t *entry; + while (iter && (entry = sentry__pathiter_next(iter)) != NULL) { + cache_count++; + time_t mtime = sentry__path_get_mtime(entry); + TEST_CHECK(now - mtime <= (5 * 24 * 60 * 60)); + } + sentry__pathiter_free(iter); + + TEST_CHECK_INT_EQUAL(cache_count, 3); + + sentry__path_free(cache_path); + sentry_close(); +} + +SENTRY_TEST(cache_max_size_and_age) +{ + // Verify size pruning keeps newer entries, removes all older once limit + // hit. A (5KB), B (6KB), C (3KB) newest-to-oldest, max_size=10KB A+B=11KB + // exceeds limit -> B pruned, C (older) also pruned + SENTRY_TEST_OPTIONS_NEW(options); + sentry_options_set_dsn(options, "https://foo@sentry.invalid/42"); + sentry_options_set_cache_keep(options, true); + sentry_options_set_cache_max_size(options, 10 * 1024); // 10 KB + sentry_init(options); + + sentry_path_t *cache_path + = sentry__path_join_str(options->database_path, "cache"); + TEST_ASSERT(!!cache_path); + TEST_ASSERT(sentry__path_remove_all(cache_path) == 0); + TEST_ASSERT(sentry__path_create_dir_all(cache_path) == 0); + + time_t now = time(NULL); + + // A (newest, 5KB), B (middle, 6KB), C (oldest, 3KB) + struct { + const char *name; + size_t size; + time_t age; + } files[] = { + { "a.envelope", 5 * 1024, 0 }, // newest + { "b.envelope", 6 * 1024, 60 }, // 1 min old + { "c.envelope", 3 * 1024, 120 }, // 2 min old + }; + + for (size_t i = 0; i < sizeof(files) / sizeof(files[0]); i++) { + sentry_path_t *filepath + = sentry__path_join_str(cache_path, files[i].name); + TEST_ASSERT(!!filepath); + + sentry_filewriter_t *fw = sentry__filewriter_new(filepath); + TEST_ASSERT(!!fw); + for (size_t j = 0; j < files[i].size / 10; j++) { + sentry__filewriter_write(fw, "0123456789", 10); + } + sentry__filewriter_free(fw); + + TEST_ASSERT(set_file_mtime(filepath, now - files[i].age) == 0); + sentry__path_free(filepath); + } + + sentry__cleanup_cache(options); + + // Verify: A kept, B and C removed (once limit hit, all older removed) + sentry_path_t *a_path = sentry__path_join_str(cache_path, "a.envelope"); + sentry_path_t *b_path = sentry__path_join_str(cache_path, "b.envelope"); + sentry_path_t *c_path = sentry__path_join_str(cache_path, "c.envelope"); + + TEST_CHECK(sentry__path_is_file(a_path)); // newest, kept + TEST_CHECK(!sentry__path_is_file(b_path)); // size-pruned + TEST_CHECK(!sentry__path_is_file(c_path)); // older than B, also pruned + + sentry__path_free(a_path); + sentry__path_free(b_path); + sentry__path_free(c_path); + sentry__path_free(cache_path); + sentry_close(); +} diff --git a/tests/unit/tests.inc b/tests/unit/tests.inc index 835285c00..c3b1530b1 100644 --- a/tests/unit/tests.inc +++ b/tests/unit/tests.inc @@ -27,6 +27,10 @@ XX(basic_write_envelope_to_file) XX(bgworker_flush) XX(breadcrumb_without_type_or_message_still_valid) XX(build_id_parser) +XX(cache_keep) +XX(cache_max_age) +XX(cache_max_size) +XX(cache_max_size_and_age) XX(capture_minidump_basic) XX(capture_minidump_invalid_path) XX(capture_minidump_null_path)