From 0ea1924fdb95a9dd72c5d72ba7855ace9f8897f3 Mon Sep 17 00:00:00 2001 From: Floris Bruynooghe Date: Wed, 23 Dec 2020 18:20:01 +0100 Subject: [PATCH] feat(difs): unwind info in dif candidates (#324) This adds the unwind info to dif candidates. For this the cficache actor also exposes DIF info and the symbolication actor now correctly merges the various DIF candidates info together. --- CHANGELOG.md | 2 +- src/actors/cficaches.rs | 99 ++++---- src/actors/common/cache.rs | 4 +- ...ymbolication__tests__minidump_windows.snap | 44 ++++ src/actors/symbolication.rs | 52 +++-- src/actors/symcaches.rs | 20 +- src/types/mod.rs | 16 +- src/types/objects.rs | 213 +++++++++++++++++- tests/integration/test_minidump.py | 4 + 9 files changed, 357 insertions(+), 97 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 05fc69583..a3a282e2c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ ### Features -- Symbolication responses include download information about all DIF object files which were looked up on the available object sources. ([#309](https://github.com/getsentry/symbolicator/pull/309) [#316](https://github.com/getsentry/symbolicator/pull/316)) +- Symbolication responses include download, debug and unwind information about all DIF object files which were looked up on the available object sources. ([#309](https://github.com/getsentry/symbolicator/pull/309) [#316](https://github.com/getsentry/symbolicator/pull/316) [#324](https://github.com/getsentry/symbolicator/pull/324)) ### Bug Fixes diff --git a/src/actors/cficaches.rs b/src/actors/cficaches.rs index be86c16bc..ebe8b8203 100644 --- a/src/actors/cficaches.rs +++ b/src/actors/cficaches.rs @@ -5,8 +5,7 @@ use std::sync::Arc; use std::time::Duration; use futures::compat::Future01CompatExt; -use futures::future::{self, Either}; -use futures::{FutureExt, TryFutureExt}; +use futures::prelude::*; use sentry::{configure_scope, Hub, SentryFutureExt}; use symbolic::{ common::ByteView, @@ -20,7 +19,9 @@ use crate::actors::objects::{ }; use crate::cache::{Cache, CacheKey, CacheStatus}; use crate::sources::{FileType, SourceConfig}; -use crate::types::{ObjectFeatures, ObjectId, ObjectType, Scope}; +use crate::types::{ + AllObjectCandidates, ObjectFeatures, ObjectId, ObjectType, ObjectUseInfo, Scope, +}; use crate::utils::futures::{BoxedFuture, ThreadPool}; use crate::utils::sentry::WriteSentryScope; @@ -72,6 +73,7 @@ pub struct CfiCacheFile { features: ObjectFeatures, status: CacheStatus, path: CachePath, + candidates: AllObjectCandidates, } impl CfiCacheFile { @@ -89,13 +91,19 @@ impl CfiCacheFile { pub fn path(&self) -> &Path { self.path.as_ref() } + + /// Returns all the DIF object candidates. + pub fn candidates(&self) -> &AllObjectCandidates { + &self.candidates + } } #[derive(Clone, Debug)] struct FetchCfiCacheInternal { request: FetchCfiCache, objects_actor: ObjectsActor, - object_meta: Arc, + meta_handle: Arc, + candidates: AllObjectCandidates, threadpool: ThreadPool, } @@ -104,7 +112,7 @@ impl CacheItemRequest for FetchCfiCacheInternal { type Error = CfiCacheError; fn get_cache_key(&self) -> CacheKey { - self.object_meta.cache_key() + self.meta_handle.cache_key() } /// Extracts the Call Frame Information (CFI) from an object file. @@ -115,12 +123,12 @@ impl CacheItemRequest for FetchCfiCacheInternal { let path = path.to_owned(); let object = self .objects_actor - .fetch(self.object_meta.clone()) + .fetch(self.meta_handle.clone()) .map_err(CfiCacheError::Fetching); let threadpool = self.threadpool.clone(); let result = object.and_then(move |object| { - let future = future::lazy(move |_| { + let future = async move { if object.status() != CacheStatus::Positive { return Ok(object.status()); } @@ -135,7 +143,7 @@ impl CacheItemRequest for FetchCfiCacheInternal { }; Ok(status) - }); + }; threadpool .spawn_handle(future.bind_hub(Hub::current())) @@ -168,14 +176,22 @@ impl CacheItemRequest for FetchCfiCacheInternal { data: ByteView<'static>, path: CachePath, ) -> Self::Item { + let mut candidates = self.candidates.clone(); + candidates.set_unwind( + self.meta_handle.source().clone(), + self.meta_handle.location(), + ObjectUseInfo::from_derived_status(status, self.meta_handle.status()), + ); + CfiCacheFile { object_type: self.request.object_type, identifier: self.request.identifier.clone(), scope, data, - features: self.object_meta.features(), + features: self.meta_handle.features(), status, path, + candidates, } } } @@ -196,11 +212,11 @@ impl CfiCacheActor { /// debug filename (the basename). To do this it looks in the existing cache with the /// given scope and if it does not yet exist in cached form will fetch the required DIFs /// and compute the required CFI cache file. - pub fn fetch( + pub async fn fetch( &self, request: FetchCfiCache, - ) -> BoxedFuture, Arc>> { - let object = self + ) -> Result, Arc> { + let found_result = self .objects .clone() .find(FindObject { @@ -210,41 +226,32 @@ impl CfiCacheActor { scope: request.scope.clone(), purpose: ObjectPurpose::Unwind, }) - .map_err(|e| Arc::new(CfiCacheError::Fetching(e))); - - let cficaches = self.cficaches.clone(); - let threadpool = self.threadpool.clone(); - let objects = self.objects.clone(); - - let object_type = request.object_type; - let identifier = request.identifier.clone(); - let scope = request.scope.clone(); - - object - .and_then(move |object| { - object - .meta - .map(move |object_meta| { - Either::Left(cficaches.compute_memoized(FetchCfiCacheInternal { - request, - objects_actor: objects, - object_meta, - threadpool, - })) - }) - .unwrap_or_else(move || { - Either::Right(future::ok(Arc::new(CfiCacheFile { - object_type, - identifier, - scope, - data: ByteView::from_slice(b""), - features: ObjectFeatures::default(), - status: CacheStatus::Negative, - path: CachePath::new(), - }))) + .map_err(|e| Arc::new(CfiCacheError::Fetching(e))) + .await?; + + match found_result.meta { + Some(meta_handle) => { + self.cficaches + .compute_memoized(FetchCfiCacheInternal { + request, + objects_actor: self.objects.clone(), + meta_handle, + threadpool: self.threadpool.clone(), + candidates: found_result.candidates, }) - }) - .boxed_local() + .await + } + None => Ok(Arc::new(CfiCacheFile { + object_type: request.object_type, + identifier: request.identifier, + scope: request.scope, + data: ByteView::from_slice(b""), + features: ObjectFeatures::default(), + status: CacheStatus::Negative, + path: CachePath::new(), + candidates: found_result.candidates, + })), + } } } diff --git a/src/actors/common/cache.rs b/src/actors/common/cache.rs index 8ecc7631c..e1a40ca27 100644 --- a/src/actors/common/cache.rs +++ b/src/actors/common/cache.rs @@ -16,7 +16,7 @@ use crate::types::Scope; use crate::utils::futures::{BoxedFuture, CallOnDrop}; /// Result from [`Cacher::compute_memoized`]. -type CacheResult = BoxedFuture, Arc>>; +type CacheResultFuture = BoxedFuture, Arc>>; // Inner result necessary because `futures::Shared` won't give us `Arc`s but its own custom // newtype around it. @@ -311,7 +311,7 @@ impl Cacher { /// occurs the error result is returned, **however** in this case nothing is written /// into the cache and the next call to the same cache item will attempt to re-compute /// the cache. - pub fn compute_memoized(&self, request: T) -> CacheResult { + pub fn compute_memoized(&self, request: T) -> CacheResultFuture { let key = request.get_cache_key(); let name = self.config.name(); diff --git a/src/actors/snapshots/symbolicator__actors__symbolication__tests__minidump_windows.snap b/src/actors/snapshots/symbolicator__actors__symbolication__tests__minidump_windows.snap index 7c34e5be6..7e417e01d 100644 --- a/src/actors/snapshots/symbolicator__actors__symbolication__tests__minidump_windows.snap +++ b/src/actors/snapshots/symbolicator__actors__symbolication__tests__minidump_windows.snap @@ -190,6 +190,8 @@ modules: has_unwind_info: true has_symbols: true has_sources: false + unwind: + status: ok debug: status: ok - source: local @@ -256,6 +258,27 @@ modules: debug_file: dbgcore.pdb image_addr: 0x70b70000 image_size: 151552 + candidates: + - source: local + location: dbgcore.dll/57898DAB25000/dbgcore.dl_ + download: + status: notfound + - source: local + location: dbgcore.dll/57898DAB25000/dbgcore.dll + download: + status: notfound + - source: local + location: dbgcore.pdb/AEC7EF2FDF4B4642A4714C3E5FE8760A1/dbgcore.pd_ + download: + status: notfound + - source: local + location: dbgcore.pdb/AEC7EF2FDF4B4642A4714C3E5FE8760A1/dbgcore.pdb + download: + status: notfound + - source: local + location: dbgcore.pdb/AEC7EF2FDF4B4642A4714C3E5FE8760A1/dbgcore.sym + download: + status: notfound - debug_status: unused unwind_status: unused features: @@ -412,6 +435,27 @@ modules: debug_file: wrpcrt4.pdb image_addr: 0x75810000 image_size: 790528 + candidates: + - source: local + location: rpcrt4.dll/5A49BB75c1000/rpcrt4.dl_ + download: + status: notfound + - source: local + location: rpcrt4.dll/5A49BB75c1000/rpcrt4.dll + download: + status: notfound + - source: local + location: wrpcrt4.pdb/AE131C6727A74FA19916B5A4AEF411901/wrpcrt4.pd_ + download: + status: notfound + - source: local + location: wrpcrt4.pdb/AE131C6727A74FA19916B5A4AEF411901/wrpcrt4.pdb + download: + status: notfound + - source: local + location: wrpcrt4.pdb/AE131C6727A74FA19916B5A4AEF411901/wrpcrt4.sym + download: + status: notfound - debug_status: unused unwind_status: unused features: diff --git a/src/actors/symbolication.rs b/src/actors/symbolication.rs index 50060c61e..702cd8763 100644 --- a/src/actors/symbolication.rs +++ b/src/actors/symbolication.rs @@ -686,8 +686,7 @@ impl SymCacheLookup { if let Some(ref symcache) = symcache { entry.object_info.arch = symcache.arch(); entry.object_info.features.merge(symcache.features()); - - entry.object_info.candidates = symcache.candidates(); // TODO(flub): merge! + entry.object_info.candidates.merge(symcache.candidates()); } entry.symcache = symcache; @@ -1414,22 +1413,28 @@ impl SymbolicationActor { requests: Vec<(CodeModuleId, RawObjectInfo)>, sources: Arc<[SourceConfig]>, ) -> Vec { - let cficaches = self.cficaches.clone(); + let mut futures = Vec::with_capacity(requests.len()); - let futures = requests - .into_iter() - .map(move |(code_module_id, object_info)| { - cficaches + for (code_id, object_info) in requests { + let sources = sources.clone(); + let scope = scope.clone(); + + let fut = async move { + let result = self + .cficaches .fetch(FetchCfiCache { object_type: object_info.ty, identifier: object_id_from_object_info(&object_info), - sources: sources.clone(), - scope: scope.clone(), + sources, + scope, }) - .then(move |result| future::ready((code_module_id, result))) - // Clone hub because of join_all - .bind_hub(Hub::new_from_top(Hub::current())) - }); + .await; + (code_id, result) + }; + + // Clone hub because of join_all concurrency. + futures.push(fut.bind_hub(Hub::new_from_top(Hub::current()))); + } future::join_all(futures).await } @@ -1463,6 +1468,7 @@ impl SymbolicationActor { let mut unwind_statuses = BTreeMap::new(); let mut object_features = BTreeMap::new(); let mut frame_info_map = BTreeMap::new(); + let mut dif_candidates = BTreeMap::new(); // Go through all the modules in the minidump and build a map of the modules with // missing or malformed CFI. ObjectFileStatus::Found is only added when the file is @@ -1470,7 +1476,10 @@ impl SymbolicationActor { // cache files are added to the frame_info_map. for (code_module_id, result) in &cfi_results { let cache_file = match result { - Ok(x) => x, + Ok(cfi_cache_file) => { + dif_candidates.insert(*code_module_id, cfi_cache_file.candidates().clone()); + cfi_cache_file + } Err(e) => { log::debug!("Error while fetching cficache: {}", LogError(e.as_ref())); unwind_statuses.insert(*code_module_id, (&**e).into()); @@ -1510,6 +1519,7 @@ impl SymbolicationActor { unwind_statuses, minidump.clone(), spawn_time, + procspawn::serde::Json(dif_candidates), ), |( frame_info_map, @@ -1517,8 +1527,11 @@ impl SymbolicationActor { mut unwind_statuses, minidump, spawn_time, + dif_candidates, )| - -> Result<_, ProcessMinidumpError> { + -> Result<_, ProcessMinidumpError> { + let procspawn::serde::Json(mut dif_candidates) = dif_candidates; + if let Ok(duration) = spawn_time.elapsed() { metric!(timer("minidump.stackwalk.spawn.duration") = duration); } @@ -1622,6 +1635,15 @@ impl SymbolicationActor { info.features.merge(features); + if let Some(code_id) = module_id { + // If we have the same code module mapped into the + // memory in multiple regions we would only have the + // candidates filled in for the first one. + if let Some(candidates) = dif_candidates.remove(&code_id) { + info.candidates = candidates; + } + } + info }) .collect::(); diff --git a/src/actors/symcaches.rs b/src/actors/symcaches.rs index 30acf3309..1128779d5 100644 --- a/src/actors/symcaches.rs +++ b/src/actors/symcaches.rs @@ -187,29 +187,11 @@ impl CacheItemRequest for FetchSymCacheInternal { .map(|cache| cache.arch()) .unwrap_or_default(); - // If self.object_meta.status() was != Positive than that status got passed straight - // through to our own `status` argument. - let debug = match status { - CacheStatus::Positive => ObjectUseInfo::Ok, - CacheStatus::Negative => { - if self.object_meta.status() == CacheStatus::Positive { - ObjectUseInfo::Error { - details: String::from("Object file no longer available"), - } - } else { - // No need to pretend that we were going to use this symcache if the - // original object file was already not there, that status is already - // reported. - ObjectUseInfo::None - } - } - CacheStatus::Malformed => ObjectUseInfo::Malformed, - }; let mut candidates = self.candidates.clone(); // yuk! candidates.set_debug( self.object_meta.source().clone(), self.object_meta.location(), - debug, + ObjectUseInfo::from_derived_status(status, self.object_meta.status()), ); SymCacheFile { diff --git a/src/types/mod.rs b/src/types/mod.rs index 28bc2b4ac..532f1d0c6 100644 --- a/src/types/mod.rs +++ b/src/types/mod.rs @@ -474,6 +474,14 @@ impl ObjectFeatures { } } +/// Newtype around a collection of [`ObjectCandidate`] structs. +/// +/// This abstracts away some common operations needed on this collection. +/// +/// [`CacheItemRequest`]: ../actors/common/cache/trait.CacheItemRequest.html +#[derive(Clone, Debug, Default, Serialize, Deserialize, Eq, PartialEq)] +pub struct AllObjectCandidates(Vec); + /// Information about a Debug Information File in the [`CompleteObjectInfo`]. /// /// All DIFs are backed by an [`ObjectHandle`](crate::actors::objects::ObjectHandle). But we @@ -579,14 +587,6 @@ pub enum ObjectUseInfo { None, } -/// Newtype around a collection of [`ObjectCandidate`] structs. -/// -/// This abstracts away some common operations needed on this collection. -/// -/// [`CacheItemRequest`]: ../actors/common/cache/trait.CacheItemRequest.html -#[derive(Clone, Debug, Default, Serialize, Deserialize, Eq, PartialEq)] -pub struct AllObjectCandidates(Vec); - /// Normalized [`RawObjectInfo`] with status attached. /// /// This describes an object in the modules list of a response to a symbolication request. diff --git a/src/types/objects.rs b/src/types/objects.rs index b6af70db8..c3a44162e 100644 --- a/src/types/objects.rs +++ b/src/types/objects.rs @@ -1,10 +1,12 @@ //! Implementations for the types describing DIF object files. +use crate::cache::CacheStatus; +use crate::sources::{SourceId, SourceLocation}; + use super::{ AllObjectCandidates, Arch, CodeId, CompleteObjectInfo, DebugId, ObjectCandidate, ObjectFeatures, ObjectFileStatus, ObjectUseInfo, RawObjectInfo, }; -use crate::sources::{SourceId, SourceLocation}; impl Default for ObjectUseInfo { fn default() -> Self { @@ -32,11 +34,13 @@ impl AllObjectCandidates { /// You can only request symcaches from a DIF object that was already in the metadata /// candidate list, therefore if the candidate is missing it is treated as an error. pub fn set_debug(&mut self, source: SourceId, location: SourceLocation, info: ObjectUseInfo) { - match self - .0 - .binary_search_by_key(&(source, location), |candidate| { - (candidate.source.clone(), candidate.location.clone()) - }) { + let found_pos = self.0.binary_search_by(|candidate| { + candidate + .source + .cmp(&source) + .then(candidate.location.cmp(&location)) + }); + match found_pos { Ok(index) => { if let Some(mut candidate) = self.0.get_mut(index) { candidate.debug = info; @@ -51,6 +55,32 @@ impl AllObjectCandidates { } } + /// Sets the [`ObjectCandidate::unwind`] field for the specified DIF object. + /// + /// You can only request cficaches from a DIF object that was already in the metadata + /// candidate list, therefore if the candidate is missing it is treated as an error. + pub fn set_unwind(&mut self, source: SourceId, location: SourceLocation, info: ObjectUseInfo) { + let found_pos = self.0.binary_search_by(|candidate| { + candidate + .source + .cmp(&source) + .then(candidate.location.cmp(&location)) + }); + match found_pos { + Ok(index) => { + if let Some(mut candidate) = self.0.get_mut(index) { + candidate.unwind = info; + } + } + Err(_) => { + sentry::capture_message( + "Missing ObjectCandidate in AllObjectCandidates::set_unwind", + sentry::Level::Error, + ); + } + } + } + /// Returns `true` if the collections contains no [`ObjectCandidate`]s. pub fn is_empty(&self) -> bool { self.0.is_empty() @@ -60,6 +90,40 @@ impl AllObjectCandidates { pub fn clear(&mut self) { self.0.clear() } + + /// Merge in the other collection of candidates. + /// + /// If a candidate already existed in the collection all data which is present in both + /// will be overwritten by the data in `other`. Practically that means + /// [`ObjectCandidate::download`] will be overwritten by `other` and for + /// [`ObjectCandidate::unwind`] and [`ObjectCandidate::debug`] it will be overwritten by + /// `other` if they are not [`ObjectUseInfo::None`]. + pub fn merge(&mut self, other: AllObjectCandidates) { + for other_info in other.0 { + let found_pos = self.0.binary_search_by(|candidate| { + candidate + .source + .cmp(&other_info.source) + .then(candidate.location.cmp(&other_info.location)) + }); + match found_pos { + Ok(index) => { + if let Some(mut info) = self.0.get_mut(index) { + info.download = other_info.download; + if other_info.unwind != ObjectUseInfo::None { + info.unwind = other_info.unwind; + } + if other_info.debug != ObjectUseInfo::None { + info.debug = other_info.debug; + } + } + } + Err(index) => { + self.0.insert(index, other_info); + } + } + } + } } impl From for CompleteObjectInfo { @@ -86,3 +150,140 @@ impl From for CompleteObjectInfo { } } } + +impl ObjectUseInfo { + /// Construct [`ObjectUseInfo`] for an object from a derived cache. + /// + /// The [`ObjectUseInfo`] provides information about items stored in a cache and which + /// are derived from an original object cache: the [`symcaches`] and the [`cficaches`]. + /// These caches have an edge case where if the underlying cache thought the object was + /// there but now it could not be fetched again. This is converted to an error case. + /// + /// [`symcaches`]: crate::actors::symcaches + /// [`cficaches`]: crate::actors::cficaches + pub fn from_derived_status(derived: CacheStatus, original: CacheStatus) -> Self { + match derived { + CacheStatus::Positive => ObjectUseInfo::Ok, + CacheStatus::Negative => { + if original == CacheStatus::Positive { + ObjectUseInfo::Error { + details: String::from("Object file no longer available"), + } + } else { + // If the original cache was already missing then it will already be + // reported and we do not want to report anything. + ObjectUseInfo::None + } + } + CacheStatus::Malformed => ObjectUseInfo::Malformed, + } + } +} + +#[cfg(test)] +mod tests { + use crate::types::ObjectDownloadInfo; + + use super::*; + + #[test] + fn test_all_object_info_merge_insert_new() { + // If a candidate didn't exist yet it should be inserted in order. + let src_a = ObjectCandidate { + source: SourceId::new("A"), + location: SourceLocation::new("a"), + download: ObjectDownloadInfo::Ok { + features: Default::default(), + }, + unwind: ObjectUseInfo::Ok, + debug: ObjectUseInfo::Ok, + }; + let src_b = ObjectCandidate { + source: SourceId::new("B"), + location: SourceLocation::new("b"), + download: ObjectDownloadInfo::Ok { + features: Default::default(), + }, + unwind: ObjectUseInfo::Ok, + debug: ObjectUseInfo::Ok, + }; + let src_c = ObjectCandidate { + source: SourceId::new("C"), + location: SourceLocation::new("c"), + download: ObjectDownloadInfo::Ok { + features: Default::default(), + }, + unwind: ObjectUseInfo::Ok, + debug: ObjectUseInfo::Ok, + }; + + let mut all: AllObjectCandidates = vec![src_a, src_c].into(); + let other: AllObjectCandidates = vec![src_b].into(); + all.merge(other); + assert_eq!(all.0[0].source, SourceId::new("A")); + assert_eq!(all.0[1].source, SourceId::new("B")); + assert_eq!(all.0[2].source, SourceId::new("C")); + } + + #[test] + fn test_all_object_info_merge_overwrite() { + let src0 = ObjectCandidate { + source: SourceId::new("A"), + location: SourceLocation::new("a"), + download: ObjectDownloadInfo::Ok { + features: Default::default(), + }, + unwind: ObjectUseInfo::Ok, + debug: ObjectUseInfo::None, + }; + let src1 = ObjectCandidate { + source: SourceId::new("A"), + location: SourceLocation::new("a"), + download: ObjectDownloadInfo::Ok { + features: Default::default(), + }, + unwind: ObjectUseInfo::Malformed, + debug: ObjectUseInfo::Ok, + }; + + let mut all: AllObjectCandidates = vec![src0].into(); + assert_eq!(all.0[0].unwind, ObjectUseInfo::Ok); + assert_eq!(all.0[0].debug, ObjectUseInfo::None); + + let other: AllObjectCandidates = vec![src1].into(); + all.merge(other); + assert_eq!(all.0[0].unwind, ObjectUseInfo::Malformed); + assert_eq!(all.0[0].debug, ObjectUseInfo::Ok); + } + + #[test] + fn test_all_object_info_merge_no_overwrite() { + let src0 = ObjectCandidate { + source: SourceId::new("A"), + location: SourceLocation::new("a"), + download: ObjectDownloadInfo::Ok { + features: Default::default(), + }, + unwind: ObjectUseInfo::Ok, + debug: ObjectUseInfo::Ok, + }; + let src1 = ObjectCandidate { + source: SourceId::new("A"), + location: SourceLocation::new("a"), + download: ObjectDownloadInfo::Ok { + features: Default::default(), + }, + unwind: ObjectUseInfo::None, + debug: ObjectUseInfo::None, + }; + + let mut all: AllObjectCandidates = vec![src0].into(); + assert_eq!(all.0[0].unwind, ObjectUseInfo::Ok); + assert_eq!(all.0[0].debug, ObjectUseInfo::Ok); + + let other: AllObjectCandidates = vec![src1].into(); + all.merge(other); + assert_eq!(all.0[0].unwind, ObjectUseInfo::Ok); + assert_eq!(all.0[0].debug, ObjectUseInfo::Ok); + } +} diff --git a/tests/integration/test_minidump.py b/tests/integration/test_minidump.py index f6b73ce01..737997db3 100644 --- a/tests/integration/test_minidump.py +++ b/tests/integration/test_minidump.py @@ -452,6 +452,7 @@ }, { "debug": {"status": "ok"}, + "unwind": {"status": "ok"}, "download": { "features": { "has_debug_info": True, @@ -618,6 +619,7 @@ }, { "debug": {"status": "ok"}, + "unwind": {"status": "ok"}, "download": { "features": { "has_debug_info": True, @@ -704,6 +706,7 @@ "status": "ok", }, "debug": {"status": "ok"}, + "unwind": {"status": "ok"}, }, ], }, @@ -836,6 +839,7 @@ "status": "ok", }, "debug": {"status": "ok"}, + "unwind": {"status": "ok"}, }, ], },