refactor (#266)

git-odb/src/store/general/find.rs

@@ -0,0 +1,38 @@
+use git_hash::oid;
+use git_object::Data;
+use git_pack::cache::DecodeEntry;
+use git_pack::data::entry::Location;
+use git_pack::index::Entry;
+use std::ops::Deref;
+
+impl<S> crate::pack::Find for super::Handle<S>
+where
+    S: Deref<Target = super::Store> + Clone,
+{
+    type Error = crate::compound::find::Error;
+
+    fn contains(&self, id: impl AsRef<oid>) -> bool {
+        todo!("contains")
+    }
+
+    fn try_find_cached<'a>(
+        &self,
+        id: impl AsRef<oid>,
+        buffer: &'a mut Vec<u8>,
+        pack_cache: &mut impl DecodeEntry,
+    ) -> Result<Option<(Data<'a>, Option<Location>)>, Self::Error> {
+        todo!("try find cached")
+    }
+
+    fn location_by_oid(&self, id: impl AsRef<oid>, buf: &mut Vec<u8>) -> Option<Location> {
+        todo!("location by oid")
+    }
+
+    fn index_iter_by_pack_id(&self, pack_id: u32) -> Option<Box<dyn Iterator<Item = Entry> + '_>> {
+        todo!("index iter by pack id")
+    }
+
+    fn entry_by_location(&self, location: &Location) -> Option<git_pack::find::Entry<'_>> {
+        todo!("entry by location")
+    }
+}

git-odb/src/store/general/handle.rs

@@ -0,0 +1,156 @@
+use crate::general::store;
+use git_features::threading::OwnShared;
+use std::ops::Deref;
+use std::sync::atomic::Ordering;
+use std::sync::Arc;
+
+pub(crate) mod multi_index {
+    // TODO: replace this one with an actual implementation of a multi-pack index.
+    pub type File = ();
+}
+
+pub enum SingleOrMultiIndex {
+    Single {
+        index: Arc<git_pack::index::File>,
+        data: Option<Arc<git_pack::data::File>>,
+    },
+    Multi {
+        index: Arc<multi_index::File>,
+        data: Vec<Option<Arc<git_pack::data::File>>>,
+    },
+}
+
+pub struct IndexLookup {
+    pub(crate) file: SingleOrMultiIndex,
+    pub(crate) id: store::IndexId,
+}
+
+pub struct IndexForObjectInPack {
+    /// The internal identifier of the pack itself, which either is referred to by an index or a multi-pack index.
+    pack_id: store::PackId,
+    /// The index of the object within the pack
+    object_index_in_pack: u32,
+}
+
+pub(crate) mod index_lookup {
+    use crate::general::{handle, store};
+    use git_hash::oid;
+    use std::sync::Arc;
+
+    impl handle::IndexLookup {
+        /// See if the oid is contained in this index, and return its full id for lookup possibly alongside its data file if already
+        /// loaded.
+        /// If it is not loaded, ask it to be loaded and put it into the returned mutable option for safe-keeping.
+        fn lookup(
+            &mut self,
+            object_id: &oid,
+        ) -> Option<(handle::IndexForObjectInPack, &mut Option<Arc<git_pack::data::File>>)> {
+            let id = self.id;
+            match &mut self.file {
+                handle::SingleOrMultiIndex::Single { index, data } => {
+                    index.lookup(object_id).map(|object_index_in_pack| {
+                        (
+                            handle::IndexForObjectInPack {
+                                pack_id: store::PackId {
+                                    index: id,
+                                    multipack_index: None,
+                                },
+                                object_index_in_pack,
+                            },
+                            data,
+                        )
+                    })
+                }
+                handle::SingleOrMultiIndex::Multi { index, data } => {
+                    todo!("find respective pack and return it as &mut Option<>")
+                }
+            }
+        }
+    }
+}
+
+pub(crate) enum Mode {
+    DeletedPacksAreInaccessible,
+    /// This mode signals that we should not unload packs even after they went missing.
+    KeepDeletedPacksAvailable,
+}
+
+/// Handle registration
+impl super::Store {
+    pub(crate) fn register_handle(&self) -> Mode {
+        self.num_handles_unstable.fetch_add(1, Ordering::Relaxed);
+        Mode::DeletedPacksAreInaccessible
+    }
+    pub(crate) fn remove_handle(&self, mode: Mode) {
+        match mode {
+            Mode::KeepDeletedPacksAvailable => {
+                let _lock = self.path.lock();
+                self.num_handles_stable.fetch_sub(1, Ordering::SeqCst)
+            }
+            Mode::DeletedPacksAreInaccessible => self.num_handles_unstable.fetch_sub(1, Ordering::Relaxed),
+        };
+    }
+    pub(crate) fn upgrade_handle(&self, mode: Mode) -> Mode {
+        if let Mode::DeletedPacksAreInaccessible = mode {
+            let _lock = self.path.lock();
+            self.num_handles_stable.fetch_add(1, Ordering::SeqCst);
+            self.num_handles_unstable.fetch_sub(1, Ordering::SeqCst);
+        }
+        Mode::KeepDeletedPacksAvailable
+    }
+}
+
+/// Handle creation
+impl super::Store {
+    pub fn to_handle(
+        self: &OwnShared<Self>,
+        refresh_mode: crate::RefreshMode,
+    ) -> super::Handle<OwnShared<super::Store>> {
+        let token = self.register_handle();
+        super::Handle {
+            store: self.clone(),
+            refresh_mode,
+            token: Some(token),
+        }
+    }
+}
+
+impl<S> super::Handle<S>
+where
+    S: Deref<Target = super::Store> + Clone,
+{
+    /// Call once if pack ids are stored and later used for lookup, meaning they should always remain mapped and not be unloaded
+    /// even if they disappear from disk.
+    /// This must be called if there is a chance that git maintenance is happening while a pack is created.
+    pub fn prevent_pack_unload(&mut self) {
+        self.token = self.token.take().map(|token| self.store.upgrade_handle(token));
+    }
+
+    pub fn store(&self) -> &S::Target {
+        &*self.store
+    }
+}
+
+impl<S> Drop for super::Handle<S>
+where
+    S: Deref<Target = super::Store> + Clone,
+{
+    fn drop(&mut self) {
+        if let Some(token) = self.token.take() {
+            self.store.remove_handle(token)
+        }
+    }
+}
+
+impl<S> Clone for super::Handle<S>
+where
+    S: Deref<Target = super::Store> + Clone,
+{
+    fn clone(&self) -> Self {
+        super::Handle {
+            store: self.store.clone(),
+            refresh_mode: self.refresh_mode,
+            token: self.store.register_handle().into(),
+        }
+    }
+}

git-odb/src/store/general/init.rs

@@ -0,0 +1,29 @@
+use crate::general::store;
+use crate::general::store::{MutableIndexAndPack, SlotMapIndex};
+use arc_swap::ArcSwap;
+use git_features::threading::OwnShared;
+use std::iter::FromIterator;
+use std::ops::Deref;
+use std::path::PathBuf;
+use std::sync::atomic::AtomicUsize;
+use std::sync::Arc;
+
+impl super::Store {
+    pub fn at(objects_dir: impl Into<PathBuf>) -> std::io::Result<Self> {
+        let objects_dir = objects_dir.into();
+        if !objects_dir.is_dir() {
+            return Err(std::io::Error::new(
+                std::io::ErrorKind::Other, // TODO: use NotADirectory when stabilized
+                format!("'{}' wasn't a directory", objects_dir.display()),
+            ));
+        }
+        Ok(super::Store {
+            path: parking_lot::Mutex::new(objects_dir),
+            files: Vec::from_iter(std::iter::repeat_with(MutableIndexAndPack::default).take(256)), // TODO: figure this out from the amount of files currently present
+            index: ArcSwap::new(Arc::new(SlotMapIndex::default())),
+            num_handles_stable: Default::default(),
+            num_handles_unstable: Default::default(),
+            num_disk_state_consolidation: Default::default(),
+        })
+    }
+}

git-odb/src/store/general/load_indices.rs

@@ -0,0 +1,123 @@
+use crate::general::{handle, store};
+use std::path::PathBuf;
+
+use crate::general::store::StateId;
+use crate::RefreshMode;
+use std::sync::atomic::Ordering;
+use std::sync::Arc;
+
+pub(crate) enum Outcome {
+    /// Drop all data and fully replace it with `indices`.
+    /// This happens if we have witnessed a generational change invalidating all of our ids and causing currently loaded
+    /// indices and maps to be dropped.
+    Replace {
+        indices: Vec<handle::IndexLookup>, // should probably be SmallVec to get around most allocations
+        loose_dbs: Arc<Vec<crate::loose::Store>>,
+        marker: store::SlotIndexMarker, // use to show where the caller left off last time
+    },
+    /// Despite all values being full copies, indices are still compatible to what was before. This also means
+    /// the caller can continue searching the added indices and loose-dbs.
+    /// Or in other words, new indices were only added to the known list, and what was seen before is known not to have changed.
+    /// Besides that, the full internal state can be replaced as with `Replace`.
+    ReplaceStable {
+        indices: Vec<handle::IndexLookup>, // should probably be SmallVec to get around most allocations
+        loose_dbs: Arc<Vec<crate::loose::Store>>,
+        marker: store::SlotIndexMarker, // use to show where the caller left off last time
+    },
+    /// No new indices to look at, caller should give up
+    NoMoreIndices,
+}
+
+impl super::Store {
+    pub(crate) fn load_next_indices(
+        &self,
+        refresh_mode: RefreshMode,
+        marker: Option<store::SlotIndexMarker>,
+    ) -> std::io::Result<Outcome> {
+        let index = self.index.load();
+        let state_id = index.state_id();
+        if index.loose_dbs.is_empty() {
+            // TODO: figure out what kind of refreshes we need. This one loads in the initial slot map, but I think this cost is paid
+            //       in full during instantiation.
+            return self.consolidate_with_disk_state(state_id);
+        }
+
+        Ok(match marker {
+            Some(marker) => {
+                if marker.generation != index.generation {
+                    self.collect_replace_outcome(false /*stable*/)
+                } else if marker.state_id == state_id {
+                    // Nothing changed in the mean time, try to load another index…
+
+                    // …and if that didn't yield anything new consider refreshing our disk state.
+                    match refresh_mode {
+                        RefreshMode::Never => Outcome::NoMoreIndices,
+                        RefreshMode::AfterAllIndicesLoaded => return self.consolidate_with_disk_state(state_id),
+                    }
+                } else {
+                    self.collect_replace_outcome(true /*stable*/)
+                }
+            }
+            None => self.collect_replace_outcome(false /*stable*/),
+        })
+    }
+
+    /// refresh and possibly clear out our existing data structures, causing all pack ids to be invalidated.
+    fn consolidate_with_disk_state(&self, seen: StateId) -> std::io::Result<Outcome> {
+        let objects_directory = self.path.lock();
+        if seen != self.index.load().state_id() {
+            todo!("return …")
+        }
+        self.num_disk_state_consolidation.fetch_add(1, Ordering::Relaxed);
+        let mut db_paths = crate::alternate::resolve(&*objects_directory)
+            .map_err(|err| std::io::Error::new(std::io::ErrorKind::Other, err))?;
+        // These are in addition to our objects directory
+        db_paths.insert(0, objects_directory.clone());
+        todo!()
+    }
+
+    /// If there is no handle with stable pack ids requirements, unload them.
+    /// This property also relates to us pruning our internal state/doing internal maintenance which affects ids, too.
+    ///
+    /// Note that this must be called with a lock to the relevant state held to assure these values don't change while
+    /// we are working.
+    fn may_unload_packs(&mut self, guard: &parking_lot::MutexGuard<'_, PathBuf>) -> bool {
+        self.num_handles_stable.load(Ordering::SeqCst) == 0
+    }
+
+    fn collect_replace_outcome(&self, is_stable: bool) -> Outcome {
+        let index = self.index.load();
+        let indices = index
+            .slot_indices
+            .iter()
+            .map(|idx| (*idx, &self.files[*idx]))
+            .filter_map(|(id, file)| {
+                let lookup = match (&**file.files.load()).as_ref()? {
+                    store::IndexAndPacks::Index(bundle) => handle::SingleOrMultiIndex::Single {
+                        index: bundle.index.loaded()?.clone(),
+                        data: bundle.data.loaded().cloned(),
+                    },
+                    store::IndexAndPacks::MultiIndex(multi) => handle::SingleOrMultiIndex::Multi {
+                        index: multi.multi_index.loaded()?.clone(),
+                        data: multi.data.iter().map(|f| f.loaded().cloned()).collect(),
+                    },
+                };
+                handle::IndexLookup { file: lookup, id }.into()
+            })
+            .collect();
+
+        if is_stable {
+            Outcome::ReplaceStable {
+                indices,
+                loose_dbs: Arc::clone(&index.loose_dbs),
+                marker: index.marker(),
+            }
+        } else {
+            Outcome::Replace {
+                indices,
+                loose_dbs: Arc::clone(&index.loose_dbs),
+                marker: index.marker(),
+            }
+        }
+    }
+}