Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

internal: VFS no longer stores all source files in memory #16307

Merged
merged 1 commit into from
Jan 8, 2024
Merged

internal: VFS no longer stores all source files in memory #16307

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 2 additions & 3 deletions crates/load-cargo/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -331,9 +331,8 @@ fn load_crate_graph(
}
let changes = vfs.take_changes();
for file in changes {
if file.exists() {
let contents = vfs.file_contents(file.file_id);
if let Ok(text) = std::str::from_utf8(contents) {
if let vfs::Change::Create(v) | vfs::Change::Modify(v) = file.change {
if let Ok(text) = std::str::from_utf8(&v) {
analysis_change.change_file(file.file_id, Some(text.into()))
}
}
Expand Down
83 changes: 37 additions & 46 deletions crates/rust-analyzer/src/global_state.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
//!
//! Each tick provides an immutable snapshot of the state as `WorldSnapshot`.

use std::time::Instant;
use std::{collections::hash_map::Entry, time::Instant};

use crossbeam_channel::{unbounded, Receiver, Sender};
use flycheck::FlycheckHandle;
Expand All @@ -21,7 +21,7 @@ use proc_macro_api::ProcMacroServer;
use project_model::{CargoWorkspace, ProjectWorkspace, Target, WorkspaceBuildScripts};
use rustc_hash::{FxHashMap, FxHashSet};
use triomphe::Arc;
use vfs::{AnchoredPathBuf, Vfs};
use vfs::{AnchoredPathBuf, ChangedFile, Vfs};

use crate::{
config::{Config, ConfigError},
Expand Down Expand Up @@ -217,8 +217,8 @@ impl GlobalState {
pub(crate) fn process_changes(&mut self) -> bool {
let _p = profile::span("GlobalState::process_changes");

let mut file_changes = FxHashMap::default();
let (change, changed_files, workspace_structure_change) = {
let mut file_changes = FxHashMap::<_, (bool, ChangedFile)>::default();
let (change, modified_files, workspace_structure_change) = {
let mut change = Change::new();
let mut guard = self.vfs.write();
let changed_files = guard.0.take_changes();
Expand All @@ -233,64 +233,63 @@ impl GlobalState {
// id that is followed by a delete we actually skip observing the file text from the
// earlier event, to avoid problems later on.
for changed_file in changed_files {
use vfs::ChangeKind::*;

file_changes
.entry(changed_file.file_id)
.and_modify(|(change, just_created)| {
// None -> Delete => keep
// Create -> Delete => collapse
//
match (change, just_created, changed_file.change_kind) {
use vfs::Change::*;
match file_changes.entry(changed_file.file_id) {
Entry::Occupied(mut o) => {
let (just_created, change) = o.get_mut();
match (&mut change.change, just_created, changed_file.change) {
// latter `Delete` wins
(change, _, Delete) => *change = Delete,
// merge `Create` with `Create` or `Modify`
(Create, _, Create | Modify) => {}
(Create(prev), _, Create(new) | Modify(new)) => *prev = new,
// collapse identical `Modify`es
(Modify, _, Modify) => {}
(Modify(prev), _, Modify(new)) => *prev = new,
// equivalent to `Modify`
(change @ Delete, just_created, Create) => {
*change = Modify;
(change @ Delete, just_created, Create(new)) => {
*change = Modify(new);
*just_created = true;
}
// shouldn't occur, but collapse into `Create`
(change @ Delete, just_created, Modify) => {
*change = Create;
(change @ Delete, just_created, Modify(new)) => {
*change = Create(new);
*just_created = true;
}
// shouldn't occur, but collapse into `Modify`
(Modify, _, Create) => {}
(Modify(prev), _, Create(new)) => *prev = new,
}
})
.or_insert((
changed_file.change_kind,
matches!(changed_file.change_kind, Create),
));
}
Entry::Vacant(v) => {
_ = v.insert((matches!(&changed_file.change, Create(_)), changed_file))
}
}
}

let changed_files: Vec<_> = file_changes
.into_iter()
.filter(|(_, (change_kind, just_created))| {
!matches!((change_kind, just_created), (vfs::ChangeKind::Delete, true))
.filter(|(_, (just_created, change))| {
!(*just_created && matches!(change.change, vfs::Change::Delete))
})
.map(|(file_id, (change_kind, _))| vfs::ChangedFile { file_id, change_kind })
.map(|(file_id, (_, change))| vfs::ChangedFile { file_id, ..change })
.collect();

let mut workspace_structure_change = None;
// A file was added or deleted
let mut has_structure_changes = false;
let mut bytes = vec![];
for file in &changed_files {
let mut modified_files = vec![];
for file in changed_files {
let vfs_path = &vfs.file_path(file.file_id);
if let Some(path) = vfs_path.as_path() {
let path = path.to_path_buf();
if reload::should_refresh_for_change(&path, file.change_kind) {
if reload::should_refresh_for_change(&path, file.kind()) {
workspace_structure_change = Some((path.clone(), false));
}
if file.is_created_or_deleted() {
has_structure_changes = true;
workspace_structure_change =
Some((path, self.crate_graph_file_dependencies.contains(vfs_path)));
} else {
modified_files.push(file.file_id);
}
}

Expand All @@ -299,10 +298,8 @@ impl GlobalState {
self.diagnostics.clear_native_for(file.file_id);
}

let text = if file.exists() {
let bytes = vfs.file_contents(file.file_id).to_vec();

String::from_utf8(bytes).ok().and_then(|text| {
let text = if let vfs::Change::Create(v) | vfs::Change::Modify(v) = file.change {
String::from_utf8(v).ok().and_then(|text| {
// FIXME: Consider doing normalization in the `vfs` instead? That allows
// getting rid of some locking
let (text, line_endings) = LineEndings::normalize(text);
Expand All @@ -327,11 +324,10 @@ impl GlobalState {
let roots = self.source_root_config.partition(vfs);
change.set_roots(roots);
}
(change, changed_files, workspace_structure_change)
(change, modified_files, workspace_structure_change)
};

self.analysis_host.apply_change(change);

{
let raw_database = self.analysis_host.raw_database();
// FIXME: ideally we should only trigger a workspace fetch for non-library changes
Expand All @@ -343,13 +339,12 @@ impl GlobalState {
force_crate_graph_reload,
);
}
self.proc_macro_changed =
changed_files.iter().filter(|file| !file.is_created_or_deleted()).any(|file| {
let crates = raw_database.relevant_crates(file.file_id);
let crate_graph = raw_database.crate_graph();
self.proc_macro_changed = modified_files.into_iter().any(|file_id| {
let crates = raw_database.relevant_crates(file_id);
let crate_graph = raw_database.crate_graph();

crates.iter().any(|&krate| crate_graph[krate].is_proc_macro)
});
crates.iter().any(|&krate| crate_graph[krate].is_proc_macro)
});
}

true
Expand Down Expand Up @@ -494,10 +489,6 @@ impl GlobalStateSnapshot {
})
}

pub(crate) fn vfs_memory_usage(&self) -> usize {
self.vfs_read().memory_usage()
}

pub(crate) fn file_exists(&self, file_id: FileId) -> bool {
self.vfs.read().0.exists(file_id)
}
Expand Down
25 changes: 15 additions & 10 deletions crates/rust-analyzer/src/handlers/notification.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,13 @@ pub(crate) fn handle_did_open_text_document(
if let Ok(path) = from_proto::vfs_path(&params.text_document.uri) {
let already_exists = state
.mem_docs
.insert(path.clone(), DocumentData::new(params.text_document.version))
.insert(
path.clone(),
DocumentData::new(
params.text_document.version,
params.text_document.text.clone().into_bytes(),
),
)
.is_err();
if already_exists {
tracing::error!("duplicate DidOpenTextDocument: {}", path);
Expand All @@ -76,28 +82,27 @@ pub(crate) fn handle_did_change_text_document(
let _p = profile::span("handle_did_change_text_document");

if let Ok(path) = from_proto::vfs_path(&params.text_document.uri) {
match state.mem_docs.get_mut(&path) {
let data = match state.mem_docs.get_mut(&path) {
Some(doc) => {
// The version passed in DidChangeTextDocument is the version after all edits are applied
// so we should apply it before the vfs is notified.
doc.version = params.text_document.version;
&mut doc.data
}
None => {
tracing::error!("unexpected DidChangeTextDocument: {}", path);
return Ok(());
}
};

let text = apply_document_changes(
let new_contents = apply_document_changes(
state.config.position_encoding(),
|| {
let vfs = &state.vfs.read().0;
let file_id = vfs.file_id(&path).unwrap();
std::str::from_utf8(vfs.file_contents(file_id)).unwrap().into()
},
std::str::from_utf8(data).unwrap(),
params.content_changes,
);
state.vfs.write().0.set_file_contents(path, Some(text.into_bytes()));
)
.into_bytes();
*data = new_contents.clone();
state.vfs.write().0.set_file_contents(path, Some(new_contents));
}
Ok(())
}
Expand Down
1 change: 0 additions & 1 deletion crates/rust-analyzer/src/handlers/request.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,6 @@ pub(crate) fn handle_analyzer_status(
.collect::<Vec<&AbsPath>>()
);
}
format_to!(buf, "\nVfs memory usage: {}\n", profile::Bytes::new(snap.vfs_memory_usage() as _));
buf.push_str("\nAnalysis:\n");
buf.push_str(
&snap
Expand Down
35 changes: 16 additions & 19 deletions crates/rust-analyzer/src/lsp/utils.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -168,7 +168,7 @@ impl GlobalState {

pub(crate) fn apply_document_changes(
encoding: PositionEncoding,
file_contents: impl FnOnce() -> String,
file_contents: &str,
mut content_changes: Vec<lsp_types::TextDocumentContentChangeEvent>,
) -> String {
// If at least one of the changes is a full document change, use the last
Expand All @@ -179,7 +179,7 @@ pub(crate) fn apply_document_changes(
let text = mem::take(&mut content_changes[idx].text);
(text, &content_changes[idx + 1..])
}
None => (file_contents(), &content_changes[..]),
None => (file_contents.to_owned(), &content_changes[..]),
};
if content_changes.is_empty() {
return text;
Expand Down Expand Up @@ -276,61 +276,58 @@ mod tests {
}

let encoding = PositionEncoding::Wide(WideEncoding::Utf16);
let text = apply_document_changes(encoding, || String::new(), vec![]);
let text = apply_document_changes(encoding, "", vec![]);
assert_eq!(text, "");
let text = apply_document_changes(
encoding,
|| text,
&text,
vec![TextDocumentContentChangeEvent {
range: None,
range_length: None,
text: String::from("the"),
}],
);
assert_eq!(text, "the");
let text = apply_document_changes(encoding, || text, c![0, 3; 0, 3 => " quick"]);
let text = apply_document_changes(encoding, &text, c![0, 3; 0, 3 => " quick"]);
assert_eq!(text, "the quick");
let text =
apply_document_changes(encoding, || text, c![0, 0; 0, 4 => "", 0, 5; 0, 5 => " foxes"]);
apply_document_changes(encoding, &text, c![0, 0; 0, 4 => "", 0, 5; 0, 5 => " foxes"]);
assert_eq!(text, "quick foxes");
let text = apply_document_changes(encoding, || text, c![0, 11; 0, 11 => "\ndream"]);
let text = apply_document_changes(encoding, &text, c![0, 11; 0, 11 => "\ndream"]);
assert_eq!(text, "quick foxes\ndream");
let text = apply_document_changes(encoding, || text, c![1, 0; 1, 0 => "have "]);
let text = apply_document_changes(encoding, &text, c![1, 0; 1, 0 => "have "]);
assert_eq!(text, "quick foxes\nhave dream");
let text = apply_document_changes(
encoding,
|| text,
&text,
c![0, 0; 0, 0 => "the ", 1, 4; 1, 4 => " quiet", 1, 16; 1, 16 => "s\n"],
);
assert_eq!(text, "the quick foxes\nhave quiet dreams\n");
let text = apply_document_changes(
encoding,
|| text,
c![0, 15; 0, 15 => "\n", 2, 17; 2, 17 => "\n"],
);
let text =
apply_document_changes(encoding, &text, c![0, 15; 0, 15 => "\n", 2, 17; 2, 17 => "\n"]);
assert_eq!(text, "the quick foxes\n\nhave quiet dreams\n\n");
let text = apply_document_changes(
encoding,
|| text,
&text,
c![1, 0; 1, 0 => "DREAM", 2, 0; 2, 0 => "they ", 3, 0; 3, 0 => "DON'T THEY?"],
);
assert_eq!(text, "the quick foxes\nDREAM\nthey have quiet dreams\nDON'T THEY?\n");
let text =
apply_document_changes(encoding, || text, c![0, 10; 1, 5 => "", 2, 0; 2, 12 => ""]);
apply_document_changes(encoding, &text, c![0, 10; 1, 5 => "", 2, 0; 2, 12 => ""]);
assert_eq!(text, "the quick \nthey have quiet dreams\n");

let text = String::from("❤️");
let text = apply_document_changes(encoding, || text, c![0, 0; 0, 0 => "a"]);
let text = apply_document_changes(encoding, &text, c![0, 0; 0, 0 => "a"]);
assert_eq!(text, "a❤️");

let text = String::from("a\nb");
let text =
apply_document_changes(encoding, || text, c![0, 1; 1, 0 => "\nțc", 0, 1; 1, 1 => "d"]);
apply_document_changes(encoding, &text, c![0, 1; 1, 0 => "\nțc", 0, 1; 1, 1 => "d"]);
assert_eq!(text, "adcb");

let text = String::from("a\nb");
let text =
apply_document_changes(encoding, || text, c![0, 1; 1, 0 => "ț\nc", 0, 2; 0, 2 => "c"]);
apply_document_changes(encoding, &text, c![0, 1; 1, 0 => "ț\nc", 0, 2; 0, 2 => "c"]);
assert_eq!(text, "ațc\ncb");
}

Expand Down
2 changes: 2 additions & 0 deletions crates/rust-analyzer/src/main_loop.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -576,6 +576,8 @@ impl GlobalState {
let vfs = &mut self.vfs.write().0;
for (path, contents) in files {
let path = VfsPath::from(path);
// if the file is in mem docs, it's managed by the client via notifications
// so only set it if its not in there
if !self.mem_docs.contains(&path) {
vfs.set_file_contents(path, contents);
}
Expand Down
5 changes: 3 additions & 2 deletions crates/rust-analyzer/src/mem_docs.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,10 +62,11 @@ impl MemDocs {
#[derive(Debug, Clone)]
pub(crate) struct DocumentData {
pub(crate) version: i32,
pub(crate) data: Vec<u8>,
}

impl DocumentData {
pub(crate) fn new(version: i32) -> Self {
DocumentData { version }
pub(crate) fn new(version: i32, data: Vec<u8>) -> Self {
DocumentData { version, data }
}
}
7 changes: 3 additions & 4 deletions crates/rust-analyzer/src/reload.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -503,10 +503,9 @@ impl GlobalState {
match vfs.file_id(&vfs_path) {
Some(file_id) => Some(file_id),
None => {
if !self.mem_docs.contains(&vfs_path) {
let contents = loader.handle.load_sync(path);
vfs.set_file_contents(vfs_path.clone(), contents);
}
// FIXME: Consider not loading this here?
let contents = loader.handle.load_sync(path);
vfs.set_file_contents(vfs_path.clone(), contents);
vfs.file_id(&vfs_path)
}
}
Expand Down
Loading