Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: codesign for deno compile binaries #24604

Merged
merged 3 commits into from
Aug 1, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 34 additions & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions cli/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,7 @@ deno_semver = "=0.5.7"
deno_task_shell = "=0.17.0"
deno_terminal.workspace = true
eszip = "=0.73.0"
libsui = "0.1.0"
napi_sym.workspace = true
node_resolver.workspace = true

Expand Down
4 changes: 1 addition & 3 deletions cli/mainrt.rs
Original file line number Diff line number Diff line change
Expand Up @@ -82,9 +82,7 @@ fn load_env_vars(env_vars: &HashMap<String, String>) {

fn main() {
let args: Vec<_> = env::args_os().collect();
let current_exe_path = current_exe().unwrap();
let standalone =
standalone::extract_standalone(&current_exe_path, Cow::Owned(args));
let standalone = standalone::extract_standalone(Cow::Owned(args));
let future = async move {
match standalone {
Ok(Some(future)) => {
Expand Down
108 changes: 54 additions & 54 deletions cli/standalone/binary.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ use std::collections::VecDeque;
use std::env::current_exe;
use std::ffi::OsString;
use std::fs;
use std::fs::File;
use std::future::Future;
use std::io::Read;
use std::io::Seek;
Expand Down Expand Up @@ -106,16 +107,19 @@ pub struct Metadata {
}

pub fn load_npm_vfs(root_dir_path: PathBuf) -> Result<FileBackedVfs, AnyError> {
let file_path = current_exe().unwrap();
let mut file = std::fs::File::open(file_path)?;
file.seek(SeekFrom::End(-(TRAILER_SIZE as i64)))?;
let mut trailer = [0; TRAILER_SIZE];
file.read_exact(&mut trailer)?;
let trailer = Trailer::parse(&trailer)?.unwrap();
file.seek(SeekFrom::Start(trailer.npm_vfs_pos))?;
let mut vfs_data = vec![0; trailer.npm_vfs_len() as usize];
file.read_exact(&mut vfs_data)?;
let mut dir: VirtualDirectory = serde_json::from_slice(&vfs_data)?;
let data = libsui::find_section("d3n0l4nd").unwrap();

// We do the first part sync so it can complete quickly
let trailer: [u8; TRAILER_SIZE] = data[0..TRAILER_SIZE].try_into().unwrap();
let trailer = match Trailer::parse(&trailer)? {
None => panic!("Could not find trailer"),
Some(trailer) => trailer,
};
let data = &data[TRAILER_SIZE..];

let vfs_data =
&data[trailer.npm_vfs_pos as usize..trailer.npm_files_pos as usize];
let mut dir: VirtualDirectory = serde_json::from_slice(vfs_data)?;

// align the name of the directory with the root dir
dir.name = root_dir_path
Expand All @@ -129,66 +133,69 @@ pub fn load_npm_vfs(root_dir_path: PathBuf) -> Result<FileBackedVfs, AnyError> {
root_path: root_dir_path,
start_file_offset: trailer.npm_files_pos,
};
Ok(FileBackedVfs::new(file, fs_root))
Ok(FileBackedVfs::new(data.to_vec(), fs_root))
Copy link
Member Author

@littledivy littledivy Jul 17, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

For future: FileBackedVfs can be made a zero-copy view on data because sui returns a slice :)

}

fn write_binary_bytes(
writer: &mut impl Write,
mut file_writer: File,
original_bin: Vec<u8>,
metadata: &Metadata,
eszip: eszip::EszipV2,
npm_vfs: Option<&VirtualDirectory>,
npm_files: &Vec<Vec<u8>>,
compile_flags: &CompileFlags,
) -> Result<(), AnyError> {
let metadata = serde_json::to_string(metadata)?.as_bytes().to_vec();
let npm_vfs = serde_json::to_string(&npm_vfs)?.as_bytes().to_vec();
let eszip_archive = eszip.into_bytes();

writer.write_all(&original_bin)?;
writer.write_all(&eszip_archive)?;
writer.write_all(&metadata)?;
writer.write_all(&npm_vfs)?;
for file in npm_files {
writer.write_all(file)?;
}
let mut writer = Vec::new();

// write the trailer, which includes the positions
// of the data blocks in the file
writer.write_all(&{
let eszip_pos = original_bin.len() as u64;
let metadata_pos = eszip_pos + (eszip_archive.len() as u64);
let metadata_pos = eszip_archive.len() as u64;
let npm_vfs_pos = metadata_pos + (metadata.len() as u64);
let npm_files_pos = npm_vfs_pos + (npm_vfs.len() as u64);
Trailer {
eszip_pos,
eszip_pos: 0,
metadata_pos,
npm_vfs_pos,
npm_files_pos,
}
.as_bytes()
})?;

writer.write_all(&eszip_archive)?;
writer.write_all(&metadata)?;
writer.write_all(&npm_vfs)?;
for file in npm_files {
writer.write_all(file)?;
}

let target = compile_flags.resolve_target();
if target.contains("linux") {
libsui::Elf::new(&original_bin).append(&writer, &mut file_writer)?;
} else if target.contains("windows") {
libsui::PortableExecutable::from(&original_bin)?
.write_resource("d3n0l4nd", writer)?
.build(&mut file_writer)?;
} else if target.contains("darwin") {
libsui::Macho::from(original_bin)?
.write_section("d3n0l4nd", writer)?
.build(&mut file_writer)?;
}
Ok(())
}

pub fn is_standalone_binary(exe_path: &Path) -> bool {
let Ok(mut output_file) = std::fs::File::open(exe_path) else {
return false;
};
if output_file
.seek(SeekFrom::End(-(TRAILER_SIZE as i64)))
.is_err()
{
// This seek may fail because the file is too small to possibly be
// `deno compile` output.
return false;
}
let mut trailer = [0; TRAILER_SIZE];
if output_file.read_exact(&mut trailer).is_err() {
let Ok(data) = std::fs::read(exe_path) else {
return false;
};
let (magic_trailer, _) = trailer.split_at(8);
magic_trailer == MAGIC_TRAILER

libsui::utils::is_elf(&data)
| libsui::utils::is_pe(&data)
| libsui::utils::is_macho(&data)
}

/// This function will try to run this binary as a standalone binary
Expand All @@ -197,40 +204,32 @@ pub fn is_standalone_binary(exe_path: &Path) -> bool {
/// then checking for the magic trailer string `d3n0l4nd`. If found,
/// the bundle is executed. If not, this function exits with `Ok(None)`.
pub fn extract_standalone(
exe_path: &Path,
cli_args: Cow<Vec<OsString>>,
) -> Result<
Option<impl Future<Output = Result<(Metadata, eszip::EszipV2), AnyError>>>,
AnyError,
> {
let Some(data) = libsui::find_section("d3n0l4nd") else {
return Ok(None);
};

// We do the first part sync so it can complete quickly
let mut file = std::fs::File::open(exe_path)?;
file.seek(SeekFrom::End(-(TRAILER_SIZE as i64)))?;
let mut trailer = [0; TRAILER_SIZE];
file.read_exact(&mut trailer)?;
let trailer = match Trailer::parse(&trailer)? {
let trailer = match Trailer::parse(&data[0..TRAILER_SIZE])? {
None => return Ok(None),
Some(trailer) => trailer,
};

file.seek(SeekFrom::Start(trailer.eszip_pos))?;

let cli_args = cli_args.into_owned();
// If we have an eszip, read it out
Ok(Some(async move {
let bufreader =
deno_core::futures::io::BufReader::new(AllowStdIo::new(file));
deno_core::futures::io::BufReader::new(&data[TRAILER_SIZE..]);

let (eszip, loader) = eszip::EszipV2::parse(bufreader)
.await
.context("Failed to parse eszip header")?;

let mut bufreader =
loader.await.context("Failed to parse eszip archive")?;

bufreader
.seek(SeekFrom::Start(trailer.metadata_pos))
.await?;
let bufreader = loader.await.context("Failed to parse eszip archive")?;

let mut metadata = String::new();

Expand Down Expand Up @@ -405,7 +404,7 @@ impl<'a> DenoCompileBinaryWriter<'a> {

pub async fn write_bin(
&self,
writer: &mut impl Write,
writer: File,
eszip: eszip::EszipV2,
root_dir_url: EszipRelativeFileBaseUrl<'_>,
entrypoint: &ModuleSpecifier,
Expand Down Expand Up @@ -518,7 +517,7 @@ impl<'a> DenoCompileBinaryWriter<'a> {
#[allow(clippy::too_many_arguments)]
fn write_standalone_binary(
&self,
writer: &mut impl Write,
writer: File,
original_bin: Vec<u8>,
mut eszip: eszip::EszipV2,
root_dir_url: EszipRelativeFileBaseUrl<'_>,
Expand Down Expand Up @@ -654,6 +653,7 @@ impl<'a> DenoCompileBinaryWriter<'a> {
eszip,
npm_vfs.as_ref(),
&npm_files,
compile_flags,
)
}

Expand Down
25 changes: 16 additions & 9 deletions cli/standalone/virtual_fs.rs
Original file line number Diff line number Diff line change
Expand Up @@ -748,12 +748,12 @@ impl deno_io::fs::File for FileBackedVfsFile {

#[derive(Debug)]
pub struct FileBackedVfs {
file: Mutex<File>,
file: Mutex<Vec<u8>>,
fs_root: VfsRoot,
}

impl FileBackedVfs {
pub fn new(file: File, fs_root: VfsRoot) -> Self {
pub fn new(file: Vec<u8>, fs_root: VfsRoot) -> Self {
Self {
file: Mutex::new(file),
fs_root,
Expand Down Expand Up @@ -836,11 +836,18 @@ impl FileBackedVfs {
pos: u64,
buf: &mut [u8],
) -> std::io::Result<usize> {
let mut fs_file = self.file.lock();
fs_file.seek(SeekFrom::Start(
self.fs_root.start_file_offset + file.offset + pos,
))?;
fs_file.read(buf)
let data = self.file.lock();
let start = self.fs_root.start_file_offset + file.offset + pos;
let end = start + buf.len() as u64;
if end > data.len() as u64 {
return Err(std::io::Error::new(
std::io::ErrorKind::UnexpectedEof,
"unexpected EOF",
));
}

buf.copy_from_slice(&data[start as usize..end as usize]);
Ok(buf.len())
}

pub fn dir_entry(&self, path: &Path) -> std::io::Result<&VirtualDirectory> {
Expand Down Expand Up @@ -1016,12 +1023,12 @@ mod test {
file.write_all(file_data).unwrap();
}
}
let file = std::fs::File::open(&virtual_fs_file).unwrap();
let dest_path = temp_dir.path().join("dest");
let data = std::fs::read(&virtual_fs_file).unwrap();
(
dest_path.to_path_buf(),
FileBackedVfs::new(
file,
data,
VfsRoot {
dir: root_dir,
root_path: dest_path.to_path_buf(),
Expand Down
6 changes: 3 additions & 3 deletions cli/tools/compile.rs
Original file line number Diff line number Diff line change
Expand Up @@ -124,12 +124,13 @@ pub async fn compile(
));
let temp_path = output_path.with_file_name(temp_filename);

let mut file = std::fs::File::create(&temp_path).with_context(|| {
let file = std::fs::File::create(&temp_path).with_context(|| {
format!("Opening temporary file '{}'", temp_path.display())
})?;

let write_result = binary_writer
.write_bin(
&mut file,
file,
eszip,
root_dir_url,
&module_specifier,
Expand All @@ -140,7 +141,6 @@ pub async fn compile(
.with_context(|| {
format!("Writing temporary file '{}'", temp_path.display())
});
drop(file);

// set it as executable
#[cfg(unix)]
Expand Down
Loading