Skip to content

Commit

Permalink
Calling turbopack from the next build CLI (#46602)
Browse files Browse the repository at this point in the history
Close WEB-661
  • Loading branch information
Brooooooklyn authored Mar 21, 2023
1 parent 17e1cc7 commit 509ed00
Show file tree
Hide file tree
Showing 14 changed files with 223 additions and 14 deletions.
2 changes: 2 additions & 0 deletions .github/workflows/build_test_deploy.yml
Original file line number Diff line number Diff line change
Expand Up @@ -1498,6 +1498,8 @@ jobs:
with:
envs: DEBUG RUSTUP_HOME CARGO_HOME RUSTUP_IO_THREADS CARGO_PROFILE_RELEASE_LTO NAPI_CLI_VERSION RUST_TOOLCHAIN PNPM_VERSION VM_RELEASE
usesh: true
sync: rsync
copyback: false
mem: 6000
prepare: |
pkg install -y -f curl node libnghttp2
Expand Down
14 changes: 14 additions & 0 deletions packages/next-swc/Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions packages/next-swc/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ members = [
"crates/napi",
"crates/wasm",
"crates/next-binding",
"crates/next-build",
"crates/next-core",
"crates/next-dev",
"crates/next-dev-tests",
Expand Down
1 change: 1 addition & 0 deletions packages/next-swc/crates/napi/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ turbo-tasks = { workspace = true }
turbo-tasks-memory = { workspace = true }
next-binding = { path = "../next-binding", features = [
"__swc_core_binding_napi",
"__turbo_next_build",
"__turbo_next_dev_server",
"__turbo_node_file_trace",
"__feature_mdx_rs",
Expand Down
103 changes: 102 additions & 1 deletion packages/next-swc/crates/napi/src/turbopack.rs
Original file line number Diff line number Diff line change
@@ -1,9 +1,110 @@
use std::convert::TryFrom;

use crate::util::MapErr;
use napi::bindgen_prelude::*;
use next_binding::turbo::next_dev::{devserver_options::DevServerOptions, start_server};
use next_binding::turbo::{
next_build::{next_build as turbo_next_build, NextBuildOptions},
next_dev::{devserver_options::DevServerOptions, start_server},
};

#[napi]
pub async fn start_turbo_dev(options: Buffer) -> napi::Result<()> {
let options: DevServerOptions = serde_json::from_slice(&options)?;
start_server(&options).await.convert_err()
}

#[napi(object, object_to_js = false)]
#[derive(Debug)]
pub struct NextBuildContext {
pub dir: Option<String>,
pub app_dir: Option<String>,
pub pages_dir: Option<String>,
pub rewrites: Option<Rewrites>,
pub original_rewrites: Option<Rewrites>,
pub original_redirects: Option<Vec<Redirect>>,
}

#[napi(object, object_to_js = false)]
#[derive(Debug)]
pub struct Rewrites {
pub fallback: Vec<Rewrite>,
pub after_files: Vec<Rewrite>,
pub before_files: Vec<Rewrite>,
}

#[napi(object, object_to_js = false)]
#[derive(Debug)]
pub struct Rewrite {
pub source: String,
pub destination: String,
}

#[napi(object, object_to_js = false)]
#[derive(Debug)]
pub struct Redirect {
pub source: String,
pub destination: String,
pub permanent: Option<bool>,
pub status_code: Option<u32>,
pub has: Option<RouteHas>,
pub missing: Option<RouteHas>,
}

#[derive(Debug)]
pub struct RouteHas {
pub r#type: RouteType,
pub key: Option<String>,
pub value: Option<String>,
}

#[derive(Debug)]
pub enum RouteType {
Header,
Query,
Cookie,
Host,
}

impl TryFrom<String> for RouteType {
type Error = napi::Error;

fn try_from(value: String) -> Result<Self> {
match value.as_str() {
"header" => Ok(RouteType::Header),
"query" => Ok(RouteType::Query),
"cookie" => Ok(RouteType::Cookie),
"host" => Ok(RouteType::Host),
_ => Err(napi::Error::new(
napi::Status::InvalidArg,
"Invalid route type",
)),
}
}
}

impl FromNapiValue for RouteHas {
unsafe fn from_napi_value(env: sys::napi_env, napi_val: sys::napi_value) -> Result<Self> {
let object = Object::from_napi_value(env, napi_val)?;
let r#type = object.get_named_property::<String>("type")?;
Ok(RouteHas {
r#type: RouteType::try_from(r#type)?,
key: object.get("key")?,
value: object.get("value")?,
})
}
}

impl From<NextBuildContext> for NextBuildOptions {
fn from(value: NextBuildContext) -> Self {
Self {
dir: value.dir,
memory_limit: None,
full_stats: None,
}
}
}

#[napi]
pub async fn next_build(ctx: NextBuildContext) -> napi::Result<()> {
turbo_next_build(ctx.into()).await.convert_err()
}
4 changes: 4 additions & 0 deletions packages/next-swc/crates/next-binding/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,7 @@ __swc_core_binding_wasm_plugin = ["swc_core/plugin_transform_host_js"]
__swc_core_testing_transform = ["swc_core/testing_transform"]

__turbo = []
__turbo_next_build = ["__turbo", "next-build"]
__turbo_next_dev_server = ["__turbo", "next-dev/serializable"]
__turbo_node_file_trace = ["__turbo", "node-file-trace/node-api"]

Expand Down Expand Up @@ -102,6 +103,9 @@ __swc_testing = ["__swc", "testing"]
[dependencies]
mdxjs = { optional = true, workspace = true }
modularize_imports = { optional = true, workspace = true }
next-build = { optional = true, path = "../next-build", default-features = false, features = [
"custom_allocator",
] }
# TODO: Not sure what's going on, but using `workspace = true` noops `default-features = false`?
next-dev = { optional = true, path = "../next-dev", default-features = false, features = [
"custom_allocator",
Expand Down
2 changes: 2 additions & 0 deletions packages/next-swc/crates/next-binding/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@ pub mod swc {

#[cfg(feature = "__turbo")]
pub mod turbo {
#[cfg(feature = "__turbo_next_build")]
pub use next_build;
#[cfg(feature = "__turbo_next_dev_server")]
pub use next_dev;
#[cfg(feature = "__turbo_node_file_trace")]
Expand Down
27 changes: 27 additions & 0 deletions packages/next-swc/crates/next-build/Cargo.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
[package]
name = "next-build"
version = "0.1.0"
description = "TBD"
license = "MPL-2.0"
edition = "2021"
autobenches = false

[features]
next-font-local = ["next-core/next-font-local"]
native-tls = ["next-core/native-tls"]
rustls-tls = ["next-core/rustls-tls"]
custom_allocator = ["turbo-malloc/custom_allocator"]

[dependencies]
anyhow = "1.0.47"
next-core = { workspace = true }
turbo-malloc = { workspace = true, default-features = false }
turbo-tasks = { workspace = true }
turbo-tasks-memory = { workspace = true }

[build-dependencies]
turbo-tasks-build = { workspace = true }
vergen = { version = "7.3.2", default-features = false, features = [
"cargo",
"build",
] }
13 changes: 13 additions & 0 deletions packages/next-swc/crates/next-build/build.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
use turbo_tasks_build::{generate_register, rerun_if_glob};

use vergen::{vergen, Config};

fn main() {
generate_register();

rerun_if_glob("tests/integration/*/*", "tests/integration");

// Attempt to collect some build time env values but will skip if there are any
// errors.
let _ = vergen(Config::default());
}
33 changes: 33 additions & 0 deletions packages/next-swc/crates/next-build/src/lib.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
use turbo_tasks::{NothingVc, StatsType, TurboTasks, TurboTasksBackendApi};
use turbo_tasks_memory::MemoryBackend;

pub fn register() {
turbo_tasks::register();
include!(concat!(env!("OUT_DIR"), "/register.rs"));
}

pub struct NextBuildOptions {
pub dir: Option<String>,
pub memory_limit: Option<usize>,
pub full_stats: Option<bool>,
}

pub async fn next_build(options: NextBuildOptions) -> anyhow::Result<()> {
register();
let tt = TurboTasks::new(MemoryBackend::new(
options.memory_limit.map_or(usize::MAX, |l| l * 1024 * 1024),
));
let stats_type = match options.full_stats {
Some(true) => StatsType::Full,
_ => StatsType::Essential,
};
tt.set_stats_type(stats_type);
let task = tt.spawn_root_task(move || {
Box::pin(async move {
// run next build here
Ok(NothingVc::new().into())
})
});
tt.wait_task_completion(task, true).await?;
Ok(())
}
24 changes: 15 additions & 9 deletions packages/next/src/build/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -255,7 +255,8 @@ export default async function build(
debugOutput = false,
runLint = true,
noMangling = false,
appDirOnly = false
appDirOnly = false,
turboNextBuild = false
): Promise<void> {
try {
const nextBuildSpan = trace('next-build', undefined, {
Expand Down Expand Up @@ -1010,8 +1011,17 @@ export default async function build(
ignore: [] as string[],
}))

let binding = (await loadBindings()) as any

async function turbopackBuild() {
const turboNextBuildStart = process.hrtime()
await binding.turbo.nextBuild(NextBuildContext)
const [duration] = process.hrtime(turboNextBuildStart)
return { duration, turbotraceContext: null }
}

const { duration: webpackBuildDuration, turbotraceContext } =
await webpackBuild()
turboNextBuild ? await turbopackBuild() : await webpackBuild()

telemetry.record(
eventBuildCompleted(pagesPaths, {
Expand All @@ -1026,7 +1036,6 @@ export default async function build(
if (!turbotraceContext) {
return
}
let binding = (await loadBindings()) as any
if (
!binding?.isWasm &&
typeof binding.turbo.startTrace === 'function'
Expand Down Expand Up @@ -1069,11 +1078,9 @@ export default async function build(
if (filesTracedFromEntries.length) {
// The turbo trace doesn't provide the traced file type and reason at present
// let's write the traced files into the first [entry].nft.json
// @ts-expect-error types
const [[, entryName]] = Array.from(entryNameMap.entries()).filter(
// @ts-expect-error types
([k]) => k.startsWith(turbotraceContextAppDir)
)
const [[, entryName]] = Array.from<[string, string]>(
entryNameMap.entries()
).filter(([k]) => k.startsWith(turbotraceContextAppDir))
const traceOutputPath = path.join(
outputPath,
`../${entryName}.js.nft.json`
Expand Down Expand Up @@ -1797,7 +1804,6 @@ export default async function build(
} else if (config.outputFileTracing) {
let nodeFileTrace: any
if (config.experimental.turbotrace) {
let binding = (await loadBindings()) as any
if (!binding?.isWasm) {
nodeFileTrace = binding.turbo.startTrace
}
Expand Down
3 changes: 3 additions & 0 deletions packages/next/src/build/swc/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -471,6 +471,9 @@ function loadNative(isCustomTurbopack = false) {
require(__INTERNAL_CUSTOM_TURBOPACK_BINDINGS).startDev(devOptions)
}
},
nextBuild: (options: unknown) => {
return bindings.nextBuild(options)
},
startTrace: (options = {}, turboTasks: unknown) =>
bindings.runTurboTracing(
toBuffer({ exact: true, ...options }),
Expand Down
6 changes: 3 additions & 3 deletions packages/next/src/build/webpack-build.ts
Original file line number Diff line number Diff line change
Expand Up @@ -409,7 +409,7 @@ async function webpackBuildWithWorker() {

const combinedResult = {
duration: 0,
turbotraceContext: {} as any,
turbotraceContext: {} as TurbotraceContext,
}
// order matters here
const ORDERED_COMPILER_NAMES = [
Expand Down Expand Up @@ -447,9 +447,9 @@ async function webpackBuildWithWorker() {
if (curResult.turbotraceContext?.entriesTrace) {
combinedResult.turbotraceContext = curResult.turbotraceContext

const { entryNameMap } = combinedResult.turbotraceContext.entriesTrace
const { entryNameMap } = combinedResult.turbotraceContext.entriesTrace!
if (entryNameMap) {
combinedResult.turbotraceContext.entriesTrace.entryNameMap = new Map(
combinedResult.turbotraceContext.entriesTrace!.entryNameMap = new Map(
entryNameMap
)
}
Expand Down
4 changes: 3 additions & 1 deletion packages/next/src/cli/next-build.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ const nextBuild: CliCommand = (argv) => {
'--no-lint': Boolean,
'--no-mangling': Boolean,
'--experimental-app-only': Boolean,
'--experimental-turbo': Boolean,
// Aliases
'-h': '--help',
'-d': '--debug',
Expand Down Expand Up @@ -76,7 +77,8 @@ const nextBuild: CliCommand = (argv) => {
args['--debug'] || process.env.NEXT_DEBUG_BUILD,
!args['--no-lint'],
args['--no-mangling'],
args['--experimental-app-only']
args['--experimental-app-only'],
args['--experimental-turbo']
).catch((err) => {
console.error('')
if (
Expand Down

0 comments on commit 509ed00

Please sign in to comment.