Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
203 changes: 200 additions & 3 deletions cli/tools/bundle/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ use deno_bundle_runtime::BundleFormat;
use deno_bundle_runtime::BundlePlatform;
use deno_bundle_runtime::PackageHandling;
use deno_bundle_runtime::SourceMapType;
use deno_cache_dir::file_fetcher::File;
use deno_config::workspace::TsTypeLib;
use deno_core::anyhow::Context as _;
use deno_core::error::AnyError;
Expand Down Expand Up @@ -209,12 +210,14 @@ pub async fn prepare_inputs(
pub async fn bundle_init(
mut flags: Arc<Flags>,
bundle_flags: &BundleFlags,
memory_files: &[File],
) -> Result<EsbuildBundler, AnyError> {
{
let flags_mut = Arc::make_mut(&mut flags);
flags_mut.unstable_config.sloppy_imports = true;
}
let factory = CliFactory::from_flags(flags.clone());
let file_fetcher = factory.file_fetcher()?;

let esbuild_path = ensure_esbuild_downloaded(&factory).await?;

Expand All @@ -229,13 +232,41 @@ pub async fn bundle_init(
let module_graph_container =
factory.main_module_graph_container().await?.clone();

// Register any entrypoints that were pre-read into memory (e.g. `/dev/stdin`)
// so esbuild's loader doesn't re-read (and drain) the underlying pipe.
//
// The bundle graph is keyed by `resolve_entrypoints`' `resolver.resolve()`
// output, which an import map or workspace member can remap away from the URL
// the file was pre-read under; register the content under the resolved URL as
// well, or the lookup misses and esbuild reopens the drained pipe. Only when
// it still names the same file though — a mapping that points the entrypoint
// at some *other* file must keep that file's own content.
let init_cwd_url = deno_path_util::url_from_directory_path(&init_cwd)?;
for file in memory_files {
file_fetcher.insert_memory_files(file.clone());
if let Ok(resolved) = resolver.resolve(
file.url.as_str(),
&init_cwd_url,
Position::new(0, 0),
ResolutionMode::Import,
NodeResolutionKind::Execution,
) && resolved != file.url
&& is_same_file(&resolved, &file.url)
{
file_fetcher.insert_memory_files(File {
url: resolved,
..file.clone()
});
}
}

let (on_end_tx, on_end_rx) = tokio::sync::mpsc::channel(10);
#[allow(
clippy::arc_with_non_send_sync,
reason = "fine because only used in output positions"
)]
let mut plugin_handler = Arc::new(DenoPluginHandler {
file_fetcher: factory.file_fetcher()?.clone(),
file_fetcher: file_fetcher.clone(),
resolver: resolver.clone(),
module_load_preparer,
resolved_roots: Arc::new(RwLock::new(Arc::new(IndexSet::new()))),
Expand Down Expand Up @@ -311,11 +342,24 @@ pub async fn bundle(
let flags_mut = Arc::make_mut(&mut flags);
flags_mut.unstable_config.sloppy_imports = true;
}
// Read non-regular entrypoints (e.g. `/dev/stdin`) exactly once and share the
// content with every factory below, so the pipe isn't read twice under
// `--check` (denoland/deno#36162). Done for both flows so that an
// extensionless entrypoint gets the same media type with and without
// `--check`. This is the same cwd `CliOptions` resolves for the factories.
let memory_files = read_nonregular_entrypoints(
&bundle_flags,
&crate::util::env::resolve_cwd(flags.initial_cwd.as_deref())?,
)?;
// `--check[=all]` is documented in `bundle --help`; honour it before we
// hand the graph to esbuild so type errors surface alongside (and not
// after) the bundle output (denoland/deno#30159).
if !matches!(flags.type_check_mode, TypeCheckMode::None) {
let check_factory = CliFactory::from_flags(flags.clone());
let file_fetcher = check_factory.file_fetcher()?;
for file in &memory_files {
file_fetcher.insert_memory_files(file.clone());
}
let main_graph_container =
check_factory.main_module_graph_container().await?;
let specifiers = main_graph_container.collect_specifiers(
Expand All @@ -334,7 +378,8 @@ pub async fn bundle(
)
.await?;
}
let bundler = bundle_init(flags.clone(), &bundle_flags).await?;
let bundler =
bundle_init(flags.clone(), &bundle_flags, &memory_files).await?;
let init_cwd = bundler.cwd.clone();
let start = std::time::Instant::now();
let response = bundler.build().await?;
Expand Down Expand Up @@ -877,7 +922,7 @@ pub async fn bundle_for_compile(
flags_mut.internal.force_bundle_mode = true;
}

let bundler = bundle_init(flags, &bundle_flags).await?;
let bundler = bundle_init(flags, &bundle_flags, &[]).await?;
let response = bundler.build().await?;

handle_esbuild_errors_and_warnings(
Expand Down Expand Up @@ -2482,6 +2527,147 @@ fn media_type_to_loader(
}
}

/// Whether `metadata` describes a non-regular local file (fifo/char device/
/// socket, e.g. `/dev/stdin` when stdin is a pipe).
///
/// `std::fs::metadata` follows symlinks, so a symlink to a FIFO is reported as
/// non-regular too.
fn is_nonregular(metadata: &std::fs::Metadata) -> bool {
!metadata.is_file() && !metadata.is_dir()
}

/// Identifies the file a path actually points at, so aliases of the same pipe
/// (`/dev/stdin` and `/dev/fd/0`, or a symlink passed alongside its target) are
/// recognized as one file. Falls back to the path where the platform has no
/// stable identity.
#[cfg(unix)]
type FileIdentity = (u64, u64);
#[cfg(not(unix))]
type FileIdentity = PathBuf;

#[cfg(unix)]
fn file_identity(_path: &Path, metadata: &std::fs::Metadata) -> FileIdentity {
use std::os::unix::fs::MetadataExt;
(metadata.dev(), metadata.ino())
}

#[cfg(not(unix))]
fn file_identity(path: &Path, _metadata: &std::fs::Metadata) -> FileIdentity {
path.to_path_buf()
}

/// Whether two `file:` URLs name the same underlying file (symlinks followed).
fn is_same_file(a: &Url, b: &Url) -> bool {
let (Ok(a), Ok(b)) = (
deno_path_util::url_to_file_path(a),
deno_path_util::url_to_file_path(b),
) else {
return false;
};
let (Ok(a_meta), Ok(b_meta)) = (std::fs::metadata(&a), std::fs::metadata(&b))
else {
return false;
};
file_identity(&a, &a_meta) == file_identity(&b, &b_meta)
}

/// The local path an entrypoint refers to, or `None` for a specifier that isn't
/// a local file (`https:`, `npm:`, `jsr:`, ...).
fn entrypoint_local_path(entrypoint: &str, init_cwd: &Path) -> Option<PathBuf> {
if deno_path_util::specifier_has_uri_scheme(entrypoint) {
let url = Url::parse(entrypoint).ok()?;
if url.scheme() != "file" {
return None;
}
url.to_file_path().ok()
} else {
let path = init_cwd.join(entrypoint);
Some(deno_path_util::normalize_path(Cow::Owned(path)).into_owned())
}
}

/// Read entrypoints that resolve to a non-regular local file (e.g. `/dev/stdin`
/// when stdin is a pipe) into memory once.
///
/// A pipe can be read only once, but `deno bundle --check` would otherwise read
/// the entrypoint twice — once for the type-check module graph and once for
/// esbuild — using two independent `CliFactory` instances. The second read hits
/// EOF and esbuild silently emits nothing (denoland/deno#36162). We read such
/// entrypoints exactly once here and register the resulting `File`s in every
/// factory's file fetcher (which consults its in-memory files before touching
/// the filesystem), so neither path re-reads the drained pipe.
fn read_nonregular_entrypoints(
bundle_flags: &BundleFlags,
init_cwd: &Path,
) -> Result<Vec<File>, AnyError> {
let mut files = Vec::new();
// Distinct entrypoints can name the same pipe: the same argument passed twice
// (duplicate CLI args are valid — they collapse to one bundle), but also
// `/dev/stdin` alongside `/dev/fd/0`, or a symlink alongside its target.
// Reading such a pipe a second time drains it (EOF overwrites the first
// source, or the second `open()` blocks forever), so read once per underlying
// file and hand the same bytes to every URL that aliases it.
let mut sources: std::collections::HashMap<FileIdentity, Arc<[u8]>> =
Default::default();
let mut seen_urls = std::collections::HashSet::new();
for entrypoint in &bundle_flags.entrypoints {
let Some(path) = entrypoint_local_path(entrypoint, init_cwd) else {
continue;
};
let metadata = match std::fs::metadata(&path) {
Ok(metadata) => metadata,
// A missing entrypoint isn't ours to report: leave it to the module
// graph, which names the path in its "Module not found" error.
Err(err) if err.kind() == std::io::ErrorKind::NotFound => continue,
// Anything else (EACCES, EINTR, ...) would silently put us back on the
// read-the-pipe-twice path, so surface it instead of skipping.
Err(err) => {
return Err(err)
.with_context(|| format!("Reading metadata for {}", path.display()));
}
};
if !is_nonregular(&metadata) {
continue;
}
// Deliberately not canonicalized — see `resolve_url_or_path_absolute`.
let url = deno_path_util::url_from_file_path(&path)?;
if !seen_urls.insert(url.clone()) {
continue;
}
let source = match sources.entry(file_identity(&path, &metadata)) {
std::collections::hash_map::Entry::Occupied(entry) => entry.get().clone(),
std::collections::hash_map::Entry::Vacant(entry) => {
let source: Arc<[u8]> = std::fs::read(&path)
.with_context(|| format!("Reading {}", path.display()))?
.into();
entry.insert(source.clone());
source
}
};
// An extensionless entrypoint like `/dev/stdin` has no media type of its
// own, and deno_graph assumes JavaScript for such a root — which would make
// `--check` quietly not type-check piped TypeScript. Stamp a TypeScript
// content type instead, matching `deno run -` (which names the stdin module
// `$deno$stdin.mts`). An entrypoint that does carry an extension (a FIFO
// named `foo.js`) keeps its own media type.
let maybe_headers = (MediaType::from_specifier(&url) == MediaType::Unknown)
.then(|| {
std::collections::HashMap::from([(
"content-type".to_string(),
"application/typescript".to_string(),
)])
});
files.push(File {
url,
mtime: None,
maybe_headers,
source,
loaded_from: deno_cache_dir::file_fetcher::LoadedFrom::Local,
});
}
Ok(files)
}

fn resolve_url_or_path_absolute(
specifier: &str,
current_dir: &Path,
Expand All @@ -2491,6 +2677,17 @@ fn resolve_url_or_path_absolute(
} else {
let path = current_dir.join(specifier);
let path = deno_path_util::normalize_path(Cow::Owned(path));
// Non-regular files (fifo/char device/socket, e.g. `/dev/stdin` when stdin
// is a pipe, or a symlink to a FIFO) must NOT be canonicalized: on Linux
// canonicalize errors (realpath -> nonexistent `/proc/PID/fd/pipe:[inode]`);
// on macOS it rewrites `/dev/stdin` to `/dev/fd/0`; a symlink rewrites to
// its target. Any of these makes the bundle flow's specifier diverge from
// the check flow's `collect_specifiers` specifier (which never
// canonicalizes), so the shared in-memory file is missed and the drained
// pipe is reopened (denoland/deno#36162).
if std::fs::metadata(&path).is_ok_and(|m| is_nonregular(&m)) {
return Ok(deno_path_util::url_from_file_path(&path)?);
}
let path = canonicalize_path(&path)?;
Ok(deno_path_util::url_from_file_path(&path)?)
}
Expand Down
3 changes: 2 additions & 1 deletion cli/tools/bundle/provider.rs
Original file line number Diff line number Diff line change
Expand Up @@ -132,7 +132,8 @@ impl BundleProvider for CliBundleProvider {
std::thread::spawn(move || {
deno_runtime::tokio_util::create_and_run_current_thread(async move {
let flags = Arc::new(flags_clone);
let bundler = match super::bundle_init(flags, &bundle_flags).await {
let bundler = match super::bundle_init(flags, &bundle_flags, &[]).await
{
Ok(bundler) => bundler,
Err(e) => {
log::trace!("bundle_init error: {e:?}");
Expand Down
3 changes: 2 additions & 1 deletion tests/integration/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -59,4 +59,5 @@ uuid = { workspace = true, features = ["serde"] }
zeromq.workspace = true

[target.'cfg(unix)'.dev-dependencies]
nix.workspace = true
# `fs` for `mkfifo`/`open` in the bundle FIFO regression tests.
nix = { workspace = true, features = ["fs"] }
Loading
Loading