From 81b0b683d2c1e2156b1be99492e0a66c57985f15 Mon Sep 17 00:00:00 2001 From: jeparlefrancais Date: Mon, 1 Jun 2026 13:58:44 -0400 Subject: [PATCH 1/4] Create content loaders Add content loaders that can be mapped to files using globs/patterns in the config file. - luau: parses files as Luau (the default for .luau/.lua) - copy: simply copies files - skip: ignore files - string: create a Lua module that returns the content as a string. - buffer: create a Lua module that returns the content as a buffer. - bytes: create a Lua module that returns the content as a bytes array. - data loaders (json, toml, yaml): converts data to a Lua module. --- Cargo.lock | 7 + Cargo.toml | 1 + src/frontend/configuration.rs | 115 +++++++- src/frontend/content_loader.rs | 270 ++++++++++++++++++ src/frontend/error.rs | 62 ++-- src/frontend/mod.rs | 7 +- src/frontend/resources.rs | 61 ++-- src/frontend/work_item.rs | 23 ++ src/frontend/worker.rs | 101 +++++-- src/frontend/worker_tree.rs | 4 +- src/nodes/expressions/mod.rs | 73 ++--- src/rules/bundle/mod.rs | 19 +- src/rules/bundle/path_require_mode/mod.rs | 131 +++------ src/rules/convert_require/mod.rs | 17 +- src/rules/mod.rs | 28 ++ src/utils/filter_pattern.rs | 10 +- src/utils/mod.rs | 3 + src/utils/serde_vec_of_pairs.rs | 84 ++++++ tests/bundle.rs | 14 +- tests/frontend.rs | 183 ++++++++++++ tests/rule_tests/convert_require.rs | 70 +++++ ...e_lua_file_with_unsupported_extension.snap | 2 +- 22 files changed, 1067 insertions(+), 218 deletions(-) create mode 100644 src/frontend/content_loader.rs create mode 100644 src/utils/serde_vec_of_pairs.rs diff --git a/Cargo.lock b/Cargo.lock index fb0eb254..9758fdce 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -103,6 +103,12 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + [[package]] name = "bitflags" version = "2.11.1" @@ -393,6 +399,7 @@ version = "0.18.0" dependencies = [ "anstyle", "assert_cmd", + "base64", "bstr", "clap", "criterion", diff --git a/Cargo.toml b/Cargo.toml index 534ec204..e3de8959 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,6 +27,7 @@ tracing = ["dep:tracing"] [dependencies] anstyle = "1.0.14" +base64 = "0.22.1" bstr = "1.12.1" clap = { version = "4.6.1", features = ["derive"] } durationfmt = "0.1.1" diff --git a/src/frontend/configuration.rs b/src/frontend/configuration.rs index 3c23a451..457a1706 100644 --- a/src/frontend/configuration.rs +++ b/src/frontend/configuration.rs @@ -7,13 +7,14 @@ use std::{ use serde::{Deserialize, Serialize}; use crate::{ + frontend::Loader, generator::{DenseLuaGenerator, LuaGenerator, ReadableLuaGenerator, TokenBasedLuaGenerator}, nodes::Block, rules::{ bundle::{BundleRequireMode, Bundler}, get_default_rules, Rule, }, - utils::{deserialize_one_or_many, FilterPattern}, + utils::{deserialize_one_or_many, deserialize_vec_of_pairs, FilterPattern}, DarkluaError, Parser, }; @@ -47,6 +48,10 @@ pub struct Configuration { deserialize_with = "deserialize_one_or_many" )] skip_files: Vec, + #[serde(flatten)] + loaders: LoaderConfiguration, + #[serde(default, skip_serializing_if = "Option::is_none")] + lua_extension: Option, } impl Configuration { @@ -59,6 +64,8 @@ impl Configuration { location: None, apply_to_files: Vec::new(), skip_files: Vec::new(), + loaders: Default::default(), + lua_extension: None, } } @@ -132,6 +139,25 @@ impl Configuration { Ok(()) } + /// Attaches a loader to a glob pattern and returns the configuration. + pub fn with_loader(mut self, loader: Loader, pattern: &str) -> Result { + self.add_loader(loader, pattern)?; + Ok(self) + } + + /// Attaches a loader to a glob pattern. + pub fn add_loader(&mut self, loader: Loader, pattern: &str) -> Result<(), DarkluaError> { + self.loaders + .loaders + .push((FilterPattern::new(pattern.to_owned())?, loader)); + Ok(()) + } + + /// Clears all registered loaders. + pub fn clear_loaders(&mut self) { + self.loaders.loaders.clear(); + } + #[inline] pub(crate) fn rules<'a, 'b: 'a>(&'b self) -> impl Iterator { self.rules.iter().map(AsRef::as_ref) @@ -153,6 +179,7 @@ impl Configuration { self.build_parser(), bundle_config.require_mode().clone(), bundle_config.excludes(), + self.loaders.clone(), ) .with_modules_identifier(bundle_config.modules_identifier()); Some(bundler) @@ -182,6 +209,17 @@ impl Configuration { true } + + pub(crate) fn loaders(&self) -> &LoaderConfiguration { + &self.loaders + } + + pub(crate) fn preferred_lua_extension(&self) -> &'static str { + self.lua_extension + .as_ref() + .unwrap_or(&Default::default()) + .as_str() + } } impl Default for Configuration { @@ -193,6 +231,8 @@ impl Default for Configuration { location: None, apply_to_files: Vec::new(), skip_files: Vec::new(), + loaders: Default::default(), + lua_extension: None, } } } @@ -200,7 +240,12 @@ impl Default for Configuration { impl std::fmt::Debug for Configuration { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("Config") + .field("location", &self.location) .field("generator", &self.generator) + .field("apply_to_files", &self.apply_to_files) + .field("skip_files", &self.skip_files) + .field("loaders", &self.loaders) + .field("bundle", &self.bundle) .field( "rules", &self @@ -218,6 +263,45 @@ impl std::fmt::Debug for Configuration { } } +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub(crate) struct LoaderConfiguration { + #[serde( + default, + skip_serializing_if = "Vec::is_empty", + deserialize_with = "deserialize_vec_of_pairs" + )] + loaders: Vec<(FilterPattern, Loader)>, +} + +impl LoaderConfiguration { + pub(crate) fn get_loader(&self, path: &Path) -> Loader { + self.loaders + .iter() + .find(|(pattern, _)| pattern.matches(path)) + .map(|(_, loader)| *loader) + .or_else(|| Loader::from_path(path)) + .unwrap_or(Loader::Skip) + } +} + +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub(crate) enum LuaExtension { + #[default] + Lua, + Luau, +} + +impl LuaExtension { + const fn as_str(&self) -> &'static str { + match self { + Self::Lua => "lua", + Self::Luau => "luau", + } + } +} + /// Parameters for configuring the Lua code generator. /// /// This enum defines different modes for generating Lua code, each with its own @@ -561,4 +645,33 @@ mod test { ); } } + + mod loaders { + use super::*; + + #[test] + fn deserialize_custom_loaders() { + let config: Configuration = + json5::from_str("{ loaders: { '*.luau': 'luau', '*.json': 'json' } }").unwrap(); + + insta::assert_debug_snapshot!(config.loaders, @r###" + LoaderConfiguration { + loaders: [ + ( + FilterPattern { + pattern: "*.luau", + }, + Luau, + ), + ( + FilterPattern { + pattern: "*.json", + }, + Json, + ), + ], + } + "###); + } + } } diff --git a/src/frontend/content_loader.rs b/src/frontend/content_loader.rs new file mode 100644 index 00000000..0c398236 --- /dev/null +++ b/src/frontend/content_loader.rs @@ -0,0 +1,270 @@ +use std::{ffi::OsStr, path::Path}; + +use base64::{engine::general_purpose::STANDARD, Engine}; +use serde::{Deserialize, Serialize}; + +use crate::{ + nodes::{ + Block, Expression, FieldExpression, FunctionCall, Prefix, ReturnStatement, + StringExpression, TableEntry, TableExpression, + }, + process::to_expression, + utils::Timer, + DarkluaError, Parser, Resources, +}; + +/// Specifies how a file should be loaded and converted when processing or bundling. +/// +/// The default loader for a file is inferred from its extension. Custom loaders +/// can be assigned to files via glob patterns in the configuration. +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum Loader { + /// Copies the file to the output. + Copy, + /// Parses and processes the file as Lua/Luau source code. + Luau, + /// Converts the file into a Lua module that returns the content as a string + /// (`return "..."`). + String, + /// Converts the file into a Lua module that returns the content as a base64-encoded + /// string (`return "..."`). + #[serde(rename = "string/base64")] + StringBase64, + /// Converts the file into a Lua module that returns the content as a buffer + /// (`return buffer.fromstring("...")`). + Buffer, + /// Converts the file into a Lua module that returns the content as a base64-encoded + /// buffer (`return buffer.fromstring("...")`). + #[serde(rename = "buffer/base64")] + BufferBase64, + /// Converts the file into a Lua module that returns the content as a byte array + /// (`return { ... }`). + Bytes, + /// Converts the file into a Lua module that returns the content as a base64-encoded + /// byte array (`return { ... }`). + #[serde(rename = "bytes/base64")] + BytesBase64, + /// Converts the file into a Lua module that returns the JSON data. + Json, + /// Converts the file into a Lua module that returns the TOML data. + Toml, + /// Converts the file into a Lua module that returns the YAML data. + #[serde(alias = "yml")] + Yaml, + /// Ignores the file. + Skip, +} + +impl Loader { + pub(crate) fn from_extension(extension: &OsStr) -> Option { + let extension = extension.to_str()?; + + match extension { + "luau" | "lua" => Some(Self::Luau), + "json" | "json5" => Some(Self::Json), + "toml" => Some(Self::Toml), + "yaml" | "yml" => Some(Self::Yaml), + "txt" => Some(Self::String), + _ => None, + } + } + + pub(crate) fn from_path(path: &Path) -> Option { + path.extension().and_then(Self::from_extension) + } + + pub(crate) fn to_internal_loader(self) -> InternalLoader { + match self { + Self::Copy => InternalLoader::Copy, + Self::Luau => InternalLoader::Luau, + Self::String => InternalLoader::String(LoaderEncoding::None), + Self::StringBase64 => InternalLoader::String(LoaderEncoding::Base64), + Self::Buffer => InternalLoader::Buffer(LoaderEncoding::None), + Self::BufferBase64 => InternalLoader::Buffer(LoaderEncoding::Base64), + Self::Bytes => InternalLoader::Bytes(LoaderEncoding::None), + Self::BytesBase64 => InternalLoader::Bytes(LoaderEncoding::Base64), + Self::Json => InternalLoader::Json, + Self::Toml => InternalLoader::Toml, + Self::Yaml => InternalLoader::Yaml, + Self::Skip => InternalLoader::Skip, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum InternalLoader { + Copy, + Luau, + String(LoaderEncoding), + Buffer(LoaderEncoding), + Bytes(LoaderEncoding), + Json, + Toml, + Yaml, + Skip, +} + +impl InternalLoader { + pub(crate) fn outputs_lua(&self) -> bool { + match self { + Self::Luau + | Self::String(_) + | Self::Buffer(_) + | Self::Bytes(_) + | Self::Json + | Self::Toml + | Self::Yaml => true, + Self::Copy | Self::Skip => false, + } + } + + pub(crate) fn load( + &self, + source: &Path, + resources: &Resources, + parser: &Parser, + ) -> Result { + match self { + Self::Copy => { + let content = resources.get_bytes(source)?; + Ok(ContentType::Copied(content)) + } + Self::Skip => Ok(ContentType::None), + Self::Luau => { + let content = resources.get(source)?; + let parser_timer = Timer::now(); + + let block = parser + .parse(&content) + .map_err(|parser_error| DarkluaError::parser_error(source, parser_error))?; + + let parser_time = parser_timer.duration_label(); + log::debug!("parsed `{}` in {}", source.display(), parser_time); + + Ok(ContentType::Parsed { + block, + source: content, + }) + } + Self::String(encoding) => { + let content = resources.get_bytes(source)?; + + let new_module = Block::default().with_last_statement(ReturnStatement::one( + StringExpression::from_value(encoding.encode(&content)?.unwrap_or(content)), + )); + + Ok(ContentType::Block(new_module)) + } + Self::Buffer(encoding) => { + let content = resources.get_bytes(source)?; + + let new_module = Block::default().with_last_statement(ReturnStatement::one( + FunctionCall::from_prefix(FieldExpression::new( + Prefix::from_name("buffer"), + "fromstring", + )) + .with_arguments(StringExpression::from_value( + encoding.encode(&content)?.unwrap_or(content), + )), + )); + + Ok(ContentType::Block(new_module)) + } + Self::Bytes(encoding) => { + let content = resources.get_bytes(source)?; + + let new_module = Block::default().with_last_statement(ReturnStatement::one( + TableExpression::new( + encoding + .encode(&content)? + .unwrap_or(content) + .iter() + .map(TableEntry::from_value) + .collect(), + ), + )); + + Ok(ContentType::Block(new_module)) + } + Self::Json => { + let content = resources.get(source)?; + let data = + json5::from_str::(&content).map_err(DarkluaError::from)?; + + ContentType::from_data("json", data, source) + } + Self::Toml => { + let content = resources.get(source)?; + let data = toml::from_str::(&content).map_err(DarkluaError::from)?; + + ContentType::from_data("toml", data, source) + } + Self::Yaml => { + let content = resources.get(source)?; + let data = serde_yaml::from_str::(&content) + .map_err(DarkluaError::from)?; + + ContentType::from_data("yaml", data, source) + } + } + } +} + +pub(crate) enum ContentType { + None, + Copied(Vec), + Expression(Expression), + Block(Block), + Parsed { block: Block, source: String }, +} + +impl ContentType { + pub(crate) fn from_data( + label: &'static str, + value: impl Serialize, + source: &Path, + ) -> Result { + log::trace!( + "transcode {} data to Lua from `{}`", + label, + source.display() + ); + let transcode_duration = Timer::now(); + let expression = to_expression(&value).map_err(DarkluaError::from)?; + log::debug!( + "transcoded {} data to Lua from `{}` in {}", + label, + source.display(), + transcode_duration.duration_label() + ); + Ok(Self::Expression(expression)) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum LoaderEncoding { + None, + Base64, +} + +impl LoaderEncoding { + pub(crate) fn encode(&self, content: &[u8]) -> Result>, DarkluaError> { + match self { + Self::None => Ok(None), + Self::Base64 => encode_base64(content).map(Some), + } + } +} + +fn encode_base64(content: &[u8]) -> Result, DarkluaError> { + let mut encoded_content = vec![0; content.len() * 4 / 3 + 4]; + + let bytes_written = STANDARD + .encode_slice(content, &mut encoded_content) + .map_err(|err| DarkluaError::custom(format!("failed to encode base64: {}", err)))?; + + encoded_content.truncate(bytes_written); + + Ok(encoded_content) +} diff --git a/src/frontend/error.rs b/src/frontend/error.rs index a510a8ca..80dd8542 100644 --- a/src/frontend/error.rs +++ b/src/frontend/error.rs @@ -2,9 +2,10 @@ use std::{ borrow::Cow, cmp::Ordering, collections::HashSet, - ffi::{OsStr, OsString}, + ffi::OsString, fmt::{self, Display}, path::PathBuf, + str::Utf8Error, }; use crate::{process::LuaSerializerError, rules::Rule, ParserError}; @@ -23,6 +24,10 @@ enum ErrorKind { ResourceNotFound { path: PathBuf, }, + ExpectedUtf8 { + path: PathBuf, + utf8_error: Utf8Error, + }, InvalidConfiguration { path: PathBuf, }, @@ -57,8 +62,11 @@ enum ErrorKind { location: String, message: String, }, - InvalidResourceExtension { - location: PathBuf, + RequireUnknownResource { + path: PathBuf, + }, + RequireCopiedResource { + path: PathBuf, }, OsStringConversion { os_string: OsString, @@ -125,6 +133,13 @@ impl DarkluaError { Self::new(ErrorKind::ResourceNotFound { path: path.into() }) } + pub(crate) fn expected_utf8(path: impl Into, utf8_error: Utf8Error) -> Self { + Self::new(ErrorKind::ExpectedUtf8 { + path: path.into(), + utf8_error, + }) + } + pub(crate) fn invalid_configuration_file(path: impl Into) -> Self { Self::new(ErrorKind::InvalidConfiguration { path: path.into() }) } @@ -212,10 +227,12 @@ impl DarkluaError { }) } - pub(crate) fn invalid_resource_extension(path: impl Into) -> Self { - Self::new(ErrorKind::InvalidResourceExtension { - location: path.into(), - }) + pub(crate) fn require_unknown_resource(path: impl Into) -> Self { + Self::new(ErrorKind::RequireUnknownResource { path: path.into() }) + } + + pub(crate) fn require_copied_resource(path: impl Into) -> Self { + Self::new(ErrorKind::RequireCopiedResource { path: path.into() }) } pub(crate) fn os_string_conversion(os_string: impl Into) -> Self { @@ -246,6 +263,9 @@ impl From for DarkluaError { fn from(err: ResourceError) -> Self { match err { ResourceError::NotFound(path) => DarkluaError::resource_not_found(path), + ResourceError::ExpectedUtf8 { path, utf8_error } => { + DarkluaError::expected_utf8(path, utf8_error) + } ResourceError::IO { path, error } => DarkluaError::io_error(path, error), } } @@ -314,6 +334,14 @@ impl Display for DarkluaError { ErrorKind::ResourceNotFound { path } => { write!(f, "unable to find `{}`", path.display())?; } + ErrorKind::ExpectedUtf8 { path, utf8_error } => { + write!( + f, + "unable to read `{}` as valid UTF-8: {}", + path.display(), + utf8_error + )?; + } ErrorKind::InvalidConfiguration { path } => { write!(f, "invalid configuration file at `{}`", path.display())?; } @@ -408,21 +436,11 @@ impl Display for DarkluaError { location, message )?; } - ErrorKind::InvalidResourceExtension { location } => { - if let Some(extension) = location.extension().map(OsStr::to_string_lossy) { - write!( - f, - "unable to require resource with extension `{}` at `{}`", - extension, - location.display() - )?; - } else { - write!( - f, - "unable to require resource without an extension at `{}`", - location.display() - )?; - } + ErrorKind::RequireUnknownResource { path } => { + write!(f, "unable to require unknown resource at `{}` (configure content loaders to load this file)", path.display())?; + } + ErrorKind::RequireCopiedResource { path } => { + write!(f, "unable to require copied resource at `{}` (configure content loaders to copy this file)", path.display())?; } ErrorKind::OsStringConversion { os_string } => { write!( diff --git a/src/frontend/mod.rs b/src/frontend/mod.rs index 20b71187..1486af73 100644 --- a/src/frontend/mod.rs +++ b/src/frontend/mod.rs @@ -1,4 +1,5 @@ mod configuration; +mod content_loader; mod error; mod options; mod resources; @@ -8,15 +9,19 @@ mod work_item; mod worker; mod worker_tree; +pub(crate) use configuration::LoaderConfiguration; pub use configuration::{BundleConfiguration, Configuration, GeneratorParameters}; +pub(crate) use content_loader::ContentType; +pub use content_loader::Loader; pub use error::{DarkluaError, DarkluaResult}; pub use options::Options; pub use resources::Resources; -use serde::Serialize; use work_item::WorkItem; use worker::Worker; pub use worker_tree::WorkerTree; +use serde::Serialize; + use crate::{ generator::{DenseLuaGenerator, LuaGenerator}, nodes::{Block, ReturnStatement}, diff --git a/src/frontend/resources.rs b/src/frontend/resources.rs index 00d5f73d..f3decc57 100644 --- a/src/frontend/resources.rs +++ b/src/frontend/resources.rs @@ -1,10 +1,10 @@ use std::{ collections::HashMap, - ffi::OsStr, fs::{self, File}, io::{self, BufWriter, ErrorKind as IOErrorKind, Write}, iter, path::{Path, PathBuf}, + str::Utf8Error, sync::{Arc, Mutex}, }; @@ -13,7 +13,7 @@ use crate::utils::normalize_path; #[derive(Debug, Clone)] enum Source { FileSystem, - Memory(Arc>>), + Memory(Arc>>>), } impl Source { @@ -52,8 +52,15 @@ impl Source { } pub fn get(&self, location: &Path) -> ResourceResult { + self.get_bytes(location).and_then(|bytes| { + String::from_utf8(bytes) + .map_err(|err| ResourceError::expected_utf8(location, err.utf8_error())) + }) + } + + fn get_bytes(&self, location: &Path) -> ResourceResult> { match self { - Self::FileSystem => fs::read_to_string(location).map_err(|err| match err.kind() { + Self::FileSystem => fs::read(location).map_err(|err| match err.kind() { IOErrorKind::NotFound => ResourceError::not_found(location), _ => ResourceError::io_error(location, err), }), @@ -62,13 +69,17 @@ impl Source { let location = normalize_path(location); data.get(&location) - .map(String::from) + .cloned() .ok_or_else(|| ResourceError::not_found(location)) } } } pub fn write(&self, location: &Path, content: &str) -> ResourceResult<()> { + self.write_bytes(location, content.as_bytes()) + } + + fn write_bytes(&self, location: &Path, content: &[u8]) -> ResourceResult<()> { match self { Self::FileSystem => { if let Some(parent) = location.parent() { @@ -80,12 +91,12 @@ impl Source { File::create(location).map_err(|err| ResourceError::io_error(location, err))?; let mut file = BufWriter::new(file); - file.write_all(content.as_bytes()) + file.write_all(content) .map_err(|err| ResourceError::io_error(location, err)) } Self::Memory(data) => { let mut data = data.lock().unwrap(); - data.insert(normalize_path(location), content.to_string()); + data.insert(normalize_path(location), content.to_vec()); Ok(()) } } @@ -106,7 +117,7 @@ impl Source { } } - pub(crate) fn walk_all(&self, location: &Path) -> impl Iterator { + fn walk_all(&self, location: &Path) -> impl Iterator { match self { Self::FileSystem => Box::new(walk_all_file_system(location.to_path_buf())) as Box>, @@ -121,7 +132,7 @@ impl Source { } } - pub(crate) fn is_empty_directory(&self, location: &Path) -> ResourceResult { + fn is_empty_directory(&self, location: &Path) -> ResourceResult { if !self.is_directory(location)? { return Ok(false); } @@ -288,14 +299,10 @@ impl Resources { } } - /// Collects all Lua and Luau files in the specified location. + /// Collects all files in the specified location. Deprecated in favor of [Self::walk]. + #[deprecated(since = "0.19.0", note = "use `Resources::walk(location)` instead")] pub fn collect_work(&self, location: impl AsRef) -> impl Iterator { - self.source.walk(location.as_ref()).filter(|path| { - matches!( - path.extension().and_then(OsStr::to_str), - Some("lua") | Some("luau") - ) - }) + self.source.walk(location.as_ref()) } /// Checks if a path exists. @@ -318,11 +325,21 @@ impl Resources { self.source.get(location.as_ref()) } + /// Reads the contents of a file as bytes. + pub fn get_bytes(&self, location: impl AsRef) -> ResourceResult> { + self.source.get_bytes(location.as_ref()) + } + /// Writes content to a file. pub fn write(&self, location: impl AsRef, content: &str) -> ResourceResult<()> { self.source.write(location.as_ref(), content) } + /// Writes content to a file as bytes. + pub fn write_bytes(&self, location: impl AsRef, content: &[u8]) -> ResourceResult<()> { + self.source.write_bytes(location.as_ref(), content) + } + /// Removes a file or directory. pub fn remove(&self, location: impl AsRef) -> ResourceResult<()> { self.source.remove(location.as_ref()) @@ -356,6 +373,11 @@ pub(crate) enum ResourceContent { pub enum ResourceError { /// The requested resource was not found. NotFound(PathBuf), + /// The requested resource is expected to be valid UTF-8, but is not. + ExpectedUtf8 { + path: PathBuf, + utf8_error: Utf8Error, + }, /// An I/O error occurred while accessing the resource. IO { path: PathBuf, error: String }, } @@ -365,6 +387,13 @@ impl ResourceError { Self::NotFound(path.into()) } + pub(crate) fn expected_utf8(path: impl Into, utf8_error: Utf8Error) -> Self { + Self::ExpectedUtf8 { + path: path.into(), + utf8_error, + } + } + pub(crate) fn io_error(path: impl Into, error: io::Error) -> Self { Self::IO { path: path.into(), @@ -448,7 +477,7 @@ mod test { resources.write("src/test.lua", ANY_CONTENT).unwrap(); assert_eq!( - Vec::from_iter(resources.collect_work("src")), + Vec::from_iter(resources.walk("src")), vec![PathBuf::from("src/test.lua")] ); } diff --git a/src/frontend/work_item.rs b/src/frontend/work_item.rs index 1293a841..bfe3e194 100644 --- a/src/frontend/work_item.rs +++ b/src/frontend/work_item.rs @@ -156,4 +156,27 @@ impl WorkItem { self.status = WorkStatus::NotStarted; self.external_file_dependencies.clear(); } + + pub(crate) fn adjust_output_extension( + &mut self, + new_extension: &str, + valid_extensions: &[&str], + ) { + if valid_extensions.is_empty() { + return; + } + let should_change = self + .data + .output + .extension() + .and_then(|extension| extension.to_ascii_lowercase().into_string().ok()) + .map(|extension| !valid_extensions.contains(&extension.as_str())); + + match should_change { + None | Some(true) => { + self.data.output.set_extension(new_extension); + } + Some(false) => {} + } + } } diff --git a/src/frontend/worker.rs b/src/frontend/worker.rs index a336d10d..aa42c0df 100644 --- a/src/frontend/worker.rs +++ b/src/frontend/worker.rs @@ -1,7 +1,8 @@ -use std::path::Path; +use std::path::{Path, PathBuf}; use super::{ configuration::Configuration, + content_loader::ContentType, resources::Resources, utils::maybe_plural, work_cache::WorkCache, @@ -10,7 +11,7 @@ use super::{ }; use crate::{ - nodes::Block, + nodes::{Block, ReturnStatement}, rules::{bundle::Bundler, ContextBuilder, Rule, RuleConfiguration}, utils::{normalize_path, Timer}, GeneratorParameters, @@ -123,28 +124,58 @@ impl<'a> Worker<'a> { pub(crate) fn advance_work(&mut self, work_item: &mut WorkItem) -> DarkluaResult<()> { match &work_item.status { WorkStatus::NotStarted => { - let source_display = work_item.source().display(); + let loader = self + .configuration() + .loaders() + .get_loader(work_item.source()) + .to_internal_loader(); + + log::debug!( + "beginning work with loader {:?} on `{}`", + loader, + work_item.source().display(), + ); - let content = self.resources.get(work_item.source())?; + if loader.outputs_lua() { + work_item.adjust_output_extension( + self.configuration().preferred_lua_extension(), + &["lua", "luau"], + ); + } let parser = self.configuration.build_parser(); - log::debug!("beginning work on `{}`", source_display); - - let parser_timer = Timer::now(); - - let mut block = parser.parse(&content).map_err(|parser_error| { - DarkluaError::parser_error(work_item.source(), parser_error) - })?; + match loader.load(work_item.source(), self.resources, &parser)? { + ContentType::None => { + work_item.status = WorkStatus::done(); + Ok(()) + } + ContentType::Copied(data) => { + self.finalize_work_item(work_item, &data, None)?; + Ok(()) + } + ContentType::Expression(expression) => { + let block = + Block::default().with_last_statement(ReturnStatement::one(expression)); + let lua_code = self.generate_code(&block, "", work_item.source()); - let parser_time = parser_timer.duration_label(); - log::debug!("parsed `{}` in {}", source_display, parser_time); + self.finalize_work_item(work_item, lua_code.as_bytes(), None)?; + Ok(()) + } + ContentType::Block(block) => { + let lua_code = self.generate_code(&block, "", work_item.source()); - self.bundle(work_item, &mut block, &content)?; + self.finalize_work_item(work_item, lua_code.as_bytes(), None)?; + Ok(()) + } + ContentType::Parsed { mut block, source } => { + self.bundle(work_item, &mut block, &source)?; - work_item.status = WorkProgress::new(content, block).into(); + work_item.status = WorkProgress::new(source, block).into(); - self.apply_rules(work_item) + self.apply_rules(work_item) + } + } } WorkStatus::InProgress(_work_progress) => self.apply_rules(work_item), WorkStatus::Done(_) => Ok(()), @@ -331,25 +362,45 @@ impl<'a> Worker<'a> { .write(work_item.data.output(), &format!("{:#?}", progress.block()))?; } + let lua_code = + self.generate_code(progress.block(), &work_progress.content, &normalized_source); + + self.finalize_work_item(work_item, lua_code.as_bytes(), Some(normalized_source))?; + + Ok(()) + } + + fn generate_code(&self, block: &Block, source_content: &str, source: &Path) -> String { let generator_timer = Timer::now(); - let lua_code = self - .configuration - .generate_lua(progress.block(), &work_progress.content); + let lua_code = self.configuration.generate_lua(block, source_content); let generator_time = generator_timer.duration_label(); log::debug!( "generated code for `{}` in {}", - source_display, + source.display(), generator_time, ); - self.resources.write(work_item.data.output(), &lua_code)?; + lua_code + } + + fn finalize_work_item( + &mut self, + work_item: &mut WorkItem, + content: &[u8], + source: Option, + ) -> DarkluaResult<()> { + self.resources + .write_bytes(work_item.data.output(), content)?; - self.cache - .link_source_to_output(normalized_source, work_item.data.output()); + self.cache.link_source_to_output( + source.unwrap_or_else(|| normalize_path(work_item.data.source())), + work_item.data.output(), + ); work_item.status = WorkStatus::done(); + Ok(()) } @@ -358,7 +409,9 @@ impl<'a> Worker<'a> { source: &Path, original_code: &'src str, ) -> ContextBuilder<'block, 'a, 'src> { - let builder = ContextBuilder::new(normalize_path(source), self.resources, original_code); + let builder = ContextBuilder::new(normalize_path(source), self.resources, original_code) + .with_preferred_lua_extension(self.configuration.preferred_lua_extension()) + .with_loaders(self.configuration.loaders().clone()); if let Some(project_location) = self.configuration.location() { builder.with_project_location(project_location) } else { diff --git a/src/frontend/worker_tree.rs b/src/frontend/worker_tree.rs index e3d4f57e..4c680c3b 100644 --- a/src/frontend/worker_tree.rs +++ b/src/frontend/worker_tree.rs @@ -69,7 +69,7 @@ impl WorkerTree { } else { let input = normalize_path(options.input()); - for source in resources.collect_work(&input) { + for source in resources.walk(&input) { let source = normalize_path(source); let relative_path = source.strip_prefix(&input).map_err(|err| { @@ -88,7 +88,7 @@ impl WorkerTree { } else { let input = normalize_path(options.input()); - for source in resources.collect_work(input) { + for source in resources.walk(input) { self.add_source_if_missing(source, None); } } diff --git a/src/nodes/expressions/mod.rs b/src/nodes/expressions/mod.rs index 1a577690..df4f1e3d 100644 --- a/src/nodes/expressions/mod.rs +++ b/src/nodes/expressions/mod.rs @@ -217,65 +217,30 @@ impl From for Expression { } } -impl From for Expression { - fn from(value: f32) -> Self { - (value as f64).into() +impl From<&f64> for Expression { + fn from(value: &f64) -> Self { + (*value).into() } } -impl From for Expression { - fn from(value: usize) -> Self { - (value as f64).into() - } -} - -impl From for Expression { - fn from(value: u64) -> Self { - (value as f64).into() - } -} - -impl From for Expression { - fn from(value: u32) -> Self { - (value as f64).into() - } -} - -impl From for Expression { - fn from(value: u16) -> Self { - (value as f64).into() - } -} - -impl From for Expression { - fn from(value: u8) -> Self { - (value as f64).into() - } -} - -impl From for Expression { - fn from(value: i64) -> Self { - (value as f64).into() - } -} - -impl From for Expression { - fn from(value: i32) -> Self { - (value as f64).into() - } -} - -impl From for Expression { - fn from(value: i16) -> Self { - (value as f64).into() - } +macro_rules! impl_from_primitive_number { + ($($type:ty),+ $(,)?) => { + $( + impl From<$type> for Expression { + fn from(value: $type) -> Self { + (value as f64).into() + } + } + impl From<&$type> for Expression { + fn from(value: &$type) -> Self { + (*value as f64).into() + } + } + )+ + }; } -impl From for Expression { - fn from(value: i8) -> Self { - (value as f64).into() - } -} +impl_from_primitive_number!(f32, usize, u64, u32, u16, u8, i64, i32, i16, i8,); impl From for Expression { fn from(binary: BinaryExpression) -> Expression { diff --git a/src/rules/bundle/mod.rs b/src/rules/bundle/mod.rs index 23888b97..151d651d 100644 --- a/src/rules/bundle/mod.rs +++ b/src/rules/bundle/mod.rs @@ -6,6 +6,7 @@ use std::path::Path; use wax::Program; +use crate::frontend::LoaderConfiguration; use crate::nodes::Block; use crate::rules::{ Context, Rule, RuleConfiguration, RuleConfigurationError, RuleMetadata, RuleProcessResult, @@ -23,6 +24,7 @@ pub(crate) struct BundleOptions { parser: Parser, modules_identifier: String, excludes: Option>, + loaders: LoaderConfiguration, } impl BundleOptions { @@ -30,6 +32,7 @@ impl BundleOptions { parser: Parser, modules_identifier: impl Into, excludes: impl Iterator, + loaders: LoaderConfiguration, ) -> Self { let excludes: Vec<_> = excludes .filter_map(|exclusion| match wax::Glob::new(exclusion) { @@ -54,6 +57,7 @@ impl BundleOptions { .expect("exclude globs errors should be filtered and only emit a warning"); Some(any_pattern) }, + loaders, } } @@ -71,6 +75,10 @@ impl BundleOptions { .map(|any| any.is_match(require)) .unwrap_or(false) } + + fn loaders(&self) -> &LoaderConfiguration { + &self.loaders + } } /// A rule that inlines required modules @@ -86,11 +94,12 @@ impl Bundler { parser: Parser, require_mode: BundleRequireMode, excludes: impl Iterator, + loaders: LoaderConfiguration, ) -> Self { Self { metadata: RuleMetadata::default(), require_mode, - options: BundleOptions::new(parser, DEFAULT_MODULE_IDENTIFIER, excludes), + options: BundleOptions::new(parser, DEFAULT_MODULE_IDENTIFIER, excludes, loaders), } } @@ -145,11 +154,17 @@ mod test { Parser::default(), BundleRequireMode::default(), std::iter::empty(), + LoaderConfiguration::default(), ) } fn new_rule_with_require_mode(mode: impl Into) -> Bundler { - Bundler::new(Parser::default(), mode.into(), std::iter::empty()) + Bundler::new( + Parser::default(), + mode.into(), + std::iter::empty(), + LoaderConfiguration::default(), + ) } // the bundler rule should only be used internally by darklua diff --git a/src/rules/bundle/path_require_mode/mod.rs b/src/rules/bundle/path_require_mode/mod.rs index e7abac3f..3b4c9096 100644 --- a/src/rules/bundle/path_require_mode/mod.rs +++ b/src/rules/bundle/path_require_mode/mod.rs @@ -7,16 +7,11 @@ use std::ops::{Deref, DerefMut}; use std::path::{Path, PathBuf}; use std::{iter, mem}; -use serde::Serialize; - -use crate::frontend::DarkluaResult; +use crate::frontend::{ContentType, DarkluaResult}; use crate::nodes::{ - Block, DoStatement, Expression, FunctionCall, Prefix, Statement, StringExpression, - VariableAssignment, -}; -use crate::process::{ - to_expression, DefaultVisitor, IdentifierTracker, NodeProcessor, NodeVisitor, ScopeVisitor, + Block, DoStatement, Expression, FunctionCall, Prefix, Statement, VariableAssignment, }; +use crate::process::{DefaultVisitor, IdentifierTracker, NodeProcessor, NodeVisitor, ScopeVisitor}; use crate::rules::require::{is_require_call, match_path_require_call, PathLocator}; use crate::rules::{ Context, ContextBuilder, FlawlessRule, ReplaceReferencedTokens, RuleProcessResult, @@ -185,74 +180,51 @@ impl<'a, 'b, 'resources, PathLocatorImpl: PathLocator> fn require_resource(&mut self, path: impl AsRef) -> DarkluaResult { let path = path.as_ref(); log::trace!("look for resource `{}`", path.display()); - let content = self.resources.get(path).map_err(DarkluaError::from)?; - - match path.extension() { - Some(extension) => match extension.to_string_lossy().as_ref() { - "lua" | "luau" => { - let parser_timer = Timer::now(); - let mut block = - self.options - .parser() - .parse(&content) - .map_err(|parser_error| { - DarkluaError::parser_error(path.to_path_buf(), parser_error) - })?; - log::debug!( - "parsed `{}` in {}", - path.display(), - parser_timer.duration_label() - ); - - if self.options.parser().is_preserving_tokens() { - log::trace!("replacing token references of {}", path.display()); - let context = ContextBuilder::new(path, self.resources, &content).build(); - // run `replace_referenced_tokens` rule to avoid generating invalid code - // when using the token-based generator - let replace_tokens = ReplaceReferencedTokens::default(); - - let apply_replace_tokens_timer = Timer::now(); + if !self.resources.exists(path).map_err(DarkluaError::from)? { + return Err(DarkluaError::resource_not_found(path)); + } - replace_tokens.flawless_process(&mut block, &context); + let loader = self.options.loaders().get_loader(path).to_internal_loader(); - log::trace!( - "replaced token references for `{}` in {}", - path.display(), - apply_replace_tokens_timer.duration_label() - ); - } + match loader.load(path, self.resources, self.options.parser())? { + ContentType::None => Err(DarkluaError::require_unknown_resource(path)), + ContentType::Copied(_data) => Err(DarkluaError::require_copied_resource(path)), + ContentType::Block(block) => Ok(RequiredResource::Block(block)), + ContentType::Expression(expression) => Ok(RequiredResource::Expression(expression)), + ContentType::Parsed { mut block, source } => { + if self.options.parser().is_preserving_tokens() { + log::trace!("replacing token references of {}", path.display()); + let context = ContextBuilder::new(path, self.resources, &source).build(); + // run `replace_referenced_tokens` rule to avoid generating invalid code + // when using the token-based generator + let replace_tokens = ReplaceReferencedTokens::default(); - let current_source = mem::replace(&mut self.source, path.to_path_buf()); + let apply_replace_tokens_timer = Timer::now(); - let apply_processor_timer = Timer::now(); - DefaultVisitor::visit_block(&mut block, self); + replace_tokens.flawless_process(&mut block, &context); - log::debug!( - "processed `{}` into bundle in {}", + log::trace!( + "replaced token references for `{}` in {}", path.display(), - apply_processor_timer.duration_label() + apply_replace_tokens_timer.duration_label() ); + } - self.source = current_source; + let current_source = mem::replace(&mut self.source, path.to_path_buf()); - Ok(RequiredResource::Block(block)) - } - "json" | "json5" => { - transcode("json", path, json5::from_str::, &content) - } - "yml" | "yaml" => transcode( - "yaml", - path, - serde_yaml::from_str::, - &content, - ), - "toml" => transcode("toml", path, toml::from_str::, &content), - "txt" => Ok(RequiredResource::Expression( - StringExpression::from_value(content).into(), - )), - _ => Err(DarkluaError::invalid_resource_extension(path)), - }, - None => unreachable!("extension should be defined"), + let apply_processor_timer = Timer::now(); + DefaultVisitor::visit_block(&mut block, self); + + log::debug!( + "processed `{}` into bundle in {}", + path.display(), + apply_processor_timer.duration_label() + ); + + self.source = current_source; + + Ok(RequiredResource::Block(block)) + } } } } @@ -275,31 +247,6 @@ impl<'a, 'b, 'resources, PathLocatorImpl: PathLocator> DerefMut } } -fn transcode<'a, T, E>( - label: &'static str, - path: &Path, - deserialize_value: impl Fn(&'a str) -> Result, - content: &'a str, -) -> Result -where - T: Serialize, - E: Into, -{ - log::trace!("transcode {} data to Lua from `{}`", label, path.display()); - let transcode_duration = Timer::now(); - let value = deserialize_value(content).map_err(E::into)?; - let expression = to_expression(&value) - .map(RequiredResource::Expression) - .map_err(DarkluaError::from); - log::debug!( - "transcoded {} data to Lua from `{}` in {}", - label, - path.display(), - transcode_duration.duration_label() - ); - expression -} - impl<'a, 'b, 'resources, PathLocatorImpl: PathLocator> NodeProcessor for RequirePathProcessor<'a, 'b, 'resources, PathLocatorImpl> { diff --git a/src/rules/convert_require/mod.rs b/src/rules/convert_require/mod.rs index e5f5991a..2e713772 100644 --- a/src/rules/convert_require/mod.rs +++ b/src/rules/convert_require/mod.rs @@ -130,9 +130,24 @@ impl<'a> RequireConverter<'a> { } fn try_require_conversion(&mut self, call: &mut FunctionCall) -> DarkluaResult<()> { - if let Some(require_path) = self.current.find_require(call, self.context)? { + if let Some(mut require_path) = self.current.find_require(call, self.context)? { log::trace!("found require path `{}`", require_path.display()); + let file_loader = self + .context + .loaders() + .get_loader(&require_path) + .to_internal_loader(); + + if file_loader.outputs_lua() + && !matches!( + require_path.extension().and_then(OsStr::to_str), + Some("lua") | Some("luau") + ) + { + require_path.set_extension(self.context.preferred_lua_extension()); + } + if let Some(new_arguments) = self.target .generate_require(&require_path, &self.current, self.context)? diff --git a/src/rules/mod.rs b/src/rules/mod.rs index a470f75f..e31f8805 100644 --- a/src/rules/mod.rs +++ b/src/rules/mod.rs @@ -82,6 +82,7 @@ pub(crate) use shift_token_line::*; pub use unused_if_branch::*; pub use unused_while::*; +use crate::frontend::LoaderConfiguration; use crate::nodes::Block; use crate::utils::FilterPattern; use crate::{DarkluaError, Resources}; @@ -108,6 +109,8 @@ pub struct ContextBuilder<'a, 'resources, 'code> { original_code: &'code str, blocks: HashMap, project_location: Option, + loaders: Option, + preferred_lua_extension: String, } impl<'a, 'resources, 'code> ContextBuilder<'a, 'resources, 'code> { @@ -123,15 +126,28 @@ impl<'a, 'resources, 'code> ContextBuilder<'a, 'resources, 'code> { original_code, blocks: Default::default(), project_location: None, + loaders: None, + preferred_lua_extension: "lua".to_owned(), } } + pub(crate) fn with_loaders(mut self, loaders: LoaderConfiguration) -> Self { + self.loaders = Some(loaders); + self + } + /// Sets the project location for this context. pub fn with_project_location(mut self, path: impl Into) -> Self { self.project_location = Some(path.into()); self } + /// Sets the preferred Lua extension. (should be `lua` or `luau`). + pub fn with_preferred_lua_extension(mut self, extension: impl Into) -> Self { + self.preferred_lua_extension = extension.into(); + self + } + /// Builds the final context with all configured options. pub fn build(self) -> Context<'a, 'resources, 'code> { Context { @@ -141,6 +157,8 @@ impl<'a, 'resources, 'code> ContextBuilder<'a, 'resources, 'code> { blocks: self.blocks, project_location: self.project_location, dependencies: Default::default(), + loaders: self.loaders.unwrap_or_default(), + preferred_lua_extension: self.preferred_lua_extension, } } @@ -162,6 +180,8 @@ pub struct Context<'a, 'resources, 'code> { blocks: HashMap, project_location: Option, dependencies: std::cell::RefCell>, + loaders: LoaderConfiguration, + preferred_lua_extension: String, } impl Context<'_, '_, '_> { @@ -192,6 +212,14 @@ impl Context<'_, '_, '_> { self.dependencies.into_inner().into_iter() } + pub(crate) fn loaders(&self) -> &LoaderConfiguration { + &self.loaders + } + + pub(crate) fn preferred_lua_extension(&self) -> &str { + &self.preferred_lua_extension + } + fn resources(&self) -> &Resources { self.resources } diff --git a/src/utils/filter_pattern.rs b/src/utils/filter_pattern.rs index ec762f17..f89e54b4 100644 --- a/src/utils/filter_pattern.rs +++ b/src/utils/filter_pattern.rs @@ -5,13 +5,21 @@ use wax::{Glob, Program}; use crate::DarkluaError; -#[derive(Debug, Clone, Serialize)] +#[derive(Clone, Serialize)] #[serde(into = "String")] pub(crate) struct FilterPattern { original: String, glob: Glob<'static>, } +impl std::fmt::Debug for FilterPattern { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("FilterPattern") + .field("pattern", &self.original) + .finish() + } +} + impl PartialEq for FilterPattern { fn eq(&self, other: &Self) -> bool { self.original == other.original diff --git a/src/utils/mod.rs b/src/utils/mod.rs index 8978b0c7..67a3132d 100644 --- a/src/utils/mod.rs +++ b/src/utils/mod.rs @@ -6,6 +6,7 @@ mod preserve_arguments_side_effects; mod scoped_hash_map; mod serde_one_or_many; mod serde_string_or_struct; +mod serde_vec_of_pairs; mod timer; pub(crate) use expressions_as_statement::{expressions_as_expression, expressions_as_statement}; @@ -15,6 +16,8 @@ pub(crate) use preserve_arguments_side_effects::preserve_arguments_side_effects; pub(crate) use scoped_hash_map::ScopedHashMap; pub(crate) use serde_one_or_many::deserialize_one_or_many; pub(crate) use serde_string_or_struct::string_or_struct; +pub(crate) use serde_vec_of_pairs::deserialize_vec_of_pairs; + use std::{ ffi::OsStr, iter::FromIterator, diff --git a/src/utils/serde_vec_of_pairs.rs b/src/utils/serde_vec_of_pairs.rs new file mode 100644 index 00000000..274a63db --- /dev/null +++ b/src/utils/serde_vec_of_pairs.rs @@ -0,0 +1,84 @@ +use std::{cmp, fmt, marker::PhantomData}; + +use serde::{ + de::{MapAccess, Visitor}, + Deserialize, Deserializer, +}; + +pub(crate) fn deserialize_vec_of_pairs<'de, D, K, V>( + deserializer: D, +) -> Result, D::Error> +where + D: Deserializer<'de>, + K: Deserialize<'de>, + V: Deserialize<'de>, +{ + struct MapAsVecVisitor(PhantomData>); + + impl MapAsVecVisitor { + fn new() -> Self { + MapAsVecVisitor(PhantomData) + } + } + + impl<'de, K, V> Visitor<'de> for MapAsVecVisitor + where + K: Deserialize<'de>, + V: Deserialize<'de>, + { + type Value = Vec<(K, V)>; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("a map") + } + + #[inline] + fn visit_unit(self) -> Result, E> { + Ok(Vec::new()) + } + + #[inline] + fn visit_map(self, mut access: T) -> Result, T::Error> + where + T: MapAccess<'de>, + { + let mut values = Vec::with_capacity(cmp::min(access.size_hint().unwrap_or(0), 4096)); + + while let Some((key, value)) = access.next_entry()? { + values.push((key, value)); + } + + Ok(values) + } + } + + let visitor = MapAsVecVisitor::::new(); + + deserializer.deserialize_map(visitor) +} + +#[cfg(test)] +mod test { + use super::*; + use serde::Deserialize; + + #[derive(Debug, Deserialize)] + struct TestVecOfPairs { + #[serde(deserialize_with = "deserialize_vec_of_pairs")] + items: Vec<(String, usize)>, + } + + #[test] + fn deserialize_empty_map() { + let v: TestVecOfPairs = serde_json::from_str(r#"{"items": {}}"#).unwrap(); + assert_eq!(v.items, Vec::new()); + } + + #[test] + fn deserialize_map_with_two_pairs() { + let v: TestVecOfPairs = + serde_json::from_str(r#"{"items": {"key": 1, "key2": 2}}"#).unwrap(); + + assert_eq!(v.items, vec![("key".to_owned(), 1), ("key2".to_owned(), 2)]); + } +} diff --git a/tests/bundle.rs b/tests/bundle.rs index 01160c24..96e5f3e6 100644 --- a/tests/bundle.rs +++ b/tests/bundle.rs @@ -83,7 +83,19 @@ mod without_rules { .result() .unwrap(); - let main = resources.get("out.lua").unwrap(); + let out_file = resources.get("out.lua"); + + assert!( + out_file.is_ok(), + "failed to locate out.lua file. Resources found: {:#?}", + { + let mut files = resources.walk("").collect::>(); + files.sort(); + files + } + ); + + let main = out_file.unwrap(); insta::assert_snapshot!(format!("bundle_without_rules_{}", snapshot_name), main); } diff --git a/tests/frontend.rs b/tests/frontend.rs index 506edda5..3ba75d79 100644 --- a/tests/frontend.rs +++ b/tests/frontend.rs @@ -274,6 +274,189 @@ fn use_default_json5_config_in_place() { assert_eq!(resources.get("src/test.lua").unwrap(), "return 'Hello'"); } +mod loaders { + + use super::*; + + use pretty_assertions::assert_eq; + + #[test] + fn use_custom_loader_to_load_txt_extension_as_luau() { + let resources = memory_resources!( + "src/test.txt" => "return _G.VALUE", + "src/example.luau" => "return _G.VALUE", + ".darklua.json" => r#"{ + "rules": [ { "rule": "inject_global_value", "identifier": "VALUE", "value": 1 } ], + "loaders": { "**/*.txt": "luau" }, + "lua_extension": "luau", + }"#, + ); + + process(&resources, Options::new("src")) + .unwrap() + .result() + .unwrap(); + + assert_eq!(resources.get("src/test.luau").unwrap(), "return 1"); + assert_eq!(resources.get("src/example.luau").unwrap(), "return 1"); + } + + #[test] + fn use_loader_for_json_files() { + let resources = memory_resources!( + "src/test.json" => r#"{ "value": 1 }"#, + ".darklua.json" => r#"{ "rules": [] }"#, + ); + + process(&resources, Options::new("src")) + .unwrap() + .result() + .unwrap(); + + insta::assert_snapshot!(resources.get("src/test.lua").unwrap(), @"return {value=1}"); + } + + #[test] + fn use_loader_for_yaml_file() { + let resources = memory_resources!( + "src/test.yaml" => r#"value: 1"#, + "src/test2.yml" => r#"value: 2"#, + ".darklua.json" => r#"{ "rules": [] }"#, + ); + + process(&resources, Options::new("src")) + .unwrap() + .result() + .unwrap(); + + insta::assert_snapshot!(resources.get("src/test.lua").unwrap(), @"return {value=1}"); + insta::assert_snapshot!(resources.get("src/test2.lua").unwrap(), @"return {value=2}"); + } + + #[test] + fn use_loader_for_toml_file() { + let resources = memory_resources!( + "src/test.toml" => r#"value = 1"#, + ".darklua.json" => r#"{ "rules": [] }"#, + ); + + process(&resources, Options::new("src")) + .unwrap() + .result() + .unwrap(); + + insta::assert_snapshot!(resources.get("src/test.lua").unwrap(), @"return {value=1}"); + } + + #[test] + fn use_loader_for_txt_file() { + let resources = memory_resources!( + "src/test.txt" => r#"Hello"#, + ".darklua.json" => r#"{ "rules": [] }"#, + ); + + process(&resources, Options::new("src")) + .unwrap() + .result() + .unwrap(); + + insta::assert_snapshot!(resources.get("src/test.lua").unwrap(), @"return 'Hello'"); + } + + #[test] + fn use_custom_string_base64_loader_for_txt_file() { + let resources = memory_resources!( + "src/test.txt" => r#"Hello"#, + ".darklua.json" => r#"{ + "rules": [], + "loaders": { "**/*.txt": "string/base64" }, + }"#, + ); + + process(&resources, Options::new("src")) + .unwrap() + .result() + .unwrap(); + + insta::assert_snapshot!(resources.get("src/test.lua").unwrap(),@ "return 'SGVsbG8='"); + } + + #[test] + fn use_custom_buffer_base64_loader_for_txt_file() { + let resources = memory_resources!( + "src/test.txt" => r#"Hello"#, + ".darklua.json" => r#"{ + "rules": [], + "loaders": { "**/*.txt": "buffer/base64" }, + }"#, + ); + + process(&resources, Options::new("src")) + .unwrap() + .result() + .unwrap(); + + insta::assert_snapshot!( + resources.get("src/test.lua").unwrap(), + @"return buffer.fromstring'SGVsbG8='" + ); + } + + #[test] + fn use_custom_bytes_loader_for_txt_file() { + let resources = memory_resources!( + "src/test.txt" => r#"Hello"#, + ".darklua.json" => r#"{ + "rules": [], + "loaders": { "**/*.txt": "bytes" }, + }"#, + ); + + process(&resources, Options::new("src")) + .unwrap() + .result() + .unwrap(); + + insta::assert_snapshot!(resources.get("src/test.lua").unwrap(), @"return {72, 101, 108, 108, 111}"); + } + + #[test] + fn use_custom_bytes_base64_loader_for_txt_file() { + let resources = memory_resources!( + "src/test.txt" => r#"Hello"#, + ".darklua.json" => r#"{ + "rules": [], + "loaders": { "**/*.txt": "bytes/base64" }, + }"#, + ); + + process(&resources, Options::new("src")) + .unwrap() + .result() + .unwrap(); + + insta::assert_snapshot!(resources.get("src/test.lua").unwrap(), @"return {83, 71, 86, 115, 98, 71, 56, 61}"); + } + + #[test] + fn use_custom_copy_loader_for_txt_file() { + let resources = memory_resources!( + "src/test.txt" => r#"Hello"#, + ".darklua.json" => r#"{ + "rules": [], + "loaders": { "**/*.txt": "copy" }, + }"#, + ); + + process(&resources, Options::new("src")) + .unwrap() + .result() + .unwrap(); + + insta::assert_snapshot!(resources.get("src/test.txt").unwrap(), @"Hello"); + } +} + mod errors { use std::path::{Path, PathBuf}; diff --git a/tests/rule_tests/convert_require.rs b/tests/rule_tests/convert_require.rs index ca3058f7..d120c80a 100644 --- a/tests/rule_tests/convert_require.rs +++ b/tests/rule_tests/convert_require.rs @@ -21,6 +21,13 @@ test_rule!( "src/test/folder/lib.lua" => "return nil", "src/sub/lib.lua" => "return nil", "src/format.lua" => "return nil", + // data files + "src/data-json.json" => "[1, 2, 3]", + "src/data-json5.json5" => "{ hello: 1 }", + "src/data-yml.yml" => "value: 1", + "src/data-toml.toml" => "value = 1", + "src/data-text.txt" => "some text", + "project.lua" => "return nil", ), test_file_name = "src/test/runner.lua", @@ -44,6 +51,17 @@ test_rule!( => "local module = require(script.Parent.Parent.Parent:FindFirstChild('project'))", module_in_parent_with_current_dir("local module = require('.././format.lua')") => "local module = require(script.Parent.Parent:FindFirstChild('format'))", + // data file requires + json_data_module_in_parent("local module = require('../data-json.json')") + => "local module = require(script.Parent.Parent:FindFirstChild('data-json'))", + json5_data_module_in_parent("local module = require('../data-json5.json5')") + => "local module = require(script.Parent.Parent:FindFirstChild('data-json5'))", + yml_data_module_in_parent("local module = require('../data-yml.yml')") + => "local module = require(script.Parent.Parent:FindFirstChild('data-yml'))", + toml_data_module_in_parent("local module = require('../data-toml.toml')") + => "local module = require(script.Parent.Parent:FindFirstChild('data-toml'))", + text_data_module_in_parent("local module = require('../data-text.txt')") + => "local module = require(script.Parent.Parent:FindFirstChild('data-text'))", ); test_rule!( @@ -63,6 +81,13 @@ test_rule!( "src/test/folder/lib.lua" => "return nil", "src/sub/lib.lua" => "return nil", "src/format.lua" => "return nil", + // data files + "src/data-json.json" => "[1, 2, 3]", + "src/data-json5.json5" => "{ hello: 1 }", + "src/data-yml.yml" => "value: 1", + "src/data-toml.toml" => "value = 1", + "src/data-text.txt" => "some text", + "project.lua" => "return nil", ), test_file_name = "src/test/runner.lua", @@ -86,6 +111,17 @@ test_rule!( => "local module = require(script.Parent.Parent.Parent:WaitForChild('project'))", module_in_parent_with_current_dir("local module = require('.././format.lua')") => "local module = require(script.Parent.Parent:WaitForChild('format'))", + // data file requires + json_data_module_in_parent("local module = require('../data-json.json')") + => "local module = require(script.Parent.Parent:WaitForChild('data-json'))", + json5_data_module_in_parent("local module = require('../data-json5.json5')") + => "local module = require(script.Parent.Parent:WaitForChild('data-json5'))", + yml_data_module_in_parent("local module = require('../data-yml.yml')") + => "local module = require(script.Parent.Parent:WaitForChild('data-yml'))", + toml_data_module_in_parent("local module = require('../data-toml.toml')") + => "local module = require(script.Parent.Parent:WaitForChild('data-toml'))", + text_data_module_in_parent("local module = require('../data-text.txt')") + => "local module = require(script.Parent.Parent:WaitForChild('data-text'))", ); test_rule!( @@ -106,6 +142,12 @@ test_rule!( "src/sub/lib.lua" => "return nil", "src/format.lua" => "return nil", "project.lua" => "return nil", + // data files + "src/data-json.json" => "[1, 2, 3]", + "src/data-json5.json5" => "{ hello: 1 }", + "src/data-yml.yml" => "value: 1", + "src/data-toml.toml" => "value = 1", + "src/data-text.txt" => "some text", // specific to property index style tests "src/test/while.lua" => "return nil", "src/test/a module.lua" => "return nil", @@ -131,6 +173,17 @@ test_rule!( => "local module = require(script.Parent.Parent.Parent.project)", module_in_parent_with_current_dir("local module = require('.././format.lua')") => "local module = require(script.Parent.Parent.format)", + // data file requires + json_data_module_in_parent("local module = require('../data-json.json')") + => "local module = require(script.Parent.Parent['data-json'])", + json5_data_module_in_parent("local module = require('../data-json5.json5')") + => "local module = require(script.Parent.Parent['data-json5'])", + yml_data_module_in_parent("local module = require('../data-yml.yml')") + => "local module = require(script.Parent.Parent['data-yml'])", + toml_data_module_in_parent("local module = require('../data-toml.toml')") + => "local module = require(script.Parent.Parent['data-toml'])", + text_data_module_in_parent("local module = require('../data-text.txt')") + => "local module = require(script.Parent.Parent['data-text'])", // specific to property index style sibling_module_with_keyword_name("local module = require('./while.lua')") => "local module = require(script.Parent['while'])", @@ -162,6 +215,12 @@ test_rule!( "src/sub/lib.luau" => "return nil", "src/format.luau" => "return nil", "project.luau" => "return nil", + // data files + "src/data-json.json" => "[1, 2, 3]", + "src/data-json5.json5" => "{ hello: 1 }", + "src/data-yml.yml" => "value: 1", + "src/data-toml.toml" => "value = 1", + "src/data-text.txt" => "some text", // specific to alias tests "src/test/while.luau" => "return nil", "src/test/a module.luau" => "return nil", @@ -188,6 +247,17 @@ test_rule!( // Init file conversions init_file_conversion("local module = require('./init')") => "local module = require('.')", + // data file requires + json_data_module_in_parent("local module = require('../data-json.json')") + => "local module = require('../data-json')", + json5_data_module_in_parent("local module = require('../data-json5.json5')") + => "local module = require('../data-json5')", + yml_data_module_in_parent("local module = require('../data-yml.yml')") + => "local module = require('../data-yml')", + toml_data_module_in_parent("local module = require('../data-toml.toml')") + => "local module = require('../data-toml')", + text_data_module_in_parent("local module = require('../data-text.txt')") + => "local module = require('../data-text')", ); test_rule!( diff --git a/tests/snapshots/bundle__without_rules__require_lua_file_with_unsupported_extension.snap b/tests/snapshots/bundle__without_rules__require_lua_file_with_unsupported_extension.snap index 3acea18e..1f5609b5 100644 --- a/tests/snapshots/bundle__without_rules__require_lua_file_with_unsupported_extension.snap +++ b/tests/snapshots/bundle__without_rules__require_lua_file_with_unsupported_extension.snap @@ -2,4 +2,4 @@ source: tests/bundle.rs expression: "error_display.join(\"\\n\")" --- -error processing `src/main.lua` (bundler): unable to require resource with extension `error` at `src/value.error` +error processing `src/main.lua` (bundler): unable to require unknown resource at `src/value.error` (configure content loaders to load this file) From bb97a24d45503b707891206097b338451326ee85 Mon Sep 17 00:00:00 2001 From: jeparlefrancais Date: Mon, 1 Jun 2026 15:19:22 -0400 Subject: [PATCH 2/4] add entry to changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 08292333..c979edef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Unreleased +* add content loaders to the configuration file to process more than Lua or Luau files with darklua ([#354](https://github.com/seaofvoices/darklua/pull/354)) * improve file watching to handle rename events and remove empty folders after processing if they weren't present before the initial run ([#351](https://github.com/seaofvoices/darklua/pull/351)) * add a new parameter for the `rename_variables` rule so that globals can be detected automatically and then avoided in the renaming pass ([#348](https://github.com/seaofvoices/darklua/pull/348)) * add support for `const` declaration of variables and functions (e.g. `const var = true` or `const function test() end`) and add rule (`make_assignment_local`) to convert those assignments to `local` assignments ([#346](https://github.com/seaofvoices/darklua/pull/346)) From b252965b1a0808f2d5773e727e1eaab604be8ae8 Mon Sep 17 00:00:00 2001 From: jeparlefrancais Date: Mon, 1 Jun 2026 16:30:03 -0400 Subject: [PATCH 3/4] add documentation page --- site/content/docs/config/index.md | 27 +++++ site/content/docs/content-loaders/index.md | 133 +++++++++++++++++++++ 2 files changed, 160 insertions(+) create mode 100644 site/content/docs/content-loaders/index.md diff --git a/site/content/docs/config/index.md b/site/content/docs/config/index.md index 3b55c00b..e42879b4 100644 --- a/site/content/docs/config/index.md +++ b/site/content/docs/config/index.md @@ -24,6 +24,24 @@ From the directory where you run `darklua process`, darklua will attempt to read To provide a different configuration file, this subcommand also accept a specific path to a configuration file with `--config `. +## Content Loaders + +darklua chooses how to process each file based on its extension: Lua and Luau files are parsed, recognized data files (such as JSON, TOML, and YAML) are converted to Lua modules, and other files are left alone. With content loaders, you can override these defaults or tell darklua what to do with other files: copy them to the output, turn their content into a Lua module, or skip them. + +Loaders are configured with the `loaders` field, which maps glob patterns to loader names. For example, to copy Rojo model files, ignore `.yml` files and turn Markdown files into string modules: + +```json5 +{ + loaders: { + "**/*.model.json": "copy", + "**/*.yml": "skip", + "**/*.md": "string", + }, +} +``` + +See the [content loaders](/docs/content-loaders) page for the full list of loaders and how each one works. + ## Filtering It is possible to limit which files are processed using `apply_to_files` and `skip_files`. Both accept glob patterns [from this implementation](https://github.com/olson-sean-k/wax/blob/master/README.md#patterns) (same syntax as `bundle.excludes`). Each field can be a single pattern string or an array of patterns. @@ -47,6 +65,15 @@ Any missing field will be replaced with its default value. // Exclude applying rules to some files skip_files: ["**/*.test.lua"], + // Tell darklua how to process files + loaders: { + "**/*.model.json": "copy", + "**/*.md": "string", + "**/*.png": "buffer/base64", + }, + // Some content loaders create Lua modules which will use this extension + lua_extension: "lua", // or "luau" + bundle: { // Identifier used by darklua to store the bundled modules modules_identifier: "__DARKLUA_BUNDLE_MODULES", diff --git a/site/content/docs/content-loaders/index.md b/site/content/docs/content-loaders/index.md new file mode 100644 index 00000000..2d1b1c03 --- /dev/null +++ b/site/content/docs/content-loaders/index.md @@ -0,0 +1,133 @@ +--- +title: Content Loaders +description: How darklua processes files +group: Configuration +order: 7 +--- + +darklua picks a loader for each file based on its extension: Lua and Luau files are parsed, recognized data files are converted to Lua modules, and everything else is left alone. The `loaders` field lets you override these defaults or assign a loader to any extension darklua does not recognize. It maps [glob patterns](https://github.com/olson-sean-k/wax/blob/master/README.md#patterns) to loader names. Patterns are matched in order, and the first one that matches a file wins. Files that match no pattern fall back to the extension-based defaults (see [Reference](#reference)), and files with no recognized loader are skipped. + +```json5 +{ + loaders: { + "**/*.model.json": "copy", + "**/*.md": "string", + "**/*.png": "buffer/base64", + }, +} +``` + +## Copying files + +The `copy` loader writes files to the output directory unchanged. This is useful for assets that need to end up alongside your processed code, such as Rojo model files: + +```json5 +{ + loaders: { + "**/*.model.json": "copy", + }, +} +``` + +Previously, the usual workaround was to copy these files manually (for example with `cp`) and then run darklua in-place. The `copy` loader removes that extra step. + +## Processing data files + +darklua can convert structured data into a Lua module that returns the data as a table. JSON, TOML, and YAML are supported: + +```json5 +{ + loaders: { + "**/*.json": "json", + "**/*.toml": "toml", + "**/*.yaml": "yaml", + }, +} +``` + +A file like `config.json` becomes `config.lua` (or `config.luau`) that returns the converted table. This already worked when bundling; loaders make it available in the regular `process` flow too. + +The `json` loader accepts JSON5 syntax, so files may include comments, trailing commas, and unquoted keys. The `yaml` loader also accepts `yml` as an alias, so `"**/*.yml": "yaml"` and `"**/*.yml": "yml"` are equivalent. + +## Embedding file content + +These loaders read a file and create a Lua module that returns its content. They work with any file, including binary files. + +- `string`: returns the content as a string. +- `buffer`: returns the content as a buffer (`buffer.fromstring(...)`). +- `bytes`: returns the content as an array of byte values. + +```json5 +{ + loaders: { + "**/*.md": "string", + "**/*.bin": "buffer", + }, +} +``` + +Each of these also has a `/base64` variant that encodes the content as base64 before embedding it: `string/base64`, `buffer/base64`, and `bytes/base64`. This keeps binary data safe inside the generated Lua source. Note that base64 content is **not** decoded at runtime, so your code receives the encoded value. + +Unlike the data loaders (`json`, `toml`, `yaml`) and `luau`, which require valid UTF-8 input, the embedding loaders accept any file content and work correctly with binary files. + +## Skipping files + +The `skip` loader ignores matching files. Use it to exclude files that would otherwise be picked up by a default loader: + +```json5 +{ + loaders: { + "**/*.test.json": "skip", + }, +} +``` + +## Choosing the Lua extension + +Loaders that produce a Lua module write a file with the same name but a Lua extension. By default this is `.lua`. Set `lua_extension` to control which extension is used: + +```json5 +{ + loaders: { + "**/*.json": "json", + }, + lua_extension: "luau", // or "lua" +} +``` + +Files that already end in `.lua` or `.luau` keep their extension. + +## Requiring loaded files + +Loaders that produce a Lua module work with `require` during bundling. You can `require` a JSON config, an embedded Markdown template, or any other file whose loader outputs Lua, and darklua will inline the generated module. The path in the `require` call is rewritten to use the `.lua` (or `.luau`) extension automatically, so `require("./data.json")` becomes a valid require after processing. + +Files matched by `skip` or `copy` cannot be `require`d. Requiring a skipped file (or a file with no recognized loader) produces the error "configure content loaders to load this file". Requiring a copied file produces "configure content loaders to copy this file". In both cases the error message points to the relevant loader to use instead. + +## Reference + +Loaders available for use in the `loaders` field: + +| Loader | What it does | +| --- | --- | +| `luau` | Parses and processes the file as Lua/Luau code. | +| `copy` | Copies the file to the output unchanged. | +| `skip` | Ignores the file. | +| `string` | Returns the content as a string. | +| `string/base64` | Returns the content as a base64-encoded string. | +| `buffer` | Returns the content as a buffer. | +| `buffer/base64` | Returns the content as a base64-encoded buffer. | +| `bytes` | Returns the content as an array of bytes. | +| `bytes/base64` | Returns the content, base64-encoded, as an array of bytes. | +| `json` | Converts JSON data to a Lua module. | +| `toml` | Converts TOML data to a Lua module. | +| `yaml` | Converts YAML data to a Lua module. | + +Default loaders assigned by file extension when no pattern matches. Anything not listed here is skipped unless you assign a loader to it. + +| Extension | Loader | +| --- | --- | +| `.lua`, `.luau` | `luau` | +| `.json`, `.json5` | `json` | +| `.toml` | `toml` | +| `.yaml`, `.yml` | `yaml` | +| `.txt` | `string` | From 8634c2bf2d5687c3396cc7977d3cbc5009aff6eb Mon Sep 17 00:00:00 2001 From: jeparlefrancais Date: Mon, 1 Jun 2026 16:34:23 -0400 Subject: [PATCH 4/4] fix docs page formatting --- site/content/docs/content-loaders/index.md | 42 +++++++++++----------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/site/content/docs/content-loaders/index.md b/site/content/docs/content-loaders/index.md index 2d1b1c03..66f5552e 100644 --- a/site/content/docs/content-loaders/index.md +++ b/site/content/docs/content-loaders/index.md @@ -107,27 +107,27 @@ Files matched by `skip` or `copy` cannot be `require`d. Requiring a skipped file Loaders available for use in the `loaders` field: -| Loader | What it does | -| --- | --- | -| `luau` | Parses and processes the file as Lua/Luau code. | -| `copy` | Copies the file to the output unchanged. | -| `skip` | Ignores the file. | -| `string` | Returns the content as a string. | -| `string/base64` | Returns the content as a base64-encoded string. | -| `buffer` | Returns the content as a buffer. | -| `buffer/base64` | Returns the content as a base64-encoded buffer. | -| `bytes` | Returns the content as an array of bytes. | -| `bytes/base64` | Returns the content, base64-encoded, as an array of bytes. | -| `json` | Converts JSON data to a Lua module. | -| `toml` | Converts TOML data to a Lua module. | -| `yaml` | Converts YAML data to a Lua module. | +| Loader | What it does | +| --------------- | ---------------------------------------------------------- | +| `luau` | Parses and processes the file as Lua/Luau code. | +| `copy` | Copies the file to the output unchanged. | +| `skip` | Ignores the file. | +| `string` | Returns the content as a string. | +| `string/base64` | Returns the content as a base64-encoded string. | +| `buffer` | Returns the content as a buffer. | +| `buffer/base64` | Returns the content as a base64-encoded buffer. | +| `bytes` | Returns the content as an array of bytes. | +| `bytes/base64` | Returns the content, base64-encoded, as an array of bytes. | +| `json` | Converts JSON data to a Lua module. | +| `toml` | Converts TOML data to a Lua module. | +| `yaml` | Converts YAML data to a Lua module. | Default loaders assigned by file extension when no pattern matches. Anything not listed here is skipped unless you assign a loader to it. -| Extension | Loader | -| --- | --- | -| `.lua`, `.luau` | `luau` | -| `.json`, `.json5` | `json` | -| `.toml` | `toml` | -| `.yaml`, `.yml` | `yaml` | -| `.txt` | `string` | +| Extension | Loader | +| ----------------- | -------- | +| `.lua`, `.luau` | `luau` | +| `.json`, `.json5` | `json` | +| `.toml` | `toml` | +| `.yaml`, `.yml` | `yaml` | +| `.txt` | `string` |