Skip to content

Commit 4505f12

Browse files
Implement archive list and archive extract for web .data files
* Add sample .data file for testing * Add function to determine if a file is a web bundle * Rename test to disambiguate with forthcoming test * HandleExtractArchive handles .data files as well as asset bundles * Propagate specific error message to console output * Simplify IsWebBundle test to use file extension Since we also need to support gzip- and brotli-compressed web bundles, and we don't want to decompress more than once. * Check for invalid web bundle in ParseWebBundleHeader And throw the appropriate exception type (requires adding package System.IO.Packaging). * Remove unneeded `using` statement * Simplify namespace paths where possible * Support extracting gzip- and brotli-compressed web bundles * Clean up `using` statements * Extract function to facilitate error handling * Reading less than the expected number of bytes raises FileFormatException * Extract function ReadUInt32 to facilitate error handling * EndOfStream from ReadUInt32 raises FileFormatException * Rename test to disambiguate with forthcoming test * Wrap console redirection in try/finally to ensure it's restored * Whitespace * Extract function ListAssetBundle * Test `archive list` on web bundle (fails) * Log error message in `archive list` * Add function ListWebBundle * HandleListArchive branches on archive type * Move members related to `archive` command to a new file * Remove redundancy in function names * Update CLI command descriptions * Update README description of `archive` command * Normalize wording: "content" -> "contents" * Whitespace * Add .gitattributes to specify auto line endings
1 parent 53b1288 commit 4505f12

File tree

10 files changed

+349
-92
lines changed

10 files changed

+349
-92
lines changed

.gitattributes

+12
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
* text=auto
2+
3+
assetbundle binary
4+
scenes binary
5+
level* binary
6+
*.dll binary
7+
*.dylib binary
8+
*.so binary
9+
*.wav binary
10+
*.fbx binary
11+
*.tif binary
12+
*.jpg binary
2.39 MB
Binary file not shown.
675 KB
Binary file not shown.
814 KB
Binary file not shown.
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
This is NOT a web bundle.

UnityDataTool.Tests/UnityDataToolTests.cs

+104-17
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,8 @@
55
using System.Text.RegularExpressions;
66
using System.Threading.Tasks;
77
using NUnit.Framework;
8-
using UnityDataTools.TestCommon;
98
using UnityDataTools.FileSystem;
9+
using UnityDataTools.TestCommon;
1010

1111
namespace UnityDataTools.UnityDataTool.Tests;
1212

@@ -57,8 +57,22 @@ public async Task InvalidFile(
5757
Assert.AreNotEqual(0, await Program.Main(command.ToArray()));
5858
}
5959

60+
public void IsWebBundle_True()
61+
{
62+
63+
var webBundlePath = Path.Combine(Context.TestDataFolder, "WebBundles", "HelloWorld.data");
64+
Assert.IsTrue(Archive.IsWebBundle(new FileInfo(webBundlePath)));
65+
}
66+
67+
[Test]
68+
public void IsWebBundle_False()
69+
{
70+
var nonWebBundlePath = Path.Combine(Context.TestDataFolder, "WebBundles", "NotAWebBundle.txt");
71+
Assert.IsFalse(Archive.IsWebBundle(new FileInfo(nonWebBundlePath)));
72+
}
73+
6074
[Test]
61-
public async Task ArchiveExtract_FilesExtractedSuccessfully(
75+
public async Task ArchiveExtract_AssetBundle_FilesExtractedSuccessfully(
6276
[Values("", "-o archive", "--output-path archive")] string options)
6377
{
6478
var path = Path.Combine(Context.UnityDataFolder, "assetbundle");
@@ -70,32 +84,105 @@ public async Task ArchiveExtract_FilesExtractedSuccessfully(
7084
}
7185

7286
[Test]
73-
public async Task ArchiveList_ListFilesCorrectly()
87+
public async Task ArchiveExtract_WebBundle_FileExtractedSuccessfully(
88+
[Values("", "-o archive", "--output-path archive")] string options,
89+
[Values("HelloWorld.data", "HelloWorld.data.gz", "HelloWorld.data.br")] string bundlePath)
7490
{
75-
var path = Path.Combine(Context.UnityDataFolder, "assetbundle");
91+
var path = Path.Combine(Context.TestDataFolder, "WebBundles", bundlePath);
92+
string[] expectedFiles = {
93+
"boot.config",
94+
"data.unity3d",
95+
"RuntimeInitializeOnLoads.json",
96+
"ScriptingAssemblies.json",
97+
Path.Combine("Il2CppData", "Metadata", "global-metadata.dat"),
98+
Path.Combine("Resources", "unity_default_resources"),
99+
};
100+
Assert.AreEqual(0, await Program.Main(new string[] { "archive", "extract", path }.Concat(options.Split(" ", StringSplitOptions.RemoveEmptyEntries)).ToArray()));
101+
foreach (var file in expectedFiles)
102+
{
103+
Assert.IsTrue(File.Exists(Path.Combine(m_TestOutputFolder, "archive", file)));
104+
}
105+
}
76106

107+
[Test]
108+
public async Task ArchiveList_AssetBundle_ListFilesCorrectly()
109+
{
110+
var path = Path.Combine(Context.UnityDataFolder, "assetbundle");
77111
using var sw = new StringWriter();
112+
var currentOut = Console.Out;
113+
try
114+
{
115+
Console.SetOut(sw);
78116

117+
Assert.AreEqual(0, await Program.Main(new string[] { "archive", "list", path }));
118+
119+
var lines = sw.ToString().Split(sw.NewLine);
120+
121+
Assert.AreEqual("CAB-5d40f7cad7c871cf2ad2af19ac542994", lines[0]);
122+
Assert.AreEqual($" Size: {Context.ExpectedData.Get("CAB-5d40f7cad7c871cf2ad2af19ac542994-Size")}", lines[1]);
123+
Assert.AreEqual($" Flags: {(ArchiveNodeFlags)(long)Context.ExpectedData.Get("CAB-5d40f7cad7c871cf2ad2af19ac542994-Flags")}", lines[2]);
124+
125+
Assert.AreEqual("CAB-5d40f7cad7c871cf2ad2af19ac542994.resS", lines[4]);
126+
Assert.AreEqual($" Size: {Context.ExpectedData.Get("CAB-5d40f7cad7c871cf2ad2af19ac542994.resS-Size")}", lines[5]);
127+
Assert.AreEqual($" Flags: {(ArchiveNodeFlags)(long)Context.ExpectedData.Get("CAB-5d40f7cad7c871cf2ad2af19ac542994.resS-Flags")}", lines[6]);
128+
129+
Assert.AreEqual("CAB-5d40f7cad7c871cf2ad2af19ac542994.resource", lines[8]);
130+
Assert.AreEqual($" Size: {Context.ExpectedData.Get("CAB-5d40f7cad7c871cf2ad2af19ac542994.resource-Size")}", lines[9]);
131+
Assert.AreEqual($" Flags: {(ArchiveNodeFlags)(long)Context.ExpectedData.Get("CAB-5d40f7cad7c871cf2ad2af19ac542994.resource-Flags")}", lines[10]);
132+
133+
}
134+
finally
135+
{
136+
Console.SetOut(currentOut);
137+
}
138+
}
139+
140+
[Test]
141+
public async Task ArchiveList_WebBundle_ListFilesCorrectly(
142+
[Values(
143+
"HelloWorld.data",
144+
"HelloWorld.data.gz",
145+
"HelloWorld.data.br"
146+
)] string bundlePath)
147+
{
148+
var path = Path.Combine(Context.TestDataFolder, "WebBundles", bundlePath);
149+
using var sw = new StringWriter();
79150
var currentOut = Console.Out;
80-
Console.SetOut(sw);
151+
try
152+
{
153+
Console.SetOut(sw);
154+
155+
Assert.AreEqual(0, await Program.Main(new string[] { "archive", "list", path }));
81156

82-
Assert.AreEqual(0, await Program.Main(new string[] { "archive", "list", path }));
157+
var actualOutput = sw.ToString();
158+
var expectedOutput = (
159+
@"data.unity3d
160+
Size: 253044
83161
84-
var lines = sw.ToString().Split(sw.NewLine);
162+
RuntimeInitializeOnLoads.json
163+
Size: 700
85164
86-
Assert.AreEqual("CAB-5d40f7cad7c871cf2ad2af19ac542994", lines[0]);
87-
Assert.AreEqual($" Size: {Context.ExpectedData.Get("CAB-5d40f7cad7c871cf2ad2af19ac542994-Size")}", lines[1]);
88-
Assert.AreEqual($" Flags: {(ArchiveNodeFlags)(long)Context.ExpectedData.Get("CAB-5d40f7cad7c871cf2ad2af19ac542994-Flags")}", lines[2]);
165+
ScriptingAssemblies.json
166+
Size: 3060
89167
90-
Assert.AreEqual("CAB-5d40f7cad7c871cf2ad2af19ac542994.resS", lines[4]);
91-
Assert.AreEqual($" Size: {Context.ExpectedData.Get("CAB-5d40f7cad7c871cf2ad2af19ac542994.resS-Size")}", lines[5]);
92-
Assert.AreEqual($" Flags: {(ArchiveNodeFlags)(long)Context.ExpectedData.Get("CAB-5d40f7cad7c871cf2ad2af19ac542994.resS-Flags")}", lines[6]);
168+
boot.config
169+
Size: 93
93170
94-
Assert.AreEqual("CAB-5d40f7cad7c871cf2ad2af19ac542994.resource", lines[8]);
95-
Assert.AreEqual($" Size: {Context.ExpectedData.Get("CAB-5d40f7cad7c871cf2ad2af19ac542994.resource-Size")}", lines[9]);
96-
Assert.AreEqual($" Flags: {(ArchiveNodeFlags)(long)Context.ExpectedData.Get("CAB-5d40f7cad7c871cf2ad2af19ac542994.resource-Flags")}", lines[10]);
171+
Il2CppData/Metadata/global-metadata.dat
172+
Size: 1641180
97173
98-
Console.SetOut(currentOut);
174+
Resources/unity_default_resources
175+
Size: 607376
176+
177+
"
178+
);
179+
180+
Assert.AreEqual(expectedOutput, actualOutput);
181+
}
182+
finally
183+
{
184+
Console.SetOut(currentOut);
185+
}
99186
}
100187

101188
[Test]

UnityDataTool/Archive.cs

+217
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,217 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using System.IO;
4+
using System.IO.Compression;
5+
using System.Linq;
6+
using System.Text;
7+
using UnityDataTools.FileSystem;
8+
9+
namespace UnityDataTools.UnityDataTool;
10+
11+
public static class Archive
12+
{
13+
private static readonly byte[] WebBundlePrefix = Encoding.UTF8.GetBytes("UnityWebData1.0\0");
14+
15+
public static int HandleExtract(FileInfo filename, DirectoryInfo outputFolder)
16+
{
17+
try
18+
{
19+
if (IsWebBundle(filename))
20+
{
21+
ExtractWebBundle(filename, outputFolder);
22+
}
23+
else
24+
{
25+
ExtractAssetBundle(filename, outputFolder);
26+
}
27+
}
28+
catch (Exception err) when (
29+
err is NotSupportedException
30+
|| err is FileFormatException)
31+
{
32+
Console.Error.WriteLine("Error opening archive");
33+
Console.Error.WriteLine(err.Message);
34+
return 1;
35+
}
36+
return 0;
37+
}
38+
39+
public static int HandleList(FileInfo filename)
40+
{
41+
try
42+
{
43+
if (IsWebBundle(filename))
44+
{
45+
ListWebBundle(filename);
46+
}
47+
else
48+
{
49+
ListAssetBundle(filename);
50+
}
51+
}
52+
catch (Exception err) when (
53+
err is NotSupportedException
54+
|| err is FileFormatException)
55+
{
56+
Console.Error.WriteLine("Error opening archive");
57+
Console.Error.WriteLine(err.Message);
58+
return 1;
59+
}
60+
61+
return 0;
62+
}
63+
64+
65+
public static bool IsWebBundle(FileInfo filename)
66+
{
67+
var path = filename.ToString();
68+
return (
69+
path.EndsWith(".data")
70+
|| path.EndsWith(".data.gz")
71+
|| path.EndsWith(".data.br")
72+
);
73+
}
74+
75+
struct WebBundleFileDescription
76+
{
77+
public uint ByteOffset;
78+
public uint Size;
79+
public string Path;
80+
}
81+
82+
static void ExtractWebBundle(FileInfo filename, DirectoryInfo outputFolder) {
83+
Console.WriteLine($"Extracting web bundle: {filename}");
84+
using var fileStream = File.Open(filename.ToString(), FileMode.Open);
85+
using var stream = GetStream(filename, fileStream);
86+
using var reader = new BinaryReader(stream, Encoding.UTF8);
87+
var fileDescriptions = ParseWebBundleHeader(reader);
88+
foreach (var description in fileDescriptions)
89+
{
90+
ExtractFileFromWebBundle(description, reader, outputFolder);
91+
}
92+
}
93+
94+
static Stream GetStream(FileInfo filename, FileStream fileStream) {
95+
var fileExtension = Path.GetExtension(filename.ToString());
96+
return fileExtension switch
97+
{
98+
".data" => fileStream,
99+
".gz" => new GZipStream(fileStream, CompressionMode.Decompress),
100+
".br" => new BrotliStream(fileStream, CompressionMode.Decompress),
101+
_ => throw new FileFormatException("Incorrect file extension for web bundle"),
102+
};
103+
}
104+
105+
static List<WebBundleFileDescription> ParseWebBundleHeader(BinaryReader reader)
106+
{
107+
var result = new List<WebBundleFileDescription>();
108+
var prefix = ReadBytes(reader, WebBundlePrefix.Length);
109+
if (!prefix.SequenceEqual(WebBundlePrefix)) {
110+
throw new FileFormatException("File is not a valid web bundle.");
111+
}
112+
uint headerSize = ReadUInt32(reader);
113+
// Advance offset past prefix string and header size uint.
114+
var currentByteOffset = WebBundlePrefix.Length + sizeof(uint);
115+
while (currentByteOffset < headerSize)
116+
{
117+
var fileByteOffset = ReadUInt32(reader);
118+
var fileSize = ReadUInt32(reader);
119+
var filePathLength = ReadUInt32(reader);
120+
var filePath = Encoding.UTF8.GetString(ReadBytes(reader, (int) filePathLength));
121+
result.Add(new WebBundleFileDescription() {
122+
ByteOffset = fileByteOffset,
123+
Size = fileSize,
124+
Path = filePath,
125+
});
126+
// Advance byte offset, so we keep track of the position (to know when we're done reading the header).
127+
currentByteOffset += 3 * sizeof(uint) + filePath.Length;
128+
}
129+
return result;
130+
}
131+
132+
static void ExtractFileFromWebBundle(WebBundleFileDescription description, BinaryReader reader, DirectoryInfo outputFolder)
133+
{
134+
// This function assumes `reader` is at the start of the binary data representing the file contents.
135+
Console.WriteLine($"... Extracting {description.Path}");
136+
var path = Path.Combine(outputFolder.ToString(), description.Path);
137+
Directory.CreateDirectory(Path.GetDirectoryName(path));
138+
File.WriteAllBytes(path, ReadBytes(reader, (int) description.Size));
139+
}
140+
141+
static uint ReadUInt32(BinaryReader reader)
142+
{
143+
try {
144+
return reader.ReadUInt32();
145+
}
146+
catch (EndOfStreamException)
147+
{
148+
throw new FileFormatException("File data is corrupt.");
149+
}
150+
}
151+
152+
static byte[] ReadBytes(BinaryReader reader, int count)
153+
{
154+
var result = reader.ReadBytes(count);
155+
if (result.Length != count)
156+
{
157+
throw new FileFormatException("File data is corrupt.");
158+
}
159+
return result;
160+
}
161+
162+
static void ExtractAssetBundle(FileInfo filename, DirectoryInfo outputFolder)
163+
{
164+
Console.WriteLine($"Extracting asset bundle: {filename}");
165+
using var archive = UnityFileSystem.MountArchive(filename.FullName, "/");
166+
foreach (var node in archive.Nodes)
167+
{
168+
Console.WriteLine($"... Extracting {node.Path}");
169+
CopyFile("/" + node.Path, Path.Combine(outputFolder.FullName, node.Path));
170+
}
171+
}
172+
173+
static void ListAssetBundle(FileInfo filename)
174+
{
175+
using var archive = UnityFileSystem.MountArchive(filename.FullName, "/");
176+
foreach (var node in archive.Nodes)
177+
{
178+
Console.WriteLine($"{node.Path}");
179+
Console.WriteLine($" Size: {node.Size}");
180+
Console.WriteLine($" Flags: {node.Flags}");
181+
Console.WriteLine();
182+
}
183+
}
184+
185+
static void ListWebBundle(FileInfo filename)
186+
{
187+
using var fileStream = File.Open(filename.ToString(), FileMode.Open);
188+
using var stream = GetStream(filename, fileStream);
189+
using var reader = new BinaryReader(stream, Encoding.UTF8);
190+
var fileDescriptions = ParseWebBundleHeader(reader);
191+
foreach (var description in fileDescriptions)
192+
{
193+
Console.WriteLine($"{description.Path}");
194+
Console.WriteLine($" Size: {description.Size}");
195+
Console.WriteLine();
196+
}
197+
}
198+
199+
static void CopyFile(string source, string dest)
200+
{
201+
using var sourceFile = UnityFileSystem.OpenFile(source);
202+
// Create the containing directory if it doesn't exist.
203+
Directory.CreateDirectory(Path.GetDirectoryName(dest));
204+
using var destFile = new FileStream(dest, FileMode.Create);
205+
206+
const int blockSize = 256 * 1024;
207+
var buffer = new byte[blockSize];
208+
long actualSize;
209+
210+
do
211+
{
212+
actualSize = sourceFile.Read(blockSize, buffer);
213+
destFile.Write(buffer, 0, (int)actualSize);
214+
}
215+
while (actualSize == blockSize);
216+
}
217+
}

0 commit comments

Comments
 (0)