Skip to content

feat: parallel compression for LZ4 #158

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
ParaN3xus wants to merge 2 commits into
base: main
Choose a base branch
from
Open

feat: parallel compression for LZ4 #158

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
ca12983
feat: parallel compress for lz4
ParaN3xus Feb 20, 2026
207a574
refactor: add check for batch size set
ParaN3xus Feb 20, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
94 changes: 92 additions & 2 deletions AssetTools.NET/Standard/AssetsBundleFileFormat/AssetBundleFile.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,9 @@
using System;
using System.Collections.Generic;
using System.IO;
#if !NET35
using System.Threading.Tasks;
#endif

namespace AssetsTools.NET
{
Expand All @@ -27,6 +30,23 @@ public class AssetBundleFile
/// </summary>
public bool DataIsCompressed { get; set; }

private static int _lz4ParallelPackBatchSize = 32;
/// <summary>
/// Number of 0x20000 blocks processed per parallel batch in LZ4/LZ4Fast Pack/Unpack.
/// Used on non-NET35 targets only. Values less than 1 are treated as 1.
/// </summary>
public static int Lz4ParallelPackBatchSize
{
get => _lz4ParallelPackBatchSize;
set
{
if (value <= 0)
{
throw new ArgumentOutOfRangeException(nameof(value), "Lz4ParallelPackBatchSize must be greater than 0.");
}
_lz4ParallelPackBatchSize = value;
}
}
public AssetsFileReader Reader;

/// <summary>
Expand Down Expand Up @@ -485,6 +505,7 @@ public void Pack(AssetsFileWriter writer, AssetBundleCompressionType compType,
case AssetBundleCompressionType.LZ4Fast:
{
// compress into 0x20000 blocks
const int blockSize = 0x20000;
BinaryReader bundleDataReader = new BinaryReader(bundleDataStream);

Stream writeStream;
Expand All @@ -493,7 +514,75 @@ public void Pack(AssetsFileWriter writer, AssetBundleCompressionType compType,
else
writeStream = GetTempFileStream();

byte[] uncompressedBlock = bundleDataReader.ReadBytes(0x20000);
#if !NET35
int totalBlockCount = (int)((bundleDataReader.BaseStream.Length + blockSize - 1) / blockSize);
int completedBlockCount = 0;

while (true)
{
List<byte[]> uncompressedBlocks = new List<byte[]>(_lz4ParallelPackBatchSize);
for (int i = 0; i < _lz4ParallelPackBatchSize; i++)
{
byte[] block = bundleDataReader.ReadBytes(blockSize);
if (block.Length == 0)
break;
uncompressedBlocks.Add(block);
}

if (uncompressedBlocks.Count == 0)
break;

// each outputBlock is a byte[]
byte[][] outputBlocks = new byte[uncompressedBlocks.Count][];
AssetBundleBlockInfo[] outputBlockInfos = new AssetBundleBlockInfo[uncompressedBlocks.Count];

Parallel.For(0, uncompressedBlocks.Count, i =>
{
byte[] uncompressedBlock = uncompressedBlocks[i];
byte[] compressedBlock = compType == AssetBundleCompressionType.LZ4Fast
? LZ4Codec.Encode32(uncompressedBlock, 0, uncompressedBlock.Length)
: LZ4Codec.Encode32HC(uncompressedBlock, 0, uncompressedBlock.Length);

if (compressedBlock.Length > uncompressedBlock.Length)
{
outputBlocks[i] = uncompressedBlock;
outputBlockInfos[i] = new AssetBundleBlockInfo()
{
CompressedSize = (uint)uncompressedBlock.Length,
DecompressedSize = (uint)uncompressedBlock.Length,
Flags = 0x00
};
}
else
{
outputBlocks[i] = compressedBlock;
outputBlockInfos[i] = new AssetBundleBlockInfo()
{
CompressedSize = (uint)compressedBlock.Length,
DecompressedSize = (uint)uncompressedBlock.Length,
Flags = 0x03
};
}
});

for (int i = 0; i < outputBlocks.Length; i++)
{
byte[] outputBlock = outputBlocks[i];
AssetBundleBlockInfo blockInfo = outputBlockInfos[i];

writeStream.Write(outputBlock, 0, outputBlock.Length);
totalCompressedSize += blockInfo.CompressedSize;
newBlocks.Add(blockInfo);
}

completedBlockCount += outputBlocks.Length;
if (progress != null && totalBlockCount > 0)
{
progress.SetProgress((float)completedBlockCount / totalBlockCount);
}
}
#else
byte[] uncompressedBlock = bundleDataReader.ReadBytes(blockSize);
while (uncompressedBlock.Length != 0)
{
byte[] compressedBlock = compType == AssetBundleCompressionType.LZ4Fast
Expand Down Expand Up @@ -536,8 +625,9 @@ public void Pack(AssetsFileWriter writer, AssetBundleCompressionType compType,
newBlocks.Add(blockInfo);
}

uncompressedBlock = bundleDataReader.ReadBytes(0x20000);
uncompressedBlock = bundleDataReader.ReadBytes(blockSize);
}
#endif

if (!blockDirAtEnd)
newStreams.Add(writeStream);
Expand Down