Merge pull request #375 from LossyDragon/optimizations

Optimizations

Author: LossyDragon

src/main/java/in/dragonbra/javasteam/types/DepotManifest.kt

@@ -16,6 +16,7 @@ import `in`.dragonbra.javasteam.util.stream.MemoryStream
 import java.io.File
 import java.io.InputStream
 import java.io.OutputStream
+import java.security.MessageDigest
 import java.time.Instant
 import java.util.*
 import javax.crypto.Cipher
@@ -46,24 +47,15 @@ class DepotManifest {
          * @param stream  Raw depot manifest stream to deserialize.
          */
         @JvmStatic
-        fun deserialize(stream: InputStream): DepotManifest {
-            val manifest = DepotManifest()
-            manifest.internalDeserialize(stream)
-            return manifest
-        }
+        fun deserialize(stream: InputStream): DepotManifest = DepotManifest().apply { internalDeserialize(stream) }
 
         /**
          * Initializes a new instance of the [DepotManifest] class.
          * Depot manifests may come from the Steam CDN or from Steam/depotcache/ manifest files.
          * @param data Raw depot manifest data to deserialize.
          */
         @JvmStatic
-        fun deserialize(data: ByteArray): DepotManifest {
-            val ms = MemoryStream(data)
-            val manifest = deserialize(ms)
-            ms.close()
-            return manifest
-        }
+        fun deserialize(data: ByteArray): DepotManifest = MemoryStream(data).use { deserialize(it) }
 
         /**
          * Loads binary manifest from a file and deserializes it.
@@ -311,8 +303,8 @@ class DepotManifest {
         }
 
         if (payload != null && metadata != null && signature != null) {
-            parseProtobufManifestMetadata(metadata!!)
-            parseProtobufManifestPayload(payload!!)
+            parseProtobufManifestMetadata(metadata)
+            parseProtobufManifestPayload(payload)
         } else {
             throw NoSuchElementException("Missing ContentManifest sections required for parsing depot manifest")
         }
@@ -419,6 +411,9 @@ class DepotManifest {
                 get() = items.size
         }
 
+        // Reuse instance.
+        val sha1Digest = MessageDigest.getInstance("SHA-1", CryptoHelper.SEC_PROV)
+
         files.forEach { file ->
             val protofile = ContentManifestPayload.FileMapping.newBuilder().apply {
                 this.size = file.totalSize
@@ -433,6 +428,7 @@ class DepotManifest {
                 protofile.filename = file.fileName.replace('/', '\\')
                 protofile.shaFilename = ByteString.copyFrom(
                     CryptoHelper.shaHash(
+                        sha1Digest,
                         file.fileName
                             .replace('/', '\\')
                             .lowercase(Locale.getDefault())

src/main/java/in/dragonbra/javasteam/types/KeyValue.kt

@@ -34,23 +34,17 @@ class KeyValue @JvmOverloads constructor(
     /**
      * Gets the children of this instance.
      */
-    var children: MutableList<KeyValue> = mutableListOf()
+    var children: MutableList<KeyValue> = ArrayList(4) // Give an initial capacity for optimization.
 
     /**
      * Gets the child [KeyValue] with the specified key.
      * If no child with the given key exists, [KeyValue.INVALID] is returned.
      * @param key key
      * @return the child [KeyValue]
      */
-    operator fun get(key: String): KeyValue {
-        children.forEach { c ->
-            if (c.name?.equals(key, ignoreCase = true) == true) {
-                return c
-            }
-        }
-
-        return INVALID
-    }
+    operator fun get(key: String): KeyValue = children.find {
+        it.name?.equals(key, ignoreCase = true) == true
+    } ?: INVALID
 
     /**
      * Sets the child [KeyValue] with the specified key.
@@ -329,7 +323,7 @@ class KeyValue @JvmOverloads constructor(
      * @param asBinary If set to <c>true</c>, saves this instance as binary.
      */
     fun saveToFile(file: File, asBinary: Boolean) {
-        FileOutputStream(file, false).use { f -> saveToStream(f, asBinary) }
+        FileOutputStream(file, false).buffered().use { f -> saveToStream(f, asBinary) }
     }
 
     /**
@@ -338,7 +332,7 @@ class KeyValue @JvmOverloads constructor(
      * @param asBinary If set to <c>true</c>, saves this instance as binary.
      */
     fun saveToFile(path: String, asBinary: Boolean) {
-        FileOutputStream(path, false).use { f -> saveToStream(f, asBinary) }
+        FileOutputStream(path, false).buffered().use { f -> saveToStream(f, asBinary) }
     }
 
     /**
@@ -479,7 +473,7 @@ class KeyValue @JvmOverloads constructor(
             }
 
             try {
-                FileInputStream(file).use { input ->
+                FileInputStream(file).buffered(8192).use { input ->
                     val kv = KeyValue()
 
                     if (asBinary) {

src/main/java/in/dragonbra/javasteam/util/Utils.java

@@ -13,7 +13,6 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.zip.CRC32;
-import java.util.zip.Checksum;
 
 /**
  * @author lngtr
@@ -173,7 +172,7 @@ private static String getSystemProperty(final String property) {
     }
 
     /**
-     * Convenience method for calculating the CRC2 checksum of a string.
+     * Convenience method for calculating the CRC32 checksum of a string.
      *
      * @param s the string
      * @return long value of the CRC32
@@ -183,14 +182,26 @@ public static long crc32(String s) {
     }
 
     /**
-     * Convenience method for calculating the CRC2 checksum of a byte array.
+     * Convenience method for calculating the CRC32 checksum of a byte array.
      *
      * @param bytes the byte array
      * @return long value of the CRC32
      */
     public static long crc32(byte[] bytes) {
-        Checksum checksum = new CRC32();
-        checksum.update(bytes, 0, bytes.length);
+        return crc32(bytes, 0, bytes.length);
+    }
+
+    /**
+     * Convenience method for calculating the CRC32 checksum of a byte array with offset and length.
+     *
+     * @param bytes  the byte array
+     * @param offset the offset to start from
+     * @param length the number of bytes to checksum
+     * @return long value of the CRC32
+     */
+    public static long crc32(byte[] bytes, int offset, int length) {
+        var checksum = new CRC32();
+        checksum.update(bytes, offset, length);
         return checksum.getValue();
     }
 

src/main/java/in/dragonbra/javasteam/util/VZipUtil.kt

@@ -1,17 +1,12 @@
 package `in`.dragonbra.javasteam.util
 
 import `in`.dragonbra.javasteam.util.compat.readNBytesCompat
-import `in`.dragonbra.javasteam.util.crypto.CryptoHelper
 import `in`.dragonbra.javasteam.util.log.LogManager
 import `in`.dragonbra.javasteam.util.stream.BinaryReader
-import `in`.dragonbra.javasteam.util.stream.BinaryWriter
 import `in`.dragonbra.javasteam.util.stream.MemoryStream
 import `in`.dragonbra.javasteam.util.stream.SeekOrigin
-import org.tukaani.xz.LZMA2Options
 import org.tukaani.xz.LZMAInputStream
-import org.tukaani.xz.LZMAOutputStream
-import java.io.ByteArrayOutputStream
-import java.util.zip.DataFormatException
+import java.util.zip.*
 import kotlin.math.max
 
 @Suppress("SpellCheckingInspection", "unused")
@@ -26,6 +21,11 @@ object VZipUtil {
 
     private const val VERSION: Byte = 'a'.code.toByte()
 
+    // Thread-local window buffer pool to avoid repeated allocations
+    private val windowBufferPool = ThreadLocal.withInitial {
+        ByteArray(1 shl 23) // 8MB max size
+    }
+
     @JvmStatic
     fun decompress(ms: MemoryStream, destination: ByteArray, verifyChecksum: Boolean = true): Int {
         try {
@@ -67,7 +67,12 @@ object VZipUtil {
 
                 // If the value of dictionary size in properties is smaller than (1 << 12),
                 // the LZMA decoder must set the dictionary size variable to (1 << 12).
-                val windowBuffer = ByteArray(max(1 shl 12, dictionarySize))
+                val windowSize = max(1 shl 12, dictionarySize)
+                val windowBuffer = if (windowSize <= (1 shl 23)) {
+                    windowBufferPool.get() // Reuse thread-local buffer
+                } else {
+                    ByteArray(windowSize) // Fallback for unusually large windows
+                }
                 val bytesRead = LZMAInputStream(
                     ms,
                     sizeDecompressed.toLong(),
@@ -78,8 +83,11 @@ object VZipUtil {
                     lzmaInput.readNBytesCompat(destination, 0, sizeDecompressed)
                 }
 
-                if (verifyChecksum && Utils.crc32(destination).toInt() != outputCrc) {
-                    throw DataFormatException("CRC does not match decompressed data. VZip data may be corrupted.")
+                if (verifyChecksum) {
+                    val actualCrc = Utils.crc32(destination, 0, bytesRead).toInt()
+                    if (actualCrc != outputCrc) {
+                        throw DataFormatException("CRC does not match decompressed data. VZip data may be corrupted.")
+                    }
                 }
 
                 return bytesRead
@@ -93,46 +101,16 @@ object VZipUtil {
         }
     }
 
-    /**
-     * Ported from SteamKit2 and is untested, use at your own risk
-     */
     @JvmStatic
     fun compress(buffer: ByteArray): ByteArray {
-        try {
-            ByteArrayOutputStream().use { ms ->
-                BinaryWriter(ms).use { writer ->
-                    val crc = CryptoHelper.crcHash(buffer)
-                    writer.writeShort(VZIP_HEADER)
-                    writer.writeByte(VERSION)
-                    writer.write(crc)
-
-                    // Configure LZMA options to match SteamKit2's settings
-                    val options = LZMA2Options().apply {
-                        dictSize = 1 shl 23 // 8MB dictionary
-                        setPreset(2) // Algorithm setting
-                        niceLen = 128 // numFastBytes equivalent
-                        matchFinder = LZMA2Options.MF_BT4
-                        mode = LZMA2Options.MODE_NORMAL
-                    }
-
-                    // Write LZMA-compressed data
-                    LZMAOutputStream(ms, options, false).use { lzmaStream ->
-                        lzmaStream.write(buffer)
-                    }
-
-                    writer.write(crc)
-                    writer.writeInt(buffer.size)
-                    writer.writeShort(VZIP_FOOTER)
-
-                    return ms.toByteArray()
-                }
-            }
-        } catch (e: NoClassDefFoundError) {
-            logger.error("Missing implementation of org.tukaani:xz")
-            throw e
-        } catch (e: ClassNotFoundException) {
-            logger.error("Missing implementation of org.tukaani:xz")
-            throw e
-        }
+        throw Exception("VZipUtil.compress is not implemented.")
+        // try {
+        // } catch (e: NoClassDefFoundError) {
+        //     logger.error("Missing implementation of org.tukaani:xz")
+        //     throw e
+        // } catch (e: ClassNotFoundException) {
+        //     logger.error("Missing implementation of org.tukaani:xz")
+        //     throw e
+        // }
     }
 }

src/main/java/in/dragonbra/javasteam/util/VZstdUtil.kt

@@ -5,33 +5,42 @@ import `in`.dragonbra.javasteam.util.log.LogManager
 import java.io.IOException
 import java.nio.ByteBuffer
 import java.nio.ByteOrder
-import java.util.zip.CRC32
 
 object VZstdUtil {
 
     private const val VZSTD_HEADER: Int = 0x615A5356
+    private const val HEADER_SIZE = 8
+    private const val FOOTER_SIZE = 15
 
     private val logger = LogManager.getLogger<VZstdUtil>()
 
     @Throws(IOException::class, IllegalArgumentException::class)
     @JvmStatic
     @JvmOverloads
     fun decompress(buffer: ByteArray, destination: ByteArray, verifyChecksum: Boolean = false): Int {
+        if (buffer.size < HEADER_SIZE + FOOTER_SIZE) {
+            throw IOException("Buffer too small to contain VZstd header and footer")
+        }
+
         val byteBuffer = ByteBuffer.wrap(buffer).order(ByteOrder.LITTLE_ENDIAN) // Convert the buffer.
 
         val header = byteBuffer.getInt(0)
         if (header != VZSTD_HEADER) {
             throw IOException("Expecting VZstdHeader at start of stream")
         }
 
-        val crc32 = byteBuffer.getInt(4)
-        val crc32Footer = byteBuffer.getInt(buffer.size - 15)
-        val sizeDecompressed = byteBuffer.getInt(buffer.size - 11)
+        // val crc32 = byteBuffer.getInt(4)
 
-        if (crc32 == crc32Footer) {
-            // They write CRC32 twice?
-            logger.debug("CRC32 appears to be written twice in the data")
-        }
+        // Read footer
+        val footerOffset = buffer.size - FOOTER_SIZE
+        val crc32Footer = byteBuffer.getInt(footerOffset)
+        val sizeDecompressed = byteBuffer.getInt(footerOffset + 4)
+
+        // This part gets spammed a lot, so we'll mute this.
+        // if (crc32 == crc32Footer) {
+        //     // They write CRC32 twice?
+        //     logger.debug("CRC32 appears to be written twice in the data")
+        // }
 
         if (buffer[buffer.size - 3] != 'z'.code.toByte() ||
             buffer[buffer.size - 2] != 's'.code.toByte() ||
@@ -44,7 +53,7 @@ object VZstdUtil {
             throw IllegalArgumentException("The destination buffer is smaller than the decompressed data size.")
         }
 
-        val compressedData = buffer.copyOfRange(8, buffer.size - 15)
+        val compressedData = buffer.copyOfRange(HEADER_SIZE, buffer.size - FOOTER_SIZE) // :( allocations
 
         try {
             val bytesDecompressed = Zstd.decompress(destination, compressedData)
@@ -54,9 +63,7 @@ object VZstdUtil {
             }
 
             if (verifyChecksum) {
-                val crc = CRC32()
-                crc.update(destination, 0, sizeDecompressed)
-                val calculatedCrc = crc.value.toInt()
+                val calculatedCrc = Utils.crc32(destination, 0, sizeDecompressed).toInt()
                 if (calculatedCrc != crc32Footer) {
                     throw IOException("CRC does not match decompressed data. VZstd data may be corrupted.")
                 }

src/main/java/in/dragonbra/javasteam/util/ZipUtil.kt

@@ -24,7 +24,7 @@ object ZipUtil {
                 throw IllegalArgumentException("Given stream should only contain one zip entry")
             }
 
-            if (verifyChecksum && Utils.crc32(destination.sliceArray(0 until sizeDecompressed)) != entry.crc) {
+            if (verifyChecksum && Utils.crc32(destination, 0, bytesRead) != entry.crc) {
                 throw Exception("Checksum validation failed for decompressed file")
             }
 

src/main/java/in/dragonbra/javasteam/util/compat/ByteArrayOutputStreamCompat.kt

@@ -1,6 +1,8 @@
 package `in`.dragonbra.javasteam.util.compat
 
 import java.io.ByteArrayOutputStream
+import java.nio.charset.Charset
+import java.nio.charset.StandardCharsets
 
 /**
  * Compatibility class to provide compatibility with Java ByteArrayOutputStream.
@@ -10,7 +12,26 @@ import java.io.ByteArrayOutputStream
  */
 object ByteArrayOutputStreamCompat {
 
+    /**
+     * Converts ByteArrayOutputStream to String using UTF-8 encoding.
+     * Compatible with all Android API levels.
+     * @param byteArrayOutputStream the stream to convert
+     * @return UTF-8 decoded string
+     */
     @JvmStatic
-    fun toString(byteArrayOutputStream: ByteArrayOutputStream): String =
-        String(byteArrayOutputStream.toByteArray(), 0, byteArrayOutputStream.size())
+    fun toString(byteArrayOutputStream: ByteArrayOutputStream): String = toString(byteArrayOutputStream, StandardCharsets.UTF_8)
+
+    /**
+     * Converts ByteArrayOutputStream to String using specified charset.
+     * Compatible with all Android API levels.
+     *
+     * @param byteArrayOutputStream the stream to convert
+     * @param charset the charset to use for decoding
+     * @return decoded string
+     */
+    @JvmStatic
+    fun toString(byteArrayOutputStream: ByteArrayOutputStream, charset: Charset): String {
+        val bytes = byteArrayOutputStream.toByteArray()
+        return String(bytes, 0, byteArrayOutputStream.size(), charset)
+    }
 }