From 96a95ca348c450b37a0dae2172228c1175297951 Mon Sep 17 00:00:00 2001 From: Roman Zaycev Date: Mon, 24 Jun 2024 00:28:38 +0700 Subject: [PATCH] BoC deserialization fixes - Performance improvements --- composer.json | 2 +- src/Olifanton/Interop/Address.php | 31 +- src/Olifanton/Interop/Boc/BitString.php | 34 +- src/Olifanton/Interop/Boc/Builder.php | 2 +- src/Olifanton/Interop/Boc/Cell.php | 179 +- .../Interop/Boc/Helpers/BocMagicPrefix.php | 10 +- .../Interop/Boc/Helpers/CellType.php | 12 + .../Interop/Boc/Helpers/CellTypeResolver.php | 30 + .../Interop/Boc/Helpers/LevelMask.php | 59 + .../Interop/Boc/Helpers/MaskResolver.php | 66 + src/Olifanton/Interop/Boc/Slice.php | 75 +- src/Olifanton/Interop/Bytes.php | 43 +- src/Olifanton/Interop/Checksum.php | 48 +- src/Olifanton/Interop/Helpers/Math.php | 24 + .../Olifanton/Interop/Tests/Boc/CellTest.php | 69 + .../Boc/Helpers/TypedArrayHelperTest.php | 41 + .../Olifanton/Interop/Tests/Boc/SliceTest.php | 76 + .../Interop/Tests/Helpers/MathTest.php | 31 + tests/bootstrap.php | 2 + tests/stub-data/boc/block1.base64.txt | 1 + tests/stub-data/boc/block1.expected.json | 4341 +++++++++++++++++ 21 files changed, 5038 insertions(+), 138 deletions(-) create mode 100644 src/Olifanton/Interop/Boc/Helpers/CellType.php create mode 100644 src/Olifanton/Interop/Boc/Helpers/CellTypeResolver.php create mode 100644 src/Olifanton/Interop/Boc/Helpers/LevelMask.php create mode 100644 src/Olifanton/Interop/Boc/Helpers/MaskResolver.php create mode 100644 src/Olifanton/Interop/Helpers/Math.php create mode 100644 tests/Olifanton/Interop/Tests/Helpers/MathTest.php create mode 100644 tests/stub-data/boc/block1.base64.txt create mode 100644 tests/stub-data/boc/block1.expected.json diff --git a/composer.json b/composer.json index a14b2d5..0fb57af 100644 --- a/composer.json +++ b/composer.json @@ -34,7 +34,7 @@ "require": { "php": ">=8.1", "ext-mbstring": "*", - "olifanton/typed-arrays": "^1", + "olifanton/typed-arrays": "^1.0.1", "myclabs/deep-copy": "^1.11", "brick/math": "^0.10" }, diff --git a/src/Olifanton/Interop/Address.php b/src/Olifanton/Interop/Address.php index 93ca308..8c7e8d3 100644 --- a/src/Olifanton/Interop/Address.php +++ b/src/Olifanton/Interop/Address.php @@ -125,13 +125,25 @@ public function toString(?bool $isUserFriendly = null, } $addr = new Uint8Array(34); - $addr[0] = $tag; - $addr[1] = $this->wc; - $addr->set($this->hashPart, 2); + $addr->fSet(0, $tag); + $addr->fSet(1, $this->wc); + + for ($i = 2; $i < $addr->length; $i++) { + $addr->fSet($i, $this->hashPart->fGet($i - 2)); + } $addressWithChecksum = new Uint8Array(36); - $addressWithChecksum->set($addr); - $addressWithChecksum->set(Checksum::crc16($addr), 34); + + for ($i = 0; $i < $addr->length; $i++) { + $addressWithChecksum->fSet($i, $addr->fGet($i)); + } + + $crc16 = Checksum::crc16($addr); + + for ($i = 0; $i < $crc16->length; $i++) { + $addressWithChecksum->fSet($i + 34, $crc16->fGet($i)); + } + $addressBase64 = base64_encode(Bytes::arrayToBytes($addressWithChecksum)); if ($isUrlSafe) { @@ -209,6 +221,15 @@ public function isUrlSafe(): bool return $this->isUrlSafe; } + public function isEqual(Address|string $other): bool + { + if (is_string($other)) { + $other = new Address($other); + } + + return Bytes::compareBytes($this->hashPart, $other->hashPart); + } + public function __toString(): string { return $this->toString(); diff --git a/src/Olifanton/Interop/Boc/BitString.php b/src/Olifanton/Interop/Boc/BitString.php index 20d3135..4b00745 100644 --- a/src/Olifanton/Interop/Boc/BitString.php +++ b/src/Olifanton/Interop/Boc/BitString.php @@ -37,11 +37,7 @@ class BitString implements \Stringable public function __construct(int $length) { $this->length = $length; - $this->array = new Uint8Array(array_fill( - 0, - self::getUint8ArrayLength($length), - 0 - )); + $this->array = new Uint8Array(self::getUint8ArrayLength($length)); } public static function empty(): self @@ -82,7 +78,7 @@ public function get(int $n): bool { $this->checkRange($n); - return ($this->array[(int)($n / 8) | 0] & (1 << (7 - ($n % 8)))) > 0; + return ($this->array->fGet((int)($n / 8) | 0) & (1 << (7 - ($n % 8)))) > 0; } /** @@ -93,7 +89,9 @@ public function get(int $n): bool public function on(int $n): void { $this->checkRange($n); - $this->array[(int)($n / 8) | 0] |= 1 << (7 - ($n % 8)); + $key = (int)($n / 8) | 0; + $cV = $this->array->fGet($key); + $this->array->fSet($key, $cV | 1 << (7 - ($n % 8))); $this->invalidateCell(); } @@ -105,7 +103,9 @@ public function on(int $n): void public function off(int $n): void { $this->checkRange($n); - $this->array[(int)($n / 8) | 0] &= ~(1 << (7 - ($n % 8))); + $key = (int)($n / 8) | 0; + $cV = $this->array->fGet($key); + $this->array->fSet($key, $cV & ~(1 << (7 - ($n % 8)))); $this->invalidateCell(); } @@ -117,7 +117,9 @@ public function off(int $n): void public function toggle(int $n): void { $this->checkRange($n); - $this->array[(int)($n / 8) | 0] ^= 1 << (7 - ($n % 8)); + $key = (int)($n / 8) | 0; + $cV = $this->array->fGet($key); + $this->array->fSet($key, $cV ^ 1 << (7 - ($n % 8))); $this->invalidateCell(); } @@ -281,7 +283,14 @@ public function writeBytes(Uint8Array $ui8): self */ public function writeString(string $value): self { - return $this->writeBytes(new Uint8Array(array_values(unpack('C*', $value)))); + $values = array_values(unpack('C*', $value)); + $arr = new Uint8Array(count($values)); + + foreach ($values as $i => $v) { + $arr->fSet($i, $v); + } + + return $this->writeBytes($arr); } /** @@ -543,7 +552,10 @@ private static function incLength(BitString $bitString, int $newLength): BitStri $bitString->length = $newLength; $tmpArr = $bitString->array; $bitString->array = new Uint8Array(self::getUint8ArrayLength($newLength)); - $bitString->array->set($tmpArr); + + for ($i = 0; $i < $tmpArr->length; $i++) { + $bitString->array->fSet($i, $tmpArr->fGet($i)); + } return $bitString; } diff --git a/src/Olifanton/Interop/Boc/Builder.php b/src/Olifanton/Interop/Boc/Builder.php index 814b0c3..e9f7bee 100644 --- a/src/Olifanton/Interop/Boc/Builder.php +++ b/src/Olifanton/Interop/Boc/Builder.php @@ -223,7 +223,7 @@ public function writeBitString(BitString $anotherBitString): self public function writeRef(Cell $cell): self { - if (count($cell->refs) === 4) { + if (count($this->cell->refs) === 4) { // @phpstan-ignore-line throw new \RuntimeException("Refs overflow"); } diff --git a/src/Olifanton/Interop/Boc/Cell.php b/src/Olifanton/Interop/Boc/Cell.php index db6c359..5653502 100644 --- a/src/Olifanton/Interop/Boc/Cell.php +++ b/src/Olifanton/Interop/Boc/Cell.php @@ -5,6 +5,11 @@ use JetBrains\PhpStorm\ArrayShape; use Olifanton\Interop\Boc\Exceptions\BitStringException; use Olifanton\Interop\Boc\Exceptions\CellException; +use Olifanton\Interop\Boc\Exceptions\SliceException; +use Olifanton\Interop\Boc\Helpers\CellType; +use Olifanton\Interop\Boc\Helpers\CellTypeResolver; +use Olifanton\Interop\Boc\Helpers\LevelMask; +use Olifanton\Interop\Boc\Helpers\MaskResolver; use Olifanton\Interop\Boc\Helpers\TypedArrayHelper; use Olifanton\Interop\Boc\Helpers\BocMagicPrefix; use Olifanton\Interop\Bytes; @@ -20,7 +25,7 @@ * `Cell` is a class that implements the concept of [TVM Cells](https://ton.org/docs/learn/overviews/Cells) in PHP. To create new and process received messages from the blockchain, you will work with instances of the Cell class. * * @property-read BitString $bits - * @property \ArrayObject $refs + * @property \ArrayObject & Cell[] $refs */ class Cell { @@ -40,6 +45,8 @@ class Cell private bool $isExotic = false; + private CellType $type = CellType::ORDINARY; + protected ?Uint8Array $_hash = null; /** @@ -135,18 +142,19 @@ public function getMaxDepth(): int public function getRefsDescriptor(): Uint8Array { - return new Uint8Array([ - count($this->_refs) + (int)$this->isExotic * 8 + $this->getMaxLevel() * 32, - ]); + $result = new Uint8Array(1); + $result->fSet(0, count($this->_refs) + (int)$this->isExotic * 8 + $this->getMaxLevel() * 32,); + + return $result; } public function getBitsDescriptor(): Uint8Array { $usedBits = $this->bits->getUsedBits(); + $result = new Uint8Array(1); + $result->fSet(0, (int)ceil($usedBits / 8) + (int)floor($usedBits / 8)); - return new Uint8Array([ - (int)ceil($usedBits / 8) + (int)floor($usedBits / 8), - ]); + return $result; } /** @@ -177,6 +185,14 @@ public function getDataWithDescriptors(): Uint8Array */ public function getRepr(): Uint8Array { + if ($this->isExotic) { + if ($this->type === CellType::MERKLE_PROOF || $this->type === CellType::MERKLE_UPDATE) { + throw new \LogicException( + "Hash calculation for Merkle proof / Merkle update cells currently not supported", + ); + } + } + $reprArray = [ $this->getDataWithDescriptors(), ]; @@ -258,11 +274,6 @@ public function print(string $indent = ''): string return $s; } - public function isExplicitlyStoredHashes(): int - { - return 0; - } - /** * Create BoC Byte array * @@ -361,6 +372,18 @@ public function beginParse(): Slice ); } + /** + * @throws CellException + */ + public function getLevelMask(): LevelMask + { + try { + return MaskResolver::get($this->bits, $this->type, $this->refs); + } catch (SliceException $e) { + throw new CellException($e->getMessage(), $e->getCode(), $e); + } + } + public function __get(string $name) { if ($name === "bits") { @@ -377,9 +400,9 @@ public function __get(string $name) private function getMaxDepthAsArray(): Uint8Array { $maxDepth = $this->getMaxDepth(); - $d = new Uint8Array([0, 0]); - $d[0] = (int)floor($maxDepth / 256); - $d[1] = $maxDepth % 256; + $d = new Uint8Array(2); + $d->fSet(0, (int)floor($maxDepth / 256)); + $d->fSet(1, $maxDepth % 256); return $d; } @@ -394,10 +417,6 @@ private function serializeForBoc(array $cellsIndex): Uint8Array $this->getDataWithDescriptors(), ]; - if ($this->isExplicitlyStoredHashes()) { - throw new CellException("Cell hashes explicit storing is not implemented"); - } - foreach ($this->_refs as $ref) { $refHash = $ref->hash(); $refIndexInt = $cellsIndex[Bytes::arrayToBytes($refHash)]; @@ -465,7 +484,7 @@ private static function deserializeBoc(string|Uint8Array $serializedBoc, bool $i for ($ci = 0; $ci < $header["cells_num"]; $ci++) { try { $dd = self::deserializeCellData($cells_data, $header["size_bytes"]); - } catch (BitStringException $e) { + } catch (BitStringException|SliceException $e) { throw new CellException( "Cell data deserialization error: " . $e->getMessage() . "; cell_num idx: " . $ci, $e->getCode(), @@ -523,67 +542,61 @@ private static function deserializeBoc(string|Uint8Array $serializedBoc, bool $i ])] private static function parseBocHeader(Uint8Array $serializedBoc): array { - if ($serializedBoc->length < 4 + 1) { + if ($serializedBoc->length < 5) { throw new CellException("Not enough bytes for magic prefix"); } + /** @var Uint8Array $inputData */ $inputData = deep_copy($serializedBoc); - $prefix = self::slice($serializedBoc, 0, 4); - $serializedBoc = self::slice($serializedBoc, 4); + $serializedBocReader = new Slice( + $serializedBoc, + $serializedBoc->length * 8, + [], + usedBits: $serializedBoc->length * 8, + ); - $size_bytes = $has_idx = $hash_crc32 = $has_cache_bits = $flags = 0; + $prefix = $serializedBocReader->loadUint(4 * 8)->toBase(16); + $size_bytes = $has_idx = $has_crc32 = $has_cache_bits = $flags = 0; - if (Bytes::compareBytes($prefix, BocMagicPrefix::reachBocMagicPrefix())) { - $flags_byte = $serializedBoc[0]; + if ($prefix === BocMagicPrefix::REACH_BOC_MAGIC_PREFIX) { + $flags_byte = $serializedBocReader->loadUint(8)->toInt(); $has_idx = $flags_byte & 128; - $hash_crc32 = $flags_byte & 64; + $has_crc32 = $flags_byte & 64; $has_cache_bits = $flags_byte & 32; $flags = ($flags_byte & 16) * 2 + ($flags_byte & 8); $size_bytes = $flags_byte % 8; - } elseif (Bytes::compareBytes($prefix, BocMagicPrefix::leanBocMagicPrefix())) { + } elseif ($prefix === BocMagicPrefix::LEAN_BOC_MAGIC_PREFIX) { $has_idx = 1; - $hash_crc32 = 0; + $has_crc32 = 0; $has_cache_bits = 0; $flags = 0; - $size_bytes = $serializedBoc[0]; - } elseif (Bytes::compareBytes($prefix, BocMagicPrefix::leanBocMagicPrefixCRC())) { + $size_bytes = $serializedBocReader->loadUint(8)->toInt(); + } elseif ($prefix === BocMagicPrefix::LEAN_BOC_MAGIC_PREFIX_CRC) { $has_idx = 1; - $hash_crc32 = 1; + $has_crc32 = 1; $has_cache_bits = 0; $flags = 0; - $size_bytes = $serializedBoc[0]; + $size_bytes = $serializedBocReader->loadUint(8)->toInt(); } - $serializedBoc = self::slice($serializedBoc, 1); - - if ($serializedBoc->length < 1 + 5 * $size_bytes) { + if ($serializedBocReader->getFreeBytes() < 1 + 5 * $size_bytes) { throw new CellException("Not enough bytes for encoding cells counters"); } - $offset_bytes = $serializedBoc[0]; - $serializedBoc = self::slice($serializedBoc, 1); - - $cells_num = Bytes::readNBytesUIntFromArray($size_bytes, $serializedBoc); - $serializedBoc = self::slice($serializedBoc, $size_bytes); - - $roots_num = Bytes::readNBytesUIntFromArray($size_bytes, $serializedBoc); - $serializedBoc = self::slice($serializedBoc, $size_bytes); - - $absent_num = Bytes::readNBytesUIntFromArray($size_bytes, $serializedBoc); - $serializedBoc = self::slice($serializedBoc, $size_bytes); - - $tot_cells_size = Bytes::readNBytesUIntFromArray($offset_bytes, $serializedBoc); - $serializedBoc = self::slice($serializedBoc, $offset_bytes); + $offset_bytes = $serializedBocReader->loadUint(8)->toInt(); + $cells_num = $serializedBocReader->loadUint($size_bytes * 8)->toInt(); + $roots_num = $serializedBocReader->loadUint($size_bytes * 8)->toInt(); + $absent_num = $serializedBocReader->loadUint($size_bytes * 8)->toInt(); + $tot_cells_size = $serializedBocReader->loadUint($offset_bytes * 8)->toInt(); - if ($serializedBoc->length < $roots_num * $size_bytes) { + if ($serializedBocReader->getFreeBytes() < $roots_num * $size_bytes) { throw new CellException("Not enough bytes for encoding root cells hashes"); } $root_list = []; for ($c = 0; $c < $roots_num; $c++) { - $root_list[] = Bytes::readNBytesUIntFromArray($size_bytes, $serializedBoc); - $serializedBoc = self::slice($serializedBoc, $size_bytes); + $root_list[] = $serializedBocReader->loadUint($size_bytes * 8)->toInt(); } $index = false; @@ -591,44 +604,41 @@ private static function parseBocHeader(Uint8Array $serializedBoc): array if ($has_idx) { $index = []; - if ($serializedBoc->length < $offset_bytes * $cells_num) { + if ($serializedBocReader->getFreeBytes() < $offset_bytes * $cells_num) { throw new CellException("Not enough bytes for index encoding"); } for ($c = 0; $c < $cells_num; $c++) { - $index[] = Bytes::readNBytesUIntFromArray($offset_bytes, $serializedBoc); - $serializedBoc = self::slice($serializedBoc, $offset_bytes); + $index[] = $serializedBocReader->loadUint($offset_bytes * 8)->toInt(); } } - if ($serializedBoc->length < $tot_cells_size) { + if ($serializedBocReader->getFreeBytes() < $tot_cells_size) { throw new CellException("Not enough bytes for cells data"); } - $cells_data = self::slice($serializedBoc, 0, $tot_cells_size); - $serializedBoc = self::slice($serializedBoc, $tot_cells_size); + $cells_data = self::slice($inputData, $serializedBocReader->getReadBytes(), /*$tot_cells_size*/); + $serializedBocReader->skipBits($tot_cells_size * 8); - if ($hash_crc32) { - if ($serializedBoc->length < 4) { + if ($has_crc32) { + if ($serializedBocReader->getFreeBytes() < 4) { throw new CellException("Not enough bytes for crc32c checksum"); } $length = $inputData->length; - if (!Bytes::compareBytes(Checksum::crc32c(self::slice($inputData, 0, $length - 4)), self::slice($serializedBoc, 0, 4))) { + if (!Bytes::compareBytes(Checksum::crc32c(self::slice($inputData, 0, $length - 4)), $serializedBocReader->loadBits(32))) { throw new CellException("Crc32c checksum mismatch"); } - - $serializedBoc = self::slice($serializedBoc, 4); } - if ($serializedBoc->length > 0) { + if ($serializedBocReader->getFreeBits() > 0) { throw new CellException("Too much bytes in BoC serialization"); } return [ "has_idx" => $has_idx, - "hash_crc32" => $hash_crc32, + "hash_crc32" => $has_crc32, "has_cache_bits" => $has_cache_bits, "flags" => $flags, "size_bytes" => $size_bytes, @@ -644,7 +654,7 @@ private static function parseBocHeader(Uint8Array $serializedBoc): array } /** - * @throws CellException|BitStringException + * @throws CellException|BitStringException|SliceException */ #[ArrayShape([ "cell" => "Olifanton\\Boc\\Cell", @@ -659,18 +669,30 @@ private static function deserializeCellData(Uint8Array $cellData, int $reference $d1 = $cellData[0]; $d2 = $cellData[1]; - $cellData = self::slice($cellData, 2); - $isExotic = $d1 & 8; $refNum = $d1 % 8; $dataByteSize = (int)ceil($d2 / 2); $fulfilledBytes = !($d2 % 2); + $levelMask = $d1 >> 5; + $hasHashes = ($d1 & 16) !== 0; $cell = new Cell(); $cell->isExotic = (bool)$isExotic; + $offset = 2; + + if ($hasHashes) { + $hashesCount = self::getHashesCountFromMask($levelMask & 7); + $offset += $hashesCount * 32 + $hashesCount * 2; + } + + $cellData = self::slice($cellData, $offset); if ($cellData->length < $dataByteSize + $referenceIndexSize * $refNum) { - throw new CellException("Not enough bytes to encode cell data"); + throw new CellException(sprintf( + "Not enough bytes to encode cell data, needed: %d, cell data bytes: %d", + $dataByteSize + $referenceIndexSize * $refNum, + $cellData->length, + )); } $cell @@ -679,6 +701,11 @@ private static function deserializeCellData(Uint8Array $cellData, int $reference self::slice($cellData, 0, $dataByteSize), $fulfilledBytes, ); + + if ($isExotic) { + $cell->type = CellTypeResolver::get($cell->bits); + } + $cellData = self::slice($cellData, $dataByteSize); for ($r = 0; $r < $refNum; $r++) { @@ -777,4 +804,16 @@ private static function isBase64String(string $base64String): bool { return (bool) preg_match('/^[a-zA-Z0-9\/\r\n+]*={0,2}$/', $base64String); } + + private static function getHashesCountFromMask(int $mask): int + { + $n = 0; + + for ($i = 0; $i < 3; $i++) { + $n += ($mask & 1); + $mask = $mask >> 1; + } + + return $n + 1; + } } diff --git a/src/Olifanton/Interop/Boc/Helpers/BocMagicPrefix.php b/src/Olifanton/Interop/Boc/Helpers/BocMagicPrefix.php index 782fb32..27d2cd3 100644 --- a/src/Olifanton/Interop/Boc/Helpers/BocMagicPrefix.php +++ b/src/Olifanton/Interop/Boc/Helpers/BocMagicPrefix.php @@ -13,10 +13,14 @@ final class BocMagicPrefix private static ?Uint8Array $leanBocMagicPrefixCRC = null; + public const REACH_BOC_MAGIC_PREFIX = "b5ee9c72"; + public const LEAN_BOC_MAGIC_PREFIX = "68ff65f3"; + public const LEAN_BOC_MAGIC_PREFIX_CRC = "acc3a728"; + public static function reachBocMagicPrefix(): Uint8Array { if (!self::$reachBocMagicPrefix) { - self::$reachBocMagicPrefix = Bytes::hexStringToBytes("b5ee9c72"); + self::$reachBocMagicPrefix = Bytes::hexStringToBytes(self::REACH_BOC_MAGIC_PREFIX); } return self::$reachBocMagicPrefix; @@ -25,7 +29,7 @@ public static function reachBocMagicPrefix(): Uint8Array public static function leanBocMagicPrefix(): Uint8Array { if (!self::$leanBocMagicPrefix) { - self::$leanBocMagicPrefix = Bytes::hexStringToBytes("68ff65f3"); + self::$leanBocMagicPrefix = Bytes::hexStringToBytes(self::LEAN_BOC_MAGIC_PREFIX); } return self::$leanBocMagicPrefix; @@ -34,7 +38,7 @@ public static function leanBocMagicPrefix(): Uint8Array public static function leanBocMagicPrefixCRC(): Uint8Array { if (!self::$leanBocMagicPrefixCRC) { - self::$leanBocMagicPrefixCRC = Bytes::hexStringToBytes("acc3a728"); + self::$leanBocMagicPrefixCRC = Bytes::hexStringToBytes(self::LEAN_BOC_MAGIC_PREFIX_CRC); } return self::$leanBocMagicPrefixCRC; diff --git a/src/Olifanton/Interop/Boc/Helpers/CellType.php b/src/Olifanton/Interop/Boc/Helpers/CellType.php new file mode 100644 index 0000000..d81b17f --- /dev/null +++ b/src/Olifanton/Interop/Boc/Helpers/CellType.php @@ -0,0 +1,12 @@ +getImmutableArray(), + $bytes->getLength(), + [], + ); + + $typeId = $reader->preloadUint(8)->toInt(); + $type = CellType::tryFrom($typeId); + + if (!$type) { + throw new \InvalidArgumentException("Unknown exotic cell type with id: " . $typeId); + } + + return $type; + } +} diff --git a/src/Olifanton/Interop/Boc/Helpers/LevelMask.php b/src/Olifanton/Interop/Boc/Helpers/LevelMask.php new file mode 100644 index 0000000..c20618c --- /dev/null +++ b/src/Olifanton/Interop/Boc/Helpers/LevelMask.php @@ -0,0 +1,59 @@ +hashIndex = self::countSetBits($this->mask); + $this->hashCount = $this->hashIndex + 1; + } + + public function getValue(): int + { + return $this->mask; + } + + public function getLevel(): int + { + return 32 - Math::clz32($this->mask); + } + + public function getHashIndex() + { + return $this->hashIndex; + } + + public function getHashCount() + { + return $this->hashCount; + } + + public function apply(int $level): self + { + return new self( + $this->mask & ((1 << $level) - 1), + ); + } + + public function isSignificant(int $level): bool + { + return $level === 0 || ($this->mask >> ($level - 1)) % 2 !== 0; + } + + private static function countSetBits(int $n): int + { + $n = $n - (($n >> 1) & 0x55555555); + $n = ($n & 0x33333333) + (($n >> 2) & 0x33333333); + + return (($n + ($n >> 4) & 0xF0F0F0F) * 0x1010101) >> 24; + } +} diff --git a/src/Olifanton/Interop/Boc/Helpers/MaskResolver.php b/src/Olifanton/Interop/Boc/Helpers/MaskResolver.php new file mode 100644 index 0000000..494670a --- /dev/null +++ b/src/Olifanton/Interop/Boc/Helpers/MaskResolver.php @@ -0,0 +1,66 @@ + $refs + * @throws SliceException|CellException + */ + public static function get(BitString $bits, CellType $type, \ArrayObject $refs): LevelMask + { + if ($type === CellType::ORDINARY) { + $mask = 0; + + foreach ($refs as $ref) { + /** @var Cell $ref */ + $mask = $mask | $ref->getLevelMask()->getValue(); + } + + return new LevelMask($mask); + } + + if ($type === CellType::PRUNED_BRANCH) { + $reader = new Slice( + $bits->getImmutableArray(), + $bits->getLength(), + [], + ); + $reader->skipBits(8); // type + + if ($bits->getLength() === 280) { + return new LevelMask(1); + } + + return new LevelMask($reader->loadUint(8)->toInt()); + } + + if ($type === CellType::LIBRARY) { + return new LevelMask(0); + } + + if ($type === CellType::MERKLE_PROOF) { + return new LevelMask($refs[0]->getLevelMask()->getValue() >> 1); + } + + if ($type === CellType::MERKLE_UPDATE) { + /** @var Cell $r0 */ + $r0 = $refs[0]; + /** @var Cell $r1 */ + $r1 = $refs[1]; + + return new LevelMask( + $r0->getLevelMask()->getValue() | $r1->getLevelMask()->getValue() >> 1, + ); + } + + throw new \RuntimeException("Unsupported cell type: " . $type->name); // @phpstan-ignore-line + } +} diff --git a/src/Olifanton/Interop/Boc/Slice.php b/src/Olifanton/Interop/Boc/Slice.php index 1c7ff76..8b83196 100644 --- a/src/Olifanton/Interop/Boc/Slice.php +++ b/src/Olifanton/Interop/Boc/Slice.php @@ -47,6 +47,16 @@ public function getUsedBits(): int return $this->usedBits - $this->readCursor; } + public function getFreeBytes(): int + { + return (int)ceil($this->getFreeBits() / 8); + } + + public function getReadBytes(): int + { + return (int)ceil($this->readCursor / 8); + } + /** * @return Cell[] */ @@ -63,7 +73,7 @@ public function get(int $n): bool { $this->checkRange($n); - return ($this->array[(int)($n / 8) | 0] & (1 << (7 - ($n % 8)))) > 0; + return ($this->array->fGet((int)($n / 8) | 0) & (1 << (7 - ($n % 8)))) > 0; } /** @@ -81,6 +91,16 @@ public function loadBit(): bool return $result; } + /** + * @return bool + * @throws SliceException + * @phpstan-impure + */ + public function preloadBit(): bool + { + return $this->get($this->readCursor); + } + /** * Reads bit array * @@ -104,6 +124,27 @@ public function loadBits(int $bitLength): Uint8Array return $result->getImmutableArray(); } + /** + * @throws SliceException + * @phpstan-impure + */ + public function preloadBits(int $bitLength): Uint8Array + { + $result = new BitString($bitLength); + + try { + for ($i = 0; $i < $bitLength; $i++) { + $result->writeBit($this->get($this->readCursor + $i)); + } + // @codeCoverageIgnoreStart + } catch (BitStringException $e) { + throw new SliceException($e->getMessage(), $e->getCode(), $e); + } + // @codeCoverageIgnoreEnd + + return $result->getImmutableArray(); + } + /** * Reads unsigned integer * @@ -125,6 +166,25 @@ public function loadUint(int $bitLength): BigInteger return BigInteger::fromBase($s, 2); } + /** + * @throws SliceException + * @phpstan-impure + */ + public function preloadUint(int $bitLength): BigInteger + { + if ($bitLength < 1) { + throw new SliceException("Incorrect bitLength: $bitLength"); + } + + $s = ""; + + for ($i = 0; $i < $bitLength; $i++) { + $s .= ($this->get($this->readCursor + $i) ? "1" : "0"); + } + + return BigInteger::fromBase($s, 2); + } + /** * Reads signed integer * @@ -232,6 +292,19 @@ public function loadRef(): Cell return $result; } + /** + * @throws SliceException + * @phpstan-impure + */ + public function loadMaybeRef(): ?Cell + { + if ($this->loadBit()) { + return $this->loadRef(); + } + + return null; + } + /** * @throws SliceException * @phpstan-impure diff --git a/src/Olifanton/Interop/Bytes.php b/src/Olifanton/Interop/Bytes.php index 78d447e..69f91c5 100644 --- a/src/Olifanton/Interop/Bytes.php +++ b/src/Olifanton/Interop/Bytes.php @@ -25,7 +25,7 @@ public static final function readNBytesUIntFromArray(int $n, Uint8Array $uint8Ar for ($i = 0; $i < $n; $i++) { $res *= 256; - $res += $uint8Array[$i]; + $res += $uint8Array->fGet($i); } return $res; @@ -44,15 +44,7 @@ public static final function compareBytes(Uint8Array $a, Uint8Array $b): bool */ public static final function arraySlice(Uint8Array $bytes, int $start, int $end): Uint8Array { - $result = new Uint8Array($end - $start); - $j = 0; - - for ($i = $start; $i < $end; $i++) { - $result[$j] = $bytes[$i]; - $j++; - } - - return $result; + return new Uint8Array($bytes->buffer->slice($start, $end)); } /** @@ -60,16 +52,18 @@ public static final function arraySlice(Uint8Array $bytes, int $start, int $end) */ public static final function concatBytes(Uint8Array $a, Uint8Array $b): Uint8Array { - $c = new Uint8Array($a->length + $b->length); + $aLength = $a->length; + $bLength = $b->length; + $c = new Uint8Array($aLength + $bLength); $i = 0; - for ($j = 0; $j < $a->length; $j++) { - $c[$i] = $a[$j]; + for ($j = 0; $j < $aLength; $j++) { + $c->fSet($i, $a->fGet($j)); $i++; } - for ($j = 0; $j < $b->length; $j++) { - $c[$i] = $b[$j]; + for ($j = 0; $j < $bLength; $j++) { + $c->fSet($i, $b->fGet($j)); $i++; } @@ -138,8 +132,7 @@ public static final function hexStringToBytes(string $str): Uint8Array $result = new Uint8Array($length); for ($i = 0; $i < $length; $i++) { - $b = substr($str, $i * 2, 2); - $result[$i] = hexdec($b); + $result->fSet($i, hexdec(substr($str, $i * 2, 2))); } return $result; @@ -159,7 +152,7 @@ public static final function bytesToHexString(Uint8Array $bytes): string $result = []; for ($i = 0; $i < $bytes->length; $i++) { - $result[] = str_pad(dechex($bytes[$i]), 2, "0", STR_PAD_LEFT); + $result[] = str_pad(dechex($bytes->fGet($i)), 2, "0", STR_PAD_LEFT); } return implode("", $result); @@ -170,7 +163,7 @@ public static final function bytesToArray(string $bytes): Uint8Array $arr = new Uint8Array(strlen($bytes)); foreach (str_split($bytes) as $i => $byte) { - $arr->offsetSet($i, unpack("C", $byte)[1]); + $arr->fSet($i, unpack("C", $byte)[1]); } return $arr; @@ -199,9 +192,19 @@ public static final function base64ToBytes(string $base64): Uint8Array $bytes = new Uint8Array($length); for ($i = 0; $i < $length; $i++) { - $bytes[$i] = ord($binaryString[$i]); + $bytes->fSet($i, ord($binaryString[$i])); } return $bytes; } + + public static function copyArrayToArray(Uint8Array $target, Uint8Array $source, int $targetStart = 0): void + { + $end = $source->length + $targetStart; + + for ($pos = $targetStart, $i = 0; $pos < $end; $pos++) { + $target->fSet($pos, $source->fGet($i)); + $i++; + } + } } diff --git a/src/Olifanton/Interop/Checksum.php b/src/Olifanton/Interop/Checksum.php index d404ac0..95d5f72 100644 --- a/src/Olifanton/Interop/Checksum.php +++ b/src/Olifanton/Interop/Checksum.php @@ -17,20 +17,14 @@ class Checksum */ public static final function crc32c(Uint8Array $bytes): Uint8Array { - $intCrc = self::crc32cInternal(0, $bytes); - $arr = new Uint8Array(4); - $arr[0] = $intCrc >> 24; - $arr[1] = $intCrc >> 16; - $arr[2] = $intCrc >> 8; - $arr[3] = $intCrc; - - $tmpArray = []; - - for ($i = 0; $i < 4; $i++) { - $tmpArray[] = $arr[$i]; - } - - return new Uint8Array(array_reverse($tmpArray)); + $intCrc = self::crc32cInternal($bytes); + $result = new Uint8Array(4); + $result->fSet(0, $intCrc); + $result->fSet(1, $intCrc >> 8); + $result->fSet(2, $intCrc >> 16); + $result->fSet(3, $intCrc >> 24); + + return $result; } /** @@ -40,10 +34,13 @@ public static final function crc16(Uint8Array $bytes): Uint8Array { $reg = 0; $message = new Uint8Array($bytes->length + 2); - $message->set($bytes); + + for ($i = 0; $i < $bytes->length; $i++) { + $message->fSet($i, $bytes->fGet($i)); + } for ($i = 0; $i < $message->length; $i++) { - $byte = $message[$i]; + $byte = $message->fGet($i); $mask = 0x80; while ($mask > 0) { @@ -65,23 +62,22 @@ public static final function crc16(Uint8Array $bytes): Uint8Array return new Uint8Array([(int)floor($reg / 256), $reg % 256]); } - private static function crc32cInternal(int $crc, Uint8Array $bytes): int + private static function crc32cInternal(Uint8Array $bytes): int { - $crc ^= 0xffffffff; + $crc = 0 ^ 0xffffffff; + $length = $bytes->length; - for ($n = 0; $n < $bytes->length; $n++) { - $crc ^= $bytes[$n]; + for ($n = 0; $n < $length; $n++) { + $crc ^= $bytes->fGet($n); for ($i = 0; $i < 8; $i++) { - $crc = $crc & 1 ? (self::rrr($crc, 1)) ^ self::POLY_32 : self::rrr($crc, 1); + $rrr = ($crc & 0xffffffff) >> 1; + $crc = $crc & 1 + ? $rrr ^ self::POLY_32 + : $rrr; } } return $crc ^ 0xffffffff; } - - private static function rrr(int $v, int $n): int - { - return ($v & 0xffffffff) >> ($n & 0x1f); - } } diff --git a/src/Olifanton/Interop/Helpers/Math.php b/src/Olifanton/Interop/Helpers/Math.php new file mode 100644 index 0000000..80c77ee --- /dev/null +++ b/src/Olifanton/Interop/Helpers/Math.php @@ -0,0 +1,24 @@ +> ($n & 0x1f); + } +} diff --git a/tests/Olifanton/Interop/Tests/Boc/CellTest.php b/tests/Olifanton/Interop/Tests/Boc/CellTest.php index e6467eb..0b2e6c2 100644 --- a/tests/Olifanton/Interop/Tests/Boc/CellTest.php +++ b/tests/Olifanton/Interop/Tests/Boc/CellTest.php @@ -320,4 +320,73 @@ public function testCachedHash(): void $h2 = Bytes::bytesToHexString($cell->hash());; $this->assertNotEquals($h1, $h2); } + + /** + * @throws \Throwable + */ + public function testFromBocWithHashes(): void + { + // https://github.com/tonkeeper/tongo/tree/b199665da34dd8ff8b51fa51c2d90f2f2dbf82b8/tlb/testdata/block-1 + $b64Boc = trim(file_get_contents(STUB_DATA_DIR . "/boc/block1.base64.txt")); + //$expectedData = json_decode(trim(file_get_contents(STUB_DATA_DIR . "/boc/block1.expected.json")), true); + + $cell = Cell::oneFromBoc($b64Boc, isBase64: true); + $slice = $cell->beginParse(); + + $this->assertEquals("11ef55aa", $slice->loadUint(32)->toBase(16)); + + $globalId = $slice->loadInt(32)->toInt(); + $this->assertEquals(-239, $globalId); + + $blockInfo = $slice->loadRef()->beginParse(); + $this->assertEquals("9bc7a987", $blockInfo->loadUint(32)->toBase(16)); + + $valueFlow = $slice->loadRef()->beginParse(); + $sumType = $valueFlow->loadUint(32)->toBase(16); + + $this->assertTrue( + in_array( + $sumType, + [ + "b8e48dfb", + "3ebf98b7", + ], + true, + ), + ); + + // stateUpdate + $slice->skipRef(); + + // + $blockExtra = $slice->loadRef(); + $blockExtraReader = $blockExtra->beginParse(); + $this->assertEquals("4a33f6fd", $blockExtraReader->loadUint(32)->toBase(16)); + + /** @var Cell $inMsgDescrCell */ + $inMsgDescrCell = $blockExtra->refs[0]; + /** @var Cell $outMsgDescrCell */ + $outMsgDescrCell = $blockExtra->refs[1]; + + $inMsgDict = $inMsgDescrCell->beginParse()->loadDict(256); + + $inMsgCount = 0; + foreach ($inMsgDict->getIterator() as $inMsg) { + $inMsgCount++; + } + + $this->assertEquals(329, $inMsgCount); + } + + /** + * @throws \Throwable + */ + public function testWithMerkleProof(): void + { + $boc = "b5ee9c7201021c010004260003b5792fb2fb7884d2a79f8e5b1279264597682fd7e56cf3ccfebea767db7173526f100000a2261348e01ab0389959f7f3c33161c3e4bf3a5901c38958667d64b5603ea04397c1d44279400000a1f24f06c056453860b0003476245d680102030201e00405008272d96846fe22c11b2cbc067eea6a82f1b332efa12da7070d4e90ee6c9bd56388009f339073f094314d4a2b696c2face70a1a07882e875bd28aa243d0a0538e291002110cae650619760604401a1b01df880125f65f6f109a54f3f1cb624f24c8b2ed05fafcad9e799fd7d4ecfb6e2e6a4de2044942e0fdde60708999830ca7800441f5cc83bdacc4b308ea56a28d39cd0d82e2c8ecfd45ccaf81a95d04b896c13c3583a8dcabf41812ba9d50018e917836c81000000003229c3178000000d01c060101df07018032000f1e41cb30becd660a374c510bcd742b99682d17958ca64e1f9d598b6ae48f65202faf080000000000000000000000000000419d5d4d00000000000000000801cf280125f65f6f109a54f3f1cb624f24c8b2ed05fafcad9e799fd7d4ecfb6e2e6a4de300078f20e5985f66b3051ba62885e6ba15ccb4168bcac653270fceacc5b57247b29017d7840070eb8b0678525200001444c2691c04c8a70c1620ceaea68000000000000000400809460329fe4b78e00eea1a217eb3fe13cddfedab08022cf926f82a08343cff3be3342e0008092201200a0b284801018eeca88229bd7b563d72ba57749cd8c63f8efa7d47f3e4f74bc7c51847ec45be00072201200c0d2201200e0f28480101fe18b21f54a2802d6fef56513063a78c4af471bf0e24c31e00793949c91aac39000622012010112848010155cdeed7850ef4313f673c311f5c39bec3c161940c0a71cadde5a031f7db13a7000528480101b988fbf55f0ef7e992d36862abf33933601f50e1eb72c71762d5225bb843c87e0004220120121328480101960d9b2f2590c46bca66ac776d6048598c153f8cf05c980a1042e8003f24928300032201201415284801016604e5bef768c9ed879cdb892c0cf2076208d4d4a41b3aecec00b045eefb6c530002220120161722012018192848010155fae57e9a2b8351802cb998e175e2b93b3dc247592edb1901d07d0097902140000128480101f142b2da4d0e106b131f3640bd8f3cad72b53a3d6153c668fe75db1de12ec1ff0000008118f1e3ac53631fcb4844506477b86e3fffbef1c88e09633956a96a6112ff7d612513ceba691b3a6a0a29131524aa081cdc20820a40cb3ae22b01445499b7618c20009d417f03138800000000000000000e8000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000020006fc9f0ccf44c78519c000000000002000000000002c3a7bf1a1987b4997fd6e16077ca4c5a9c62dd0bde5c7cd809ef35f2cbfaf24444d07b1c"; + $cell = Cell::oneFromBoc($boc); + $this->expectException(\LogicException::class); + $this->expectExceptionMessage("Hash calculation for Merkle proof / Merkle update cells currently not supported"); + $cell->hash(); + } } diff --git a/tests/Olifanton/Interop/Tests/Boc/Helpers/TypedArrayHelperTest.php b/tests/Olifanton/Interop/Tests/Boc/Helpers/TypedArrayHelperTest.php index 17da288..40e8f11 100644 --- a/tests/Olifanton/Interop/Tests/Boc/Helpers/TypedArrayHelperTest.php +++ b/tests/Olifanton/Interop/Tests/Boc/Helpers/TypedArrayHelperTest.php @@ -4,6 +4,8 @@ use Olifanton\Interop\Boc\Builder; use Olifanton\Interop\Boc\Helpers\TypedArrayHelper; +use Olifanton\Interop\Bytes; +use Olifanton\TypedArrays\Uint8Array; use PHPUnit\Framework\TestCase; class TypedArrayHelperTest extends TestCase @@ -17,8 +19,47 @@ public function testSlice(): void $sliced = TypedArrayHelper::sliceUint8Array($cell->bits->getImmutableArray(), 0); $this->assertEquals( + Bytes::bytesToBase64($cell->bits->getImmutableArray()), + Bytes::bytesToBase64($sliced), + ); + $this->assertNotSame( $cell->bits->getImmutableArray(), $sliced, ); } + + /** + * @throws \Throwable + */ + public function testSliceHalf(): void + { + $arr = new Uint8Array([1, 0, 1, 1]); + + $slice0 = TypedArrayHelper::sliceUint8Array($arr, 0, 2); + $slice1 = TypedArrayHelper::sliceUint8Array($arr, 2); + + $this->assertEquals(2, $slice0->length); + $this->assertEquals(2, $slice1->length); + + $this->assertEquals(1, $slice0[0]); + $this->assertEquals(0, $slice0[1]); + + $this->assertEquals(1, $slice1[0]); + $this->assertEquals(1, $slice1[1]); + } + + public function testSliceNotSame(): void + { + $arr = new Uint8Array([1, 0, 1, 1]); + $slice = TypedArrayHelper::sliceUint8Array($arr, 0); + + $this->assertEquals(4, $arr->length); + $this->assertEquals(4, $slice->length); + + $this->assertEquals( + Bytes::bytesToBase64($arr), + Bytes::bytesToBase64($slice), + ); + $this->assertNotSame($arr, $slice); + } } diff --git a/tests/Olifanton/Interop/Tests/Boc/SliceTest.php b/tests/Olifanton/Interop/Tests/Boc/SliceTest.php index b60ddc0..62fa15f 100644 --- a/tests/Olifanton/Interop/Tests/Boc/SliceTest.php +++ b/tests/Olifanton/Interop/Tests/Boc/SliceTest.php @@ -7,6 +7,7 @@ use Olifanton\Interop\Boc\Cell; use Olifanton\Interop\Boc\HashmapE; use Olifanton\Interop\Boc\Slice; +use Olifanton\Interop\Bytes; use Olifanton\Interop\Tests\Stubs\CellFactory; use PHPUnit\Framework\TestCase; @@ -257,4 +258,79 @@ public function testLoadFilledDict(): void $dict->get([0, 1])->beginParse()->loadInt(32), ); } + + /** + * @throws \Throwable + */ + public function testPreloadBit(): void + { + $slice = (new Builder()) + ->writeBit(1) + ->writeBit(0) + ->writeBit(1) + ->cell() + ->beginParse(); + + $this->assertTrue($slice->preloadBit()); + $this->assertTrue($slice->preloadBit()); + $this->assertTrue($slice->preloadBit()); + $this->assertTrue($slice->preloadBit()); + + $b0 = $slice->loadBit(); + $this->assertTrue($b0); + + $this->assertFalse($slice->preloadBit()); + $this->assertFalse($slice->preloadBit()); + $this->assertFalse($slice->preloadBit()); + $this->assertFalse($slice->preloadBit()); + + $b1 = $slice->loadBit(); + $this->assertFalse($b1); + + $this->assertTrue($slice->preloadBit()); + $this->assertTrue($slice->preloadBit()); + $this->assertTrue($slice->preloadBit()); + $this->assertTrue($slice->preloadBit()); + + $b2 = $slice->loadBit(); + $this->assertTrue($b2); + } + + /** + * @throws \Throwable + */ + public function testPreloadBits(): void + { + $slice = (new Builder()) + ->writeBit(1) + ->writeBit(0) + ->writeBit(1) + ->writeBit(0) + ->cell() + ->beginParse(); + + $bits = $slice->preloadBits(4); + $this->assertEquals( + "a0", + Bytes::bytesToHexString($bits), + ); + + $this->assertTrue($slice->loadBit()); + } + + /** + * @throws \Throwable + */ + public function testPreloadUint(): void + { + $slice = (new Builder()) + ->writeUint(999, 32) + ->cell() + ->beginParse(); + + $this->assertEquals(999, $slice->preloadUint(32)->toInt()); + $this->assertEquals(999, $slice->preloadUint(32)->toInt()); + $this->assertEquals(999, $slice->preloadUint(32)->toInt()); + $this->assertEquals(999, $slice->preloadUint(32)->toInt()); + } } diff --git a/tests/Olifanton/Interop/Tests/Helpers/MathTest.php b/tests/Olifanton/Interop/Tests/Helpers/MathTest.php new file mode 100644 index 0000000..961cce8 --- /dev/null +++ b/tests/Olifanton/Interop/Tests/Helpers/MathTest.php @@ -0,0 +1,31 @@ + 32, + 1 => 31, + 2 => 30, + 10 => 28, + -1 => 0, + -100 => 0, + 256 => 23, + 10000 => 18, + ]; + + foreach ($cases as $num => $expected) { + $this->assertEquals( + $expected, + Math::clz32($num), + "Num: " . $num, + ); + } + } +} diff --git a/tests/bootstrap.php b/tests/bootstrap.php index dfbb2ec..1515b03 100644 --- a/tests/bootstrap.php +++ b/tests/bootstrap.php @@ -1,3 +1,5 @@