Skip to content

Commit

Permalink
Update Test data matches
Browse files Browse the repository at this point in the history
Should probably automate that at some point
  • Loading branch information
ModernMAK committed Oct 19, 2023
1 parent a331954 commit fd909ce
Show file tree
Hide file tree
Showing 3 changed files with 33 additions and 32 deletions.
55 changes: 28 additions & 27 deletions src/relic/sga/v2/serialization.py
Original file line number Diff line number Diff line change
Expand Up @@ -149,7 +149,7 @@ def assemble_meta(_: BinaryIO, header: MetaBlock, __: None) -> Dict[str, object]


def disassemble_meta(
_: BinaryIO, metadata: Dict[str, object]
_: BinaryIO, metadata: Dict[str, object]
) -> Tuple[MetaBlock, None]:
"""Converts the archive's metadata dictionary into a MetaBlock class the Serializer can use."""
meta = MetaBlock(
Expand Down Expand Up @@ -197,7 +197,7 @@ def assemble_file(self, parent_dir: FS, file_def: FileDef) -> None:

# Still hate this, but might as well reuse it
_HEADER_SIZE = (
256 + 8
256 + 8
) # 256 string buffer (likely 256 cause 'max path' on windows used to be 256), and 4 byte unk, and 4 byte checksum (crc32)
lazy_data_header = FileLazyInfo(
jump_to=self.ptrs.data_pos + file_def.data_pos - _HEADER_SIZE,
Expand All @@ -213,15 +213,15 @@ def assemble_file(self, parent_dir: FS, file_def: FileDef) -> None:
unpacked_size=file_def.length_on_disk,
stream=self.stream,
decompress=file_def.storage_type
!= StorageType.STORE, # self.decompress_files,
!= StorageType.STORE, # self.decompress_files,
)

def _generate_crc32() -> bytes:
return zlib.crc32(lazy_info_decomp.read()).to_bytes(
4, "little", signed=False
)

def _set_info(_name: str, _modified: bytes, _crc32: bytes) -> None:
def _set_info(_name: str, _modified: int, _crc32: bytes) -> None:
essence_info: Dict[str, Any] = dict(
parent_dir.getinfo(_name, [ESSENCE_NAMESPACE]).raw[ESSENCE_NAMESPACE]
)
Expand All @@ -236,14 +236,14 @@ def _generate_metadata() -> None:

info = parent_dir.getinfo(name, ["details"])
timestamp = info.get("details", "modified", time.time())
modified = int(timestamp).to_bytes(4, "little", signed=False)
modified = int(timestamp) # .to_bytes(4, "little", signed=False)

crc32 = _generate_crc32()
_set_info(name, modified, crc32)

if (
lazy_data_header.jump_to < 0
or lazy_data_header.jump_to >= lazy_info_decomp.jump_to
lazy_data_header.jump_to < 0
or lazy_data_header.jump_to >= lazy_info_decomp.jump_to
):
# Ignore checksum / name ~ Archive does not have this metadata
# Recalculate it
Expand All @@ -259,23 +259,24 @@ def _generate_metadata() -> None:
if name != expected_name:
_generate_metadata() # assume invalid metadata block
else:
checksum1 = data_header[256:260]
checksum2 = data_header[260:264]
checksum2_gen = _generate_crc32()
modified_buffer: bytes = data_header[256:260]
modified = int.from_bytes(
modified_buffer, "little", signed=False
)
crc32 = data_header[260:264]
crc32_generated = _generate_crc32()

if checksum2 != checksum2_gen:
raise MismatchError(
"CRC Checksum", checksum2_gen, checksum2
)
if crc32 != crc32_generated:
raise MismatchError("CRC Checksum", crc32_generated, crc32)

_set_info(name, checksum1, checksum2)
_set_info(name, modified, crc32)
except UnicodeDecodeError:
_generate_metadata()


class _DisassassemblerV2(FSDisassembler[FileDef]):
_HEADER_SIZE = (
256 + 8
256 + 8
) # 256 string buffer (likely 256 cause 'max path' on windows used to be 256), and 8 byte checksum

def disassemble_file(self, container_fs: FS, file_name: str) -> FileDef:
Expand Down Expand Up @@ -308,14 +309,15 @@ def disassemble_file(self, container_fs: FS, file_name: str) -> FileDef:
)

name_buffer = bytearray(b"\0" * 256)
name_buffer[0 : len(file_name)] = file_name.encode("ascii")
name_buffer[0: len(file_name)] = file_name.encode("ascii")
_name_buffer_pos = _write_data(name_buffer, self.data_stream)
uncompressed_crc = zlib.crc32(data)
# compressed_crc = zlib.crc32(store_data)
if (
"modified" in metadata and metadata["modified"] != b"UNK\0"
if "modified" in metadata and metadata["modified"] != int.from_bytes(
b"UNK\0", "little", signed=False
): # handle my unknown case ~ UNK\0 resolves to 1970, so I don't think we need to worry about that
timestamp_buffer: bytes = metadata["modified"] # type: ignore
timestamp: int = metadata["modified"] # type: ignore
timestamp_buffer = timestamp.to_bytes(4, "little", signed=True)

# if creation/modification are different, use the new timestamp
# Cumbersome, but allows header MD5s to invalidate
Expand All @@ -328,10 +330,9 @@ def disassemble_file(self, container_fs: FS, file_name: str) -> FileDef:

else:
info = container_fs.getinfo(file_name, ["details"])
timestamp = info.get("details", "modified", time.time())
timestamp: float = info.get("details", "modified", time.time()) # type: ignore
timestamp_buffer = int(timestamp).to_bytes(4, "little", signed=False)
if len(timestamp_buffer) != 4:
raise ValueError("SGA-V2 Metadata `Timestamp` was not a 4 bytes value!")

_unk_buffer_pos = _write_data(timestamp_buffer, self.data_stream)

_crc_buffer_pos = _write_data(
Expand All @@ -348,10 +349,10 @@ class EssenceFSSerializer(_s.EssenceFSSerializer[FileDef, MetaBlock, None]):
"""

def __init__(
self,
toc_serializer: StreamSerializer[TocBlock],
meta_serializer: StreamSerializer[MetaBlock],
toc_serialization_info: TOCSerializationInfo[FileDef],
self,
toc_serializer: StreamSerializer[TocBlock],
meta_serializer: StreamSerializer[MetaBlock],
toc_serialization_info: TOCSerializationInfo[FileDef],
):
super().__init__(
version=version,
Expand Down
Binary file modified tests/data/SampleSGA-v2-Oct-15-2023.sga
Binary file not shown.
10 changes: 5 additions & 5 deletions tests/test_opener.py
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ def test_read(self, v2_file_path):
# Hack to get "SampleSGA-v2" from the sample data
_sample_path_on_disk = [f for f in v2_test_files if "SampleSGA-v2.sga" in f][0]
_sample_drives = ["test"]
_UNK = b"UNK\0"
_MODIFIED = 1697739416
_sample_data = b"Ready to unleash 11 barrels of lead.\nWhere's that artillery?!?!\nOrks are da biggust and da strongest.\nFix bayonets!\nFear me, but follow!\nCall for an earth-shaker?\nMy mind is too weary to fight on...\nWe'll be off as soon as the fuel arrives.\nWhere are those tech priests.\nFire until they see the glow of our barrels!"
_CRC32 = zlib.crc32(_sample_data).to_bytes(4, "little", signed=False)
_store_txt = {
Expand All @@ -75,7 +75,7 @@ def test_read(self, v2_file_path):
"essence": {
"storage_type": int(StorageType.STORE),
"name": "STORE.txt",
"unk": _UNK,
"modified": _MODIFIED,
"crc32": _CRC32,
},
},
Expand All @@ -89,7 +89,7 @@ def test_read(self, v2_file_path):
"essence": {
"storage_type": int(StorageType.BUFFER_COMPRESS),
"name": "BUFFER.txt",
"unk": _UNK,
"modified": _MODIFIED,
"crc32": _CRC32,
},
},
Expand All @@ -103,7 +103,7 @@ def test_read(self, v2_file_path):
"essence": {
"storage_type": int(StorageType.STREAM_COMPRESS),
"name": "STREAM.txt",
"unk": _UNK,
"modified": _MODIFIED,
"crc32": _CRC32,
},
},
Expand All @@ -121,7 +121,7 @@ def test_read(self, v2_file_path):
_sample_meta = [
{
"name": "SampleSGA-v2",
"file_md5": "c2e05c8d56e6ea19956b5ec1dbb28ab8",
"file_md5": "facecd6df030c790e8bee02bcd7728a6",
"header_md5": "ba4d641eb08a4c9fc95e4b3bf61cf390",
"version": {"major": 2, "minor": 0},
}
Expand Down

0 comments on commit fd909ce

Please sign in to comment.