Skip to content

Commit

Permalink
[build_scripts] extraced file download and extraction functionality o…
Browse files Browse the repository at this point in the history 
…ut of generate_project_files.py and into file_utilities.py, this way it the binaries themselves can download the assets
  • Loading branch information
@PanosK92
PanosK92 committed Oct 15, 2024
1 parent d608817 commit d5ef91f
Show file tree
Hide file tree
Showing 3 changed files with 90 additions and 85 deletions.
67 changes: 67 additions & 0 deletions build_scripts/file_utilities.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
#Copyright(c) 2016-2024 Panos Karabelas
#
#Permission is hereby granted, free of charge, to any person obtaining a copy
#of this software and associated documentation files (the "Software"), to deal
#in the Software without restriction, including without limitation the rights
#to use, copy, modify, merge, publish, distribute, sublicense, and / or sell
#copies of the Software, and to permit persons to whom the Software is furnished
#to do so, subject to the following conditions :
#
#The above copyright notice and this permission notice shall be included in
#all copies or substantial portions of the Software.
#
#THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
#IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
#FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE AUTHORS OR
#COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
#IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
#CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

import os
from pathlib import Path
import hashlib
import requests
from tqdm import tqdm

def calculate_file_hash(file_path, hash_type="sha256"):
hash_func = hashlib.new(hash_type)
with open(file_path, "rb") as f:
for chunk in iter(lambda: f.read(4096), b""):
hash_func.update(chunk)
return hash_func.hexdigest()

def download_file(url, destination, expected_hash):
if os.path.exists(destination):
if calculate_file_hash(destination) == expected_hash:
print(f"File {destination} already exists with the correct hash. Skipping download.")
return

print(f"\nDownloading {destination}...")
os.makedirs(os.path.dirname(destination), exist_ok=True)
response = requests.get(url, stream=True)
total_size = int(response.headers.get('content-length', 0))
block_size = 1024
t = tqdm(total=total_size, unit='iB', unit_scale=True)

with open(destination, 'wb') as f:
for chunk in response.iter_content(block_size):
t.update(len(chunk))
f.write(chunk)
t.close()

if total_size != 0 and t.n != total_size:
print("ERROR, something went wrong during download")
return

if calculate_file_hash(destination) != expected_hash:
print(f"ERROR, hash mismatch for {destination}")
return

def extract_archive(archive_path, destination_path, is_windows):
cmd = (
f"build_scripts\\7z.exe x {archive_path} -o{destination_path} -aoa"
if is_windows
else f"7za x {archive_path} -o{destination_path} -aoa"
)
print(f"Extracting {archive_path} to {destination_path}...")
os.system(cmd)
106 changes: 22 additions & 84 deletions build_scripts/generate_project_files.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import importlib
import stat
import hashlib
import file_utilities

paths = {
"binaries": {
Expand Down Expand Up @@ -69,30 +70,6 @@ def calculate_file_hash(file_path, hash_type="sha256"):
hash_func.update(chunk)
return hash_func.hexdigest()

def download_file(url, destination, expected_hash):
# downloads a file from the specified URL to the given destination with a progress bar and hash check."""
if not os.path.exists(destination):
print(f"\nFile {destination} doesn't exist. Downloading...")
elif calculate_file_hash(destination) != expected_hash:
print(f"\nHash of {destination} is outdated. Downloading new version...")
else:
print(f"\nFile {destination} already exists with the correct hash. Skipping download.")
return

os.makedirs(os.path.dirname(destination), exist_ok=True) # ensure the directory exists
response = requests.get(url, stream=True)
total_size = int(response.headers.get('content-length', 0))
block_size = 1024 # 1 kilobyte
t = tqdm(total=total_size, unit='iB', unit_scale=True)
with open(destination, 'wb') as f:
for chunk in response.iter_content(block_size):
t.update(len(chunk))
f.write(chunk)
t.close()
if total_size != 0 and t.n != total_size:
print("ERROR, something went wrong during download")
return

def is_directory(path):
if not os.path.exists(path):
return os.path.splitext(path)[1] == ""
Expand Down Expand Up @@ -123,43 +100,7 @@ def on_rm_error(func, path, exc_info):
return False
return True

def extract_third_party_dependencies():
print("\n1. Extracting third-party dependencies...")
cmd = (
"build_scripts\\7z.exe e third_party\\libraries\\libraries.7z -othird_party\\libraries\\ -aoa"
if sys.argv[1] == "vs2022"
else "7za e third_party/libraries/libraries.7z -othird_party/libraries/ -aoa"
)
os.system(cmd)

def extract_assets():
print("1.5 Extracting assets...")
cmd = (
"build_scripts\\7z.exe x assets\\assets.7z -oassets\\ -aoa"
if sys.argv[1] == "vs2022"
else "7za x assets/assets.7z -oassets/ -aoa"
)
os.system(cmd)

if not os.path.exists("assets\\models"):
print("Warning: assets\\models directory not found after extraction.")

def create_binaries_folder():
print("\n2. Copying required data to the binaries directory..")
copy("data", paths["binaries"]["data"])

def copy_dlls():
print("\n3. Copying required DLLs to the binary directory...")
for lib in paths["third_party_libs"].values():
copy(lib, Path("binaries"))

def copy_assets():
print("\n4. Copying some assets to the project directory...")
for asset_type, asset_path in paths["assets"].items():
copy(asset_path, paths["binaries"][asset_type])

def generate_project_files():
print("\n5. Generating project files...")
cmd = (
f"build_scripts\\premake5.exe --file=build_scripts\\premake.lua {sys.argv[1]} {sys.argv[2]}"
if sys.argv[1] == "vs2022"
Expand Down Expand Up @@ -190,41 +131,38 @@ def print_local_file_hashes():
print(f"{name}: File not found")

def main():
# skip asset downloads when running in CI
is_ci = "ci" in sys.argv

print_local_file_hashes()


print("\n1. Create binaries folder with the required data files...\n")
copy("data", paths["binaries"]["data"])

print("\n2. Download and extract libraries...\n")
library_url = 'https://www.dropbox.com/scl/fi/6behqi6a1ymt3claptq8c/libraries.7z?rlkey=wq6ac6ems9oq9j8qhd0dbtich&st=tdakenrt&dl=1'
library_destination = 'third_party/libraries/libraries.7z'
library_expected_hash = '3aff247046a474d2ad6a30865803639fabe38b229c0d8d9f5bac2d44c4e7a562'

assets_url = 'https://www.dropbox.com/scl/fi/hagxxndy0dnq7pu0ufkxh/assets.7z?rlkey=gmwlxlhf6q3eubh7r50q2xp27&st=60lavvyz&dl=1'
assets_destination = 'assets/assets.7z'
assets_expected_hash = '59cd3b52b0aa84ed3f9bfc9fdef7af945d3f42e134e8bc8bded2bc9519380b8a'
file_utilities.download_file(library_url, library_destination, library_expected_hash)
file_utilities.extract_archive("third_party/libraries/libraries.7z", "third_party/libraries/", sys.argv[1] == "vs2022")

# download libraries regardless
download_file(library_url, library_destination, library_expected_hash)

# skip asset download if running in CI
if not is_ci:
download_file(assets_url, assets_destination, assets_expected_hash)

# extract the downloaded files (libraries always extracted)
extract_third_party_dependencies()

if not is_ci:
extract_assets()

create_binaries_folder()
copy_dlls()
print("\n3. Copying required DLLs to the binary directory...")
for lib in paths["third_party_libs"].values():
copy(lib, Path("binaries"))

if not is_ci:
copy_assets()

print("\n4. Download and extract assets...\n")
assets_url = 'https://www.dropbox.com/scl/fi/hagxxndy0dnq7pu0ufkxh/assets.7z?rlkey=gmwlxlhf6q3eubh7r50q2xp27&st=60lavvyz&dl=1'
assets_destination = 'assets/assets.7z'
assets_expected_hash = '59cd3b52b0aa84ed3f9bfc9fdef7af945d3f42e134e8bc8bded2bc9519380b8a'
file_utilities.download_file(assets_url, assets_destination, assets_expected_hash)
file_utilities.extract_archive("assets/assets.7z", "assets/", sys.argv[1] == "vs2022")
print("\n5.Copying some assets to the project directory...")
for asset_type, asset_path in paths["assets"].items():
copy(asset_path, paths["binaries"][asset_type])

print("\n6. Generate project files...\n")
generate_project_files()

# allow humans to observe the output
if not is_ci:
input("\nPress any key to continue...")

Expand Down
2 changes: 1 addition & 1 deletion readme.md
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@

<img align="center" padding="2" src="https://raw.githubusercontent.com/PanosK92/SpartanEngine/master/data/textures/banner.bmp"/>

<p>Spartan Engine is one of the most advanced game engines out there, pushing the limits of real-time solutions and innovation. What started as a portfolio project has evolved into a cutting-edge platform for developers to explore, learn, and contribute. This isn't an engine for the average user, it's designed for advanced research and experimentation, ideal for industry veterans looking to push boundaries, not to build a game (yet). With a thriving Discord community of over 430 members, including industry veterans, and contribution perks that you won't believe when you see, it's one of the most unique projects you'll ever come across.</p>
<p>Spartan Engine is one of the most advanced one-man game engines out there, pushing the limits of real-time solutions and innovation. What started as a portfolio project has evolved into a cutting-edge platform for developers to explore, learn, and contribute. This isn't an engine for the average user, it's designed for advanced research and experimentation, ideal for industry veterans looking to push boundaries, not to build a game (yet). With a thriving Discord community of over 430 members, including industry veterans, and contribution perks that you won't believe when you see, it's one of the most unique projects you'll ever come across.</p>

- <img align="left" width="32" src="https://i.pinimg.com/736x/99/65/5e/99655e9fe24eb0a7ea38de683cedb735.jpg"/>For occasional updates regarding the project's development, you can follow me on <a href="https://twitter.com/panoskarabelas1?ref_src=twsrc%5Etfw">X</a>.

Expand Down

0 comments on commit d5ef91f

Please sign in to comment.