Optimizations, improvements, fix bugs and additional builtins (#514)

* optimization

* fix std path in build

* some small opt

* additional optimizations

* remove import

* remove import

* small opt

* tmp chang

* tmp opt

* tmp opt

* tmp

* perf opt

* bench rm proof

* fix mrg

* small opt

* fixed integration tests

* small opt

* rm ref

* improved opt + other benchmarks

* small tests

* wrked benchmarks

* rm unecesary test.json

* opts

* sm imp

* some opt

* upd to last version of zig

* optimisations of memory cell

* Update build.zig

fix build

* small opt in usort

* tst

* last optimisation

* test compare output rm json from repo

* git ignore compiled programs

* update version of rust cairo-vm

* clear makefile and add requirements.txt

* upgrade zig cli and update compability

* integration test through make

* test

* rm clean

* test

* update

* test

* test ci

* bump starknet lib

* some opt

* fix tests

* fix test, commit hard tests to late

* comment test

* ci fix

* ci

* fix integration test & ci test

* test ci

* test ci

* fix test + make check

* fix

* fix ci

* nondet builtin

* change ci

* compile cairo programs in another script

* fix secp

* update cairo programs and fix ci

* build rm profile

* fix hash

* fix max_path_bytes deprecated, make dep on commit link

* fix ci

.github/workflows/build_release.yml

@@ -17,10 +17,10 @@ jobs:
         with:
           zig-version: master
 
-      - name: Setup Rust
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
+      # - name: Setup Rust
+      #   uses: actions-rs/toolchain@v1
+      #   with:
+      #     toolchain: stable
 
       - name: Build ReleaseFast
         run: |

.github/workflows/test.yml

@@ -17,11 +17,6 @@ jobs:
         with:
           zig-version: master
 
-      - name: Setup Rust
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-
       - name: Build
         run: make build
 
@@ -48,10 +43,10 @@ jobs:
         with:
           zig-version: master
 
-      - name: Setup Rust
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
+      # - name: Setup Rust
+      #   uses: actions-rs/toolchain@v1
+      #   with:
+      #     toolchain: stable
 
       - name: Test
         run: |
@@ -68,8 +63,27 @@ jobs:
         with:
           zig-version: master
 
+      # - name: Setup Rust
+      #   uses: actions-rs/toolchain@v1
+      #   with:
+      #     toolchain: stable
+
+      - name: Install python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.9'
+
+        # NOTE: we do it in one step because GitHub cleans the PATH between steps
+      - name: Install dependencies and compile project
+        run: |
+          source scripts/install/install-ubuntu.sh
+          make check
+
+      - name: Compile cairo programs
+        run: |
+          make compile-cairo-programs
+
       - name: Integration test
         run: |
           zig version
-          make build-integration-test
-          ./zig-out/bin/integration_test
+          zig build -Doptimize=ReleaseFast integration_test

.gitignore

@@ -7,6 +7,7 @@ target/
 
 # Zig-specific build artifacts
 zig-cache/
+.zig-cache/
 zig-out/
 /release/
 /debug/
@@ -16,6 +17,11 @@ zig-out/
 docsite/
 cairo-vm-env
 cairo-vm
+tmp/
+
+# we dont need compiled cairo programs in repo
+/cairo_programs/*.json
+/cairo_programs/benchmarks/*.json
 
 # Misc
-.vscode/
+.vscode/

Makefile

@@ -3,12 +3,15 @@
 CAIRO_VM_CLI:=cairo-vm/target/release/cairo-vm-cli
 
 $(CAIRO_VM_CLI):
-	git clone --depth 1 -b v0.9.1 https://github.com/lambdaclass/cairo-vm
+	git clone --depth 1 https://github.com/lambdaclass/cairo-vm
 	cd cairo-vm; cargo b --release --bin cairo-vm-cli
 
 build_cairo_vm_cli: | $(CAIRO_VM_CLI)
 BENCH_DIR=cairo_programs/benchmarks
 
+
+check:
+
 # Creates a pyenv and installs cairo-lang
 deps:
 	pyenv install  -s 3.9.15
@@ -19,6 +22,7 @@ deps:
 # Creates a pyenv and installs cairo-lang
 deps-macos:
 	brew install gmp pyenv
+	brew install python@3.9
 	pyenv install -s 3.9.15
 	PYENV_VERSION=3.9.15 /opt/homebrew/bin/python3.9 -m venv cairo-vm-env
 	. cairo-vm-env/bin/activate ; \
@@ -36,32 +40,22 @@ test:
 test-filter:
 	@zig build test --summary all -Dtest-filter="$(FILTER)"
 
-build-integration-test:
-	@zig build integration_test
-
 run-integration-test:
-	@zig build integration_test
-	./zig-out/bin/integration_test
+	@zig build -Doptimize=ReleaseFast integration_test
 
 run-integration-test-filter:
-	@zig build integration_test
-	./zig-out/bin/integration_test $(FILTER)
-
-build-and-run-pedersen-table-gen:
-	@zig build pedersen_table_gen
-	> ./src/math/crypto/pedersen/gen/constants.zig
-	./zig-out/bin/pedersen_table_gen
-	@zig fmt ./src/math/crypto/pedersen/gen/constants.zig
-
-build-and-run-poseidon-consts-gen:
-	@zig build poseidon_consts_gen
-	> ./src/math/crypto/poseidon/gen/constants.zig
-	./zig-out/bin/poseidon_consts_gen
-	@zig fmt ./src/math/crypto/poseidon/gen/constants.zig
-
-build-compare-benchmarks: build_cairo_vm_cli build
+	@zig build -Doptimize=ReleaseFast integration_test $(FILTER)
+
+build-compare-benchmarks: build_cairo_vm_cli build-optimize
 	cd scripts; sh benchmarks.sh
 
+
+compile-cairo-programs:
+	cd scripts; sh build_cairo_programs.sh
+
+build-compare-output: build_cairo_vm_cli build-optimize
+	cd scripts; sh test_compare_output.sh
+
 clean:
-	@rm -rf zig-cache
+	@rm -rf ./zig-cache
 	@rm -rf zig-out

build.zig

@@ -9,12 +9,19 @@ const external_dependencies = [_]build_helpers.Dependency{
         .name = "zig-cli",
         .module_name = "zig-cli",
     },
+    .{
+        .name = "starknet",
+        .module_name = "ziggy-starkdust",
+    },
 };
 
 // Although this function looks imperative, note that its job is to
 // declaratively construct a build graph that will be executed by an external
 // runner.
 pub fn build(b: *std.Build) void {
+    const options = b.addOptions();
+    options.addOption(bool, "extensive", false);
+
     // Standard target options allows the person running `zig build` to choose
     // what target to build for. Here we do not override the defaults, which
     // means any target is allowed, and the default is native. Other options
@@ -46,7 +53,7 @@ pub fn build(b: *std.Build) void {
     // **************************************************************
     // expose ziggy-starkdust as a module
     _ = b.addModule(package_name, .{
-        .root_source_file = .{ .path = package_path },
+        .root_source_file = b.path(package_path),
         .imports = deps,
     });
 
@@ -57,7 +64,7 @@ pub fn build(b: *std.Build) void {
         .name = "ziggy-starkdust",
         // In this case the main source file is merely a path, however, in more
         // complicated build scripts, this could be a generated file.
-        .root_source_file = .{ .path = "src/lib.zig" },
+        .root_source_file = b.path("src/lib.zig"),
         .target = target,
         .optimize = optimize,
     });
@@ -78,10 +85,16 @@ pub fn build(b: *std.Build) void {
         .name = "ziggy-starkdust",
         // In this case the main source file is merely a path, however, in more
         // complicated build scripts, this could be a generated file.
-        .root_source_file = .{ .path = "src/main.zig" },
+        .root_source_file = b.path("src/main.zig"),
         .target = target,
         .optimize = optimize,
+
+        .link_libc = true,
+        .omit_frame_pointer = if (optimize == .ReleaseFast) null else false,
+        .strip = if (optimize == .ReleaseFast) true else null,
     });
+    exe.root_module.addOptions("cfg", options);
+
     // Add dependency modules to the executable.
     for (deps) |mod| exe.root_module.addImport(
         mod.name,
@@ -109,15 +122,7 @@ pub fn build(b: *std.Build) void {
         run_cmd.addArgs(args);
     }
 
-    integration_test(b, optimize, target);
-
-    // This creates a build step. It will be visible in the `zig build --help` menu,
-    // and can be selected like this: `zig build poseidon_consts_gen`
-    poseidon_consts_gen(b, optimize, target);
-
-    // This creates a build step. It will be visible in the `zig build --help` menu,
-    // and can be selected like this: `zig build pedersen_table_gen`
-    pedersen_table_gen(b, optimize, target);
+    integration_test(b, optimize, target, deps, options);
 
     // This creates a build step. It will be visible in the `zig build --help` menu,
     // and can be selected like this: `zig build run`
@@ -137,12 +142,12 @@ pub fn build(b: *std.Build) void {
     // Creates a step for unit testing. This only builds the test executable
     // but does not run it.
     const unit_tests = b.addTest(.{
-        .root_source_file = .{ .path = "src/tests.zig" },
+        .root_source_file = b.path("src/tests.zig"),
         .target = target,
         .optimize = optimize,
         .filter = test_filter,
-        .single_threaded = false,
     });
+    unit_tests.root_module.addOptions("cfg", options);
 
     // Add dependency modules to the tests.
     for (deps) |mod| unit_tests.root_module.addImport(
@@ -167,55 +172,31 @@ fn integration_test(
     b: *std.Build,
     mode: std.builtin.Mode,
     target: std.Build.ResolvedTarget,
+    deps: []std.Build.Module.Import,
+    options: *std.Build.Step.Options,
 ) void {
     const binary = b.addExecutable(.{
         .name = "integration_test",
-        .root_source_file = .{ .path = "src/integration_tests.zig" },
+        .root_source_file = b.path("src/integration_tests.zig"),
         .target = target,
         .optimize = mode,
+        .omit_frame_pointer = if (mode == .ReleaseFast) true else false,
+        .strip = true,
     });
 
+    binary.root_module.addOptions("cfg", options);
+
+    // Add dependency modules to the executable.
+    for (deps) |mod| binary.root_module.addImport(
+        mod.name,
+        mod.module,
+    );
+
     const integration_test_build = b.step("integration_test", "Build cli integration tests");
     integration_test_build.dependOn(&binary.step);
 
-    const install_step = b.addInstallArtifact(binary, .{});
+    const install_step = b.addRunArtifact(
+        binary,
+    );
     integration_test_build.dependOn(&install_step.step);
 }
-
-fn poseidon_consts_gen(
-    b: *std.Build,
-    mode: std.builtin.Mode,
-    target: std.Build.ResolvedTarget,
-) void {
-    const binary = b.addExecutable(.{
-        .name = "poseidon_consts_gen",
-        .root_source_file = .{ .path = "src/poseidon_consts_gen.zig" },
-        .target = target,
-        .optimize = mode,
-    });
-
-    const poseidon_consts_gen_build = b.step("poseidon_consts_gen", "Cli: poseidon consts generator");
-    poseidon_consts_gen_build.dependOn(&binary.step);
-
-    const install_step = b.addInstallArtifact(binary, .{});
-    poseidon_consts_gen_build.dependOn(&install_step.step);
-}
-
-fn pedersen_table_gen(
-    b: *std.Build,
-    mode: std.builtin.Mode,
-    target: std.Build.ResolvedTarget,
-) void {
-    const binary = b.addExecutable(.{
-        .name = "pedersen_table_gen",
-        .root_source_file = .{ .path = "src/pedersen_table_gen.zig" },
-        .target = target,
-        .optimize = mode,
-    });
-
-    const pedersen_table_gen_build = b.step("pedersen_table_gen", "Cli: pedersen table generator");
-    pedersen_table_gen_build.dependOn(&binary.step);
-
-    const install_step = b.addInstallArtifact(binary, .{});
-    pedersen_table_gen_build.dependOn(&install_step.step);
-}

build.zig.zon

@@ -10,8 +10,12 @@
     },
     .dependencies = .{
         .@"zig-cli" = .{
-            .url = "https://github.com/sam701/zig-cli/archive/refs/heads/main.tar.gz",
-            .hash = "1220c008492d9460c3be2b209600a948181e6efb3bf0d79a1633def499632e708f4b",
+            .url = "https://github.com/sam701/zig-cli/archive/9a94c4803a52e54c26b198096d63fb5bde752da2.zip",
+            .hash = "1220ab73fb7cc11b2308edc3364988e05efcddbcac31b707f55e6216d1b9c0da13f1",
+        },
+        .starknet = .{
+            .url = "https://github.com/StringNick/starknet-zig/archive/8cfb4286ffda4ad2781647c3d96b2aec8ccfeb32.zip",
+            .hash = "122026eaa24834fd2e2cc7e8b6c4eefb03dda08158a2844615f189758fa24d32fc44",
         },
     },
 }

cairo_programs/_keccak.cairo

@@ -0,0 +1,44 @@
+%builtins output range_check bitwise
+
+from starkware.cairo.common.cairo_keccak.keccak import _keccak
+from starkware.cairo.common.cairo_builtins import BitwiseBuiltin
+from starkware.cairo.common.alloc import alloc
+from starkware.cairo.common.serialize import serialize_word
+
+func fill_array(array: felt*, base: felt, array_length: felt, iterator: felt) {
+    if (iterator == array_length) {
+        return ();
+    }
+
+    assert array[iterator] = base;
+
+    return fill_array(array, base, array_length, iterator + 1);
+}
+
+func main{output_ptr: felt*, range_check_ptr, bitwise_ptr: BitwiseBuiltin*}() {
+    alloc_locals;
+
+    let (output: felt*) = alloc();
+    let keccak_output = output;
+
+    let (inputs: felt*) = alloc();
+    let inputs_start = inputs;
+    fill_array(inputs, 9, 3, 0);
+
+    let (state: felt*) = alloc();
+    let state_start = state;
+    fill_array(state, 5, 25, 0);
+
+    let n_bytes = 24;
+
+    let (res: felt*) = _keccak{keccak_ptr=keccak_output}(
+        inputs=inputs_start, n_bytes=n_bytes, state=state_start
+    );
+
+    serialize_word(res[0]);
+    serialize_word(res[1]);
+    serialize_word(res[2]);
+    serialize_word(res[4]);
+
+    return ();
+}