diff --git a/infra/base-images/base-builder-rust/Dockerfile b/infra/base-images/base-builder-rust/Dockerfile index afc282d3314f..61267f3c7345 100644 --- a/infra/base-images/base-builder-rust/Dockerfile +++ b/infra/base-images/base-builder-rust/Dockerfile @@ -26,7 +26,7 @@ ENV OSSFUZZ_RUSTPATH /rust # manually specifying what toolchain to use. Note that this environment variable # is additionally used by `install_rust.sh` as the toolchain to install. # cf https://rust-lang.github.io/rustup/overrides.html -ENV RUSTUP_TOOLCHAIN nightly-2024-02-12 +ENV RUSTUP_TOOLCHAIN nightly-2024-07-12 # Configure the linker used by default for x86_64 linux to be `clang` instead of # rustc's default of `cc` which is able to find custom-built libraries like diff --git a/infra/base-images/base-clang/Dockerfile b/infra/base-images/base-clang/Dockerfile index 047d5ea27a61..757ea3de5a59 100644 --- a/infra/base-images/base-clang/Dockerfile +++ b/infra/base-images/base-clang/Dockerfile @@ -60,8 +60,19 @@ ENV CCC "clang++" # warning, to allow compiling legacy code. # See https://releases.llvm.org/16.0.0/tools/clang/docs/ReleaseNotes.html#potentially-breaking-changes # Same for deprecated-declarations, int-conversion, -# incompatible-function-pointer-types, enum-constexpr-conversion +# incompatible-function-pointer-types, enum-constexpr-conversion, +# vla-cxx-extension -ENV CFLAGS "-O1 -fno-omit-frame-pointer -gline-tables-only -Wno-error=enum-constexpr-conversion -Wno-error=incompatible-function-pointer-types -Wno-error=int-conversion -Wno-error=deprecated-declarations -Wno-error=implicit-function-declaration -Wno-error=implicit-int -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION" +ENV CFLAGS -O1 \ + -fno-omit-frame-pointer \ + -gline-tables-only \ + -Wno-error=enum-constexpr-conversion \ + -Wno-error=incompatible-function-pointer-types \ + -Wno-error=int-conversion \ + -Wno-error=deprecated-declarations \ + -Wno-error=implicit-function-declaration \ + -Wno-error=implicit-int \ + -Wno-error=vla-cxx-extension \ + -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION ENV CXXFLAGS_EXTRA "-stdlib=libc++" ENV CXXFLAGS "$CFLAGS $CXXFLAGS_EXTRA" diff --git a/infra/base-images/base-clang/checkout_build_install_llvm.sh b/infra/base-images/base-clang/checkout_build_install_llvm.sh index 9de954049a04..e5ab30d04a4c 100755 --- a/infra/base-images/base-clang/checkout_build_install_llvm.sh +++ b/infra/base-images/base-clang/checkout_build_install_llvm.sh @@ -54,7 +54,7 @@ apt-get update && apt-get install -y $LLVM_DEP_PACKAGES --no-install-recommends # languages, projects, ...) is needed. # Check CMAKE_VERSION infra/base-images/base-clang/Dockerfile was released # recently enough to fully support this clang version. -OUR_LLVM_REVISION=llvmorg-18-init-4631-gd50b56d1 +OUR_LLVM_REVISION=llvmorg-18.1.8 mkdir $SRC/chromium_tools cd $SRC/chromium_tools @@ -116,6 +116,7 @@ cmake -G "Ninja" \ -DLLVM_TARGETS_TO_BUILD="$TARGET_TO_BUILD" \ -DLLVM_ENABLE_PROJECTS="clang;lld" \ -DLLVM_BINUTILS_INCDIR="/usr/include/" \ + -DLIBCXXABI_USE_LLVM_UNWINDER=OFF \ $LLVM_SRC/llvm ninja -j $NPROC @@ -202,6 +203,7 @@ function cmake_libcxx { -DLIBCXX_ENABLE_SHARED=OFF \ -DLIBCXX_ENABLE_STATIC_ABI_LIBRARY=ON \ -DLIBCXXABI_ENABLE_SHARED=OFF \ + -DLIBCXXABI_USE_LLVM_UNWINDER=OFF \ -DCMAKE_BUILD_TYPE=Release \ -DLLVM_ENABLE_PIC=ON \ -DLLVM_TARGETS_TO_BUILD="$TARGET_TO_BUILD" \ diff --git a/infra/base-images/base-runner/coverage b/infra/base-images/base-runner/coverage index 35cf236d0739..e11cb103ea6d 100755 --- a/infra/base-images/base-runner/coverage +++ b/infra/base-images/base-runner/coverage @@ -115,7 +115,7 @@ function run_fuzz_target { # Extract fuzztest binary name from fuzztest wrapper script. target=(${target//@/ }[0]) fi - profraw_update.py $OUT/$target $profraw_file_mask $profraw_file_mask + profraw_update.py $OUT/$target -i $profraw_file_mask llvm-profdata merge -j=1 -sparse $profraw_file_mask -o $profdata_file # Delete unnecessary and (potentially) large .profraw files. diff --git a/infra/base-images/base-runner/profraw_update.py b/infra/base-images/base-runner/profraw_update.py index bd2bf41cc3eb..d1ce1fd86396 100755 --- a/infra/base-images/base-runner/profraw_update.py +++ b/infra/base-images/base-runner/profraw_update.py @@ -22,10 +22,11 @@ import sys HeaderGeneric = namedtuple('HeaderGeneric', 'magic version') -HeaderVersion7 = namedtuple( - 'HeaderVersion7', +HeaderVersion9 = namedtuple( + 'HeaderVersion9', 'BinaryIdsSize DataSize PaddingBytesBeforeCounters CountersSize \ - PaddingBytesAfterCounters NamesSize CountersDelta NamesDelta ValueKindLast') + PaddingBytesAfterCounters NumBitmapBytes PaddingBytesAfterBitmapBytes NamesSize CountersDelta BitmapDelta NamesDelta ValueKindLast' +) PROFRAW_MAGIC = 0xff6c70726f667281 @@ -39,6 +40,10 @@ def relativize_address(data, offset, databegin, sect_prf_cnts, sect_prf_data): value = struct.pack('Q', value) for i in range(8): data[offset + i] = value[i] + # address was made relative + return True + # no changes done + return False def upgrade(data, sect_prf_cnts, sect_prf_data): @@ -46,58 +51,89 @@ def upgrade(data, sect_prf_cnts, sect_prf_data): generic_header = HeaderGeneric._make(struct.unpack('QQ', data[:16])) if generic_header.magic != PROFRAW_MAGIC: raise Exception('Bad magic.') + base_version = generic_header.version + + if base_version >= 9: + # Nothing to do. + return data + if base_version < 5 or base_version == 6: + raise Exception('Unhandled version.') + if generic_header.version == 5: generic_header = generic_header._replace(version=7) # Upgrade from version 5 to 7 by adding binaryids field. data = data[:8] + struct.pack('Q', generic_header.version) + struct.pack( 'Q', 0) + data[16:] - if generic_header.version < 7: - raise Exception('Unhandled version.') if generic_header.version == 7: # cf https://reviews.llvm.org/D111123 generic_header = generic_header._replace(version=8) data = data[:8] + struct.pack('Q', generic_header.version) + data[16:] - v7_header = HeaderVersion7._make(struct.unpack('QQQQQQQQQ', data[16:88])) + if generic_header.version == 8: + # see https://reviews.llvm.org/D138846 + generic_header = generic_header._replace(version=9) + # Upgrade from version 8 to 9 by adding NumBitmapBytes, PaddingBytesAfterBitmapBytes and BitmapDelta fields. + data = data[:8] + struct.pack( + 'Q', generic_header.version) + data[16:56] + struct.pack( + 'QQ', 0, 0) + data[56:72] + struct.pack('Q', 0) + data[72:] + + v9_header = HeaderVersion9._make(struct.unpack('QQQQQQQQQQQQ', data[16:112])) - if v7_header.BinaryIdsSize % 8 != 0: + if base_version <= 8 and v9_header.BinaryIdsSize % 8 != 0: # Adds padding for binary ids. # cf commit b9f547e8e51182d32f1912f97a3e53f4899ea6be # cf https://reviews.llvm.org/D110365 - padlen = 8 - (v7_header.BinaryIdsSize % 8) - v7_header = v7_header._replace(BinaryIdsSize=v7_header.BinaryIdsSize + + padlen = 8 - (v9_header.BinaryIdsSize % 8) + v7_header = v9_header._replace(BinaryIdsSize=v9_header.BinaryIdsSize + padlen) - data = data[:16] + struct.pack('Q', v7_header.BinaryIdsSize) + data[24:] - data = data[:88 + v7_header.BinaryIdsSize] + bytes( - padlen) + data[88 + v7_header.BinaryIdsSize:] - - if v7_header.CountersDelta != (sect_prf_cnts - - sect_prf_data) & 0xffffffffffffffff: - # Rust linking seems to add an offset... - sect_prf_data = v7_header.CountersDelta - sect_prf_cnts + sect_prf_data - sect_prf_cnts = v7_header.CountersDelta - + data = data[:16] + struct.pack('Q', v9_header.BinaryIdsSize) + data[24:] + data = data[:112 + v9_header.BinaryIdsSize] + bytes( + padlen) + data[112 + v9_header.BinaryIdsSize:] + + if base_version <= 8: + offset = 112 + v9_header.BinaryIdsSize + for d in range(v9_header.DataSize): + # Add BitmapPtr and aligned u32(NumBitmapBytes) + data = data[:offset + 3 * 8] + struct.pack( + 'Q', 0) + data[offset + 3 * 8:offset + 6 * 8] + struct.pack( + 'Q', 0) + data[offset + 6 * 8:] + value = struct.unpack('Q', + data[offset + 2 * 8:offset + 3 * 8])[0] - 16 * d + data = data[:offset + 2 * 8] + struct.pack('Q', + value) + data[offset + 3 * 8:] + offset += 8 * 8 + + if base_version >= 8: + # Nothing more to do. + return data + + # Last changes are relaed to bump from 7 to version 8 making CountersPtr relative. dataref = sect_prf_data - relativize_address(data, 64, dataref, sect_prf_cnts, sect_prf_data) + # 80 is offset of CountersDelta. + if not relativize_address(data, 80, dataref, sect_prf_cnts, sect_prf_data): + return data - offset = 88 + v7_header.BinaryIdsSize + offset = 112 + v9_header.BinaryIdsSize # This also works for C+Rust binaries compiled with # clang-14/rust-nightly-clang-13. - for _ in range(v7_header.DataSize): + for _ in range(v9_header.DataSize): # 16 is the offset of CounterPtr in ProfrawData structure. relativize_address(data, offset + 16, dataref, sect_prf_cnts, sect_prf_data) # We need this because of CountersDelta -= sizeof(*SrcData); # seen in __llvm_profile_merge_from_buffer. - dataref += 44 + 2 * (v7_header.ValueKindLast + 1) + dataref += 44 + 2 * (v9_header.ValueKindLast + 1) + if was8: + #profraw9 added RelativeBitmapPtr and NumBitmapBytes (8+4 rounded up to 16) + dataref -= 16 # This is the size of one ProfrawData structure. - offset += 44 + 2 * (v7_header.ValueKindLast + 1) + offset += 44 + 2 * (v9_header.ValueKindLast + 1) return data def main(): """Helper script for upgrading a profraw file to latest version.""" - if len(sys.argv) != 4: - sys.stderr.write('Usage: %s \n' % sys.argv[0]) + if len(sys.argv) < 3: + sys.stderr.write('Usage: %s options? ...\n' % sys.argv[0]) return 1 # First find llvm profile sections addresses in the elf, quick and dirty. @@ -113,14 +149,31 @@ def main(): elif b'__llvm_prf_data' in line: sect_prf_data = int(line.split()[3], 16) - # Then open and read the input profraw file. - with open(sys.argv[2], 'rb') as input_file: - profraw_base = bytearray(input_file.read()) - # Do the upgrade, returning a bytes object. - profraw_latest = upgrade(profraw_base, sect_prf_cnts, sect_prf_data) - # Write the output to the file given to the command line. - with open(sys.argv[3], 'wb') as output_file: - output_file.write(profraw_latest) + out_name = "default.profup" + in_place = False + start = 2 + if sys.argv[2] == "-i": + in_place = True + start = start + 1 + elif sys.argv[2] == "-o": + out_name = sys.argv[3] + start = 4 + + if len(sys.argv) < start: + sys.stderr.write('Usage: %s options ...\n' % sys.argv[0]) + return 1 + + for i in range(start, len(sys.argv)): + # Then open and read the input profraw file. + with open(sys.argv[i], 'rb') as input_file: + profraw_base = bytearray(input_file.read()) + # Do the upgrade, returning a bytes object. + profraw_latest = upgrade(profraw_base, sect_prf_cnts, sect_prf_data) + # Write the output to the file given to the command line. + if in_place: + out_name = sys.argv[i] + with open(out_name, 'wb') as output_file: + output_file.write(profraw_latest) return 0 diff --git a/projects/elfutils/Dockerfile b/projects/elfutils/Dockerfile index 8f5a3b798c80..a8b641324557 100644 --- a/projects/elfutils/Dockerfile +++ b/projects/elfutils/Dockerfile @@ -14,7 +14,10 @@ # ################################################################################ -FROM gcr.io/oss-fuzz-base/base-builder +FROM gcr.io/oss-fuzz-base/base-builder@sha256:56905c98ae0083d14da0e7371184e694560a74750533f321ac0e9145af0e8d2e +# ! This project was pinned after a clang bump. Please remove the pin, Try to fix any build warnings and errors, as well as runtime errors +# see https://github.com/google/oss-fuzz/pull/12365 + RUN apt-get update && \ apt-get install -y pkg-config make autoconf autopoint zlib1g-dev zlib1g-dev:i386 flex gawk bison RUN git clone --depth 1 https://sourceware.org/git/elfutils.git diff --git a/projects/envoy/Dockerfile b/projects/envoy/Dockerfile index 2b0ec708aa7a..ab5d3b844590 100644 --- a/projects/envoy/Dockerfile +++ b/projects/envoy/Dockerfile @@ -15,7 +15,10 @@ ################################################################################ -FROM gcr.io/oss-fuzz-base/base-builder +FROM gcr.io/oss-fuzz-base/base-builder@sha256:56905c98ae0083d14da0e7371184e694560a74750533f321ac0e9145af0e8d2e +# ! This project was pinned after a clang bump. Please remove the pin, Try to fix any build warnings and errors, as well as runtime errors +# see https://github.com/google/oss-fuzz/pull/12365 + RUN apt-get update && apt-get -y install \ build-essential \ diff --git a/projects/icu/project.yaml b/projects/icu/project.yaml index 60f5e7890da5..4ef9f92fba78 100644 --- a/projects/icu/project.yaml +++ b/projects/icu/project.yaml @@ -17,5 +17,10 @@ sanitizers: - undefined # Disabled MSAN because of https://github.com/google/oss-fuzz/issues/6294 # - memory +fuzzing_engines: + - libfuzzer + - honggfuzz + - afl + # - centipede disabled due to https://github.com/google/oss-fuzz/pull/12365 clang 18 update main_repo: 'https://github.com/unicode-org/icu.git' diff --git a/projects/librawspeed/build.sh b/projects/librawspeed/build.sh index 5cdc48366bf2..610fffbd361c 100755 --- a/projects/librawspeed/build.sh +++ b/projects/librawspeed/build.sh @@ -15,4 +15,6 @@ # ################################################################################ +export CFLAGS="$CFLAGS -Wno-error=nan-infinity-disabled" +export CXXFLAGS="$CXXFLAGS -Wno-error=nan-infinity-disabled" $SRC/librawspeed/.ci/oss-fuzz.sh diff --git a/projects/rust-lexical/Dockerfile b/projects/rust-lexical/Dockerfile index f7229355d471..426e9111c968 100644 --- a/projects/rust-lexical/Dockerfile +++ b/projects/rust-lexical/Dockerfile @@ -13,7 +13,10 @@ # limitations under the License. # ################################################################################ -FROM gcr.io/oss-fuzz-base/base-builder-rust +FROM gcr.io/oss-fuzz-base/base-builder-rust@sha256:b9a45fecf0d9be6559fca019e90577632242be120ee2d97cec5c2045c1440710 +# ! This project was pinned after a clang bump. Please remove the pin, Try to fix any build warnings and errors, as well as runtime errors +# /usr/bin/ld: /src/rust-lexical/fuzz/target/x86_64-unknown-linux-gnu/release/deps/parse_integer_u16-53e4bc89ab30e724.parse_integer_u16.9056e4c0a19617b4-cgu.0.rcgu.o: in function `asan.module_dtor.204': +# parse_integer_u16.9056e4c0a19617b4-cgu.0:(.text.asan.module_dtor.204[asan.module_dtor]+0x6): undefined reference to `__sancov_gen_.998' RUN git clone --depth 1 https://github.com/Alexhuszagh/rust-lexical COPY build.sh $SRC/ diff --git a/projects/samba/build.sh b/projects/samba/build.sh index e066a972508b..2f7ee4d41486 100755 --- a/projects/samba/build.sh +++ b/projects/samba/build.sh @@ -15,6 +15,6 @@ # ################################################################################ -export CFLAGS="$CFLAGS -Wno-error=strict-prototypes" +export CFLAGS="$CFLAGS -Wno-error=strict-prototypes -Wno-error=format-truncation" # The real script is maintained in the Samba repo exec lib/fuzzing/oss-fuzz/build_samba.sh diff --git a/projects/suricata/Dockerfile b/projects/suricata/Dockerfile index 3433f3a76ae9..c5aa504a68f1 100644 --- a/projects/suricata/Dockerfile +++ b/projects/suricata/Dockerfile @@ -29,13 +29,9 @@ RUN git clone --depth=1 https://github.com/catenacyber/fuzzpcap ADD https://rules.emergingthreats.net/open/suricata/emerging.rules.zip emerging.rules.zip -ENV RUSTUP_TOOLCHAIN nightly RUN cargo install --force cbindgen -# TODO remove once we have clang with coverage version 9 as rustc -ENV RUSTUP_TOOLCHAIN nightly-2024-02-12 RUN git clone --depth 1 https://github.com/OISF/suricata.git suricata -RUN git clone --depth 1 --branch master-6.0.x https://github.com/OISF/suricata.git suricata6 RUN git clone --depth 1 --branch main-7.0.x https://github.com/OISF/suricata.git suricata7 RUN git clone --depth 1 https://github.com/OISF/libhtp.git libhtp RUN git clone --depth 1 https://github.com/OISF/suricata-verify suricata-verify diff --git a/projects/suricata/build.sh b/projects/suricata/build.sh index 6e42816a3a7c..3eb2a0331602 100755 --- a/projects/suricata/build.sh +++ b/projects/suricata/build.sh @@ -160,7 +160,6 @@ cat $t/*.rules > corpus/$i || true; echo -ne '\0' >> corpus/$i; cat $t/*.pcap >> done set -x zip -q -r $OUT/fuzz_sigpcap_seed_corpus.zip corpus -cp $OUT/fuzz_sigpcap_seed_corpus.zip $OUT/fuzz_sigpcap6_seed_corpus.zip rm -Rf corpus mkdir corpus set +x @@ -170,7 +169,6 @@ echo -ne '\0' >> corpus/$i; python3 $SRC/fuzzpcap/tcptofpc.py $t/*.pcap >> corpu done set -x zip -q -r $OUT/fuzz_sigpcap_aware_seed_corpus.zip corpus -cp $OUT/fuzz_sigpcap_aware_seed_corpus.zip $OUT/fuzz_sigpcap_aware6_seed_corpus.zip rm -Rf corpus mkdir corpus set +x @@ -180,4 +178,3 @@ python3 $SRC/fuzzpcap/tcptofpc.py $t/*.pcap >> corpus/$i || rm corpus/$i; i=$((i done set -x zip -q -r $OUT/fuzz_predefpcap_aware_seed_corpus.zip corpus -cp $OUT/fuzz_predefpcap_aware_seed_corpus.zip $OUT/fuzz_predefpcap_aware6_seed_corpus.zip diff --git a/projects/wasmer/Dockerfile b/projects/wasmer/Dockerfile index 88abfcbe5264..9193eb2964f9 100644 --- a/projects/wasmer/Dockerfile +++ b/projects/wasmer/Dockerfile @@ -27,4 +27,7 @@ RUN mkdir -p $SRC/.llvm && curl --proto '=https' --tlsv1.2 -sSf \ WORKDIR wasmer +# dead code warnings with nightly-2024-07-12 +ENV RUSTUP_TOOLCHAIN nightly-2024-02-12 + COPY build.sh default.options $SRC/