diff --git a/.github/workflows/CI-mingw.yml b/.github/workflows/CI-mingw.yml deleted file mode 100644 index 390de83d..00000000 --- a/.github/workflows/CI-mingw.yml +++ /dev/null @@ -1,141 +0,0 @@ -name: CI-mingw - -on: [push, pull_request] - -permissions: - contents: read - -defaults: - run: - shell: msys2 {0} - -jobs: - build: - - strategy: - matrix: - compiler: [g++, clang++] - # TODO: add MSYS after #556 is fixed - msystem: [MINGW32, MINGW64, CLANG64] - include: - #- msystem: MSYS - # pkg-prefix: '' - - msystem: MINGW32 - pkg-prefix: 'mingw-w64-i686-' - - msystem: MINGW64 - pkg-prefix: 'mingw-w64-x86_64-' - - msystem: CLANG64 - pkg-prefix: 'mingw-w64-clang-x86_64-' - - compiler: g++ - compiler-pkg: gcc - - compiler: clang++ - compiler-pkg: clang - exclude: - - msystem: CLANG64 - compiler: g++ - # the mingw-w64-i686-clang package is no longer available - - msystem: MINGW32 - compiler: clang++ - fail-fast: false - - runs-on: windows-2025 - - env: - CXX: ${{ matrix.compiler }} - - steps: - - uses: actions/checkout@v4 - with: - persist-credentials: false - - - name: Set up MSYS2 - uses: msys2/setup-msys2@v2 - with: - release: false # use pre-installed - msystem: ${{ matrix.msystem }} - # TODO: install mingw-w64-x86_64-make and use mingw32.make instead - currently fails with "Windows Subsystem for Linux has no installed distributions." - # TODO: also run tests with non-prefixed Python? - install: >- - make - ${{ matrix.pkg-prefix }}cmake - ${{ matrix.pkg-prefix }}python - ${{ matrix.pkg-prefix }}python-pytest - - - name: install compiler - run: | - pacman -S --noconfirm ${{ matrix.pkg-prefix }}${{ matrix.compiler-pkg }} - ${CXX} -v - - - name: make simplecpp - run: | - make -j$(nproc) CXXOPTS="-Werror" - - # gcc *and* clang are required to run-tests.py - # install it at this point since it has gcc as dependency which might interfere with the build - - name: install compiler (clang) - if: matrix.compiler == 'g++' - run: | - pacman -S --noconfirm clang - - - name: install compiler (gcc) - if: matrix.compiler == 'clang++' - run: | - pacman -S --noconfirm gcc - - - name: make test - run: | - # TODO: run tests with Windows paths - make -j$(nproc) test - - - name: selfcheck - run: | - # TODO: run tests with Windows paths - make -j$(nproc) selfcheck - - - name: make (c++14) - run: | - make clean - make -j$(nproc) CXXOPTS="-Werror -std=c++14" - - - name: make (c++17) - run: | - make clean - make -j$(nproc) CXXOPTS="-Werror -std=c++17" - - - name: make (c++20) - run: | - make clean - make -j$(nproc) CXXOPTS="-Werror -std=c++20" - - - name: make (c++23) - run: | - make clean - make -j$(nproc) CXXOPTS="-Werror -std=c++23" - - - name: Run CMake - run: | - cmake -S . -B cmake.output -DCMAKE_COMPILE_WARNING_AS_ERROR=On - - - name: CMake simplecpp - run: | - cmake --build cmake.output --target simplecpp -- -j $(nproc) - - - name: CMake testrunner - run: | - cmake --build cmake.output --target testrunner -- -j $(nproc) - - - name: Run testrunner - run: | - ./cmake.output/testrunner - - - name: Run with libstdc++ debug mode - if: matrix.compiler == 'g++' - run: | - make clean - make -j$(nproc) test selfcheck CXXOPTS="-Werror -g3 -D_GLIBCXX_DEBUG" - - - name: Run with libc++ hardening mode - if: matrix.compiler == 'clang++' && matrix.msystem == 'CLANG64' - run: | - make clean - make -j$(nproc) test selfcheck CXXOPTS="-Werror -stdlib=libc++ -g3 -D_LIBCPP_HARDENING_MODE=_LIBCPP_HARDENING_MODE_DEBUG" LDOPTS="-lc++" diff --git a/.github/workflows/CI-unixish.yml b/.github/workflows/CI-unixish.yml deleted file mode 100644 index 1a4ce649..00000000 --- a/.github/workflows/CI-unixish.yml +++ /dev/null @@ -1,194 +0,0 @@ -name: CI-unixish - -on: [push, pull_request] - -permissions: - contents: read - -jobs: - build: - - strategy: - matrix: - os: [ubuntu-22.04, ubuntu-22.04-arm, ubuntu-24.04, ubuntu-24.04-arm, macos-14, macos-15, macos-15-intel, macos-26, macos-26-intel] - compiler: [clang++] - include: - - os: ubuntu-22.04 - compiler: g++ - - os: ubuntu-24.04 - compiler: g++ - fail-fast: false - - runs-on: ${{ matrix.os }} - - env: - CXX: ${{ matrix.compiler }} - - steps: - - uses: actions/checkout@v6 - with: - persist-credentials: false - - # the man-db trigger causes package installations to stall for several minutes at times. so just drop the package. - # see https://github.com/actions/runner/issues/4030 - - name: Remove man-db package on ubuntu - if: matrix.os == 'ubuntu-24.04' - run: | - sudo apt-get update - sudo apt-get remove man-db - - - name: Install missing software on ubuntu - if: matrix.os == 'ubuntu-24.04' - run: | - sudo apt-get update - sudo apt-get install valgrind - - # llvm contains llvm-profdata - - name: Install missing software on ubuntu (clang++) - if: contains(matrix.os, 'ubuntu') && matrix.compiler == 'clang++' - run: | - sudo apt-get update - sudo apt-get install libc++-dev llvm - - # coreutils contains "nproc" - - name: Install missing software on macos - if: contains(matrix.os, 'macos') - run: | - brew install coreutils - - - name: Install missing Python packages - run: | - python3 -m pip config set global.break-system-packages true - python3 -m pip install pytest - - - name: make simplecpp - run: make -j$(nproc) CXXOPTS="-Werror" - - - name: make test - run: make -j$(nproc) test CXXOPTS="-Werror" - - - name: selfcheck - run: | - make -j$(nproc) selfcheck - - - name: make (c++14) - run: | - make clean - make -j$(nproc) CXXOPTS="-Werror -std=c++14" - - - name: make (c++17) - run: | - make clean - make -j$(nproc) CXXOPTS="-Werror -std=c++17" - - - name: make (c++20) - run: | - make clean - make -j$(nproc) CXXOPTS="-Werror -std=c++20" - - - name: make (c++23) - run: | - make clean - # ubuntu-22.04 and macos-14 do not support c++23 yet - make -j$(nproc) CXXOPTS="-Werror -std=c++2b" - - - name: Run CMake - run: | - cmake -S . -B cmake.output -Werror=dev --warn-uninitialized -DCMAKE_COMPILE_WARNING_AS_ERROR=On - - - name: CMake simplecpp - run: | - cmake --build cmake.output --target simplecpp -- -j $(nproc) - - - name: CMake testrunner - run: | - cmake --build cmake.output --target testrunner -- -j $(nproc) - ./cmake.output/testrunner - # Re-run tests from within the build directory to validate that - # SIMPLECPP_TEST_SOURCE_DIR is correctly defined and resolved - (cd cmake.output && ./testrunner) - - - name: Run valgrind - if: matrix.os == 'ubuntu-24.04' - run: | - make clean - make -j$(nproc) CXXOPTS="-O1" - valgrind --leak-check=full --num-callers=50 --show-reachable=yes --track-origins=yes --gen-suppressions=all --error-exitcode=42 ./testrunner - # TODO: run Python tests with valgrind - VALGRIND_TOOL=memcheck ./selfcheck.sh - - - name: Run with libstdc++ debug mode - if: matrix.os == 'ubuntu-24.04' && matrix.compiler == 'g++' - run: | - make clean - make -j$(nproc) test selfcheck CXXOPTS="-Werror -g3 -D_GLIBCXX_DEBUG" - - - name: Run with libc++ hardening mode - if: matrix.os == 'ubuntu-24.04' && matrix.compiler == 'clang++' - run: | - make clean - make -j$(nproc) test selfcheck CXXOPTS="-Werror -stdlib=libc++ -g3 -D_LIBCPP_HARDENING_MODE=_LIBCPP_HARDENING_MODE_DEBUG" LDOPTS="-lc++" - - - name: Run AddressSanitizer - if: matrix.os == 'ubuntu-24.04' || matrix.os == 'macos-26' - run: | - make clean - make -j$(nproc) test selfcheck CXXOPTS="-Werror -O2 -g3 -fsanitize=address" LDOPTS="-fsanitize=address" - env: - ASAN_OPTIONS: detect_stack_use_after_return=1 - - - name: Run UndefinedBehaviorSanitizer - if: matrix.os == 'ubuntu-24.04' || matrix.os == 'macos-26' - run: | - make clean - make -j$(nproc) test selfcheck CXXOPTS="-Werror -O2 -g3 -fsanitize=undefined -fno-sanitize=signed-integer-overflow" LDOPTS="-fsanitize=undefined -fno-sanitize=signed-integer-overflow" - env: - UBSAN_OPTIONS: print_stacktrace=1:halt_on_error=1:report_error_type=1 - - # TODO: requires instrumented libc++ - - name: Run MemorySanitizer - if: false && matrix.os == 'ubuntu-24.04' && matrix.compiler == 'clang++' - run: | - make clean - make -j$(nproc) test selfcheck CXXOPTS="-Werror -O2 -g3 -stdlib=libc++ -fsanitize=memory" LDOPTS="-lc++ -fsanitize=memory" - - - name: Run callgrind - if: matrix.os == 'ubuntu-24.04' - run: | - wget https://github.com/danmar/simplecpp/archive/refs/tags/1.5.1.tar.gz - tar xvf 1.5.1.tar.gz - rm -f 1.5.1.tar.gz - - make clean - make -j$(nproc) CXXOPTS="-O2 -g3" simplecpp - VALGRIND_TOOL=callgrind SIMPLECPP_PATH=simplecpp-1.5.1 ./selfcheck.sh >callgrind.log || (cat callgrind.log && false) - cat callgrind.log - - # PGO - start - make clean - make -j$(nproc) CXXOPTS="-O2 -g3 -fprofile-generate" LDOPTS="-fprofile-generate" simplecpp - SIMPLECPP_PATH=simplecpp-1.5.1 ./selfcheck.sh >/dev/null - - if compgen -G "default_*.profraw" > /dev/null; then - llvm-profdata merge -output=default.profdata default_*.profraw - fi - - make clean - make -j$(nproc) CXXOPTS="-O2 -g3 -fprofile-use" LDOPTS="-fprofile-use" simplecpp - VALGRIND_TOOL=callgrind SIMPLECPP_PATH=simplecpp-1.5.1 ./selfcheck.sh >callgrind_pgo.log || (cat callgrind_pgo.log && false) - cat callgrind_pgo.log - # PGO - end - - for f in callgrind.out.*; - do - callgrind_annotate --auto=no $f > $f.annotated.log - head -50 $f.annotated.log - done - rm -rf simplecpp-1.5.1 - - - uses: actions/upload-artifact@v4 - if: matrix.os == 'ubuntu-24.04' - with: - name: Callgrind Output - ${{ matrix.compiler }} - path: | - ./callgrind.* diff --git a/.github/workflows/CI-windows.yml b/.github/workflows/CI-windows.yml deleted file mode 100644 index cfced0d2..00000000 --- a/.github/workflows/CI-windows.yml +++ /dev/null @@ -1,82 +0,0 @@ -# Some convenient links: -# - https://github.com/actions/virtual-environments/blob/master/images/win/Windows2019-Readme.md -# - -name: CI-windows - -on: [push,pull_request] - -permissions: - contents: read - -defaults: - run: - shell: cmd - -jobs: - - build: - strategy: - matrix: - os: [windows-2022, windows-2025, windows-11-arm] - config: [Release, Debug] - fail-fast: false - - runs-on: ${{ matrix.os }} - - steps: - - uses: actions/checkout@v6 - with: - persist-credentials: false - - - name: Setup msbuild.exe - uses: microsoft/setup-msbuild@v2 - - - name: Set up Python - uses: actions/setup-python@v6 - with: - python-version: '3.14' - check-latest: true - - - name: Install missing Python packages - run: | - python -m pip install pip --upgrade || exit /b !errorlevel! - python -m pip install pytest || exit /b !errorlevel! - - - name: Run CMake - run: | - cmake -G "Visual Studio 17 2022" -A x64 -Werror=dev --warn-uninitialized -DCMAKE_COMPILE_WARNING_AS_ERROR=On . || exit /b !errorlevel! - - - name: Build - run: | - msbuild -m simplecpp.sln /p:Configuration=${{ matrix.config }} /p:Platform=x64 || exit /b !errorlevel! - - - name: Test - run: | - .\${{ matrix.config }}\testrunner.exe || exit /b !errorlevel! - - - name: Selfcheck - run: | - .\${{ matrix.config }}\simplecpp.exe simplecpp.cpp -e || exit /b !errorlevel! - - - name: integration test - run: | - set SIMPLECPP_EXE_PATH=.\${{ matrix.config }}\simplecpp.exe - python -m pytest integration_test.py -vv || exit /b !errorlevel! - - - name: Run CMake (c++17) - run: | - cmake -S . -B build.cxx17 -G "Visual Studio 17 2022" -A x64 -Werror=dev --warn-uninitialized -DCMAKE_CXX_STANDARD=17 -DCMAKE_COMPILE_WARNING_AS_ERROR=On || exit /b !errorlevel! - - - name: Build (c++17) - run: | - msbuild -m build.cxx17\simplecpp.sln /p:Configuration=${{ matrix.config }} /p:Platform=x64 || exit /b !errorlevel! - - - name: Run CMake (c++20) - run: | - cmake -S . -B build.cxx20 -G "Visual Studio 17 2022" -A x64 -Werror=dev --warn-uninitialized -DCMAKE_CXX_STANDARD=20 -DCMAKE_COMPILE_WARNING_AS_ERROR=On || exit /b !errorlevel! - - - name: Build (c++20) - run: | - msbuild -m build.cxx20\simplecpp.sln /p:Configuration=${{ matrix.config }} /p:Platform=x64 || exit /b !errorlevel! - diff --git a/.github/workflows/clang-tidy.yml b/.github/workflows/clang-tidy.yml deleted file mode 100644 index 333672d7..00000000 --- a/.github/workflows/clang-tidy.yml +++ /dev/null @@ -1,51 +0,0 @@ -# Syntax reference https://help.github.com/en/actions/reference/workflow-syntax-for-github-actions -# Environment reference https://help.github.com/en/actions/reference/virtual-environments-for-github-hosted-runners -name: clang-tidy - -on: [push, pull_request] - -permissions: - contents: read - -jobs: - build: - - runs-on: ubuntu-24.04 - - steps: - - uses: actions/checkout@v6 - with: - persist-credentials: false - - # the man-db trigger causes package installations to stall for several minutes at times. so just drop the package. - # see https://github.com/actions/runner/issues/4030 - - name: Remove man-db package - run: | - sudo apt-get update - sudo apt-get remove man-db - - - name: Install missing software - run: | - sudo apt-get update - sudo apt-get install -y cmake make - - - name: Install clang - run: | - wget https://apt.llvm.org/llvm.sh - chmod +x llvm.sh - sudo ./llvm.sh 22 - sudo apt-get install clang-tidy-22 - - - name: Verify clang-tidy configuration - run: | - clang-tidy-22 --verify-config - - - name: Prepare CMake - run: | - cmake -S . -B cmake.output -Werror=dev --warn-uninitialized -DCMAKE_CXX_STANDARD=23 -DCMAKE_COMPILE_WARNING_AS_ERROR=On -DCMAKE_EXPORT_COMPILE_COMMANDS=ON - env: - CXX: clang-22 - - - name: Clang-Tidy - run: | - run-clang-tidy-22 -q -j $(nproc) -enable-check-profile -p=cmake.output diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml deleted file mode 100644 index e9f1361d..00000000 --- a/.github/workflows/format.yml +++ /dev/null @@ -1,62 +0,0 @@ -# Syntax reference https://help.github.com/en/actions/reference/workflow-syntax-for-github-actions -# Environment reference https://help.github.com/en/actions/reference/virtual-environments-for-github-hosted-runners -name: format - -on: - push: - branches: - - 'master' - - 'releases/**' - - '1.*' - tags: - - '1.*' - pull_request: - -permissions: - contents: read - -jobs: - format: - - runs-on: ubuntu-22.04 - - defaults: - run: - shell: bash -euo pipefail {0} - - env: - UNCRUSTIFY_INSTALL_DIR: ${{ github.workspace }}/runformat-uncrustify - - steps: - - uses: actions/checkout@v6 - with: - persist-credentials: false - - - name: Determine uncrustify version - id: get-uncrustify-version - run: | - version="$(./runformat --expected-uncrustify-version)" - echo "Expected uncrustify version: $version" - echo "version=$version" >> "$GITHUB_OUTPUT" - - - name: Set UNCRUSTIFY_VERSION env variable - run: | - version=$(./runformat --expected-uncrustify-version) - echo "version [$version]" - echo "UNCRUSTIFY_VERSION=${version}" >> "$GITHUB_ENV" - - - name: Cache uncrustify - uses: actions/cache@v4 - id: cache-uncrustify - with: - path: ${{ env.UNCRUSTIFY_INSTALL_DIR }} - key: ${{ runner.os }}-uncrustify-${{ steps.get-uncrustify-version.outputs.version }} - - - name: Install uncrustify - if: steps.cache-uncrustify.outputs.cache-hit != 'true' - run: | - ./runformat --install --install-dir "${UNCRUSTIFY_INSTALL_DIR}" - - - name: Uncrustify check - run: | - ./runformat diff --git a/.github/workflows/fuzz.yml b/.github/workflows/fuzz.yml new file mode 100644 index 00000000..5bead117 --- /dev/null +++ b/.github/workflows/fuzz.yml @@ -0,0 +1,82 @@ +# Syntax reference https://help.github.com/en/actions/reference/workflow-syntax-for-github-actions +# Environment reference https://help.github.com/en/actions/reference/virtual-environments-for-github-hosted-runners +name: fuzz +on: [pull_request] + +permissions: + contents: read + +jobs: + fuzz: + runs-on: ubuntu-24.04 + if: ${{ github.repository_owner == 'danmar' }} + + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + + # the man-db trigger causes package installations to stall for several minutes at times. so just drop the package. + # see https://github.com/actions/runner/issues/4030 + - name: Remove man-db package + run: | + sudo apt-get update + sudo apt-get remove man-db + + - name: Install missing software + run: | + sudo apt-get update + sudo apt-get install -y make + + - name: Install clang + run: | + wget https://apt.llvm.org/llvm.sh + chmod +x llvm.sh + sudo ./llvm.sh 22 + + - name: Generate corpus + run: | + mkdir corpus_test + make testrunner CXXOPTS="-DSTORE_INPUT_DIR=\"\\\"$(pwd)/corpus_test\\\"\"" + ./testrunner + + - name: Upload corpus (testrunner) + uses: actions/upload-artifact@v6 + with: + name: corpus_test + path: ./corpus_test + + - name: Build fuzzer + id: build + run: | + make clean + # TODO: test O/LTO for best speed + # TODO: use -stdlib=libc++ -lc++ + make -j$(nproc) CXXOPTS="-O3 -flto -fno-omit-frame-pointer -g -gline-tables-only -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -fsanitize=address,undefined -fsanitize-address-use-after-scope -fno-sanitize=integer -fno-sanitize-recover=undefined" LDOPTS="-flto" LIB_FUZZING_ENGINE="-fsanitize=fuzzer" fuzz + env: + CXX: clang++-22 + + - name: Run fuzzer + run: | + mkdir corpus + mkdir artifacts + ./fuzz -only_ascii=1 -timeout=5 -fork=$(nproc) -use_value_profile=1 -reduce_inputs=0 -timeout_exitcode=0 -max_total_time=60 -artifact_prefix=./artifacts/ corpus corpus_test + # if a crash happens with a file from the corpus the fuzzer will not fail - so fail if any artifacts have been written + test $(find ./artifacts -type f | wc -l) -eq 0 + env: + ASAN_OPTIONS: detect_stack_use_after_return=1 + UBSAN_OPTIONS: print_stacktrace=1:halt_on_error=1:report_error_type=1 + + - name: Upload corpus + if: success() || failure() + uses: actions/upload-artifact@v6 + with: + name: corpus + path: ./corpus + + - name: Upload artifacts + if: success() || failure() + uses: actions/upload-artifact@v6 + with: + name: artifacts + path: ./artifacts diff --git a/Makefile b/Makefile index b7b54597..5488bfb4 100644 --- a/Makefile +++ b/Makefile @@ -12,21 +12,32 @@ test.o: CPPFLAGS += $(TEST_CPPFLAGS) test.o: CXXFLAGS += -Wno-multichar %.o: %.cpp simplecpp.h - $(CXX) $(CPPFLAGS) $(CXXFLAGS) -c $< + $(CXX) $(CPPFLAGS) $(CXXFLAGS) -c $< $(LIB_FUZZING_ENGINE) + +fuzz_no.o: fuzz.cpp + $(CXX) $(CPPFLAGS) $(CXXFLAGS) -DNO_FUZZ -c -o $@ $^ testrunner: test.o simplecpp.o - $(CXX) $(LDFLAGS) simplecpp.o test.o -o testrunner + $(CXX) $(LDFLAGS) -o $@ $^ test: testrunner simplecpp ./testrunner python3 run-tests.py python3 -m pytest integration_test.py -vv +fuzz: fuzz.o simplecpp.o + # TODO: use -stdlib=libc++ -lc++ + # make fuzz CXX=clang++ CXXOPTS="-O2 -fno-omit-frame-pointer -g -gline-tables-only -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -fsanitize=address,undefined -fsanitize-address-use-after-scope -fno-sanitize=integer -fno-sanitize-recover=undefined" LIB_FUZZING_ENGINE="-fsanitize=fuzzer" + $(CXX) $(LDFLAGS) $(CXXFLAGS) -o $@ $^ $(LIB_FUZZING_ENGINE) + +no-fuzz: fuzz_no.o simplecpp.o + $(CXX) $(LDFLAGS) $(CXXFLAGS) -o $@ $^ + selfcheck: simplecpp CXX=$(CXX) ./selfcheck.sh simplecpp: main.o simplecpp.o - $(CXX) $(LDFLAGS) main.o simplecpp.o -o simplecpp + $(CXX) $(LDFLAGS) -o $@ $^ clean: - rm -f testrunner simplecpp *.o + rm -f testrunner fuzz no-fuzz simplecpp *.o diff --git a/fuzz.cpp b/fuzz.cpp new file mode 100644 index 00000000..88b69e3e --- /dev/null +++ b/fuzz.cpp @@ -0,0 +1,70 @@ +/* + * simplecpp - A simple and high-fidelity C/C++ preprocessor library + * Copyright (C) 2016-2024 simplecpp team + */ + +#include "simplecpp.h" + +#include + +#ifdef NO_FUZZ +#include +#include +#include +#include +#endif + +/* + 0 - store in corpus + -1 - omit from corpus +*/ +static int doProcess(const uint8_t *data, size_t dataSize) +{ + simplecpp::OutputList outputList; + std::vector files; + simplecpp::TokenList rawtokens(data, dataSize, files, "test.cpp", &outputList); + + simplecpp::TokenList outputTokens(files); + simplecpp::FileDataCache filedata; + const simplecpp::DUI dui; + std::list macroUsage; + std::list ifCond; + simplecpp::preprocess(outputTokens, rawtokens, files, filedata, dui, &outputList, ¯oUsage, &ifCond); + + simplecpp::cleanup(filedata); + + return 0; +} + +#ifndef NO_FUZZ +extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t dataSize); + +int LLVMFuzzerTestOneInput(const uint8_t *data, size_t dataSize) +{ + return doProcess(data, dataSize); +} +#else +int main(int argc, char * argv[]) +{ + if (argc < 2 || argc > 3) + return EXIT_FAILURE; + + std::ifstream f(argv[1]); + if (!f.is_open()) + return EXIT_FAILURE; + + std::ostringstream oss; + oss << f.rdbuf(); + + if (!f.good()) + return EXIT_FAILURE; + + const int cnt = (argc == 3) ? std::stoi(argv[2]) : 1; + + const std::string code = oss.str(); + for (int i = 0; i < cnt; ++i) + doProcess(reinterpret_cast(code.data()), code.size()); + + return EXIT_SUCCESS; +} +#endif diff --git a/simplecpp.cpp b/simplecpp.cpp index a0e8caa8..89bb7420 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -53,8 +53,10 @@ #ifdef _WIN32 # include +using mode_t = unsigned short; #else # include +# include #endif static bool isHex(const std::string &s) @@ -3044,9 +3046,11 @@ static std::string openHeaderDirect(std::ifstream &f, const std::string &path) if (nonExistingFilesCache.contains(path)) return ""; // file is known not to exist, skip expensive file open call #endif - f.open(path.c_str()); - if (f.is_open()) - return path; + if (simplecpp::isFile(path)) { + f.open(path.c_str()); + if (f.is_open()) + return path; + } #ifdef SIMPLECPP_WINDOWS nonExistingFilesCache.add(path); #endif @@ -3180,6 +3184,9 @@ bool simplecpp::FileDataCache::getFileId(const std::string &path, FileID &id) if (stat(path.c_str(), &statbuf) != 0) return false; + if ((statbuf.st_mode & S_IFMT) != S_IFREG) + return false; + id.dev = statbuf.st_dev; id.ino = statbuf.st_ino; @@ -3967,3 +3974,21 @@ std::string simplecpp::getCppStdString(const std::string &std) { return getCppStdString(getCppStd(std)); } + +static mode_t file_type(const std::string &path) +{ + struct stat file_stat; + if (stat(path.c_str(), &file_stat) == -1) + return 0; + return file_stat.st_mode & S_IFMT; +} + +bool simplecpp::isFile(const std::string &path) +{ + return file_type(path) == S_IFREG; +} + +bool simplecpp::isDirectory(const std::string &path) +{ + return file_type(path) == S_IFDIR; +} diff --git a/simplecpp.h b/simplecpp.h index b744e62d..4758ee93 100644 --- a/simplecpp.h +++ b/simplecpp.h @@ -443,6 +443,20 @@ namespace simplecpp { bool removeComments{}; /** remove comment tokens from included files */ }; + /** + * @brief Checks if given path is a file + * @param path Path to be checked + * @return true if given path is a file + */ + SIMPLECPP_LIB bool isFile(const std::string &path); + + /** + * @brief Checks if a given path is a directory + * @param path Path to be checked + * @return true if given path is a directory + */ + SIMPLECPP_LIB bool isDirectory(const std::string &path); + struct SIMPLECPP_LIB FileData { /** The canonical filename associated with this data */ std::string filename; diff --git a/test.cpp b/test.cpp index bc8cb4d5..74fc7ad3 100644 --- a/test.cpp +++ b/test.cpp @@ -81,9 +81,28 @@ static void testcase(const std::string &name, void (*f)(), int argc, char * cons #define TEST_CASE(F) (testcase(#F, F, argc, argv)) +#ifdef STORE_INPUT_DIR +// make testrunner CXXOPTS="-DSTORE_INPUT_DIR=\"\\\"/home/user/simple_corpus\\\"\"" +#include +#include + +static void storeInput(const std::string& str) +{ + static std::atomic_uint64_t num(0); + { + std::ofstream out(STORE_INPUT_DIR "/" + std::to_string(num++)); + out << str; + } +} +#endif + static simplecpp::TokenList makeTokenList(const char code[], std::size_t size, std::vector &filenames, const std::string &filename=std::string(), simplecpp::OutputList *outputList=nullptr) { - std::istringstream istr(std::string(code, size)); + const std::string str(code, size); +#ifdef STORE_INPUT_DIR + storeInput(str); +#endif + std::istringstream istr(str); return {istr,filenames,filename,outputList}; } @@ -691,6 +710,181 @@ static void define13() "}", preprocess(code)); } +static void define14() // #296 +{ + const char code[] = "#define bar(x) x % 2\n" + "#define foo(x) printf(#x \"\\n\")\n" + "\n" + " foo(bar(3));\n"; + ASSERT_EQUALS("\n" + "\n" + "\n" + "printf ( \"bar(3)\" \"\\n\" ) ;", preprocess(code)); +} + +static void define15() // #231 +{ + const char code[] = "#define CAT(a, b) CAT2(a, b)\n" + "#define CAT2(a, b) a ## b\n" + "#define FOO x\n" + "#define BAR() CAT(F, OO)\n" + "#define BAZ CAT(B, AR)()\n" + "BAZ\n"; + ASSERT_EQUALS("\n" + "\n" + "\n" + "\n" + "\n" + "x", preprocess(code)); +} + +static void define16() // #201 +{ + const char code[] = "#define ALL_COLORS(warm_colors) \\\n" + " X(Blue) \\\n" + " X(Green) \\\n" + " X(Purple) \\\n" + " warm_colors\n" + "\n" + "#define WARM_COLORS \\\n" + " X(Red) \\\n" + " X(Yellow) \\\n" + " X(Orange)\n" + "\n" + "#define COLOR_SET ALL_COLORS(WARM_COLORS)\n" + "\n" + "#define X(color) #color,\n" + "\n" + "COLOR_SET\n"; + ASSERT_EQUALS("\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\"Blue\" , \"Green\" , \"Purple\" , \"Red\" , \"Yellow\" , \"Orange\" ,", preprocess(code)); +} + +static void define17() // #185 +{ + const char code[] = "#define at(x, y) x##y\n" + "#define b(...) \\\n" + "aa(__VA_ARGS__, , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , \\\n" + ", , , , , , , , 2)\n" + "#define aa(c, d, a, b, e, f, g, h, ab, ac, i, ad, j, k, l, m, n, o, p, ae, q, \\\n" + "r, s, t, u, v, w, x, y, z, af, ag, ah, ai, aj, ak, al, am, an, ao, \\\n" + "ap) \\\n" + "ap\n" + "#define aq(...) ar(b(__VA_ARGS__), __VA_ARGS__) static_assert(true, \" \")\n" + "#define ar(ap, ...) at(I_, ap)(__VA_ARGS__)\n" + "#define I_2(as, a)\n" + "aq(a, array);\n"; + ASSERT_EQUALS("\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "static_assert ( true , \" \" ) ;", preprocess(code)); +} + +static void define18() // #130 +{ + const char code[] = "#define MAC2STR(x) x[0],x[1],x[2],x[3],x[4],x[5]\n" + "#define FT_DEBUG(fmt, args...) if(pGlobalCtx && pGlobalCtx->debug_level>=2) printf(\"FT-dbg: \"fmt, ##args)\n" + "\n" + "FT_DEBUG(\" %02x:%02x:%02x:%02x:%02x:%02x\\n\", MAC2STR(pCtx->wlan_intf_addr[i]));\n"; + ASSERT_EQUALS("\n" + "\n" + "\n" + "if ( pGlobalCtx && pGlobalCtx -> debug_level >= 2 ) printf ( \"FT-dbg: \" \" %02x:%02x:%02x:%02x:%02x:%02x\\n\" , pCtx -> wlan_intf_addr [ i ] [ 0 ] , pCtx -> wlan_intf_addr [ i ] [ 1 ] , pCtx -> wlan_intf_addr [ i ] [ 2 ] , pCtx -> wlan_intf_addr [ i ] [ 3 ] , pCtx -> wlan_intf_addr [ i ] [ 4 ] , pCtx -> wlan_intf_addr [ i ] [ 5 ] ) ;", preprocess(code)); +} + +static void define19() // #124 +{ + const char code[] = "#define CONCAT(tok) tok##suffix\n" + "\n" + "CONCAT(Test);\n" + "CONCAT(const Test);\n"; + ASSERT_EQUALS("\n" + "\n" + "Testsuffix ;\n" + "const Testsuffix ;", preprocess(code)); +} + +static void define20() // #113 +{ + const char code[] = "#define TARGS4 T1,T2,T3,T4\n" + "#define FOOIMPL(T__CLASS, TARGS) void foo(const T__CLASS& x) { }\n" + "#define FOOIMPL_4(T__CLASS) FOOIMPL(T__CLASS, TARGS4)\n" + "FOOIMPL_4(y)\n"; + ASSERT_EQUALS("\n" + "\n" + "\n" + "void foo ( const y < T1 , T2 , T3 , T4 > & x ) { }", preprocess(code)); +} + +static void define21() // #66 +{ + const char code[] = "#define GETMYID(a) ((a))+1\n" + "#define FIGHT_FOO(c, ...) foo(c, ##__VA_ARGS__)\n" + "FIGHT_FOO(1, GETMYID(a));\n"; + ASSERT_EQUALS("\n" + "\n" + "foo ( 1 , ( ( a ) ) + 1 ) ;", preprocess(code)); +} + +static void define22() // #40 +{ + const char code[] = "#define COUNTER_NAME(NAME, ...) NAME##Count\n" + "#define COMMA ,\n" + "\n" + "#define DECLARE_COUNTERS(LIST) unsigned long LIST(COUNTER_NAME, COMMA);\n" + "\n" + "#define ACTUAL_LIST(FUNCTION, SEPARATOR) \\\n" + "FUNCTION(event1, int, foo) SEPARATOR \\\n" + "FUNCTION(event2, char, bar)\n" + "\n" + "DECLARE_COUNTERS(ACTUAL_LIST)\n"; + ASSERT_EQUALS("\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "unsigned long event1Count , event2Count ;", preprocess(code)); +} + +static void define23() // #40 +{ + const char code[] = "#define COMMA ,\n" + "#define MULTI(SEPARATOR) A SEPARATOR B\n" + "\n" + "#define VARS MULTI(COMMA)\n" + "unsigned VARS;\n"; + ASSERT_EQUALS("\n" + "\n" + "\n" + "\n" + "unsigned A , B ;", preprocess(code)); +} static void define_invalid_1() @@ -1153,6 +1347,15 @@ static void pragma_backslash() ASSERT_EQUALS("", preprocess(code, &outputList)); } +static void pragma_backslash_2() // #217 +{ + const char code[] = "#pragma comment(linker, \"foo \\\n" + "bar\")\n"; + + simplecpp::OutputList outputList; + ASSERT_EQUALS("", preprocess(code, &outputList)); +} + static void dollar() { ASSERT_EQUALS("$ab", readfile("$ab")); @@ -2247,6 +2450,44 @@ static void missingHeader4() ASSERT_EQUALS("file0,1,syntax_error,No header in #include\n", toString(outputList)); } +#ifndef _WIN32 +static void missingHeader5() +{ + // this is a directory + const char code[] = "#include \"/\"\n"; + simplecpp::OutputList outputList; + ASSERT_EQUALS("", preprocess(code, &outputList)); + ASSERT_EQUALS("file0,1,missing_header,Header not found: \"/\"\n", toString(outputList)); +} + +static void missingHeader6() +{ + // this is a directory + const char code[] = "#include \"/usr\"\n"; + simplecpp::OutputList outputList; + ASSERT_EQUALS("", preprocess(code, &outputList)); + ASSERT_EQUALS("file0,1,missing_header,Header not found: \"/usr\"\n", toString(outputList)); +} + +static void missingHeader7() +{ + // this is a directory + const char code[] = "#include \n"; + simplecpp::OutputList outputList; + ASSERT_EQUALS("", preprocess(code, &outputList)); + ASSERT_EQUALS("file0,1,missing_header,Header not found: \n", toString(outputList)); +} + +static void missingHeader8() +{ + // this is a directory + const char code[] = "#include \n"; + simplecpp::OutputList outputList; + ASSERT_EQUALS("", preprocess(code, &outputList)); + ASSERT_EQUALS("file0,1,missing_header,Header not found: \n", toString(outputList)); +} +#endif + static void nestedInclude() { const char code[] = "#include \"test.h\"\n"; @@ -3617,6 +3858,14 @@ static void leak() "#define e\n"; (void)preprocess(code, simplecpp::DUI()); } + { + const char code[] = "#include\n" + "#include\n"; + simplecpp::OutputList outputList; + ASSERT_EQUALS("", preprocess(code, &outputList)); + ASSERT_EQUALS("file0,1,missing_header,Header not found: \n" + "file0,2,missing_header,Header not found: \n", toString(outputList)); + } } int main(int argc, char **argv) @@ -3655,6 +3904,16 @@ int main(int argc, char **argv) TEST_CASE(define11); TEST_CASE(define12); TEST_CASE(define13); + TEST_CASE(define14); // #296 + TEST_CASE(define15); // #231 + TEST_CASE(define16); // #201 + TEST_CASE(define17); // #185 + TEST_CASE(define18); // #130 + TEST_CASE(define19); // #124 + TEST_CASE(define20); // #113 + TEST_CASE(define21); // #66 + TEST_CASE(define22); // #40 + TEST_CASE(define23); // #40 TEST_CASE(define_invalid_1); TEST_CASE(define_invalid_2); TEST_CASE(define_define_1); @@ -3697,6 +3956,7 @@ int main(int argc, char **argv) TEST_CASE(define_va_opt_9); // #632 TEST_CASE(pragma_backslash); // multiline pragma directive + TEST_CASE(pragma_backslash_2); // #217 // UB: #ifdef as macro parameter TEST_CASE(define_ifdef); @@ -3801,6 +4061,12 @@ int main(int argc, char **argv) TEST_CASE(missingHeader2); TEST_CASE(missingHeader3); TEST_CASE(missingHeader4); +#ifndef _WIN32 + TEST_CASE(missingHeader5); + TEST_CASE(missingHeader6); + TEST_CASE(missingHeader7); + TEST_CASE(missingHeader8); +#endif TEST_CASE(nestedInclude); TEST_CASE(systemInclude); TEST_CASE(circularInclude);