Merge pull request #209 from boostorg/develop

Merge 1.85 fixes
boostorg · Apr 1, 2024 · c11757d · c11757d
2 parents e02cda6 + 8f22cdc
commit c11757d
Show file tree

Hide file tree

Showing 14 changed files with 142 additions and 18 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -65,17 +65,29 @@ jobs:
             llvm_ver: ""
             toolset: "gcc-7"
             cxxstd: "11,14,17"
-          - name: "TOOLSET=clang CXXSTD=11,14,1z Job 4"
+          - name: "TOOLSET=gcc-9 CXXSTD=11,14,17,20 Job 6"
+            buildtype: "boost"
+            packages: "g++-9"
+            packages_to_remove: ""
+            os: "ubuntu-20.04"
+            container: "ubuntu:16.04"
+            cxx: "g++-9"
+            sources: ""
+            llvm_os: ""
+            llvm_ver: ""
+            toolset: "gcc-9"
+            cxxstd: "11,14,17,20"
+          - name: "TOOLSET=clang CXXSTD=11,14,17,20 Job 4"
             buildtype: "boost"
             packages: ""
             packages_to_remove: ""
-            os: "ubuntu-18.04"
+            os: "ubuntu-20.04"
             cxx: "clang++"
             sources: ""
             llvm_os: ""
             llvm_ver: ""
             toolset: "clang"
-            cxxstd: "11,14,1z"
+            cxxstd: "11,14,17,20"
 
     runs-on: ${{ matrix.os }}
     container: ${{ matrix.container }}
@@ -178,17 +190,17 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - name: "TOOLSET=clang CXXSTD=11,14,1z Job 5"
+          - name: "TOOLSET=clang CXXSTD=11,14,17,20 Job 5"
             buildtype: "boost"
             packages: ""
-            os: "macos-10.15"
+            os: "macos-12"
             cxx: "clang++"
             sources: ""
             llvm_os: ""
             llvm_ver: ""
-            xcode_version: 11.7
+            xcode_version: 14.2.0
             toolset: "clang"
-            cxxstd: "11,14,1z"
+            cxxstd: "11,14,17,20"
 
     runs-on: ${{ matrix.os }}
 

diff --git a/ChangeLog b/ChangeLog
@@ -18,6 +18,10 @@ TODO (known issues):
 
 CHANGELOG
 
+Boost V1.85:
+  - Fixed #200: Emitted pragmas lack terminating newline
+  - Fixed #202: YYMARKER not updated when fill called on BOOST_WAVE_BSIZE boundary
+
 Boost V1.84:
   - Fixed #188: Segmentation fault when "#pragma\n" is encountered
 

diff --git a/appveyor.yml b/appveyor.yml
@@ -25,6 +25,9 @@ environment:
     - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2019
       TOOLSET: msvc-14.2
       CXXSTD: 14,17
+    - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2022
+      TOOLSET: msvc-14.3
+      CXXSTD: 14,17,20
 ##
 ## cygwin and mingw disabled for now, failing to build
 ## for reasons unrelated to wave

diff --git a/include/boost/wave/cpplexer/re2clex/cpp_re.hpp b/include/boost/wave/cpplexer/re2clex/cpp_re.hpp
@@ -18,6 +18,8 @@
 #include <boost/wave/wave_config.hpp>
 #include <boost/wave/token_ids.hpp>
 #include <boost/wave/cpplexer/cpplexer_exceptions.hpp>
+#include <boost/wave/cpplexer/re2clex/aq.hpp>
+#include <boost/wave/cpplexer/re2clex/scanner.hpp>
 
 // this must occur after all of the includes and before any code appears
 #ifdef BOOST_HAS_ABI_HEADERS
@@ -38,8 +40,10 @@
 #define YYMARKER  marker
 #define YYFILL(n)                                                             \
     {                                                                         \
+        s->ptr = marker;                                                      \
         cursor = uchar_wrapper(fill(s, cursor), cursor.column);               \
         limit = uchar_wrapper (s->lim);                                       \
+        marker = uchar_wrapper(s->ptr);                                       \
     }                                                                         \
     /**/
 
@@ -235,10 +239,13 @@ uchar *fill(Scanner<Iterator> *s, uchar *cursor)
         /* backslash-newline erasing time */
 
         /* first scan for backslash-newline and erase them */
-        for (p = s->lim; p < s->lim + cnt - 2; ++p)
+        /* a backslash-newline combination can be 2 (regular) or 4 (trigraph backslash) chars */
+        /* start checking 3 chars within the old buffer, if possible */
+        for (p = (std::max)(s->lim - 3, s->cur); p < s->lim + cnt - 2; ++p)
         {
             int len = 0;
-            if (is_backslash(p, s->lim + cnt, len))
+            /* is there a backslash, and room afterwards for a newline? */
+            if (is_backslash(p, s->lim + cnt, len) && ((p + len) < (s->lim + cnt)))
             {
                 if (*(p+len) == '\n')
                 {
@@ -250,7 +257,8 @@ uchar *fill(Scanner<Iterator> *s, uchar *cursor)
                 }
                 else if (*(p+len) == '\r')
                 {
-                    if (*(p+len+1) == '\n')
+                    /* is there also room for a newline, and is one present? */
+                    if (((p + len + 1) < s->lim + cnt) && (*(p+len+1) == '\n'))
                     {
                         int offset = len + 2;
                         memmove(p, p + offset, s->lim + cnt - p - offset);
@@ -341,7 +349,6 @@ uchar *fill(Scanner<Iterator> *s, uchar *cursor)
     }
     return cursor;
 }
-#undef BOOST_WAVE_BSIZE
 
 ///////////////////////////////////////////////////////////////////////////////
 //  Special wrapper class holding the current cursor position

diff --git a/include/boost/wave/cpplexer/re2clex/cpp_re2c_lexer.hpp b/include/boost/wave/cpplexer/re2clex/cpp_re2c_lexer.hpp
@@ -273,7 +273,8 @@ lexer<IteratorT, PositionT, TokenT>::get(TokenT& result)
     case T_ANY_TRIGRAPH:
         if (boost::wave::need_convert_trigraphs(language)) {
             value = impl::convert_trigraph(
-                string_type((char const *)scanner.tok));
+                string_type((char const *)scanner.tok,
+                            scanner.cur-scanner.tok));
         }
         else {
             value = string_type((char const *)scanner.tok,

diff --git a/include/boost/wave/util/cpp_iterator.hpp b/include/boost/wave/util/cpp_iterator.hpp
@@ -2455,6 +2455,7 @@ pp_iterator_functor<ContextT>::on_pragma(
 #endif
         }
     }
+    expanded.push_back(result_type(T_NEWLINE, "\n", act_token.get_position()));
 
     // the queues should be empty at this point
     BOOST_ASSERT(unput_queue.empty());

diff --git a/include/boost/wave/util/filesystem_compatibility.hpp b/include/boost/wave/util/filesystem_compatibility.hpp
@@ -16,6 +16,7 @@
 #include <boost/version.hpp>
 #include <boost/filesystem/path.hpp>
 #include <boost/filesystem/operations.hpp>
+#include <boost/filesystem/directory.hpp>
 
 namespace boost { namespace wave { namespace util
 {
@@ -206,6 +207,17 @@ namespace boost { namespace wave { namespace util
             return true;
         return boost::filesystem::create_directories(p);
     }
+
+    // starting with Boost 1.85 no_push was renamed to disable_recursion_pending for consistency
+    // with std::filesystem (deprecated some point before that)
+    inline void no_push(boost::filesystem::recursive_directory_iterator & it)
+    {
+#if BOOST_VERSION >= 108400 && BOOST_FILESYSTEM_VERSION >= 3
+        it.disable_recursion_pending();
+#else
+        it.no_push();
+#endif
+    }
 }}}
 
 #endif
diff --git a/samples/check_macro_naming/check_macro_naming.cpp b/samples/check_macro_naming/check_macro_naming.cpp
@@ -170,7 +170,7 @@ int main(int argc, char *argv[])
                        exclude_dirs.end(),
                        fs::canonical(it->path())) != exclude_dirs.end())) {
             // skip recursion here
-            it.no_push();
+            boost::wave::util::no_push(it);
         }
     }
 

diff --git a/samples/waveidl/idl.cpp b/samples/waveidl/idl.cpp
@@ -15,6 +15,7 @@
 #include <boost/assert.hpp>
 #include <boost/program_options.hpp>
 #include <boost/filesystem/path.hpp>
+#include <boost/wave/util/filesystem_compatibility.hpp>
 
 ///////////////////////////////////////////////////////////////////////////////
 //  Include Wave itself
@@ -475,7 +476,7 @@ main (int argc, char *argv[])
     // started from. If this exists, treat it as a wave config file
     fs::path filename = boost::wave::util::create_path(argv[0]);
 
-        filename = filename.branch_path() / "waveidl.cfg";
+        filename = boost::wave::util::branch_path(filename) / "waveidl.cfg";
         cmd_line_util::read_config_file_options(filename.string(), 
             desc_overall_cfgfile, vm, true);
 

diff --git a/test/build/Jamfile.v2 b/test/build/Jamfile.v2
@@ -243,5 +243,14 @@ test-suite wave
                 /boost/thread//boost_thread
                 /boost/filesystem//boost_filesystem
         ]
+
+        [
+            run
+            # sources
+                ../testwave/fill_boundary.cpp
+                /boost/wave//boost_wave
+                /boost/thread//boost_thread
+                /boost/filesystem//boost_filesystem
+        ]
     ;
 
diff --git a/test/testwave/fill_boundary.cpp b/test/testwave/fill_boundary.cpp
@@ -0,0 +1,74 @@
+// Copyright 2024 Jeff Trull.
+//
+// Distributed under the Boost Software License, Version 1.0.
+//
+// See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt
+
+// There are some tricky corner cases that occur when Wave handles large inputs.
+// Tokens at, near, or crossing the boundary at which a "fill" occurs can
+// interact in unexpected ways. This test is intended to increase code
+// coverage for such situations.
+
+#include <boost/wave/cpplexer/re2clex/cpp_re.hpp>   // for BOOST_WAVE_BSIZE
+
+#include <boost/wave.hpp>
+#include <boost/wave/token_ids.hpp>
+#include <boost/wave/cpplexer/cpp_lex_token.hpp>
+#include <boost/wave/cpplexer/cpp_lex_iterator.hpp>
+
+#include <string>
+
+using token_t = boost::wave::cpplexer::lex_token<>;
+using lex_iter_t = boost::wave::cpplexer::lex_iterator<token_t>;
+using ctx_t = boost::wave::context<
+    std::string::iterator, lex_iter_t,
+    boost::wave::iteration_context_policies::load_file_to_string,
+    boost::wave::context_policies::default_preprocessing_hooks>;
+
+
+int main() {
+    using namespace boost::wave;
+
+    // construct input with a trigraph in a tricky spot
+    constexpr std::size_t space_count = BOOST_WAVE_BSIZE - 18;
+    auto inp_txt = std::string(space_count, ' ') + ";";
+    // add in pound trigraph one character at a time
+    // otherwise, the string literal may get translated by the compiler
+    inp_txt.push_back('?');
+    inp_txt.push_back('?');
+    inp_txt.push_back('=');
+    inp_txt += "                  auto foo = bar;\n";
+
+    ctx_t ctx( inp_txt.begin(), inp_txt.end(), "longfile.cpp" );
+
+    auto it = ctx.begin();
+
+    if (token_id(*it) != T_SPACE)
+        return 1;
+
+    if (it->get_value().size() != space_count)
+        return 2;
+
+    ++it;
+
+    if (token_id(*it) != T_SEMICOLON)
+        return 3;
+
+    ++it;
+
+    if (token_id(*it) != T_POUND_TRIGRAPH)
+        return 4;
+
+    ++it;
+
+    if (token_id(*it) != T_SPACE)
+        return 5;
+
+    ++it;
+
+    if (token_id(*it) != T_AUTO)
+        return 6;
+
+    return 0;
+}
diff --git a/test/testwave/testfiles/t_2_006.cpp b/test/testwave/testfiles/t_2_006.cpp
@@ -24,9 +24,11 @@
 //R #pragma STDC preprocessed pragma body
 #pragma STDC PRAGMA_BODY
 // also test pragma without a body (allowed per https://eel.is/c++draft/cpp.pragma)
-//R #line 29 "t_2_006.cpp"
+//R #line 30 "t_2_006.cpp"
 //R #pragma
+//R auto x = 0;
 #pragma
+auto x = 0;
 
 //H 10: t_2_006.cpp(12): #define
 //H 08: t_2_006.cpp(12): PRAGMA_BODY=preprocessed pragma body
@@ -40,4 +42,4 @@
 //H 01: t_2_006.cpp(12): PRAGMA_BODY
 //H 02: preprocessed pragma body
 //H 03: preprocessed pragma body
-//H 10: t_2_006.cpp(29): #pragma
+//H 10: t_2_006.cpp(30): #pragma
diff --git a/tool/cpp.cpp b/tool/cpp.cpp
@@ -17,7 +17,6 @@
 ///////////////////////////////////////////////////////////////////////////////
 // Include additional Boost libraries
 #include <boost/filesystem/path.hpp>
-#include <boost/filesystem/convenience.hpp>
 #include <boost/filesystem/fstream.hpp>
 #include <boost/timer/timer.hpp>
 #include <boost/any.hpp>

diff --git a/tool/trace_macro_expansion.hpp b/tool/trace_macro_expansion.hpp
@@ -22,7 +22,6 @@
 #include <boost/config.hpp>
 #include <boost/filesystem/path.hpp>
 #include <boost/filesystem/operations.hpp>
-#include <boost/filesystem/convenience.hpp>
 
 #include <boost/wave/token_ids.hpp>
 #include <boost/wave/util/macro_helpers.hpp>
-Original file line number
+Diff line change
@@ Expand Up / @@ -2455,6 +2455,7 @@ pp_iterator_functor<ContextT>::on_pragma( @@
     #endif
             }
         }
+        expanded.push_back(result_type(T_NEWLINE, "\n", act_token.get_position()));
         // the queues should be empty at this point
         BOOST_ASSERT(unput_queue.empty());
@@ Expand Down @@