Merge pull request commonmark#11 from compnerd/merge-2019-10-09

merge from upstream (2019/10/09)

CMakeLists.txt

@@ -1,31 +1,24 @@
-cmake_minimum_required(VERSION 2.8.9)
-
-# prevent ugly developer warnings because version is set directly, not through project()
-# it should be redone properly by using VERSION in project() if on CMake 3.x
-if(CMAKE_MAJOR_VERSION GREATER 2)
-    cmake_policy(SET CMP0048 OLD)
-endif()
-
-project(cmark)
+cmake_minimum_required(VERSION 3.0)
+project(cmark VERSION 0.29.0)
 
 include("FindAsan.cmake")
 
 if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_BINARY_DIR}")
     message(FATAL_ERROR "Do not build in-source.\nPlease remove CMakeCache.txt and the CMakeFiles/ directory.\nThen: mkdir build ; cd build ; cmake .. ; make")
 endif()
 
-set(PROJECT_NAME "cmark")
-
-set(PROJECT_VERSION_MAJOR 0)
-set(PROJECT_VERSION_MINOR 28)
-set(PROJECT_VERSION_PATCH 3)
-set(PROJECT_VERSION ${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}.${PROJECT_VERSION_PATCH} )
-
 option(CMARK_TESTS "Build cmark tests and enable testing" ON)
 option(CMARK_STATIC "Build static libcmark library" ON)
 option(CMARK_SHARED "Build shared libcmark library" ON)
 option(CMARK_LIB_FUZZER "Build libFuzzer fuzzing harness" OFF)
 
+# The Linux modules distributed with CMake add "-rdynamic" to the build flags
+# which is incompatible with static linking under certain configurations.
+# Unsetting CMAKE_SHARED_LIBRARY_LINK_C_FLAGS ensures this does not happen.
+if(CMARK_STATIC AND "${CMAKE_SYSTEM_NAME}" STREQUAL "Linux")
+  SET(CMAKE_SHARED_LIBRARY_LINK_C_FLAGS)
+endif()
+
 add_subdirectory(src)
 if(CMARK_TESTS AND (CMARK_SHARED OR CMARK_STATIC))
   add_subdirectory(api_test)

Makefile

@@ -73,7 +73,7 @@ prof:
 afl:
 	@[ -n "$(AFL_PATH)" ] || { echo '$$AFL_PATH not set'; false; }
 	mkdir -p $(BUILDDIR)
-	cd $(BUILDDIR) && cmake .. -DCMAKE_C_COMPILER=$(AFL_PATH)/afl-clang
+	cd $(BUILDDIR) && cmake .. -DCMARK_TESTS=0 -DCMAKE_C_COMPILER=$(AFL_PATH)/afl-clang
 	$(MAKE)
 	$(AFL_PATH)/afl-fuzz \
 	    -i test/afl_test_cases \

README.md

@@ -156,14 +156,14 @@ be found in the man pages in the `man` subdirectory.
 Security
 --------
 
-By default, the library will pass through raw HTML and potentially
+By default, the library will scrub raw HTML and potentially
 dangerous links (`javascript:`, `vbscript:`, `data:`, `file:`).
 
-It is recommended that users either disable this potentially unsafe
-feature by using the option `CMARK_OPT_SAFE` (or `--safe` with the
-command-line program), or run the output through an HTML sanitizer
-to protect against
-[XSS attacks](http://en.wikipedia.org/wiki/Cross-site_scripting).
+To allow these, use the option `CMARK_OPT_UNSAFE` (or
+`--unsafe`) with the command line program. If doing so, we
+recommend you use a HTML sanitizer specific to your needs to
+protect against [XSS
+attacks](http://en.wikipedia.org/wiki/Cross-site_scripting).
 
 Contributing
 ------------

api_test/main.c

@@ -177,7 +177,8 @@ static void accessors(test_batch_runner *runner) {
   OK(runner, cmark_node_set_literal(string, literal + sizeof("prefix")),
      "set_literal suffix");
 
-  char *rendered_html = cmark_render_html(doc, CMARK_OPT_DEFAULT);
+  char *rendered_html = cmark_render_html(doc,
+		          CMARK_OPT_DEFAULT | CMARK_OPT_UNSAFE);
   static const char expected_html[] =
       "<h3>Header</h3>\n"
       "<ol start=\"3\">\n"
@@ -859,7 +860,7 @@ static void test_safe(test_batch_runner *runner) {
                                  "a>\n[link](JAVAscript:alert('hi'))\n![image]("
                                  "file:my.js)\n";
   char *html = cmark_markdown_to_html(raw_html, sizeof(raw_html) - 1,
-                                      CMARK_OPT_DEFAULT | CMARK_OPT_SAFE);
+                                      CMARK_OPT_DEFAULT);
   STR_EQ(runner, html, "<!-- raw HTML omitted -->\n<p><!-- raw HTML omitted "
                        "-->hi<!-- raw HTML omitted -->\n<a "
                        "href=\"\">link</a>\n<img src=\"\" alt=\"image\" "

appveyor.yml

@@ -13,5 +13,9 @@ install:
 build_script:
   - 'tools\appveyor-build.bat'
 
+artifacts:
+  - path: build/src/cmark.exe
+    name: cmark.exe
+
 test_script:
   - 'nmake test'

changelog.txt

@@ -1,3 +1,127 @@
+[0.29.0]
+
+  * Update spec to 0.29.
+  * Make rendering safe by default (#239, #273).
+    Adds `CMARK_OPT_UNSAFE` and make `CMARK_OPT_SAFE` a no-op (for API
+    compatibility).  The new default behavior is to suppress raw HTML and
+    potentially dangerous links.  The `CMARK_OPT_UNSAFE` option has to be set
+    explicitly to prevent this.
+    **NOTE:** This change will require modifications in bindings for cmark
+    and in most libraries and programs that use cmark.
+    Borrows heavily from @kivikakk's patch in github/cmark-gfm#123.
+  * Add sourcepos info for inlines (Yuki Izumi).
+  * Disallow more than 32 nested balanced parens in a link (Yuki Izumi).
+  * Resolve link references before creating setext header.
+    A setext header line after a link reference should not
+    create a header, according to the spec.
+  * commonmark renderer: improve escaping.
+    URL-escape special characters when escape mode is URL, and not otherwise.
+    Entity-escape control characters (< 0x20) in non-literal escape modes.
+  * render:  only emit actual newline when escape mode is LITERAL.
+    For markdown content, e.g., in other contexts we want some
+    kind of escaping, not a literal newline.
+  * Update code span normalization to conform with spec change.
+  * Allow empty `<>` link destination in reference link.
+  * Remove leftover includes of `memory.h` (#290).
+  * A link destination can't start with `<` unless it is
+    an angle-bracket link that also ends with `>` (#289).
+    (If your URL really starts with `<`, URL-escape it.)
+  * Allow internal delimiter runs to match if both have lengths that are
+    multiples of 3.  See commonmark/commonmark#528.
+  * Include `references.h` in `parser.h` (#287).
+  * Fix `[link](<foo\>)`.
+  * Use hand-rolled scanner for thematic break (see #284).
+    Keep track of the last position where a thematic break
+    failed to match on a line, to avoid rescanning unnecessarily.
+  * Rename `ends_with_blank_line` with `S_` prefix.
+  * Add `CMARK_NODE__LAST_LINE_CHECKED` flag (#284).
+    Use this to avoid unnecessary recursion in `ends_with_blank_line`.
+  * In `ends_with_blank_line`, call `S_set_last_line_blank`
+    to avoid unnecessary repetition (#284).  Once we settle whether a list
+    item ends in a blank line, we don't need to revisit this in considering
+    parent list items.
+  * Disallow unescaped `(` in parenthesized link title.
+  * Copy line/col info straight from opener/closer (Ashe Connor).
+    We can't rely on anything in `subj` since it's been modified while parsing
+    the subject and could represent line info from a future line.  This is
+    simple and works.
+  * `render.c`: reset `last_breakable` after cr.  Fixes jgm/pandoc#5033.
+  * Fix a typo in `houdini_href_e.c` (Felix Yan).
+  * commonmark writer: use `~~~` fences if info string contains backtick.
+    This is needed for round-trip tests.
+  * Update scanners for new info string rules.
+  * Add XSLT stylesheet to convert cmark XML back to Commonmark
+    (Nick Wellnhofer, #264).  Initial version of an XSLT stylesheet that
+    converts the XML format produced by `cmark -t xml` back to Commonmark.
+  * Check for whitespace before reference title (#263).
+  * Bump CMake to version 3 (Jonathan Müller).
+  * Build: Remove deprecated call to `add_compiler_export_flags()`
+    (Jonathan Müller).  It is deprecated in CMake 3.0, the replacement is to
+    set the `CXX_VISIBILITY_PRESET` (or in our case `C_VISIBILITY_PRESET`) and
+    `VISIBILITY_INLINES_HIDDEN` properties of the target.  We're already
+    setting them by setting the CMake variables anyway, so the call can be
+    removed.
+  * Build: only attempt to install MSVC system libraries on Windows
+    (Saleem Abdulrasool).  Newer versions of CMake attempt to query the system
+    for information about the VS 2017 installation.  Unfortunately, this query
+    fails on non-Windows systems when cross-compiling:
+    `cmake_host_system_information does not recognize <key> VS_15_DIR`.
+    CMake will not find these system libraries on non-Windows hosts anyways,
+    and we were silencing the warnings, so simply omit the installation when
+    cross-compiling to Windows.
+  * Simplify code normalization, in line with spec change.
+  * Implement code span spec changes.  These affect both parsing and writing
+    commonmark.
+  * Add link parsing corner cases to regressions (Ashe Connor).
+  * Add `xml:space="preserve"` in XML output when appropriate
+    (Nguyễn Thái Ngọc Duy).
+    (For text, code, code_block, html_inline and html_block tags.)
+  * Removed meta from list of block tags.  Added regression test.
+    See commonmark/CommonMark#527.
+  * `entity_tests.py` - omit noisy success output.
+  * `pathological_tests.py`: make tests run faster.
+    Commented out the (already ignored) "many references" test, which
+    times out.  Reduced the iterations for a couple other tests.
+  * `pathological_tests.py`: added test for deeply nested lists.
+  * Optimize `S_find_first_nonspace`.  We were needlessly redoing things we'd
+    already done.  Now we skip the work if the first nonspace is greater than
+    the current offset.  This fixes pathological slowdown with deeply nested
+    lists (#255).  For N = 3000, the time goes from over 17s to about 0.7s.
+    Thanks to Martin Mitas for diagnosing the problem.
+  * Allow spaces in link destination delimited with pointy brackets.
+  * Adjust max length of decimal/numeric entities.
+    See commonmark/CommonMark#487.
+  * Fix inline raw HTML parsing.
+    This fixes a recently added failing spec test case.  Previously spaces
+    were being allowed in unquoted attribute values; no we forbid them.
+  * Don't allow list markers to be indented >= 4 spaces.
+    See commonmark/CommonMark#497.
+  * Check for empty buffer when rendering (Phil Turnbull).
+    For empty documents, `->size` is zero so
+    `renderer.buffer->ptr[renderer.buffer->size - 1]` will cause an
+    out-of-bounds read. Empty buffers always point to the global
+    `cmark_strbuf__initbuf` buffer so we read `cmark_strbuf__initbuf[-1]`.
+  * Also run API tests with `CMARK_SHARED=OFF` (Nick Wellnhofer).
+  * Rename roundtrip and entity tests (Nick Wellnhofer).
+    Rename the tests to reflect that they use the library, not the
+    executable.
+  * Generate export header for static-only build (#247, Nick Wellnhofer).
+  * Fuzz width parameter too (Phil Turnbull).  Allow the `width` parameter to
+    be generated too so we get better fuzz-coverage.
+  * Don't discard empty fuzz test-cases (Phil Turnbull).  We currently discard
+    fuzz test-cases that are empty but empty inputs are valid markdown. This
+    improves the fuzzing coverage slightly.
+  * Fixed exit code for pathological tests.
+  * Add allowed failures to `pathological_tests.py`.
+    This allows us to include tests that we don't yet know how to pass.
+  * Add timeout to `pathological_tests.py`.
+    Tests must complete in 8 seconds or are errors.
+  * Add more pathological tests (Martin Mitas).
+    These tests target the issues #214, #218, #220.
+  * Use pledge(2) on OpenBSD (Ashe Connor).
+  * Update the Racket wrapper (Eli Barzilay).
+  * Makefile: For afl target, don't build tests.
+
 [0.28.3]
 
   * Include GNUInstallDirs in src/CMakeLists.txt (Nick Wellnhofer, #240).

man/man3/cmark.3

@@ -1,4 +1,4 @@
-.TH cmark 3 "June 02, 2017" "LOCAL" "Library Functions Manual"
+.TH cmark 3 "March 19, 2019" "LOCAL" "Library Functions Manual"
 .SH
 NAME
 .PP
@@ -727,11 +727,25 @@ Render \f[C]softbreak\f[] elements as hard line breaks.
 .fi
 
 .PP
-Suppress raw HTML and unsafe links (\f[C]javascript:\f[],
+\f[C]CMARK_OPT_SAFE\f[] is defined here for API compatibility, but it no
+longer has any effect. "Safe" mode is now the default: set
+\f[C]CMARK_OPT_UNSAFE\f[] to disable it.
+
+.PP
+.nf
+\fC
+.RS 0n
+#define CMARK_OPT_UNSAFE (1 << 17)
+.RE
+\f[]
+.fi
+
+.PP
+Render raw HTML and unsafe links (\f[C]javascript:\f[],
 \f[C]vbscript:\f[], \f[C]file:\f[], and \f[C]data:\f[], except for
 \f[C]image/png\f[], \f[C]image/gif\f[], \f[C]image/jpeg\f[], or
-\f[C]image/webp\f[] mime types). Raw HTML is replaced by a placeholder
-HTML comment. Unsafe links are replaced by empty strings.
+\f[C]image/webp\f[] mime types). By default, raw HTML is replaced by a
+placeholder HTML comment. Unsafe links are replaced by empty strings.
 
 .PP
 .nf

src/CMakeLists.txt

@@ -46,10 +46,7 @@ set(LIBRARY_SOURCES
   )
 
 set(PROGRAM "cmark")
-set(PROGRAM_SOURCES
-  ${LIBRARY_SOURCES}
-  main.c
-  )
+set(PROGRAM_SOURCES main.c)
 
 include_directories(. ${CMAKE_CURRENT_BINARY_DIR})
 
@@ -59,11 +56,15 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/cmark_version.h.in
 include (GenerateExportHeader)
 
 add_executable(${PROGRAM} ${PROGRAM_SOURCES})
-add_compiler_export_flags()
 
-# Disable the PUBLIC declarations when compiling the executable:
-set_target_properties(${PROGRAM} PROPERTIES
-  COMPILE_FLAGS -DCMARK_STATIC_DEFINE)
+if (CMARK_STATIC)
+  target_link_libraries(${PROGRAM} ${STATICLIBRARY})
+  # Disable the PUBLIC declarations when compiling the executable:
+  set_target_properties(${PROGRAM} PROPERTIES
+    COMPILE_FLAGS -DCMARK_STATIC_DEFINE)
+elseif (CMARK_SHARED)
+  target_link_libraries(${PROGRAM} ${LIBRARY})
+endif()
 
 # Check integrity of node structure when compiled as debug:
 set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -DCMARK_DEBUG_NODES")
@@ -72,12 +73,9 @@ set(CMAKE_LINKER_DEBUG "${CMAKE_LINKER_FLAGS_DEBUG}")
 set(CMAKE_C_FLAGS_PROFILE "${CMAKE_C_FLAGS_RELEASE} -pg")
 set(CMAKE_LINKER_PROFILE "${CMAKE_LINKER_FLAGS_RELEASE} -pg")
 
-if ((${CMAKE_MAJOR_VERSION} GREATER 1 AND ${CMAKE_MINOR_VERSION} GREATER 8) OR ${CMAKE_MAJOR_VERSION} GREATER 2)
-  set(CMAKE_C_VISIBILITY_PRESET hidden)
-  set(CMAKE_VISIBILITY_INLINES_HIDDEN 1)
-elseif(CMAKE_COMPILER_IS_GNUCC OR ${CMAKE_C_COMPILER_ID} STREQUAL "Clang")
-  set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fvisibility=hidden")
-endif ()
+# -fvisibility=hidden
+set(CMAKE_C_VISIBILITY_PRESET hidden)
+set(CMAKE_VISIBILITY_INLINES_HIDDEN 1)
 
 if (CMARK_SHARED)
   add_library(${LIBRARY} SHARED ${LIBRARY_SOURCES})