Comply with CommonMark 0.31.2

markdown.dtx

@@ -26956,7 +26956,7 @@ parsers.html_comment = Cs( parsers.html_comment_start
 
 parsers.html_inline_comment = (parsers.html_comment_start / "")
                             * -P(">") * -P("->")
-                            * Cs((V("NoSoftLineBreakEndline") + parsers.any - P("--") 
+                            * Cs((V("NoSoftLineBreakEndline") + parsers.any
                                 - parsers.nested_breaking_blank - parsers.html_comment_end)^0)
                             * (parsers.html_comment_end / "")
 
@@ -28007,6 +28007,26 @@ function M.reader.new(writer, options)
     end
   end
 
+  local punctuation_3_byte_ungrouped = 
+    lpeg.S("\226\134\184\226\134\185\226\134\143\226\134\147\226\134\186\226\134\187\z
+            \226\134\188\226\134\189\226\137\141\226\137\142\226\137\160\226\137\161\z
+            \226\137\163\226\137\164\226\137\166\226\137\168\226\137\169\226\137\170\z
+            \226\137\171\226\137\173\226\137\175\226\137\177\226\137\178\226\137\179\z
+            \226\137\180\226\137\181\226\137\182\226\137\183\226\137\184\226\137\185\z
+            \226\137\187\226\137\188\226\137\189\226\137\191")
+
+  local punctuation_3_byte_grouped  = lpeg.S("\226")
+                                    * ( lpeg.S("\138") * (lpeg.R("\129\191") - lpeg.S("\165"))
+                                      + lpeg.S("\139") * lpeg.R("\128\191")
+                                      + lpeg.S("\140") * lpeg.R("\128\191")
+                                      + lpeg.S("\141") * lpeg.R("\128\191")) 
+                                    + lpeg.S("\239\191") * lpeg.R("\140\191")
+
+  local punctuation_by_size = { lpeg.S("!\"#$%&'()*+,-./:;<=>?@[]\\^_`{|}~"),
+                                lpeg.S("\194\161\194\167\194\171\194\182\194\183\194\187\194\191\z
+                                \206\131\206\137\206\171\206\182\206\183\206\181\206\191\207\131\207\134"),
+                                punctuation_3_byte_ungrouped + punctuation_3_byte_grouped,
+                                lpeg.S("")}
 %    \end{macrocode}
 % \begin{markdown}
 %
@@ -28024,23 +28044,30 @@ function M.reader.new(writer, options)
       else
         char_length = pos + 1
       end
-      c = lpeg.match({ C(utf8_by_byte_count(char_length)) },s,i+pos)
-      if (c ~= nil) and (unicode.utf8.match(c, chartype)) then
-        return i
+
+      if (chartype == "punctuation") then
+        if lpeg.match(punctuation_by_size[char_length], s, i+pos) then
+          return i
+        end
+      else
+        c = lpeg.match({ C(utf8_by_byte_count(char_length)) },s,i+pos)
+        if (c ~= nil) and (unicode.utf8.match(c, chartype)) then
+          return i
+        end
       end
     end
   end
 
   local function check_preceding_unicode_punctuation(s, i)
-    return check_unicode_type(s, i, -4, -1, "%p")
+    return check_unicode_type(s, i, -4, -1, "punctuation")
   end
 
   local function check_preceding_unicode_whitespace(s, i)
     return check_unicode_type(s, i, -4, -1, "%s")
   end
 
   local function check_following_unicode_punctuation(s, i)
-    return check_unicode_type(s, i, 0, 3, "%p")
+    return check_unicode_type(s, i, 0, 3, "punctuation")
   end
 
   local function check_following_unicode_whitespace(s, i)

tests/testfiles/CommonMark_0.30/autolinks/002.test

@@ -1,16 +1,16 @@
 %   ---RESULT--- "example": 594,
 %   
-%   <p><a href="http://foo.bar.baz/test?q=hello&amp;id=22&amp;boolean">http://foo.bar.baz/test?q=hello&amp;id=22&amp;boolean</a></p>
+%   <p><a href="https://foo.bar.baz/test?q=hello&amp;id=22&amp;boolean">https://foo.bar.baz/test?q=hello&amp;id=22&amp;boolean</a></p>
 %   
 %   ---\RESULT---
 
 <<<
-<http://foo.bar.baz/test?q=hello&id=22&boolean>
+<https://foo.bar.baz/test?q=hello&id=22&boolean>
 >>>
 BEGIN document
 BEGIN link
-- label: http://foo.bar.baz/test?q=hello(ampersand)id=22(ampersand)boolean
-- URI: http://foo.bar.baz/test?q=hello&id=22&boolean
+- label: https://foo.bar.baz/test?q=hello(ampersand)id=22(ampersand)boolean
+- URI: https://foo.bar.baz/test?q=hello&id=22&boolean
 - title: 
 END link
 END document

tests/testfiles/CommonMark_0.30/autolinks/009.test

@@ -1,11 +1,11 @@
 %   ---RESULT--- "example": 601,
 %   
-%   <p>&lt;http://foo.bar/baz bim&gt;</p>
+%   <p>&lt;https://foo.bar/baz bim&gt;</p>
 %   
 %   ---\RESULT---
 
 <<<
-<http://foo.bar/baz bim>
+<https://foo.bar/baz bim>
 >>>
 BEGIN document
 END document

tests/testfiles/CommonMark_0.30/autolinks/010.test

@@ -1,16 +1,16 @@
 %   ---RESULT--- "example": 602,
 %   
-%   <p><a href="http://example.com/%5C%5B%5C">http://example.com/\[\</a></p>
+%   <p><a href="https://example.com/%5C%5B%5C">https://example.com/\[\</a></p>
 %   
 %   ---\RESULT---
 
 <<<
-<http://example.com/\[\>
+<https://example.com/\[\>
 >>>
 BEGIN document
 BEGIN link
-- label: http://example.com/(backslash)[(backslash)
-- URI: http://example.com/(backslash)[(backslash)
+- label: https://example.com/(backslash)[(backslash)
+- URI: https://example.com/(backslash)[(backslash)
 - title: 
 END link
 END document

tests/testfiles/CommonMark_0.30/autolinks/015.test

@@ -1,11 +1,11 @@
 %   ---RESULT--- "example": 607,
 %   
-%   <p>&lt; http://foo.bar &gt;</p>
+%   <p>&lt; https://foo.bar &gt;</p>
 %   
 %   ---\RESULT---
 
 <<<
-< http://foo.bar >
+< https://foo.bar >
 >>>
 BEGIN document
 END document

tests/testfiles/CommonMark_0.30/autolinks/018.test

@@ -1,11 +1,11 @@
 %   ---RESULT--- "example": 610,
 %   
-%   <p>http://example.com</p>
+%   <p>https://example.com</p>
 %   
 %   ---\RESULT---
 
 <<<
-http://example.com
+https://example.com
 >>>
 BEGIN document
 END document

tests/testfiles/CommonMark_0.30/backslash_escapes/009.test

@@ -1,16 +1,16 @@
 %   ---RESULT--- "example": 20,
 %   
-%   <p><a href="http://example.com?find=%5C*">http://example.com?find=\*</a></p>
+%   <p><a href="https://example.com?find=%5C*">https://example.com?find=\*</a></p>
 %   
 %   ---\RESULT---
 
 <<<
-<http://example.com?find=\*>
+<https://example.com?find=\*>
 >>>
 BEGIN document
 BEGIN link
-- label: http://example.com?find=(backslash)*
-- URI: http://example.com?find=(backslash)*
+- label: https://example.com?find=(backslash)*
+- URI: https://example.com?find=(backslash)*
 - title: 
 END link
 END document

tests/testfiles/CommonMark_0.30/code_spans/018.test

@@ -1,12 +1,12 @@
 %   ---RESULT--- "example": 345,
 %   
-%   <p><code>&lt;http://foo.bar.</code>baz&gt;`</p>
+%   <p><code>&lt;https://foo.bar.</code>baz&gt;`</p>
 %   
 %   ---\RESULT---
 
 <<<
-`<http://foo.bar.`baz>`
+`<https://foo.bar.`baz>`
 >>>
 BEGIN document
-codeSpan: <http://foo.bar.
+codeSpan: <https://foo.bar.
 END document

tests/testfiles/CommonMark_0.30/code_spans/019.test

@@ -1,16 +1,16 @@
 %   ---RESULT--- "example": 346,
 %   
-%   <p><a href="http://foo.bar.%60baz">http://foo.bar.`baz</a>`</p>
+%   <p><a href="https://foo.bar.%60baz">https://foo.bar.`baz</a>`</p>
 %   
 %   ---\RESULT---
 
 <<<
-<http://foo.bar.`baz>`
+<https://foo.bar.`baz>`
 >>>
 BEGIN document
 BEGIN link
-- label: http://foo.bar.`baz
-- URI: http://foo.bar.`baz
+- label: https://foo.bar.`baz
+- URI: https://foo.bar.`baz
 - title: 
 END link
 END document

tests/testfiles/CommonMark_0.30/emphasis_and_strong_emphasis/130.test

@@ -1,16 +1,16 @@
 %   ---RESULT--- "example": 479,
 %   
-%   <p>**a<a href="http://foo.bar/?q=**">http://foo.bar/?q=**</a></p>
+%   <p>**a<a href="https://foo.bar/?q=**">https://foo.bar/?q=**</a></p>
 %   
 %   ---\RESULT---
 
 <<<
-**a<http://foo.bar/?q=**>
+**a<https://foo.bar/?q=**>
 >>>
 BEGIN document
 BEGIN link
-- label: http://foo.bar/?q=**
-- URI: http://foo.bar/?q=**
+- label: https://foo.bar/?q=**
+- URI: https://foo.bar/?q=**
 - title: 
 END link
 END document

tests/testfiles/CommonMark_0.30/emphasis_and_strong_emphasis/131.test

@@ -1,18 +1,18 @@
 %   ---RESULT--- "example": 480,
 %   
-%   <p>__a<a href="http://foo.bar/?q=__">http://foo.bar/?q=__</a></p>
+%   <p>__a<a href="https://foo.bar/?q=__">https://foo.bar/?q=__</a></p>
 %   
 %   ---\RESULT---
 
 <<<
-__a<http://foo.bar/?q=__>
+__a<https://foo.bar/?q=__>
 >>>
 BEGIN document
 underscore
 underscore
 BEGIN link
-- label: http://foo.bar/?q=(underscore)(underscore)
-- URI: http://foo.bar/?q=__
+- label: https://foo.bar/?q=(underscore)(underscore)
+- URI: https://foo.bar/?q=__
 - title: 
 END link
 END document

tests/testfiles/CommonMark_0.30/emphasis_and_strong_emphasis/132.test

@@ -0,0 +1,20 @@
+%   ---RESULT--- "example": 354, (new)
+%   
+%   <p>*$*alpha.</p>
+%   <p>*£*bravo.</p>
+%   <p>*€*charlie.</p>
+%   
+%   ---\RESULT---
+
+<<<
+*$*alpha.
+
+*£*bravo.
+
+*€*charlie.
+>>>
+BEGIN document
+dollarSign
+paragraphSeparator
+paragraphSeparator
+END document

tests/testfiles/CommonMark_0.30/links/020.test

@@ -1,17 +1,17 @@
 %   ---RESULT--- "example": 500,
 %   
 %   <p><a href="#fragment">link</a></p>
-%   <p><a href="http://example.com#fragment">link</a></p>
-%   <p><a href="http://example.com?foo=3#frag">link</a></p>
+%   <p><a href="https://example.com#fragment">link</a></p>
+%   <p><a href="https://example.com?foo=3#frag">link</a></p>
 %   
 %   ---\RESULT---
 
 <<<
 [link](#fragment)
 
-[link](http://example.com#fragment)
+[link](https://example.com#fragment)
 
-[link](http://example.com?foo=3#frag)
+[link](https://example.com?foo=3#frag)
 >>>
 BEGIN document
 BEGIN link
@@ -22,13 +22,13 @@ END link
 paragraphSeparator
 BEGIN link
 - label: link
-- URI: http://example.com#fragment
+- URI: https://example.com#fragment
 - title: 
 END link
 paragraphSeparator
 BEGIN link
 - label: link
-- URI: http://example.com?foo=3#frag
+- URI: https://example.com?foo=3#frag
 - title: 
 END link
 END document

tests/testfiles/CommonMark_0.30/links/045.test

@@ -1,16 +1,16 @@
 %   ---RESULT--- "example": 525,
 %   
-%   <p>[foo<a href="http://example.com/?search=%5D(uri)">http://example.com/?search=](uri)</a></p>
+%   <p>[foo<a href="https://example.com/?search=%5D(uri)">https://example.com/?search=](uri)</a></p>
 %   
 %   ---\RESULT---
 
 <<<
-[foo<http://example.com/?search=](uri)>
+[foo<https://example.com/?search=](uri)>
 >>>
 BEGIN document
 BEGIN link
-- label: http://example.com/?search=](uri)
-- URI: http://example.com/?search=](uri)
+- label: https://example.com/?search=](uri)
+- URI: https://example.com/?search=](uri)
 - title: 
 END link
 END document

tests/testfiles/CommonMark_0.30/links/057.test

@@ -1,18 +1,18 @@
 %   ---RESULT--- "example": 537,
 %   
-%   <p>[foo<a href="http://example.com/?search=%5D%5Bref%5D">http://example.com/?search=][ref]</a></p>
+%   <p>[foo<a href="https://example.com/?search=%5D%5Bref%5D">https://example.com/?search=][ref]</a></p>
 %   
 %   ---\RESULT---
 
 <<<
-[foo<http://example.com/?search=][ref]>
+[foo<https://example.com/?search=][ref]>
 
 [ref]: /uri
 >>>
 BEGIN document
 BEGIN link
-- label: http://example.com/?search=][ref]
-- URI: http://example.com/?search=][ref]
+- label: https://example.com/?search=][ref]
+- URI: https://example.com/?search=][ref]
 - title: 
 END link
 END document

tests/testfiles/CommonMark_0.30/raw_html/014.test

@@ -1,11 +1,14 @@
 %   ---RESULT--- "example": 625,
 %   
-%   <p>foo &lt;!-- not a comment -- two hyphens --&gt;</p>
+%   <p>foo <!-- this is a --
+%   comment - with hyphens --></p>
 %   
 %   ---\RESULT---
 
 <<<
-foo <!-- not a comment -- two hyphens -->
+foo <!-- this is a --
+comment - with hyphens -->
 >>>
 BEGIN document
+inlineHtmlComment:  this is a -- comment - with hyphens 
 END document