From 0957c0372d9725ddd477f948e55c0dabe24045d3 Mon Sep 17 00:00:00 2001 From: Aaron Bockover Date: Fri, 16 Dec 2016 15:33:38 -0500 Subject: [PATCH] Implement YAML block additional feature This implements support for parsing YAML blocks and front-matter following the Pandoc rules: 1. Delimited by three hyphens (---) at the top 2. Delimited by three hyphens (---) or dots (...) at the bottom 3. May occur anywhere in the document, but if not at the beginning must be preceded by a blank line. BlockTag.YamlBlock is defined, and FencedCodeData is used to store the closing fence character (either `-` or `.`) and to indicate whether the closing fence has been seen in the same manner as BlockTag.FencedCode. YAML support may be enabled via two-flavors: 1. CommonMarkAdditionalFeatures.YamlBlocks: this allows for any number of YAML blocks anywhere in the document. 2. CommonMarkAdditionalFeatures.YamlFrontMatterOnly: allows for exactly one YAML block, defined on the first line of the document. The HTML formatters treat BlockTag.YamlBlock blocks the same way as BlockTag.FencedCode blocks, except that instead of writing the info string, a `class="language-yaml"` attribute will be written. --- CommonMark.Tests/CommonMark.Tests.csproj | 1 + CommonMark.Tests/YamlBlockTests.cs | 112 +++++++++++++++++++++ CommonMark/CommonMarkAdditionalFeatures.cs | 13 +++ CommonMark/CommonMarkConverter.cs | 2 +- CommonMark/Formatters/HtmlFormatter.cs | 6 ++ CommonMark/Formatters/HtmlFormatterSlim.cs | 6 ++ CommonMark/Formatters/Printer.cs | 8 ++ CommonMark/Parser/BlockMethods.cs | 41 +++++++- CommonMark/Syntax/BlockTag.cs | 10 +- 9 files changed, 194 insertions(+), 5 deletions(-) create mode 100644 CommonMark.Tests/YamlBlockTests.cs diff --git a/CommonMark.Tests/CommonMark.Tests.csproj b/CommonMark.Tests/CommonMark.Tests.csproj index f0d76f9..e0b8ac6 100644 --- a/CommonMark.Tests/CommonMark.Tests.csproj +++ b/CommonMark.Tests/CommonMark.Tests.csproj @@ -75,6 +75,7 @@ True Specs.tt + diff --git a/CommonMark.Tests/YamlBlockTests.cs b/CommonMark.Tests/YamlBlockTests.cs new file mode 100644 index 0000000..b3f2098 --- /dev/null +++ b/CommonMark.Tests/YamlBlockTests.cs @@ -0,0 +1,112 @@ +using Microsoft.VisualStudio.TestTools.UnitTesting; + +using CommonMark.Syntax; + +namespace CommonMark.Tests +{ + [TestClass] + public class YamlBlockTests + { + private const string Category = "Container blocks - YAML"; + + private static CommonMarkSettings GetSettings(bool frontMatterOnly = false) + { + var settings = CommonMarkSettings.Default.Clone(); + settings.AdditionalFeatures = frontMatterOnly + ? CommonMarkAdditionalFeatures.YamlFrontMatterOnly + : CommonMarkAdditionalFeatures.YamlBlocks; + return settings; + } + + [TestMethod] + [TestCategory(Category)] + public void YamlDisabled() + { + Helpers.ExecuteTest( + "---\nparagraph\n...", + "
\n

paragraph\n...

"); + } + + [TestMethod] + [TestCategory(Category)] + public void YamlEmpty() + { + Helpers.ExecuteTest( + "---\n\n...", + "
\n
", + GetSettings()); + } + + [TestMethod] + [TestCategory(Category)] + public void YamlFrontMatterOnly() + { + Helpers.ExecuteTest( + "---\nfrontmatter\n...\n\nparagraph\n\n---\nline 1\nline 2\n...\n", + "
frontmatter\n
\n" + + "

paragraph

\n" + + "
\n" + + "

line 1\nline 2\n...

", + GetSettings(true)); + } + + [TestMethod] + [TestCategory(Category)] + public void YamlMultipleBlocks() + { + Helpers.ExecuteTest( + "---\nfrontmatter\n...\n\nparagraph\n\n---\nline 1\nline 2\n...\n", + "
frontmatter\n
\n" + + "

paragraph

\n" + + "
line 1\nline 2\n
", + GetSettings()); + } + + [TestMethod] + [TestCategory(Category)] + public void YamlAndThematicBreak() + { + Helpers.ExecuteTest( + "----\n\nnot yaml\n---\nalso not yaml\n\n---\nbut this\nis\nyaml\n...\npara", + "
\n" + + "

not yaml

\n" + + "

also not yaml

\n" + + "
but this\nis\nyaml\n
\n" + + "

para

", + GetSettings()); + } + + [TestMethod] + [TestCategory(Category)] + public void YamlClosingFenceDash() + { + AssertYamlClosingFenceAndAst("---"); + } + + [TestMethod] + [TestCategory(Category)] + public void YamlClosingFenceDot() + { + AssertYamlClosingFenceAndAst("..."); + } + + private static void AssertYamlClosingFenceAndAst(string fence) + { + var markdown = "---\nyaml\n" + fence; + Helpers.ExecuteTest( + markdown, + "
yaml\n
", + GetSettings()); + + var doc = CommonMarkConverter.Parse(markdown, GetSettings()); + + Assert.IsNotNull(doc.FirstChild); + Assert.AreEqual(BlockTag.YamlBlock, doc.FirstChild.Tag); + Assert.IsNotNull(doc.FirstChild.FencedCodeData); + Assert.AreEqual(0, doc.FirstChild.FencedCodeData.FenceOffset); + Assert.AreEqual(-1, doc.FirstChild.FencedCodeData.FenceLength); + Assert.AreEqual(fence[0], doc.FirstChild.FencedCodeData.FenceChar); + Assert.IsNull(doc.FirstChild.FencedCodeData.Info); + } + } +} \ No newline at end of file diff --git a/CommonMark/CommonMarkAdditionalFeatures.cs b/CommonMark/CommonMarkAdditionalFeatures.cs index 8351ce2..b565499 100644 --- a/CommonMark/CommonMarkAdditionalFeatures.cs +++ b/CommonMark/CommonMarkAdditionalFeatures.cs @@ -25,6 +25,19 @@ public enum CommonMarkAdditionalFeatures /// PlaceholderBracket = 2, + /// + /// Allow YAML blocks (delimited by a line starting with exactly --- and a line ending + /// with exactly --- or .... YAML blocks will take precedence over a --- + /// that might otherwise yield a thematic break. + /// + YamlBlocks = 4, + + /// + /// Like but will yield a maximum of one block, and only if the first + /// line is exactly ---. + /// + YamlFrontMatterOnly = 8, + /// /// All additional features are enabled. /// diff --git a/CommonMark/CommonMarkConverter.cs b/CommonMark/CommonMarkConverter.cs index 6487851..768d05e 100644 --- a/CommonMark/CommonMarkConverter.cs +++ b/CommonMark/CommonMarkConverter.cs @@ -111,7 +111,7 @@ public static Syntax.Block ProcessStage1(TextReader source, CommonMarkSettings s reader.ReadLine(line); while (line.Line != null) { - BlockMethods.IncorporateLine(line, ref cur); + BlockMethods.IncorporateLine(line, ref cur, settings); reader.ReadLine(line); } } diff --git a/CommonMark/Formatters/HtmlFormatter.cs b/CommonMark/Formatters/HtmlFormatter.cs index bb40418..af5a111 100644 --- a/CommonMark/Formatters/HtmlFormatter.cs +++ b/CommonMark/Formatters/HtmlFormatter.cs @@ -219,6 +219,7 @@ protected virtual void WriteBlock(Block block, bool isOpening, bool isClosing, o case BlockTag.IndentedCode: case BlockTag.FencedCode: + case BlockTag.YamlBlock: ignoreChildNodes = true; @@ -237,6 +238,11 @@ protected virtual void WriteBlock(Block block, bool isOpening, bool isClosing, o WriteEncodedHtml(new StringPart(info, 0, x)); Write('\"'); } + else if (block.Tag == BlockTag.YamlBlock) + { + Write(" class=\"language-yaml\""); + } + Write('>'); WriteEncodedHtml(block.StringContent); WriteLine(""); diff --git a/CommonMark/Formatters/HtmlFormatterSlim.cs b/CommonMark/Formatters/HtmlFormatterSlim.cs index f322c6e..e3a795d 100644 --- a/CommonMark/Formatters/HtmlFormatterSlim.cs +++ b/CommonMark/Formatters/HtmlFormatterSlim.cs @@ -318,6 +318,7 @@ private static void BlocksToHtmlInner(HtmlTextWriter writer, Block block, Common case BlockTag.IndentedCode: case BlockTag.FencedCode: + case BlockTag.YamlBlock: writer.EnsureLine(); writer.WriteConstant("
');
                         EscapeHtml(block.StringContent, writer);
                         writer.WriteLineConstant("
"); diff --git a/CommonMark/Formatters/Printer.cs b/CommonMark/Formatters/Printer.cs index 7a3477d..644f238 100644 --- a/CommonMark/Formatters/Printer.cs +++ b/CommonMark/Formatters/Printer.cs @@ -162,6 +162,14 @@ public static void PrintBlocks(TextWriter writer, Block block, CommonMarkSetting format_str(block.StringContent.ToString(buffer), buffer)); break; + case BlockTag.YamlBlock: + writer.Write("yaml_block"); + PrintPosition(trackPositions, writer, block); + writer.Write(" closing_fence_char={0} {1}", + block.FencedCodeData.FenceChar, + format_str(block.StringContent.ToString(buffer), buffer)); + break; + case BlockTag.HtmlBlock: writer.Write("html_block"); PrintPosition(trackPositions, writer, block); diff --git a/CommonMark/Parser/BlockMethods.cs b/CommonMark/Parser/BlockMethods.cs index b844e7c..0bf6c13 100644 --- a/CommonMark/Parser/BlockMethods.cs +++ b/CommonMark/Parser/BlockMethods.cs @@ -29,7 +29,8 @@ private static bool AcceptsLines(BlockTag block_type) return (block_type == BlockTag.Paragraph || block_type == BlockTag.AtxHeading || block_type == BlockTag.IndentedCode || - block_type == BlockTag.FencedCode); + block_type == BlockTag.FencedCode || + block_type == BlockTag.YamlBlock); } private static void AddLine(Block block, LineInfo lineInfo, string ln, int offset, int remainingSpaces, int length = -1, bool isAddOffsetRequired = true) @@ -137,9 +138,12 @@ public static void Finalize(Block b, LineInfo line) break; case BlockTag.FencedCode: + case BlockTag.YamlBlock: // first line of contents becomes info var firstlinelen = b.StringContent.IndexOf('\n') + 1; - b.FencedCodeData.Info = InlineMethods.Unescape(b.StringContent.TakeFromStart(firstlinelen, true).Trim()); + var firstline = b.StringContent.TakeFromStart(firstlinelen, true); + if (b.Tag == BlockTag.FencedCode) + b.FencedCodeData.Info = InlineMethods.Unescape(firstline.Trim()); break; case BlockTag.List: // determine tight/loose status @@ -464,7 +468,7 @@ private static void AdvanceOffset(string line, int count, bool columns, ref int // Process one line at a time, modifying a block. // Returns 0 if successful. curptr is changed to point to // the currently open block. - public static void IncorporateLine(LineInfo line, ref Block curptr) + public static void IncorporateLine(LineInfo line, ref Block curptr, CommonMarkSettings settings) { var ln = line.Line; @@ -571,6 +575,7 @@ public static void IncorporateLine(LineInfo line, ref Block curptr) } case BlockTag.FencedCode: + case BlockTag.YamlBlock: { // -1 means we've seen closer if (container.FencedCodeData.FenceLength == -1) @@ -632,6 +637,7 @@ public static void IncorporateLine(LineInfo line, ref Block curptr) // unless last matched container is code block, try new container starts: while (container.Tag != BlockTag.FencedCode && container.Tag != BlockTag.IndentedCode && + container.Tag != BlockTag.YamlBlock && container.Tag != BlockTag.HtmlBlock) { @@ -670,6 +676,20 @@ public static void IncorporateLine(LineInfo line, ref Block curptr) AdvanceOffset(ln, first_nonspace + matched - offset, false, ref offset, ref column, ref remainingSpaces); + } + else if (!indented && + ((container.IsLastLineBlank && (settings.AdditionalFeatures & CommonMarkAdditionalFeatures.YamlBlocks) != 0) + || (line.LineNumber == 1 && (settings.AdditionalFeatures & (CommonMarkAdditionalFeatures.YamlFrontMatterOnly | CommonMarkAdditionalFeatures.YamlBlocks)) != 0)) + && ln == "---\n") + { + + container = CreateChildBlock(container, line, BlockTag.YamlBlock, first_nonspace); + container.FencedCodeData = new FencedCodeData(); + container.FencedCodeData.FenceChar = '-'; + container.FencedCodeData.FenceLength = 3; + + AdvanceOffset(ln, 3, false, ref offset, ref column, ref remainingSpaces); + } else if (!indented && curChar == '<' && (0 != (matched = (int)Scanner.scan_html_block_start(ln, first_nonspace, ln.Length)) @@ -795,6 +815,7 @@ public static void IncorporateLine(LineInfo line, ref Block curptr) container.Tag != BlockTag.BlockQuote && container.Tag != BlockTag.SetextHeading && container.Tag != BlockTag.FencedCode && + container.Tag != BlockTag.YamlBlock && !(container.Tag == BlockTag.ListItem && container.FirstChild == null && container.SourcePosition >= line.LineOffset)); @@ -851,6 +872,20 @@ public static void IncorporateLine(LineInfo line, ref Block curptr) AddLine(container, line, ln, offset, remainingSpaces); } + } + else if (container.Tag == BlockTag.YamlBlock) + { + + if ((curChar == '-' && ln == "---\n") || (curChar == '.' && ln == "...\n")) + { + container.FencedCodeData.FenceLength = -1; + container.FencedCodeData.FenceChar = ln[0]; + } + else + { + AddLine(container, line, ln, offset, remainingSpaces); + } + } else if (container.Tag == BlockTag.HtmlBlock) { diff --git a/CommonMark/Syntax/BlockTag.cs b/CommonMark/Syntax/BlockTag.cs index d3e034d..7579306 100644 --- a/CommonMark/Syntax/BlockTag.cs +++ b/CommonMark/Syntax/BlockTag.cs @@ -83,6 +83,14 @@ public enum BlockTag : byte /// /// A text block that contains only link reference definitions. /// - ReferenceDefinition + ReferenceDefinition, + + /// + /// A YAML metadta block (for example, ---\nyaml: metadata\n...). + /// Only present if or + /// are enabled + /// The block is structured like a block. + /// + YamlBlock } }