diff --git a/config/feature_flags/development/markdown_dollar_math.yml b/config/feature_flags/development/markdown_dollar_math.yml new file mode 100644 index 0000000000000000000000000000000000000000..842837ea688269e27b12bca98cb8d87e1b0465b0 --- /dev/null +++ b/config/feature_flags/development/markdown_dollar_math.yml @@ -0,0 +1,8 @@ +--- +name: markdown_dollar_math +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/94111 +rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/371180 +milestone: '15.4' +type: development +group: group::project management +default_enabled: false diff --git a/doc/user/markdown.md b/doc/user/markdown.md index 6f90a9f0b1fa53156ffb710b70add572e326fe77..0da0b277307e985eb6a8062b3964aa097e967286 100644 --- a/doc/user/markdown.md +++ b/doc/user/markdown.md @@ -346,10 +346,13 @@ backslash `\`. Otherwise the diff highlight does not render correctly: [View this topic in GitLab](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/user/markdown.md#math). Math written in LaTeX syntax is rendered with [KaTeX](https://github.com/KaTeX/KaTeX). +_KaTeX only supports a [subset](https://katex.org/docs/supported.html) of LaTeX._ +This syntax also works for the Asciidoctor `:stem: latexmath`. For details, see +the [Asciidoctor user manual](https://asciidoctor.org/docs/user-manual/#activating-stem-support). -Math written between dollar signs `$` is rendered inline with the text. Math written -in a [code block](#code-spans-and-blocks) with the language declared as `math` is rendered -on a separate line: +Math written between dollar signs with backticks (``$`...`$``) is rendered +inline with the text. Math written in a [code block](#code-spans-and-blocks) with +the language declared as `math` is rendered on a separate line: ````markdown This math is inline: $`a^2+b^2=c^2`$. @@ -369,10 +372,44 @@ This math is on a separate line: a^2+b^2=c^2 ``` -_KaTeX only supports a [subset](https://katex.org/docs/supported.html) of LaTeX._ +#### LaTeX-compatible fencing -This syntax also works for the Asciidoctor `:stem: latexmath`. For details, see -the [Asciidoctor user manual](https://asciidoctor.org/docs/user-manual/#activating-stem-support). +> Introduced in GitLab 15.4 [with a flag](../administration/feature_flags.md) named `markdown_dollar_math`. Disabled by default. + +[View this topic in GitLab](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/user/markdown.md#latex-compatible-fencing). + +FLAG: +On self-managed GitLab, by default this feature is not available. To make it available, +ask an administrator to [enable the feature flag](../administration/feature_flags.md) named `markdown_dollar_math`. +On GitLab.com, this feature is available. +The feature is not ready for production use. + +Math written between dollar signs (`$...$`) is rendered +inline with the text. Math written between double dollar signs (`$$...$$`) is rendered +on a separate line: + +````markdown +This math is inline: $a^2+b^2=c^2$. + +This math is on a separate line: $$a^2+b^2=c^2$$ + +This math is on a separate line: + +$$ +a^2+b^2=c^2 +$$ +```` + +<!-- Uncomment the example below when the flag is enabled on GitLab.com --> +<!-- This math is inline: $a^2+b^2=c^2$. + +This math is on a separate line: $$a^2+b^2=c^2$$ + +This math is on a separate line: + +$$ +a^2+b^2=c^2 +$$ --> ### Task lists diff --git a/lib/banzai/filter/math_filter.rb b/lib/banzai/filter/math_filter.rb index 0ac506776bece9c6e95f6019ece509376057be9b..1ca4b2c89db609d7f2abb1a14ce6e67e5092a703 100644 --- a/lib/banzai/filter/math_filter.rb +++ b/lib/banzai/filter/math_filter.rb @@ -7,7 +7,7 @@ # - app/assets/javascripts/behaviors/markdown/nodes/code_block.js module Banzai module Filter - # HTML filter that adds class="code math" and removes the dollar sign in $`2+2`$. + # HTML filter that implements our math syntax, adding class="code math" # class MathFilter < HTML::Pipeline::Filter CSS_MATH = 'pre.code.language-math' @@ -15,14 +15,42 @@ class MathFilter < HTML::Pipeline::Filter CSS_CODE = 'code' XPATH_CODE = Gitlab::Utils::Nokogiri.css_to_xpath(CSS_CODE).freeze + # These are based on the Pandoc heuristics, + # https://pandoc.org/MANUAL.html#extension-tex_math_dollars + # Note: at this time, using a dollar sign literal, `\$` inside + # a math statement does not work correctly. + # Corresponds to the "$...$" syntax + DOLLAR_INLINE_PATTERN = %r{ + (?<matched>\$(?<math>(?:\S[^$\n]*?\S|[^$\s]))\$)(?:[^\d]|$) + }x.freeze + + # Corresponds to the "$$...$$" syntax + DOLLAR_DISPLAY_INLINE_PATTERN = %r{ + (?<matched>\$\$\ *(?<math>[^$\n]+?)\ *\$\$) + }x.freeze + + # Corresponds to the $$\n...\n$$ syntax + DOLLAR_DISPLAY_BLOCK_PATTERN = %r{ + ^(?<matched>\$\$\ *\n(?<math>.*)\n\$\$\ *)$ + }x.freeze + + # Order dependent. Handle the `$$` syntax before the `$` syntax + DOLLAR_MATH_PIPELINE = [ + { pattern: DOLLAR_DISPLAY_INLINE_PATTERN, tag: :code, style: :display }, + { pattern: DOLLAR_DISPLAY_BLOCK_PATTERN, tag: :pre, style: :display }, + { pattern: DOLLAR_INLINE_PATTERN, tag: :code, style: :inline } + ].freeze + + # Do not recognize math inside these tags + IGNORED_ANCESTOR_TAGS = %w[pre code tt].to_set + # Attribute indicating inline or display math. STYLE_ATTRIBUTE = 'data-math-style' # Class used for tagging elements that should be rendered TAG_CLASS = 'js-render-math' - INLINE_CLASSES = "code math #{TAG_CLASS}" - + MATH_CLASSES = "code math #{TAG_CLASS}" DOLLAR_SIGN = '$' # Limit to how many nodes can be marked as math elements. @@ -31,8 +59,48 @@ class MathFilter < HTML::Pipeline::Filter RENDER_NODES_LIMIT = 50 def call - nodes_count = 0 + @nodes_count = 0 + + process_dollar_pipeline if Feature.enabled?(:markdown_dollar_math, group) + + process_dollar_backtick_inline + process_math_codeblock + + doc + end + + def process_dollar_pipeline + doc.xpath('descendant-or-self::text()').each do |node| + next if has_ancestor?(node, IGNORED_ANCESTOR_TAGS) + + node_html = node.to_html + next unless node_html.match?(DOLLAR_INLINE_PATTERN) || + node_html.match?(DOLLAR_DISPLAY_INLINE_PATTERN) || + node_html.match?(DOLLAR_DISPLAY_BLOCK_PATTERN) + + temp_doc = Nokogiri::HTML.fragment(node_html) + DOLLAR_MATH_PIPELINE.each do |pipeline| + temp_doc.xpath('child::text()').each do |temp_node| + html = temp_node.to_html + temp_node.content.scan(pipeline[:pattern]).each do |matched, math| + html.sub!(matched, math_html(tag: pipeline[:tag], style: pipeline[:style], math: math)) + @nodes_count += 1 + break if @nodes_count >= RENDER_NODES_LIMIT + end + + temp_node.replace(html) + + break if @nodes_count >= RENDER_NODES_LIMIT + end + end + + node.replace(temp_doc) + end + end + + # Corresponds to the "$`...`$" syntax + def process_dollar_backtick_inline doc.xpath(XPATH_CODE).each do |code| closing = code.next opening = code.previous @@ -44,22 +112,38 @@ def call closing.content.first == DOLLAR_SIGN && opening.content.last == DOLLAR_SIGN - code[:class] = INLINE_CLASSES + code[:class] = MATH_CLASSES code[STYLE_ATTRIBUTE] = 'inline' closing.content = closing.content[1..] opening.content = opening.content[0..-2] - nodes_count += 1 - break if nodes_count >= RENDER_NODES_LIMIT + @nodes_count += 1 + break if @nodes_count >= RENDER_NODES_LIMIT end end + end + # corresponds to the "```math...```" syntax + def process_math_codeblock doc.xpath(XPATH_MATH).each do |el| el[STYLE_ATTRIBUTE] = 'display' el[:class] += " #{TAG_CLASS}" end + end - doc + private + + def math_html(tag:, math:, style:) + case tag + when :code + "<code class=\"#{MATH_CLASSES}\" data-math-style=\"#{style}\">#{math}</code>" + when :pre + "<pre class=\"#{MATH_CLASSES}\" data-math-style=\"#{style}\"><code>#{math}</code></pre>" + end + end + + def group + context[:group] || context[:project]&.group end end end diff --git a/spec/features/markdown/markdown_spec.rb b/spec/features/markdown/markdown_spec.rb index 9eff02a8c1b0a7fc2aa38f8b101a42555b131f27..08f9b8eda13ff2a68eba385085d53a17005695db 100644 --- a/spec/features/markdown/markdown_spec.rb +++ b/spec/features/markdown/markdown_spec.rb @@ -263,6 +263,10 @@ def doc(html = @html) expect(doc).to parse_task_lists end + aggregate_failures 'MathFilter' do + expect(doc).to parse_math + end + aggregate_failures 'InlineDiffFilter' do expect(doc).to parse_inline_diffs end diff --git a/spec/fixtures/markdown.md.erb b/spec/fixtures/markdown.md.erb index 18cd63b7bcb3fd83ccb5296f8fad1dcf39b00816..14885813d9365c26a9c65826b1e74a96a2363b0c 100644 --- a/spec/fixtures/markdown.md.erb +++ b/spec/fixtures/markdown.md.erb @@ -283,7 +283,23 @@ References should be parseable even inside _<%= merge_request.to_reference %>_ e - [x] Complete sub-task 1 - [X] Complete task 2 -#### Gollum Tags +### Math + +- Dollar math: $a^2 + b^2 = c^2$ +- Dollar math and snippet reference: $d^2 + e^2 = f^2$ and <%= snippet.to_reference %> +- Dollar math and snippet in another project: <%= xsnippet.to_reference(project) %> and $g^2 + h^2 = i^2$ +- Not dollar math: $20,000 and $30,000 +- Dollar-backtick math: $`j^2 + k^2 = l^2`$ +- Dollar display math: $$m^2 + n^2 = o^2$$ +- Dollar display math and snippet reference: $$p^2 + q^2 = r^2$$ and <%= snippet.to_reference %> +- Dollar math and snippet in another project: <%= xsnippet.to_reference(project) %> and $$s^2 + t^2 = u^2$$ +- Display math using a block + + ```math + v^2 + w^2 = x^2 + ``` + +### Gollum Tags - [[linked-resource]] - [[link-text|linked-resource]] @@ -326,15 +342,15 @@ However the wrapping tags cannot be mixed as such: ### Colors -`#F00` -`#F00A` -`#FF0000` -`#FF0000AA` -`RGB(0,255,0)` -`RGB(0%,100%,0%)` -`RGBA(0,255,0,0.7)` -`HSL(540,70%,50%)` -`HSLA(540,70%,50%,0.7)` +`#F00` +`#F00A` +`#FF0000` +`#FF0000AA` +`RGB(0,255,0)` +`RGB(0%,100%,0%)` +`RGBA(0,255,0,0.7)` +`HSL(540,70%,50%)` +`HSLA(540,70%,50%,0.7)` ### Mermaid diff --git a/spec/lib/banzai/filter/math_filter_spec.rb b/spec/lib/banzai/filter/math_filter_spec.rb index 128f8532d3991cf04008c311505c66ac9e9a175d..dd116eb110944d2e4f1fed48609407f98b4be2b6 100644 --- a/spec/lib/banzai/filter/math_filter_spec.rb +++ b/spec/lib/banzai/filter/math_filter_spec.rb @@ -3,128 +3,179 @@ require 'spec_helper' RSpec.describe Banzai::Filter::MathFilter do + using RSpec::Parameterized::TableSyntax include FilterSpecHelper - it 'leaves regular inline code unchanged' do - input = "<code>2+2</code>" - doc = filter(input) - - expect(doc.to_s).to eq input - end - - it 'removes surrounding dollar signs and adds class code, math and js-render-math' do - doc = filter("$<code>2+2</code>$") - - expect(doc.to_s).to eq '<code class="code math js-render-math" data-math-style="inline">2+2</code>' - end - - it 'only removes surrounding dollar signs' do - doc = filter("test $<code>2+2</code>$ test") - before = doc.xpath('descendant-or-self::text()[1]').first - after = doc.xpath('descendant-or-self::text()[3]').first - - expect(before.to_s).to eq 'test ' - expect(after.to_s).to eq ' test' - end - - it 'only removes surrounding single dollar sign' do - doc = filter("test $$<code>2+2</code>$$ test") - before = doc.xpath('descendant-or-self::text()[1]').first - after = doc.xpath('descendant-or-self::text()[3]').first - - expect(before.to_s).to eq 'test $' - expect(after.to_s).to eq '$ test' - end - - it 'adds data-math-style inline attribute to inline math' do - doc = filter('$<code>2+2</code>$') - code = doc.xpath('descendant-or-self::code').first - - expect(code['data-math-style']).to eq 'inline' - end - - it 'adds class code and math to inline math' do - doc = filter('$<code>2+2</code>$') - code = doc.xpath('descendant-or-self::code').first - - expect(code[:class]).to include("code") - expect(code[:class]).to include("math") - end - - it 'adds js-render-math class to inline math' do - doc = filter('$<code>2+2</code>$') - code = doc.xpath('descendant-or-self::code').first - - expect(code[:class]).to include("js-render-math") - end - - # Cases with faulty syntax. Should be a no-op - - it 'ignores cases with missing dolar sign at the end' do - input = "test $<code>2+2</code> test" - doc = filter(input) - - expect(doc.to_s).to eq input - end - - it 'ignores cases with missing dolar sign at the beginning' do - input = "test <code>2+2</code>$ test" - doc = filter(input) - - expect(doc.to_s).to eq input - end - - it 'ignores dollar signs if it is not adjacent' do - input = '<p>We check strictly $<code>2+2</code> and <code>2+2</code>$ </p>' - doc = filter(input) - - expect(doc.to_s).to eq input - end - - it 'ignores dollar signs if they are inside another element' do - input = '<p>We check strictly <em>$</em><code>2+2</code><em>$</em></p>' - doc = filter(input) - - expect(doc.to_s).to eq input - end - - # Display math - - it 'adds data-math-style display attribute to display math' do - doc = filter('<pre class="code highlight js-syntax-highlight language-math" v-pre="true"><code>2+2</code></pre>') - pre = doc.xpath('descendant-or-self::pre').first - - expect(pre['data-math-style']).to eq 'display' - end - - it 'adds js-render-math class to display math' do - doc = filter('<pre class="code highlight js-syntax-highlight language-math" v-pre="true"><code>2+2</code></pre>') - pre = doc.xpath('descendant-or-self::pre').first - - expect(pre[:class]).to include("js-render-math") - end - - it 'ignores code blocks that are not math' do - input = '<pre class="code highlight js-syntax-highlight language-plaintext" v-pre="true"><code>2+2</code></pre>' - doc = filter(input) - - expect(doc.to_s).to eq input - end - - it 'requires the pre to contain both code and math' do - input = '<pre class="highlight js-syntax-highlight language-plaintext language-math" v-pre="true"><code>2+2</code></pre>' - doc = filter(input) - - expect(doc.to_s).to eq input - end - - it 'dollar signs around to display math' do - doc = filter('$<pre class="code highlight js-syntax-highlight language-math" v-pre="true"><code>2+2</code></pre>$') - before = doc.xpath('descendant-or-self::text()[1]').first - after = doc.xpath('descendant-or-self::text()[3]').first - - expect(before.to_s).to eq '$' - expect(after.to_s).to eq '$' + shared_examples 'inline math' do + it 'removes surrounding dollar signs and adds class code, math and js-render-math' do + doc = filter(text) + expected = result_template.gsub('<math>', '<code class="code math js-render-math" data-math-style="inline">') + expected.gsub!('</math>', '</code>') + + expect(doc.to_s).to eq expected + end + end + + shared_examples 'display math' do + let_it_be(:template_prefix_with_pre) { '<pre class="code math js-render-math" data-math-style="display"><code>' } + let_it_be(:template_prefix_with_code) { '<code class="code math js-render-math" data-math-style="display">' } + let(:use_pre_tags) { false } + + it 'removes surrounding dollar signs and adds class code, math and js-render-math' do + doc = filter(text) + + template_prefix = use_pre_tags ? template_prefix_with_pre : template_prefix_with_code + template_suffix = "</code>#{'</pre>' if use_pre_tags}" + expected = result_template.gsub('<math>', template_prefix) + expected.gsub!('</math>', template_suffix) + + expect(doc.to_s).to eq expected + end + end + + describe 'inline math using $...$ syntax' do + context 'with valid syntax' do + where(:text, :result_template) do + '$2+2$' | '<math>2+2</math>' + '$22+1$ and $22 + a^2$' | '<math>22+1</math> and <math>22 + a^2</math>' + '$22 and $2+2$' | '$22 and <math>2+2</math>' + '$2+2$ $22 and flightjs/Flight$22 $2+2$' | '<math>2+2</math> $22 and flightjs/Flight$22 <math>2+2</math>' + '$1/2$ <b>test</b>' | '<math>1/2</math> <b>test</b>' + '$a!$' | '<math>a!</math>' + '$x$' | '<math>x</math>' + end + + with_them do + it_behaves_like 'inline math' + end + end + + it 'does not handle dollar literals properly' do + doc = filter('$20+30\$$') + expected = '<code class="code math js-render-math" data-math-style="inline">20+30\\</code>$' + + expect(doc.to_s).to eq expected + end + end + + describe 'inline math using $`...`$ syntax' do + context 'with valid syntax' do + where(:text, :result_template) do + '$<code>2+2</code>$' | '<math>2+2</math>' + '$<code>22+1</code>$ and $<code>22 + a^2</code>$' | '<math>22+1</math> and <math>22 + a^2</math>' + '$22 and $<code>2+2</code>$' | '$22 and <math>2+2</math>' + '$<code>2+2</code>$ $22 and flightjs/Flight$22 $<code>2+2</code>$' | '<math>2+2</math> $22 and flightjs/Flight$22 <math>2+2</math>' + 'test $$<code>2+2</code>$$ test' | 'test $<math>2+2</math>$ test' + end + + with_them do + it_behaves_like 'inline math' + end + end + end + + describe 'inline display math using $$...$$ syntax' do + context 'with valid syntax' do + where(:text, :result_template) do + '$$2+2$$' | '<math>2+2</math>' + '$$ 2+2 $$' | '<math>2+2</math>' + '$$22+1$$ and $$22 + a^2$$' | '<math>22+1</math> and <math>22 + a^2</math>' + '$22 and $$2+2$$' | '$22 and <math>2+2</math>' + '$$2+2$$ $22 and flightjs/Flight$22 $$2+2$$' | '<math>2+2</math> $22 and flightjs/Flight$22 <math>2+2</math>' + 'flightjs/Flight$22 and $$a^2 + b^2 = c^2$$' | 'flightjs/Flight$22 and <math>a^2 + b^2 = c^2</math>' + '$$a!$$' | '<math>a!</math>' + '$$x$$' | '<math>x</math>' + '$$20,000 and $$30,000' | '<math>20,000 and</math>30,000' + end + + with_them do + it_behaves_like 'display math' + end + end + end + + describe 'block display math using $$\n...\n$$ syntax' do + context 'with valid syntax' do + where(:text, :result_template) do + "$$\n2+2\n$$" | "<math>2+2</math>" + end + + with_them do + it_behaves_like 'display math' do + let(:use_pre_tags) { true } + end + end + end + end + + describe 'display math using ```math...``` syntax' do + it 'adds data-math-style display attribute to display math' do + doc = filter('<pre class="code highlight js-syntax-highlight language-math" v-pre="true"><code>2+2</code></pre>') + pre = doc.xpath('descendant-or-self::pre').first + + expect(pre['data-math-style']).to eq 'display' + end + + it 'adds js-render-math class to display math' do + doc = filter('<pre class="code highlight js-syntax-highlight language-math" v-pre="true"><code>2+2</code></pre>') + pre = doc.xpath('descendant-or-self::pre').first + + expect(pre[:class]).to include("js-render-math") + end + + it 'ignores code blocks that are not math' do + input = '<pre class="code highlight js-syntax-highlight language-plaintext" v-pre="true"><code>2+2</code></pre>' + doc = filter(input) + + expect(doc.to_s).to eq input + end + + it 'requires the pre to contain both code and math' do + input = '<pre class="highlight js-syntax-highlight language-plaintext language-math" v-pre="true"><code>2+2</code></pre>' + doc = filter(input) + + expect(doc.to_s).to eq input + end + + it 'dollar signs around to display math' do + doc = filter('$<pre class="code highlight js-syntax-highlight language-math" v-pre="true"><code>2+2</code></pre>$') + before = doc.xpath('descendant-or-self::text()[1]').first + after = doc.xpath('descendant-or-self::text()[3]').first + + expect(before.to_s).to eq '$' + expect(after.to_s).to eq '$' + end + end + + describe 'unrecognized syntax' do + where(:text) do + [ + '<code>2+2</code>', + 'test $<code>2+2</code> test', + 'test <code>2+2</code>$ test', + '<em>$</em><code>2+2</code><em>$</em>', + '$20,000 and $30,000', + '$20,000 in $USD', + '$ a^2 $', + "test $$\n2+2\n$$", + "$\n$", + '$$$' + ] + end + + with_them do + it 'is ignored' do + expect(filter(text).to_s).to eq text + end + end + end + + it 'handles multiple styles in one text block' do + doc = filter('$<code>2+2</code>$ + $3+3$ + $$4+4$$') + + expect(doc.search('.js-render-math').count).to eq(3) + expect(doc.search('[data-math-style="inline"]').count).to eq(2) + expect(doc.search('[data-math-style="display"]').count).to eq(1) end it 'limits how many elements can be marked as math' do @@ -134,4 +185,11 @@ expect(doc.search('.js-render-math').count).to eq(2) end + + it 'does not recognize new syntax when feature flag is off' do + stub_feature_flags(markdown_dollar_math: false) + doc = filter('$1+2$') + + expect(doc.to_s).to eq '$1+2$' + end end diff --git a/spec/support/matchers/markdown_matchers.rb b/spec/support/matchers/markdown_matchers.rb index 8bec3be2535fe9e18c1459a5b7d5c53de684cde4..a80c269f9154ad624f487239650bf2a284136e56 100644 --- a/spec/support/matchers/markdown_matchers.rb +++ b/spec/support/matchers/markdown_matchers.rb @@ -134,7 +134,7 @@ def have_image(src) set_default_markdown_messages match do |actual| - expect(actual).to have_selector('a.gfm.gfm-snippet', count: 5) + expect(actual).to have_selector('a.gfm.gfm-snippet', count: 9) end end @@ -196,6 +196,16 @@ def have_image(src) end end + # MathFilter + matcher :parse_math do + set_default_markdown_messages + + match do |actual| + expect(actual).to have_selector('[data-math-style="inline"]', count: 4) + expect(actual).to have_selector('[data-math-style="display"]', count: 4) + end + end + # InlineDiffFilter matcher :parse_inline_diffs do set_default_markdown_messages