From aa60db6cee8e285a39077c567c7f59fecbd0e0a3 Mon Sep 17 00:00:00 2001
From: blackst0ne <blackst0ne.ru@gmail.com>
Date: Wed, 7 Mar 2018 23:14:41 +1100
Subject: [PATCH] Add CommonMark markdown engine
---
Gemfile | 1 +
Gemfile.lock | 5 +++
.../replace_redcarpet_with_cmark.yml | 5 +++
config/initializers/8_metrics.rb | 1 +
.../filter/markdown_engines/common_mark.rb | 45 +++++++++++++++++++
.../filter/markdown_engines/redcarpet.rb | 32 +++++++++++++
lib/banzai/filter/markdown_filter.rb | 41 ++++++++---------
lib/banzai/filter/syntax_highlight_filter.rb | 1 +
lib/banzai/renderer/common_mark/html.rb | 21 +++++++++
lib/banzai/renderer/html.rb | 13 ------
lib/banzai/renderer/redcarpet/html.rb | 15 +++++++
lib/rouge/plugins/common_mark.rb | 20 +++++++++
12 files changed, 165 insertions(+), 35 deletions(-)
create mode 100644 changelogs/unreleased/replace_redcarpet_with_cmark.yml
create mode 100644 lib/banzai/filter/markdown_engines/common_mark.rb
create mode 100644 lib/banzai/filter/markdown_engines/redcarpet.rb
create mode 100644 lib/banzai/renderer/common_mark/html.rb
delete mode 100644 lib/banzai/renderer/html.rb
create mode 100644 lib/banzai/renderer/redcarpet/html.rb
create mode 100644 lib/rouge/plugins/common_mark.rb
diff --git a/Gemfile b/Gemfile
index 6838ddbf01a8..c7d18b36269c 100644
--- a/Gemfile
+++ b/Gemfile
@@ -126,6 +126,7 @@ gem 'html-pipeline', '~> 1.11.0'
gem 'deckar01-task_list', '2.0.0'
gem 'gitlab-markup', '~> 1.6.2'
gem 'redcarpet', '~> 3.4'
+gem 'commonmarker', '~> 0.17'
gem 'RedCloth', '~> 4.3.2'
gem 'rdoc', '~> 4.2'
gem 'org-ruby', '~> 0.9.12'
diff --git a/Gemfile.lock b/Gemfile.lock
index 89b86ae02590..edb2d887c0db 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -131,6 +131,8 @@ GEM
coercible (1.0.0)
descendants_tracker (~> 0.0.1)
colorize (0.7.7)
+ commonmarker (0.17.8)
+ ruby-enum (~> 0.5)
concord (0.1.5)
adamantium (~> 0.2.0)
equalizer (~> 0.0.9)
@@ -797,6 +799,8 @@ GEM
rubocop (>= 0.51)
rubocop-rspec (1.22.1)
rubocop (>= 0.52.1)
+ ruby-enum (0.7.2)
+ i18n
ruby-fogbugz (0.2.1)
crack (~> 0.4)
ruby-prof (0.16.2)
@@ -1019,6 +1023,7 @@ DEPENDENCIES
charlock_holmes (~> 0.7.5)
chronic (~> 0.10.2)
chronic_duration (~> 0.10.6)
+ commonmarker (~> 0.17)
concurrent-ruby (~> 1.0.5)
connection_pool (~> 2.0)
creole (~> 0.5.0)
diff --git a/changelogs/unreleased/replace_redcarpet_with_cmark.yml b/changelogs/unreleased/replace_redcarpet_with_cmark.yml
new file mode 100644
index 000000000000..7ce848b0bbd7
--- /dev/null
+++ b/changelogs/unreleased/replace_redcarpet_with_cmark.yml
@@ -0,0 +1,5 @@
+---
+title: Add CommonMark markdown engine (experimental)
+merge_request: 14835
+author: blackst0ne
+type: added
diff --git a/config/initializers/8_metrics.rb b/config/initializers/8_metrics.rb
index 45b39b2a38db..7cdf49159b4d 100644
--- a/config/initializers/8_metrics.rb
+++ b/config/initializers/8_metrics.rb
@@ -94,6 +94,7 @@ def instrument_classes(instrumentation)
instrumentation.instrument_instance_methods(RepositoryCheck::SingleRepositoryWorker)
+ instrumentation.instrument_instance_methods(Rouge::Plugins::CommonMark)
instrumentation.instrument_instance_methods(Rouge::Plugins::Redcarpet)
instrumentation.instrument_instance_methods(Rouge::Formatters::HTMLGitlab)
diff --git a/lib/banzai/filter/markdown_engines/common_mark.rb b/lib/banzai/filter/markdown_engines/common_mark.rb
new file mode 100644
index 000000000000..bc9597df8945
--- /dev/null
+++ b/lib/banzai/filter/markdown_engines/common_mark.rb
@@ -0,0 +1,45 @@
+# `CommonMark` markdown engine for GitLab's Banzai markdown filter.
+# This module is used in Banzai::Filter::MarkdownFilter.
+# Used gem is `commonmarker` which is a ruby wrapper for libcmark (CommonMark parser)
+# including GitHub's GFM extensions.
+# Homepage: https://github.com/gjtorikian/commonmarker
+
+module Banzai
+ module Filter
+ module MarkdownEngines
+ class CommonMark
+ EXTENSIONS = [
+ :autolink, # provides support for automatically converting URLs to anchor tags.
+ :strikethrough, # provides support for strikethroughs.
+ :table, # provides support for tables.
+ :tagfilter # strips out several "unsafe" HTML tags from being used: https://github.github.com/gfm/#disallowed-raw-html-extension-
+ ].freeze
+
+ PARSE_OPTIONS = [
+ :FOOTNOTES, # parse footnotes.
+ :STRIKETHROUGH_DOUBLE_TILDE, # parse strikethroughs by double tildes (as redcarpet does).
+ :VALIDATE_UTF8 # replace illegal sequences with the replacement character U+FFFD.
+ ].freeze
+
+ # The `:GITHUB_PRE_LANG` option is not used intentionally because
+ # it renders a fence block with language as `<pre lang="LANG"><code>some code\n</code></pre>`
+ # while GitLab's syntax is `<pre><code lang="LANG">some code\n</code></pre>`.
+ # If in the future the syntax is about to be made GitHub-compatible, please, add `:GITHUB_PRE_LANG` render option below
+ # and remove `code_block` method from `lib/banzai/renderer/common_mark/html.rb`.
+ RENDER_OPTIONS = [
+ :DEFAULT # default rendering system. Nothing special.
+ ].freeze
+
+ def initialize
+ @renderer = Banzai::Renderer::CommonMark::HTML.new(options: RENDER_OPTIONS)
+ end
+
+ def render(text)
+ doc = CommonMarker.render_doc(text, PARSE_OPTIONS, EXTENSIONS)
+
+ @renderer.render(doc)
+ end
+ end
+ end
+ end
+end
diff --git a/lib/banzai/filter/markdown_engines/redcarpet.rb b/lib/banzai/filter/markdown_engines/redcarpet.rb
new file mode 100644
index 000000000000..ac99941fefab
--- /dev/null
+++ b/lib/banzai/filter/markdown_engines/redcarpet.rb
@@ -0,0 +1,32 @@
+# `Redcarpet` markdown engine for GitLab's Banzai markdown filter.
+# This module is used in Banzai::Filter::MarkdownFilter.
+# Used gem is `redcarpet` which is a ruby library for markdown processing.
+# Homepage: https://github.com/vmg/redcarpet
+
+module Banzai
+ module Filter
+ module MarkdownEngines
+ class Redcarpet
+ OPTIONS = {
+ fenced_code_blocks: true,
+ footnotes: true,
+ lax_spacing: true,
+ no_intra_emphasis: true,
+ space_after_headers: true,
+ strikethrough: true,
+ superscript: true,
+ tables: true
+ }.freeze
+
+ def initialize
+ html_renderer = Banzai::Renderer::Redcarpet::HTML.new
+ @renderer = ::Redcarpet::Markdown.new(html_renderer, OPTIONS)
+ end
+
+ def render(text)
+ @renderer.render(text)
+ end
+ end
+ end
+ end
+end
diff --git a/lib/banzai/filter/markdown_filter.rb b/lib/banzai/filter/markdown_filter.rb
index 9cac303e6450..c1e2b6802406 100644
--- a/lib/banzai/filter/markdown_filter.rb
+++ b/lib/banzai/filter/markdown_filter.rb
@@ -1,34 +1,31 @@
module Banzai
module Filter
class MarkdownFilter < HTML::Pipeline::TextFilter
- # https://github.com/vmg/redcarpet#and-its-like-really-simple-to-use
- REDCARPET_OPTIONS = {
- fenced_code_blocks: true,
- footnotes: true,
- lax_spacing: true,
- no_intra_emphasis: true,
- space_after_headers: true,
- strikethrough: true,
- superscript: true,
- tables: true
- }.freeze
-
def initialize(text, context = nil, result = nil)
- super text, context, result
- @text = @text.delete "\r"
+ super(text, context, result)
+
+ @renderer = renderer(context[:markdown_engine]).new
+ @text = @text.delete("\r")
end
def call
- html = self.class.renderer.render(@text)
- html.rstrip!
- html
+ @renderer.render(@text).rstrip
+ end
+
+ private
+
+ DEFAULT_ENGINE = :redcarpet
+
+ def engine(engine_from_context)
+ engine_from_context ||= DEFAULT_ENGINE
+
+ engine_from_context.to_s.classify
end
- def self.renderer
- Thread.current[:banzai_markdown_renderer] ||= begin
- renderer = Banzai::Renderer::HTML.new
- Redcarpet::Markdown.new(renderer, REDCARPET_OPTIONS)
- end
+ def renderer(engine_from_context)
+ "Banzai::Filter::MarkdownEngines::#{engine(engine_from_context)}".constantize
+ rescue NameError
+ raise NameError, "`#{engine_from_context}` is unknown markdown engine"
end
end
end
diff --git a/lib/banzai/filter/syntax_highlight_filter.rb b/lib/banzai/filter/syntax_highlight_filter.rb
index 0ac7e231b5b7..6dbf0d68fe8f 100644
--- a/lib/banzai/filter/syntax_highlight_filter.rb
+++ b/lib/banzai/filter/syntax_highlight_filter.rb
@@ -1,3 +1,4 @@
+require 'rouge/plugins/common_mark'
require 'rouge/plugins/redcarpet'
module Banzai
diff --git a/lib/banzai/renderer/common_mark/html.rb b/lib/banzai/renderer/common_mark/html.rb
new file mode 100644
index 000000000000..c7a54629f31d
--- /dev/null
+++ b/lib/banzai/renderer/common_mark/html.rb
@@ -0,0 +1,21 @@
+module Banzai
+ module Renderer
+ module CommonMark
+ class HTML < CommonMarker::HtmlRenderer
+ def code_block(node)
+ block do
+ code = node.string_content
+ lang = node.fence_info
+ lang_attr = lang.present? ? %Q{ lang="#{lang}"} : ''
+ result =
+ "<pre>" \
+ "<code#{lang_attr}>#{html_escape(code)}</code>" \
+ "</pre>"
+
+ out(result)
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/banzai/renderer/html.rb b/lib/banzai/renderer/html.rb
deleted file mode 100644
index 252caa359473..000000000000
--- a/lib/banzai/renderer/html.rb
+++ /dev/null
@@ -1,13 +0,0 @@
-module Banzai
- module Renderer
- class HTML < Redcarpet::Render::HTML
- def block_code(code, lang)
- lang_attr = lang ? %Q{ lang="#{lang}"} : ''
-
- "\n<pre>" \
- "<code#{lang_attr}>#{html_escape(code)}</code>" \
- "</pre>"
- end
- end
- end
-end
diff --git a/lib/banzai/renderer/redcarpet/html.rb b/lib/banzai/renderer/redcarpet/html.rb
new file mode 100644
index 000000000000..94df5d8b1e15
--- /dev/null
+++ b/lib/banzai/renderer/redcarpet/html.rb
@@ -0,0 +1,15 @@
+module Banzai
+ module Renderer
+ module Redcarpet
+ class HTML < ::Redcarpet::Render::HTML
+ def block_code(code, lang)
+ lang_attr = lang ? %Q{ lang="#{lang}"} : ''
+
+ "\n<pre>" \
+ "<code#{lang_attr}>#{html_escape(code)}</code>" \
+ "</pre>"
+ end
+ end
+ end
+ end
+end
diff --git a/lib/rouge/plugins/common_mark.rb b/lib/rouge/plugins/common_mark.rb
new file mode 100644
index 000000000000..8f9de061124b
--- /dev/null
+++ b/lib/rouge/plugins/common_mark.rb
@@ -0,0 +1,20 @@
+# A rouge plugin for CommonMark markdown engine.
+# Used to highlight code generated by CommonMark.
+
+module Rouge
+ module Plugins
+ module CommonMark
+ def code_block(code, language)
+ lexer = Lexer.find_fancy(language, code) || Lexers::PlainText
+
+ formatter = rouge_formatter(lexer)
+ formatter.format(lexer.lex(code))
+ end
+
+ # override this method for custom formatting behavior
+ def rouge_formatter(lexer)
+ Formatters::HTMLLegacy.new(css_class: "highlight #{lexer.tag}")
+ end
+ end
+ end
+end
--
GitLab