diff --git a/CHANGELOG b/CHANGELOG index 06dc2993b73..2095e4fc0fd 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -14,6 +14,7 @@ v 8.13.0 (unreleased) - AbstractReferenceFilter caches project_refs on RequestStore when active - Replaced the check sign to arrow in the show build view. !6501 - Add a /wip slash command to toggle the Work In Progress status of a merge request. !6259 (tbalthazar) + - Fix Error 500 when viewing old merge requests with bad diff data - Speed-up group milestones show page - Fix inconsistent options dropdown caret on mobile viewports (ClemMakesApps) - Don't include archived projects when creating group milestones. !4940 (Jeroen Jacobs) diff --git a/app/models/merge_request_diff.rb b/app/models/merge_request_diff.rb index 36b8b70870b..3f7e96186a1 100644 --- a/app/models/merge_request_diff.rb +++ b/app/models/merge_request_diff.rb @@ -6,6 +6,9 @@ class MergeRequestDiff < ActiveRecord::Base # Prevent store of diff if commits amount more then 500 COMMITS_SAFE_SIZE = 100 + # Valid types of serialized diffs allowed by Gitlab::Git::Diff + VALID_CLASSES = [Hash, Rugged::Patch, Rugged::Diff::Delta] + belongs_to :merge_request state_machine :state, initial: :empty do @@ -170,6 +173,15 @@ class MergeRequestDiff < ActiveRecord::Base private + # Old GitLab implementations may have generated diffs as ["--broken-diff"]. + # Avoid an error 500 by ignoring bad elements. See: + # https://gitlab.com/gitlab-org/gitlab-ce/issues/20776 + def valid_raw_diff?(raw) + return false unless raw.respond_to?(:each) + + raw.any? { |element| VALID_CLASSES.include?(element.class) } + end + def dump_commits(commits) commits.map(&:to_hash) end @@ -200,7 +212,7 @@ class MergeRequestDiff < ActiveRecord::Base end def load_diffs(raw, options) - if raw.respond_to?(:each) + if valid_raw_diff?(raw) if paths = options[:paths] raw = raw.select do |diff| paths.include?(diff[:old_path]) || paths.include?(diff[:new_path]) diff --git a/doc/ci/pipelines.md b/doc/ci/pipelines.md index ca9b986a060..729c1dc8c0d 100644 --- a/doc/ci/pipelines.md +++ b/doc/ci/pipelines.md @@ -31,6 +31,8 @@ project. ## Seeing build status Clicking on a pipeline will show the builds that were run for that pipeline. +Clicking on an individual build will show you its build trace, and allow you to +cancel the build, retry it, or erase the build trace. ## Badges diff --git a/doc/ci/variables/README.md b/doc/ci/variables/README.md index 22d67bd9964..a4c3a731a20 100644 --- a/doc/ci/variables/README.md +++ b/doc/ci/variables/README.md @@ -48,6 +48,7 @@ The `API_TOKEN` will take the Secure Variable value: `SECURE`. | **CI_RUNNER_ID** | 8.10 | 0.5 | The unique id of runner being used | | **CI_RUNNER_DESCRIPTION** | 8.10 | 0.5 | The description of the runner as saved in GitLab | | **CI_RUNNER_TAGS** | 8.10 | 0.5 | The defined runner tags | +| **CI_DEBUG_TRACE** | all | 1.7 | Whether [debug tracing](#debug-tracing) is enabled | | **GITLAB_USER_ID** | 8.12 | all | The id of the user who started the build | | **GITLAB_USER_EMAIL** | 8.12 | all | The email of the user who started the build | @@ -105,6 +106,39 @@ Variables can be defined at a global level, but also at a job level. More information about Docker integration can be found in [Using Docker Images](../docker/using_docker_images.md). +#### Debug tracing + +> **WARNING:** Enabling debug tracing can have severe security implications. The + output **will** contain the content of all your secure variables and any other + secrets! The output **will** be uploaded to the GitLab server and made visible + in build traces! + +By default, GitLab Runner hides most of the details of what it is doing when +processing a job. This behaviour keeps build traces short, and prevents secrets +from being leaked into the trace unless your script writes them to the screen. + +If a job isn't working as expected, this can make the problem difficult to +investigate; in these cases, you can enable debug tracing in `.gitlab-ci.yml`. +Available on GitLab Runner v1.7+, this feature enables the shell's execution +trace, resulting in a verbose build trace listing all commands that were run, +variables that were set, etc. + +Before enabling this, you should ensure builds are visible to +[team members only](../../../user/permissions.md#project-features). You should +also [erase](../pipelines.md#seeing-build-traces) all generated build traces +before making them visible again. + +To enable debug traces, set the `CI_DEBUG_TRACE` variable to `true`: + +```yaml +job1: + variables: + CI_DEBUG_TRACE: "true" +``` + +The [example project](https://gitlab.com/gitlab-examples/ci-debug-trace) +demonstrates a working configuration, including build trace examples. + ### User-defined variables (Secure Variables) **This feature requires GitLab Runner 0.4.0 or higher** diff --git a/lib/banzai/filter/emoji_filter.rb b/lib/banzai/filter/emoji_filter.rb index 2492b5213ac..23ae6dfc79a 100644 --- a/lib/banzai/filter/emoji_filter.rb +++ b/lib/banzai/filter/emoji_filter.rb @@ -1,6 +1,6 @@ module Banzai module Filter - # HTML filter that replaces :emoji: with images. + # HTML filter that replaces :emoji: and unicode with images. # # Based on HTML::Pipeline::EmojiFilter # @@ -13,14 +13,14 @@ module Banzai def call search_text_nodes(doc).each do |node| content = node.to_html - next unless content.include?(':') next if has_ancestor?(node, IGNORED_ANCESTOR_TAGS) + if content.include?(':') || node.text.match(emoji_unicode_pattern) + html = emoji_name_image_filter(content) + html = emoji_unicode_image_filter(html) + next if html == content + node.replace(html) + end - html = emoji_image_filter(content) - - next if html == content - - node.replace(html) end doc @@ -31,18 +31,34 @@ module Banzai # text - String text to replace :emoji: in. # # Returns a String with :emoji: replaced with images. - def emoji_image_filter(text) + def emoji_name_image_filter(text) text.gsub(emoji_pattern) do |match| name = $1 ":#{name}:" end end + + # Replace unicode emojis with corresponding images if they exist. + # + # text - String text to replace unicode emojis in. + # + # Returns a String with unicode emojis replaced with images. + + def emoji_unicode_image_filter(text) + text.gsub(emoji_unicode_pattern) do |moji| + ":#{Gitlab::Emoji.emojis_by_moji[moji][" + end + end # Build a regexp that matches all valid :emoji: names. def self.emoji_pattern @emoji_pattern ||= /:(#{Gitlab::Emoji.emojis_names.map { |name| Regexp.escape(name) }.join('|')}):/ end + # Build a regexp that matches all valid unicode emojis names. + def self.emoji_unicode_pattern + @emoji_unicode_pattern ||= /(#{Gitlab::Emoji.emojis_unicodes.map { |moji| Regexp.escape(moji) }.join('|')})/ + end private def emoji_url(name) @@ -60,6 +76,21 @@ module Banzai end end + def emoji_unicode_url(moji) + emoji_unicode_path = emoji_unicode_filename(moji) + + if context[:asset_host] + # Asset host is specified. + url_to_image(emoji_unicode_path) + elsif context[:asset_root] + # Gitlab url is specified + File.join(context[:asset_root], url_to_image(emoji_unicode_path)) + else + # All other cases + url_to_image(emoji_unicode_path) + end + end + def url_to_image(image) ActionController::Base.helpers.url_to_image(image) end @@ -71,6 +102,13 @@ module Banzai def emoji_filename(name) "#{Gitlab::Emoji.emoji_filename(name)}.png" end + def emoji_unicode_pattern + self.class.emoji_unicode_pattern + end + + def emoji_unicode_filename(name) + "#{Gitlab::Emoji.emoji_unicode_filename(name)}.png" + end end end end diff --git a/lib/gitlab/emoji.rb b/lib/gitlab/emoji.rb index b63213ae208..803b0c2d057 100644 --- a/lib/gitlab/emoji.rb +++ b/lib/gitlab/emoji.rb @@ -9,7 +9,9 @@ module Gitlab def emojis_by_moji Gemojione.index.instance_variable_get(:@emoji_by_moji) end - + def emojis_unicodes + emojis_by_moji.keys.sort + end def emojis_names emojis.keys.sort end @@ -17,5 +19,8 @@ module Gitlab def emoji_filename(name) emojis[name]["unicode"] end + def emoji_unicode_filename(moji) + emojis_by_moji[moji]["unicode"] + end end end diff --git a/spec/lib/banzai/filter/emoji_filter_spec.rb b/spec/lib/banzai/filter/emoji_filter_spec.rb index b5b38cf0c8c..475160bb5ec 100644 --- a/spec/lib/banzai/filter/emoji_filter_spec.rb +++ b/spec/lib/banzai/filter/emoji_filter_spec.rb @@ -12,11 +12,14 @@ describe Banzai::Filter::EmojiFilter, lib: true do ActionController::Base.asset_host = @original_asset_host end - it 'replaces supported emoji' do + it 'replaces supported name emoji' do doc = filter('

:heart:

') expect(doc.css('img').first.attr('src')).to eq 'https://foo.com/assets/2764.png' end - + it 'replaces supported unicode emoji' do + doc = filter('

❤️

') + expect(doc.css('img').first.attr('src')).to eq 'https://foo.com/assets/2764.png' + end it 'ignores unsupported emoji' do exp = act = '

:foo:

' doc = filter(act) @@ -28,46 +31,96 @@ describe Banzai::Filter::EmojiFilter, lib: true do expect(doc.css('img').first.attr('src')).to eq 'https://foo.com/assets/1F44D.png' end + it 'correctly encodes unicode to the URL' do + doc = filter('

👍

') + expect(doc.css('img').first.attr('src')).to eq 'https://foo.com/assets/1F44D.png' + end + it 'matches at the start of a string' do doc = filter(':+1:') expect(doc.css('img').size).to eq 1 end + it 'unicode matches at the start of a string' do + doc = filter("'👍'") + expect(doc.css('img').size).to eq 1 + end + it 'matches at the end of a string' do doc = filter('This gets a :-1:') expect(doc.css('img').size).to eq 1 end + it 'unicode matches at the end of a string' do + doc = filter('This gets a 👍') + expect(doc.css('img').size).to eq 1 + end + it 'matches with adjacent text' do doc = filter('+1 (:+1:)') expect(doc.css('img').size).to eq 1 end + it 'unicode matches with adjacent text' do + doc = filter('+1 (👍)') + expect(doc.css('img').size).to eq 1 + end + it 'matches multiple emoji in a row' do doc = filter(':see_no_evil::hear_no_evil::speak_no_evil:') expect(doc.css('img').size).to eq 3 end + it 'unicode matches multiple emoji in a row' do + doc = filter("'🙈🙉🙊'") + expect(doc.css('img').size).to eq 3 + end + + it 'mixed matches multiple emoji in a row' do + doc = filter("'🙈:see_no_evil:🙉:hear_no_evil:🙊:speak_no_evil:'") + expect(doc.css('img').size).to eq 6 + end + it 'has a title attribute' do doc = filter(':-1:') expect(doc.css('img').first.attr('title')).to eq ':-1:' end + it 'unicode has a title attribute' do + doc = filter("'👎'") + expect(doc.css('img').first.attr('title')).to eq ':thumbsdown:' + end + it 'has an alt attribute' do doc = filter(':-1:') expect(doc.css('img').first.attr('alt')).to eq ':-1:' end + it 'unicode has an alt attribute' do + doc = filter("'👎'") + expect(doc.css('img').first.attr('alt')).to eq ':thumbsdown:' + end + it 'has an align attribute' do doc = filter(':8ball:') expect(doc.css('img').first.attr('align')).to eq 'absmiddle' end + it 'unicode has an align attribute' do + doc = filter("'🎱'") + expect(doc.css('img').first.attr('align')).to eq 'absmiddle' + end + it 'has an emoji class' do doc = filter(':cat:') expect(doc.css('img').first.attr('class')).to eq 'emoji' end + it 'unicode has an emoji class' do + doc = filter("'🐱'") + expect(doc.css('img').first.attr('class')).to eq 'emoji' + end + it 'has height and width attributes' do doc = filter(':dog:') img = doc.css('img').first @@ -76,12 +129,26 @@ describe Banzai::Filter::EmojiFilter, lib: true do expect(img.attr('height')).to eq '20' end + it 'unicode has height and width attributes' do + doc = filter("'🐶'") + img = doc.css('img').first + + expect(img.attr('width')).to eq '20' + expect(img.attr('height')).to eq '20' + end + it 'keeps whitespace intact' do doc = filter('This deserves a :+1:, big time.') expect(doc.to_html).to match(/^This deserves a , big time\.\z/) end + it 'unicode keeps whitespace intact' do + doc = filter('This deserves a 🎱, big time.') + + expect(doc.to_html).to match(/^This deserves a , big time\.\z/) + end + it 'uses a custom asset_root context' do root = Gitlab.config.gitlab.url + 'gitlab/root' diff --git a/spec/models/merge_request_diff_spec.rb b/spec/models/merge_request_diff_spec.rb index 530a7def553..96f1f60dbc0 100644 --- a/spec/models/merge_request_diff_spec.rb +++ b/spec/models/merge_request_diff_spec.rb @@ -44,6 +44,16 @@ describe MergeRequestDiff, models: true do end end + context 'when the raw diffs have invalid content' do + before { mr_diff.update_attributes(st_diffs: ["--broken-diff"]) } + + it 'returns an empty DiffCollection' do + expect(mr_diff.raw_diffs.to_a).to be_empty + expect(mr_diff.raw_diffs).to be_a(Gitlab::Git::DiffCollection) + expect(mr_diff.raw_diffs).to be_empty + end + end + context 'when the raw diffs exist' do it 'returns the diffs' do expect(mr_diff.raw_diffs).to be_a(Gitlab::Git::DiffCollection)