From a8281ac43424e4b820286823bdb48f068b21d7d3 Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Tue, 11 Jan 2022 15:15:55 +0000 Subject: [PATCH] Add latest changes from gitlab-org/gitlab@master --- .gitlab/ci/rails.gitlab-ci.yml | 2 +- .../behaviors/markdown/render_gfm.js | 7 +- .../markdown/render_sandboxed_mermaid.js | 233 ++++++++++++++++++ app/assets/javascripts/lib/mermaid.js | 61 +++++ .../pipelines/components/header_component.vue | 4 +- .../components/crm_contacts/crm_contacts.vue | 15 +- app/assets/stylesheets/utilities.scss | 10 - .../oauth/token_info_controller.rb | 2 +- app/controllers/sandbox_controller.rb | 11 + .../clusters/agent_tokens_resolver.rb | 9 +- .../resolvers/concerns/resolves_pipelines.rb | 4 +- .../types/clusters/agent_token_status_enum.rb | 14 ++ .../types/clusters/agent_token_type.rb | 2 +- app/models/clusters/agent_token.rb | 1 + app/models/namespace_setting.rb | 15 -- app/views/sandbox/mermaid.html.erb | 9 + ...t_view_scans.yml => sandboxed_mermaid.yml} | 12 +- .../counts_28d/20210216175109_suggestions.yml | 3 +- .../counts_all/20210216175053_suggestions.yml | 3 +- config/routes.rb | 3 + config/webpack.config.js | 1 + doc/api/graphql/reference/index.md | 42 +++- doc/raketasks/backup_restore.md | 3 +- doc/user/admin_area/license.md | 2 +- lib/backup/manager.rb | 2 +- lib/backup/packages.rb | 13 + .../Jobs/Secret-Detection.gitlab-ci.yml | 14 +- .../content_security_policy/config_loader.rb | 2 +- lib/gitlab/gon_helper.rb | 1 + lib/gitlab/usage_data.rb | 6 +- lib/tasks/gitlab/backup.rake | 32 ++- locale/gitlab.pot | 6 +- package.json | 4 +- shared/packages/.gitkeep | 0 .../oauth/token_info_controller_spec.rb | 24 +- spec/factories/clusters/agent_tokens.rb | 4 + .../issues/user_comments_on_issue_spec.rb | 1 + spec/features/markdown/mermaid_spec.rb | 4 + .../markdown/sandboxed_mermaid_spec.rb | 32 +++ .../clusters/agent_tokens_resolver_spec.rb | 9 + .../concerns/resolves_pipelines_spec.rb | 20 +- .../clusters/agent_token_status_enum_spec.rb | 8 + spec/lib/backup/manager_spec.rb | 6 +- spec/lib/backup/object_backup_spec.rb | 36 +++ spec/lib/backup/terraform_state_spec.rb | 27 -- .../config_loader_spec.rb | 6 +- spec/lib/gitlab/usage_data_spec.rb | 7 +- spec/models/ci/runner_spec.rb | 2 +- spec/models/clusters/agent_token_spec.rb | 16 +- spec/models/namespace_setting_spec.rb | 53 ---- spec/requests/sandbox_controller_spec.rb | 14 ++ spec/routing/routing_spec.rb | 6 + .../cross-database-modification-allowlist.yml | 1 - spec/support/helpers/test_env.rb | 5 + spec/tasks/gitlab/backup_rake_spec.rb | 20 +- yarn.lock | 16 +- 56 files changed, 642 insertions(+), 223 deletions(-) create mode 100644 app/assets/javascripts/behaviors/markdown/render_sandboxed_mermaid.js create mode 100644 app/assets/javascripts/lib/mermaid.js create mode 100644 app/controllers/sandbox_controller.rb create mode 100644 app/graphql/types/clusters/agent_token_status_enum.rb create mode 100644 app/views/sandbox/mermaid.html.erb rename config/feature_flags/development/{dast_view_scans.yml => sandboxed_mermaid.yml} (55%) create mode 100644 lib/backup/packages.rb create mode 100644 shared/packages/.gitkeep create mode 100644 spec/features/markdown/sandboxed_mermaid_spec.rb create mode 100644 spec/graphql/types/clusters/agent_token_status_enum_spec.rb create mode 100644 spec/lib/backup/object_backup_spec.rb delete mode 100644 spec/lib/backup/terraform_state_spec.rb create mode 100644 spec/requests/sandbox_controller_spec.rb diff --git a/.gitlab/ci/rails.gitlab-ci.yml b/.gitlab/ci/rails.gitlab-ci.yml index 830c6784a5f..89e9d3bc8b6 100644 --- a/.gitlab/ci/rails.gitlab-ci.yml +++ b/.gitlab/ci/rails.gitlab-ci.yml @@ -463,7 +463,7 @@ db:backup_and_restore: script: - . scripts/prepare_build.sh - bundle exec rake db:drop db:create db:structure:load db:seed_fu - - mkdir -p tmp/tests/public/uploads tmp/tests/{artifacts,pages,lfs-objects,terraform_state,registry} + - mkdir -p tmp/tests/public/uploads tmp/tests/{artifacts,pages,lfs-objects,terraform_state,registry,packages} - bundle exec rake gitlab:backup:create - date - bundle exec rake gitlab:backup:restore diff --git a/app/assets/javascripts/behaviors/markdown/render_gfm.js b/app/assets/javascripts/behaviors/markdown/render_gfm.js index 4698fcd4d42..c4e09efe263 100644 --- a/app/assets/javascripts/behaviors/markdown/render_gfm.js +++ b/app/assets/javascripts/behaviors/markdown/render_gfm.js @@ -4,6 +4,7 @@ import initUserPopovers from '../../user_popovers'; import highlightCurrentUser from './highlight_current_user'; import renderMath from './render_math'; import renderMermaid from './render_mermaid'; +import renderSandboxedMermaid from './render_sandboxed_mermaid'; import renderMetrics from './render_metrics'; // Render GitLab flavoured Markdown @@ -13,7 +14,11 @@ import renderMetrics from './render_metrics'; $.fn.renderGFM = function renderGFM() { syntaxHighlight(this.find('.js-syntax-highlight').get()); renderMath(this.find('.js-render-math')); - renderMermaid(this.find('.js-render-mermaid')); + if (gon.features?.sandboxedMermaid) { + renderSandboxedMermaid(this.find('.js-render-mermaid')); + } else { + renderMermaid(this.find('.js-render-mermaid')); + } highlightCurrentUser(this.find('.gfm-project_member').get()); initUserPopovers(this.find('.js-user-link').get()); diff --git a/app/assets/javascripts/behaviors/markdown/render_sandboxed_mermaid.js b/app/assets/javascripts/behaviors/markdown/render_sandboxed_mermaid.js new file mode 100644 index 00000000000..92cdd1c600f --- /dev/null +++ b/app/assets/javascripts/behaviors/markdown/render_sandboxed_mermaid.js @@ -0,0 +1,233 @@ +import $ from 'jquery'; +import { once, countBy } from 'lodash'; +import { __ } from '~/locale'; +import { + getBaseURL, + relativePathToAbsolute, + setUrlParams, + joinPaths, +} from '~/lib/utils/url_utility'; +import { darkModeEnabled } from '~/lib/utils/color_utils'; +import { setAttributes } from '~/lib/utils/dom_utils'; + +// Renders diagrams and flowcharts from text using Mermaid in any element with the +// `js-render-mermaid` class. +// +// Example markup: +// +// 
+//  graph TD;
+//    A-- > B;
+//    A-- > C;
+//    B-- > D;
+//    C-- > D;
+//
+// + +const SANDBOX_FRAME_PATH = '/-/sandbox/mermaid'; +// This is an arbitrary number; Can be iterated upon when suitable. +const MAX_CHAR_LIMIT = 2000; +// Max # of mermaid blocks that can be rendered in a page. +const MAX_MERMAID_BLOCK_LIMIT = 50; +// Max # of `&` allowed in Chaining of links syntax +const MAX_CHAINING_OF_LINKS_LIMIT = 30; +// Keep a map of mermaid blocks we've already rendered. +const elsProcessingMap = new WeakMap(); +let renderedMermaidBlocks = 0; + +// Pages without any restrictions on mermaid rendering +const PAGES_WITHOUT_RESTRICTIONS = [ + // Group wiki + 'groups:wikis:show', + 'groups:wikis:edit', + 'groups:wikis:create', + + // Project wiki + 'projects:wikis:show', + 'projects:wikis:edit', + 'projects:wikis:create', + + // Project files + 'projects:show', + 'projects:blob:show', +]; + +function shouldLazyLoadMermaidBlock(source) { + /** + * If source contains `&`, which means that it might + * contain Chaining of links a new syntax in Mermaid. + */ + if (countBy(source)['&'] > MAX_CHAINING_OF_LINKS_LIMIT) { + return true; + } + + return false; +} + +function fixElementSource(el) { + // Mermaid doesn't like `
` tags, so collapse all like tags into `
`, which is parsed correctly. + const source = el.textContent?.replace(//g, '
'); + + // Remove any extra spans added by the backend syntax highlighting. + Object.assign(el, { textContent: source }); + + return { source }; +} + +function getSandboxFrameSrc() { + const path = joinPaths(gon.relative_url_root || '', SANDBOX_FRAME_PATH); + if (!darkModeEnabled()) { + return path; + } + const absoluteUrl = relativePathToAbsolute(path, getBaseURL()); + return setUrlParams({ darkMode: darkModeEnabled() }, absoluteUrl); +} + +function renderMermaidEl(el, source) { + const iframeEl = document.createElement('iframe'); + setAttributes(iframeEl, { + src: getSandboxFrameSrc(), + sandbox: 'allow-scripts', + frameBorder: 0, + scrolling: 'no', + }); + + // Add the original source into the DOM + // to allow Copy-as-GFM to access it. + const sourceEl = document.createElement('text'); + sourceEl.textContent = source; + sourceEl.classList.add('gl-display-none'); + + const wrapper = document.createElement('div'); + wrapper.appendChild(iframeEl); + wrapper.appendChild(sourceEl); + + el.closest('pre').replaceWith(wrapper); + + // Event Listeners + iframeEl.addEventListener('load', () => { + // Potential risk associated with '*' discussed in below thread + // https://gitlab.com/gitlab-org/gitlab/-/merge_requests/74414#note_735183398 + iframeEl.contentWindow.postMessage(source, '*'); + }); + + window.addEventListener( + 'message', + (event) => { + if (event.origin !== 'null' || event.source !== iframeEl.contentWindow) { + return; + } + const { h, w } = event.data; + iframeEl.width = w; + iframeEl.height = h; + }, + false, + ); +} + +function renderMermaids($els) { + if (!$els.length) return; + + const pageName = document.querySelector('body').dataset.page; + + // A diagram may have been truncated in search results which will cause errors, so abort the render. + if (pageName === 'search:show') return; + + let renderedChars = 0; + + $els.each((i, el) => { + // Skipping all the elements which we've already queued in requestIdleCallback + if (elsProcessingMap.has(el)) { + return; + } + + const { source } = fixElementSource(el); + /** + * Restrict the rendering to a certain amount of character + * and mermaid blocks to prevent mermaidjs from hanging + * up the entire thread and causing a DoS. + */ + if ( + !PAGES_WITHOUT_RESTRICTIONS.includes(pageName) && + ((source && source.length > MAX_CHAR_LIMIT) || + renderedChars > MAX_CHAR_LIMIT || + renderedMermaidBlocks >= MAX_MERMAID_BLOCK_LIMIT || + shouldLazyLoadMermaidBlock(source)) + ) { + const html = ` + + `; + + const $parent = $(el).parent(); + + if (!$parent.hasClass('lazy-alert-shown')) { + $parent.after(html); + $parent + .siblings() + .find('.js-warning-text') + .text( + __('Warning: Displaying this diagram might cause performance issues on this page.'), + ); + $parent.addClass('lazy-alert-shown'); + } + + return; + } + + renderedChars += source.length; + renderedMermaidBlocks += 1; + + const requestId = window.requestIdleCallback(() => { + renderMermaidEl(el, source); + }); + + elsProcessingMap.set(el, requestId); + }); +} + +const hookLazyRenderMermaidEvent = once(() => { + $(document.body).on('click', '.js-lazy-render-mermaid', function eventHandler() { + const parent = $(this).closest('.js-lazy-render-mermaid-container'); + const pre = parent.prev(); + + const el = pre.find('.js-render-mermaid'); + + parent.remove(); + + // sandbox update + const element = el.get(0); + const { source } = fixElementSource(element); + + renderMermaidEl(element, source); + }); +}); + +export default function renderMermaid($els) { + if (!$els.length) return; + + const visibleMermaids = $els.filter(function filter() { + return $(this).closest('details').length === 0 && $(this).is(':visible'); + }); + + renderMermaids(visibleMermaids); + + $els.closest('details').one('toggle', function toggle() { + if (this.open) { + renderMermaids($(this).find('.js-render-mermaid')); + } + }); + + hookLazyRenderMermaidEvent(); +} diff --git a/app/assets/javascripts/lib/mermaid.js b/app/assets/javascripts/lib/mermaid.js new file mode 100644 index 00000000000..d621c9ddf9e --- /dev/null +++ b/app/assets/javascripts/lib/mermaid.js @@ -0,0 +1,61 @@ +import mermaid from 'mermaid'; +import { getParameterByName } from '~/lib/utils/url_utility'; + +const setIframeRenderedSize = (h, w) => { + const { origin } = window.location; + window.parent.postMessage({ h, w }, origin); +}; + +const drawDiagram = (source) => { + const element = document.getElementById('app'); + const insertSvg = (svgCode) => { + element.innerHTML = svgCode; + + const height = parseInt(element.firstElementChild.getAttribute('height'), 10); + const width = parseInt(element.firstElementChild.style.maxWidth, 10); + setIframeRenderedSize(height, width); + }; + mermaid.mermaidAPI.render('mermaid', source, insertSvg); +}; + +const darkModeEnabled = () => getParameterByName('darkMode') === 'true'; + +const initMermaid = () => { + let theme = 'neutral'; + + if (darkModeEnabled()) { + theme = 'dark'; + } + + mermaid.initialize({ + // mermaid core options + mermaid: { + startOnLoad: false, + }, + // mermaidAPI options + theme, + flowchart: { + useMaxWidth: true, + htmlLabels: true, + }, + secure: ['secure', 'securityLevel', 'startOnLoad', 'maxTextSize', 'htmlLabels'], + securityLevel: 'strict', + }); +}; + +const addListener = () => { + window.addEventListener( + 'message', + (event) => { + if (event.origin !== window.location.origin) { + return; + } + drawDiagram(event.data); + }, + false, + ); +}; + +addListener(); +initMermaid(); +export default {}; diff --git a/app/assets/javascripts/pipelines/components/header_component.vue b/app/assets/javascripts/pipelines/components/header_component.vue index 4db6a3c9fd8..8088858f381 100644 --- a/app/assets/javascripts/pipelines/components/header_component.vue +++ b/app/assets/javascripts/pipelines/components/header_component.vue @@ -212,7 +212,9 @@ export default {