Merge branch 'import-gh-wiki' into 'master'

Import GitHub wiki into GitLab

Closes #2834 

GitHub doesn’t apply any constraints to theirs wiki slug allowing chars like ,, :, *, etc, we need to remove our constraints or some wiki pages will not be available after they are imported.

Some wikis use the Gollum's tags to link its internal/external resources like: images, urls, wiki pages, etc. So, to avoid that wiki links/images displays completely broken after they were imported, we added a new `WikiPipeline`, that for now will parse only simple links, and image tags.

##### Before `WikiPipeline`:

![Screenshot_2016-01-11_20.14.48](/uploads/46fd5dbb5acfc70aa8ecca3050b675e4/Screenshot_2016-01-11_20.14.48.png)

##### After `WikiPipeline`:

![Screenshot_2016-01-11_20.15.56](/uploads/b1d94aa852f385f867a7868c358b4257/Screenshot_2016-01-11_20.15.56.png)

See merge request !2324

CHANGELOG

@@ -50,6 +50,7 @@ v 8.4.0 (unreleased)
   - Fix Encoding::CompatibilityError bug when markdown content has some complex URL (Jason Lee)
   - Add API support for managing build variables of project
   - Allow broadcast messages to be edited
+  - Import GitHub wiki into GitLab
 
 v 8.3.4
   - Use gitlab-workhorse 0.5.4 (fixes API routing bug)

app/assets/javascripts/wikis.js.coffee

@@ -1,17 +1,18 @@
+#= require latinise
+
 class @Wikis
   constructor: ->
-    $('.build-new-wiki').bind "click", (e) ->
-      $('[data-error~=slug]').addClass("hidden")
-      $('p.hint').show()
+    $('.build-new-wiki').bind 'click', (e) =>
+      $('[data-error~=slug]').addClass('hidden')
       field = $('#new_wiki_path')
-      valid_slug_pattern = /^[\w\/-]+$/
+      slug = @slugify(field.val())
 
-      slug = field.val()
-      if slug.match valid_slug_pattern
+      if (slug.length > 0)
         path = field.attr('data-wikis-path')
-        if(slug.length > 0)
-          location.href = path + "/" + slug
-      else
-        e.preventDefault()
-        $('p.hint').hide()
-        $('[data-error~=slug]').removeClass("hidden")
+        location.href = path + '/' + slug
+
+  dasherize: (value) ->
+    value.replace(/[_\s]+/g, '-')
+
+  slugify: (value) =>
+    @dasherize(value.trim().toLowerCase().latinise())

app/helpers/gitlab_markdown_helper.rb

@@ -91,7 +91,7 @@ module GitlabMarkdownHelper
   def render_wiki_content(wiki_page)
     case wiki_page.format
     when :markdown
-      markdown(wiki_page.content)
+      markdown(wiki_page.content, pipeline: :wiki, project_wiki: @project_wiki)
     when :asciidoc
       asciidoc(wiki_page.content)
     else

app/models/project_wiki.rb

@@ -38,6 +38,10 @@ class ProjectWiki
     [Gitlab.config.gitlab.url, "/", path_with_namespace, ".git"].join('')
   end
 
+  def wiki_base_path
+    ["/", @project.path_with_namespace, "/wikis"].join('')
+  end
+
   # Returns the Gollum::Wiki object.
   def wiki
     @wiki ||= begin

app/models/wiki_page.rb

@@ -169,7 +169,7 @@ class WikiPage
   private
 
   def set_attributes
-    attributes[:slug] = @page.escaped_url_path
+    attributes[:slug] = @page.url_path
     attributes[:title] = @page.title
     attributes[:format] = @page.format
   end

app/views/projects/wikis/_new.html.haml

@@ -5,12 +5,9 @@
         %a.close{href: "#", "data-dismiss" => "modal"} ×
         %h3.page-title New Wiki Page
       .modal-body
-        = label_tag :new_wiki_path do
-          %span Page slug
-        = text_field_tag :new_wiki_path, nil, placeholder: 'how-to-setup', class: 'form-control', required: true, :'data-wikis-path' => namespace_project_wikis_path(@project.namespace, @project)
-        %p.hidden.text-danger{data: { error: "slug" }}
-          The page slug is invalid. Please don't use characters other then: a-z 0-9 _ - and /
-        %p.hint
-          Please don't use spaces.
+        .form-group
+          = label_tag :new_wiki_path do
+            %span Page slug
+          = text_field_tag :new_wiki_path, nil, placeholder: 'how-to-setup', class: 'form-control', required: true, :'data-wikis-path' => namespace_project_wikis_path(@project.namespace, @project)
         .form-actions
           = link_to 'Create Page', '#', class: 'build-new-wiki btn btn-create'

config/routes.rb

@@ -519,7 +519,7 @@ Rails.application.routes.draw do
           end
         end
 
-        WIKI_SLUG_ID = { id: /[a-zA-Z.0-9_\-\/]+/ } unless defined? WIKI_SLUG_ID
+        WIKI_SLUG_ID = { id: /\S+/ } unless defined? WIKI_SLUG_ID
 
         scope do
           # Order matters to give priority to these matches

features/project/wiki.feature

@@ -69,11 +69,6 @@ Feature: Project Wiki
     And I click on the "Pages" button
     Then I should see non-escaped link in the pages list
 
-  @javascript
-  Scenario: Creating an invalid new page
-    Given I create a New page with an invalid name
-    Then I should see an error message
-
   @javascript
   Scenario: Edit Wiki page that has a path
     Given I create a New page with paths

features/steps/project/wiki.rb

@@ -132,16 +132,6 @@ class Spinach::Features::ProjectWiki < Spinach::FeatureSteps
     expect(current_path).to include 'one/two/three'
   end
 
-  step 'I create a New page with an invalid name' do
-    click_on 'New Page'
-    fill_in 'Page slug', with: 'invalid name'
-    click_on 'Create Page'
-  end
-
-  step 'I should see an error message' do
-    expect(page).to have_content "The page slug is invalid"
-  end
-
   step 'I should see non-escaped link in the pages list' do
     expect(page).to have_xpath("//a[@href='/#{project.path_with_namespace}/wikis/one/two/three']")
   end

lib/banzai/filter/gollum_tags_filter.rb

@@ -0,0 +1,151 @@
+require 'banzai'
+require 'html/pipeline/filter'
+
+module Banzai
+  module Filter
+    # HTML Filter for parsing Gollum's tags in HTML. It's only parses the
+    # following tags:
+    #
+    # - Link to internal pages:
+    #
+    #   * [[Bug Reports]]
+    #   * [[How to Contribute|Contributing]]
+    #
+    # - Link to external resources:
+    #
+    #   * [[http://en.wikipedia.org/wiki/Git_(software)]]
+    #   * [[Git|http://en.wikipedia.org/wiki/Git_(software)]]
+    #
+    # - Link internal images, the special attributes will be ignored:
+    #
+    #   * [[images/logo.png]]
+    #   * [[images/logo.png|alt=Logo]]
+    #
+    # - Link external images, the special attributes will be ignored:
+    #
+    #   * [[http://example.com/images/logo.png]]
+    #   * [[http://example.com/images/logo.png|alt=Logo]]
+    #
+    # Based on Gollum::Filter::Tags
+    #
+    # Context options:
+    #   :project_wiki (required) - Current project wiki.
+    #
+    class GollumTagsFilter < HTML::Pipeline::Filter
+      include ActionView::Helpers::TagHelper
+
+      # Pattern to match tags content that should be parsed in HTML.
+      #
+      # Gollum's tags have been made to resemble the tags of other markups,
+      # especially MediaWiki. The basic syntax is:
+      #
+      # [[tag]]
+      #
+      # Some tags will accept attributes which are separated by pipe
+      # symbols.Some attributes must precede the tag and some must follow it:
+      #
+      # [[prefix-attribute|tag]]
+      # [[tag|suffix-attribute]]
+      #
+      # See https://github.com/gollum/gollum/wiki
+      #
+      # Rubular: http://rubular.com/r/7dQnE5CUCH
+      TAGS_PATTERN = %r{\[\[(.+?)\]\]}
+
+      # Pattern to match allowed image extensions
+      ALLOWED_IMAGE_EXTENSIONS = %r{.+(jpg|png|gif|svg|bmp)\z}i
+
+      def call
+        search_text_nodes(doc).each do |node|
+          content = node.content
+
+          next unless content.match(TAGS_PATTERN)
+
+          html = process_tag($1)
+
+          if html && html != node.content
+            node.replace(html)
+          end
+        end
+
+        doc
+      end
+
+      private
+
+      # Process a single tag into its final HTML form.
+      #
+      # tag - The String tag contents (the stuff inside the double brackets).
+      #
+      # Returns the String HTML version of the tag.
+      def process_tag(tag)
+        parts = tag.split('|')
+
+        return if parts.size.zero?
+
+        process_image_tag(parts) || process_page_link_tag(parts)
+      end
+
+      # Attempt to process the tag as an image tag.
+      #
+      # tag - The String tag contents (the stuff inside the double brackets).
+      #
+      # Returns the String HTML if the tag is a valid image tag or nil
+      # if it is not.
+      def process_image_tag(parts)
+        content = parts[0].strip
+
+        return unless image?(content)
+
+        if url?(content)
+          path = content
+        elsif file = project_wiki.find_file(content)
+          path = ::File.join project_wiki_base_path, file.path
+        end
+
+        if path
+          content_tag(:img, nil, src: path)
+        end
+      end
+
+      def image?(path)
+        path =~ ALLOWED_IMAGE_EXTENSIONS
+      end
+
+      def url?(path)
+        path.start_with?(*%w(http https))
+      end
+
+      # Attempt to process the tag as a page link tag.
+      #
+      # tag - The String tag contents (the stuff inside the double brackets).
+      #
+      # Returns the String HTML if the tag is a valid page link tag or nil
+      # if it is not.
+      def process_page_link_tag(parts)
+        if parts.size == 1
+          url = parts[0].strip
+        else
+          name, url = *parts.compact.map(&:strip)
+        end
+
+        content_tag(:a, name || url, href: url)
+      end
+
+      def project_wiki
+        context[:project_wiki]
+      end
+
+      def project_wiki_base_path
+        project_wiki && project_wiki.wiki_base_path
+      end
+
+      # Ensure that a :project_wiki key exists in context
+      #
+      # Note that while the key might exist, its value could be nil!
+      def validate
+        needs :project_wiki
+      end
+    end
+  end
+end

lib/banzai/pipeline/wiki_pipeline.rb

@@ -0,0 +1,11 @@
+require 'banzai'
+
+module Banzai
+  module Pipeline
+    class WikiPipeline < FullPipeline
+      def self.filters
+        super.insert(1, Filter::GollumTagsFilter)
+      end
+    end
+  end
+end

lib/gitlab/github_import/importer.rb

@@ -1,6 +1,8 @@
 module Gitlab
   module GithubImport
     class Importer
+      include Gitlab::ShellAdapter
+
       attr_reader :project, :client
 
       def initialize(project)
@@ -12,10 +14,7 @@ module Gitlab
       end
 
       def execute
-        import_issues
-        import_pull_requests
-
-        true
+        import_issues && import_pull_requests && import_wiki
       end
 
       private
@@ -34,6 +33,10 @@ module Gitlab
             end
           end
         end
+
+        true
+      rescue ActiveRecord::RecordInvalid
+        false
       end
 
       def import_pull_requests
@@ -48,6 +51,10 @@ module Gitlab
             import_comments_on_diff(pull_request.number, merge_request)
           end
         end
+
+        true
+      rescue ActiveRecord::RecordInvalid
+        false
       end
 
       def import_comments(issue_number, noteable)
@@ -66,6 +73,18 @@ module Gitlab
           noteable.notes.create!(comment.attributes)
         end
       end
+
+      def import_wiki
+        unless project.wiki_enabled?
+          wiki = WikiFormatter.new(project)
+          gitlab_shell.import_repository(wiki.path_with_namespace, wiki.import_url)
+          project.update_attribute(:wiki_enabled, true)
+        end
+
+        true
+      rescue Gitlab::Shell::Error
+        false
+      end
     end
   end
 end

lib/gitlab/github_import/project_creator.rb

@@ -20,7 +20,8 @@ module Gitlab
           visibility_level: repo.private ? Gitlab::VisibilityLevel::PRIVATE : Gitlab::VisibilityLevel::PUBLIC,
           import_type: "github",
           import_source: repo.full_name,
-          import_url: repo.clone_url.sub("https://", "https://#{@session_data[:github_access_token]}@")
+          import_url: repo.clone_url.sub("https://", "https://#{@session_data[:github_access_token]}@"),
+          wiki_enabled: !repo.has_wiki? # If repo has wiki we'll import it later
         ).execute
 
         project.create_import_data(data: { "github_session" => session_data } )

lib/gitlab/github_import/wiki_formatter.rb

@@ -0,0 +1,19 @@
+module Gitlab
+  module GithubImport
+    class WikiFormatter
+      attr_reader :project
+
+      def initialize(project)
+        @project = project
+      end
+
+      def path_with_namespace
+        "#{project.path_with_namespace}.wiki"
+      end
+
+      def import_url
+        project.import_url.sub(/\.git\z/, ".wiki.git")
+      end
+    end
+  end
+end

spec/features/markdown_spec.rb

@@ -175,13 +175,15 @@ describe 'GitLab Markdown', feature: true do
     end
   end
 
+  before(:all) do
+    @feat = MarkdownFeature.new
+
+    # `markdown` helper expects a `@project` variable
+    @project = @feat.project
+  end
+
   context 'default pipeline' do
     before(:all) do
-      @feat = MarkdownFeature.new
-
-      # `markdown` helper expects a `@project` variable
-      @project = @feat.project
-
       @html = markdown(@feat.raw_markdown)
     end
 
@@ -221,6 +223,57 @@ describe 'GitLab Markdown', feature: true do
     end
   end
 
+  context 'wiki pipeline' do
+    before do
+      @project_wiki = @feat.project_wiki
+
+      file = Gollum::File.new(@project_wiki.wiki)
+      expect(file).to receive(:path).and_return('images/example.jpg')
+      expect(@project_wiki).to receive(:find_file).with('images/example.jpg').and_return(file)
+
+      @html = markdown(@feat.raw_markdown, { pipeline: :wiki, project_wiki: @project_wiki })
+    end
+
+    it_behaves_like 'all pipelines'
+
+    it 'includes RelativeLinkFilter' do
+      expect(doc).not_to parse_relative_links
+    end
+
+    it 'includes EmojiFilter' do
+      expect(doc).to parse_emoji
+    end
+
+    it 'includes TableOfContentsFilter' do
+      expect(doc).to create_header_links
+    end
+
+    it 'includes AutolinkFilter' do
+      expect(doc).to create_autolinks
+    end
+
+    it 'includes all reference filters' do
+      aggregate_failures do
+        expect(doc).to reference_users
+        expect(doc).to reference_issues
+        expect(doc).to reference_merge_requests
+        expect(doc).to reference_snippets
+        expect(doc).to reference_commit_ranges
+        expect(doc).to reference_commits
+        expect(doc).to reference_labels
+        expect(doc).to reference_milestones
+      end
+    end
+
+    it 'includes TaskListFilter' do
+      expect(doc).to parse_task_lists
+    end
+
+    it 'includes GollumTagsFilter' do
+      expect(doc).to parse_gollum_tags
+    end
+  end
+
   # Fake a `current_user` helper
   def current_user
     @feat.user

spec/fixtures/markdown.md.erb

@@ -230,3 +230,12 @@ References should be parseable even inside _<%= merge_request.to_reference %>_ e
   - [ ] Incomplete sub-task 2
   - [x] Complete sub-task 1
 - [X] Complete task 2
+
+#### Gollum Tags
+
+- [[linked-resource]]
+- [[link-text|linked-resource]]
+- [[http://example.com]]
+- [[link-text|http://example.com/pdfs/gollum.pdf]]
+- [[images/example.jpg]]
+- [[http://example.com/images/example.jpg]]

spec/helpers/gitlab_markdown_helper_spec.rb

@@ -121,12 +121,13 @@ describe GitlabMarkdownHelper do
     before do
       @wiki = double('WikiPage')
       allow(@wiki).to receive(:content).and_return('wiki content')
+      helper.instance_variable_set(:@project_wiki, @wiki)
     end
 
-    it "should use GitLab Flavored Markdown for markdown files" do
+    it "should use Wiki pipeline for markdown files" do
       allow(@wiki).to receive(:format).and_return(:markdown)
 
-      expect(helper).to receive(:markdown).with('wiki content')
+      expect(helper).to receive(:markdown).with('wiki content', pipeline: :wiki, project_wiki: @wiki)
 
       helper.render_wiki_content(@wiki)
     end

spec/lib/banzai/filter/gollum_tags_filter_spec.rb

@@ -0,0 +1,89 @@
+require 'spec_helper'
+
+describe Banzai::Filter::GollumTagsFilter, lib: true do
+  include FilterSpecHelper
+
+  let(:project) { create(:project) }
+  let(:user) { double }
+  let(:project_wiki) { ProjectWiki.new(project, user) }
+
+  describe 'validation' do
+    it 'ensure that a :project_wiki key exists in context' do
+      expect { filter("See [[images/image.jpg]]", {}) }.to raise_error ArgumentError, "Missing context keys for Banzai::Filter::GollumTagsFilter: :project_wiki"
+    end
+  end
+
+  context 'linking internal images' do
+    it 'creates img tag if image exists' do
+      file = Gollum::File.new(project_wiki.wiki)
+      expect(file).to receive(:path).and_return('images/image.jpg')
+      expect(project_wiki).to receive(:find_file).with('images/image.jpg').and_return(file)
+
+      tag = '[[images/image.jpg]]'
+      doc = filter("See #{tag}", project_wiki: project_wiki)
+
+      expect(doc.at_css('img')['src']).to eq "#{project_wiki.wiki_base_path}/images/image.jpg"
+    end
+
+    it 'does not creates img tag if image does not exist' do
+      expect(project_wiki).to receive(:find_file).with('images/image.jpg').and_return(nil)
+
+      tag = '[[images/image.jpg]]'
+      doc = filter("See #{tag}", project_wiki: project_wiki)
+
+      expect(doc.css('img').size).to eq 0
+    end
+  end
+
+  context 'linking external images' do
+    it 'creates img tag for valid URL' do
+      tag = '[[http://example.com/image.jpg]]'
+      doc = filter("See #{tag}", project_wiki: project_wiki)
+
+      expect(doc.at_css('img')['src']).to eq "http://example.com/image.jpg"
+    end
+
+    it 'does not creates img tag for invalid URL' do
+      tag = '[[http://example.com/image.pdf]]'
+      doc = filter("See #{tag}", project_wiki: project_wiki)
+
+      expect(doc.css('img').size).to eq 0
+    end
+  end
+
+  context 'linking external resources' do
+    it "the created link's text will be equal to the resource's text" do
+      tag = '[[http://example.com]]'
+      doc = filter("See #{tag}", project_wiki: project_wiki)
+
+      expect(doc.at_css('a').text).to eq 'http://example.com'
+      expect(doc.at_css('a')['href']).to eq 'http://example.com'
+    end
+
+    it "the created link's text will be link-text" do
+      tag = '[[link-text|http://example.com/pdfs/gollum.pdf]]'
+      doc = filter("See #{tag}", project_wiki: project_wiki)
+
+      expect(doc.at_css('a').text).to eq 'link-text'
+      expect(doc.at_css('a')['href']).to eq 'http://example.com/pdfs/gollum.pdf'
+    end
+  end
+
+  context 'linking internal resources' do
+    it "the created link's text will be equal to the resource's text" do
+      tag = '[[wiki-slug]]'
+      doc = filter("See #{tag}", project_wiki: project_wiki)
+
+      expect(doc.at_css('a').text).to eq 'wiki-slug'
+      expect(doc.at_css('a')['href']).to eq 'wiki-slug'
+    end
+
+    it "the created link's text will be link-text" do
+      tag = '[[link-text|wiki-slug]]'
+      doc = filter("See #{tag}", project_wiki: project_wiki)
+
+      expect(doc.at_css('a').text).to eq 'link-text'
+      expect(doc.at_css('a')['href']).to eq 'wiki-slug'
+    end
+  end
+end

spec/lib/gitlab/github_import/wiki_formatter_spec.rb

@@ -0,0 +1,22 @@
+require 'spec_helper'
+
+describe Gitlab::GithubImport::WikiFormatter, lib: true do
+  let(:project) do
+    create(:project, namespace: create(:namespace, path: 'gitlabhq'),
+                     import_url: 'https://xxx@github.com/gitlabhq/sample.gitlabhq.git')
+  end
+
+  subject(:wiki) { described_class.new(project)}
+
+  describe '#path_with_namespace' do
+    it 'appends .wiki to project path' do
+      expect(wiki.path_with_namespace).to eq 'gitlabhq/gitlabhq.wiki'
+    end
+  end
+
+  describe '#import_url' do
+    it 'returns URL of the wiki repository' do
+      expect(wiki.import_url).to eq 'https://xxx@github.com/gitlabhq/sample.gitlabhq.wiki.git'
+    end
+  end
+end

spec/models/project_wiki_spec.rb

@@ -36,6 +36,13 @@ describe ProjectWiki, models: true do
     end
   end
 
+  describe "#wiki_base_path" do
+    it "returns the wiki base path" do
+      wiki_base_path = "/#{project.path_with_namespace}/wikis"
+      expect(subject.wiki_base_path).to eq(wiki_base_path)
+    end
+  end
+
   describe "#wiki" do
     it "contains a Gollum::Wiki instance" do
       expect(subject.wiki).to be_a Gollum::Wiki

spec/support/markdown_feature.rb

@@ -28,6 +28,10 @@ class MarkdownFeature
     end
   end
 
+  def project_wiki
+    @project_wiki ||= ProjectWiki.new(project, user)
+  end
+
   def issue
     @issue ||= create(:issue, project: project)
   end

spec/support/matchers/markdown_matchers.rb

@@ -66,6 +66,24 @@ module MarkdownMatchers
     end
   end
 
+  # GollumTagsFilter
+  matcher :parse_gollum_tags do
+    def have_image(src)
+      have_css("img[src$='#{src}']")
+    end
+
+    set_default_markdown_messages
+
+    match do |actual|
+      expect(actual).to have_link('linked-resource', href: 'linked-resource')
+      expect(actual).to have_link('link-text', href: 'linked-resource')
+      expect(actual).to have_link('http://example.com', href: 'http://example.com')
+      expect(actual).to have_link('link-text', href: 'http://example.com/pdfs/gollum.pdf')
+      expect(actual).to have_image('/gitlabhq/wikis/images/example.jpg')
+      expect(actual).to have_image('http://example.com/images/example.jpg')
+    end
+  end
+
   # UserReferenceFilter
   matcher :reference_users do
     set_default_markdown_messages