From a9f7fd2c1a7052247333b89f6a22a883b480370d Mon Sep 17 00:00:00 2001 From: Valery Sizov Date: Wed, 31 Dec 2014 15:07:48 +0200 Subject: [PATCH] Github Importer --- Gemfile | 2 + Gemfile.lock | 6 ++ app/controllers/github_imports_controller.rb | 75 +++++++++++++++++++ .../omniauth_callbacks_controller.rb | 2 +- app/helpers/projects_helper.rb | 16 ++++ app/views/github_imports/create.js.haml | 18 +++++ app/views/github_imports/status.html.haml | 41 ++++++++++ app/views/projects/new.html.haml | 10 ++- app/workers/repository_import_worker.rb | 8 +- config/routes.rb | 8 ++ ...135007_add_import_data_to_project_table.rb | 8 ++ db/schema.rb | 5 +- lib/gitlab/github/client.rb | 29 +++++++ lib/gitlab/github/importer.rb | 48 ++++++++++++ lib/gitlab/github/project_creator.rb | 37 +++++++++ lib/gitlab/regex.rb | 2 +- .../github_imports_controller_spec.rb | 64 ++++++++++++++++ spec/helpers/projects_helper_spec.rb | 9 +++ spec/lib/gitlab/github/project_creator.rb | 25 +++++++ 19 files changed, 408 insertions(+), 5 deletions(-) create mode 100644 app/controllers/github_imports_controller.rb create mode 100644 app/views/github_imports/create.js.haml create mode 100644 app/views/github_imports/status.html.haml create mode 100644 db/migrate/20141223135007_add_import_data_to_project_table.rb create mode 100644 lib/gitlab/github/client.rb create mode 100644 lib/gitlab/github/importer.rb create mode 100644 lib/gitlab/github/project_creator.rb create mode 100644 spec/controllers/github_imports_controller_spec.rb create mode 100644 spec/lib/gitlab/github/project_creator.rb diff --git a/Gemfile b/Gemfile index 46ba460506b..fb9df59e611 100644 --- a/Gemfile +++ b/Gemfile @@ -263,3 +263,5 @@ group :production do end gem "newrelic_rpm" + +gem 'octokit', '3.7.0' diff --git a/Gemfile.lock b/Gemfile.lock index 4d4be5674dc..cc46ad92342 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -318,6 +318,8 @@ GEM jwt (~> 0.1.4) multi_json (~> 1.0) rack (~> 1.2) + octokit (3.7.0) + sawyer (~> 0.6.0, >= 0.5.3) omniauth (1.1.4) hashie (>= 1.2, < 3) rack @@ -472,6 +474,9 @@ GEM sass (~> 3.2.0) sprockets (~> 2.8, <= 2.11.0) sprockets-rails (~> 2.0) + sawyer (0.6.0) + addressable (~> 2.3.5) + faraday (~> 0.8, < 0.10) sdoc (0.3.20) json (>= 1.1.3) rdoc (~> 3.10) @@ -671,6 +676,7 @@ DEPENDENCIES mysql2 newrelic_rpm nprogress-rails + octokit (= 3.7.0) omniauth (~> 1.1.3) omniauth-github omniauth-google-oauth2 diff --git a/app/controllers/github_imports_controller.rb b/app/controllers/github_imports_controller.rb new file mode 100644 index 00000000000..97a2637b1eb --- /dev/null +++ b/app/controllers/github_imports_controller.rb @@ -0,0 +1,75 @@ +class GithubImportsController < ApplicationController + before_filter :github_auth, except: :callback + + rescue_from Octokit::Unauthorized, with: :github_unauthorized + + def callback + token = client.auth_code.get_token(params[:code]).token + current_user.github_access_token = token + current_user.save + redirect_to status_github_import_url + end + + def status + @repos = octo_client.repos + octo_client.orgs.each do |org| + @repos += octo_client.repos(org.login) + end + + @already_added_projects = current_user.created_projects.where(import_type: "github") + already_added_projects_names = @already_added_projects.pluck(:import_source) + + @repos.reject!{|repo| already_added_projects_names.include? repo.full_name} + end + + def create + @repo_id = params[:repo_id].to_i + repo = octo_client.repo(@repo_id) + target_namespace = params[:new_namespace].presence || repo.owner.login + existing_namespace = Namespace.find_by("path = ? OR name = ?", target_namespace, target_namespace) + + if existing_namespace + if existing_namespace.owner == current_user + namespace = existing_namespace + else + @already_been_taken = true + @target_namespace = target_namespace + @project_name = repo.name + render and return + end + else + namespace = Group.create(name: target_namespace, path: target_namespace, owner: current_user) + namespace.add_owner(current_user) + end + + Gitlab::Github::ProjectCreator.new(repo, namespace, current_user).execute + end + + private + + def client + @client ||= Gitlab::Github::Client.new.client + end + + def octo_client + Octokit.auto_paginate = true + @octo_client ||= Octokit::Client.new(:access_token => current_user.github_access_token) + end + + def github_auth + if current_user.github_access_token.blank? + go_to_gihub_for_permissions + end + end + + def go_to_gihub_for_permissions + redirect_to client.auth_code.authorize_url({ + redirect_uri: callback_github_import_url, + scope: "repo, user, user:email" + }) + end + + def github_unauthorized + go_to_gihub_for_permissions + end +end diff --git a/app/controllers/omniauth_callbacks_controller.rb b/app/controllers/omniauth_callbacks_controller.rb index 3e984e5007a..442a1cf7518 100644 --- a/app/controllers/omniauth_callbacks_controller.rb +++ b/app/controllers/omniauth_callbacks_controller.rb @@ -65,7 +65,7 @@ class OmniauthCallbacksController < Devise::OmniauthCallbacksController redirect_to omniauth_error_path(oauth['provider'], error: error_message) and return end end - rescue ForbiddenAction => e + rescue Gitlab::OAuth::ForbiddenAction => e flash[:notice] = e.message redirect_to new_user_session_path end diff --git a/app/helpers/projects_helper.rb b/app/helpers/projects_helper.rb index e489d431e84..39d6be06383 100644 --- a/app/helpers/projects_helper.rb +++ b/app/helpers/projects_helper.rb @@ -237,4 +237,20 @@ module ProjectsHelper result.password = '*****' if result.password.present? result end + + def project_status_css_class(status) + case status + when "started" + "active" + when "failed" + "danger" + when "finished" + "success" + end + end + + def github_import_enabled? + Gitlab.config.omniauth.enabled && enabled_oauth_providers.include?(:github) + end end + diff --git a/app/views/github_imports/create.js.haml b/app/views/github_imports/create.js.haml new file mode 100644 index 00000000000..e354c2da4dd --- /dev/null +++ b/app/views/github_imports/create.js.haml @@ -0,0 +1,18 @@ +- if @already_been_taken + :plain + target_field = $("tr#repo_#{@repo_id} .import-target") + origin_target = target_field.text() + project_name = "#{@project_name}" + origin_namespace = "#{@target_namespace}" + target_field.empty() + target_field.append("

This namespace already been taken! Please choose another one

") + target_field.append("") + target_field.append("/" + project_name) + target_field.data("project_name", project_name) + target_field.find('input').prop("value", origin_namespace) +- else + :plain + $("table.import-jobs tbody").prepend($("tr#repo_#{@repo_id}")) + $("tr#repo_#{@repo_id}").addClass("active").find(".import-actions").text("started") + + \ No newline at end of file diff --git a/app/views/github_imports/status.html.haml b/app/views/github_imports/status.html.haml new file mode 100644 index 00000000000..6a196cae39d --- /dev/null +++ b/app/views/github_imports/status.html.haml @@ -0,0 +1,41 @@ +%h3.page-title + Import repositories from github + +%hr +%h4 + Select projects you want to import. + +%table.table.table-bordered.import-jobs + %thead + %tr + %th From GitHub + %th To GitLab + %th Status + %tbody + - @already_added_projects.each do |repo| + %tr{id: "repo_#{repo.id}", class: "#{project_status_css_class(repo.import_status)}"} + %td= repo.import_source + %td= repo.name_with_namespace + %td= repo.human_import_status_name + + - @repos.each do |repo| + %tr{id: "repo_#{repo.id}"} + %td= repo.full_name + %td.import-target + = repo.full_name + %td.import-actions + = button_tag "Add", class: "btn btn-add-to-import" + + +:coffeescript + $(".btn-add-to-import").click () -> + new_namespace = null + tr = $(this).closest("tr") + id = tr.attr("id").replace("repo_", "") + if tr.find(".import-target input").length > 0 + new_namespace = tr.find(".import-target input").prop("value") + tr.find(".import-target").empty().append(new_namespace + "/" + tr.find(".import-target").data("project_name")) + $.post "#{github_import_url}", {repo_id: id, new_namespace: new_namespace}, dataType: 'script' + + + diff --git a/app/views/projects/new.html.haml b/app/views/projects/new.html.haml index f320a2b505e..88c1f725703 100644 --- a/app/views/projects/new.html.haml +++ b/app/views/projects/new.html.haml @@ -39,7 +39,15 @@ %br The import will time out after 4 minutes. For big repositories, use a clone/push combination. For SVN repositories, check #{link_to "this migrating from SVN doc.", "http://doc.gitlab.com/ce/workflow/migrating_from_svn.html"} - %hr + + - if github_import_enabled? + .project-import.form-group + .col-sm-2 + .col-sm-10 + %i.fa.fa-bars + = link_to "Import projects from github", status_github_import_path + + %hr.prepend-botton-10 .form-group = f.label :description, class: 'control-label' do diff --git a/app/workers/repository_import_worker.rb b/app/workers/repository_import_worker.rb index 01586150cd2..0bcc42bc62c 100644 --- a/app/workers/repository_import_worker.rb +++ b/app/workers/repository_import_worker.rb @@ -10,7 +10,13 @@ class RepositoryImportWorker project.path_with_namespace, project.import_url) - if result + if project.import_type == 'github' + result_of_data_import = Gitlab::Github::Importer.new(project).execute + else + result_of_data_import = true + end + + if result && result_of_data_import project.import_finish project.save project.satellite.create unless project.satellite.exists? diff --git a/config/routes.rb b/config/routes.rb index d36540024aa..fc82926abb1 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -51,6 +51,14 @@ Gitlab::Application.routes.draw do end get "/s/:username" => "snippets#user_index", as: :user_snippets, constraints: { username: /.*/ } + # + # Github importer area + # + resource :github_import, only: [:create, :new] do + get :status + get :callback + end + # # Explroe area # diff --git a/db/migrate/20141223135007_add_import_data_to_project_table.rb b/db/migrate/20141223135007_add_import_data_to_project_table.rb new file mode 100644 index 00000000000..5db78f94cc9 --- /dev/null +++ b/db/migrate/20141223135007_add_import_data_to_project_table.rb @@ -0,0 +1,8 @@ +class AddImportDataToProjectTable < ActiveRecord::Migration + def change + add_column :projects, :import_type, :string + add_column :projects, :import_source, :string + + add_column :users, :github_access_token, :string + end +end diff --git a/db/schema.rb b/db/schema.rb index cb945e71665..b87b7d05509 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -314,6 +314,8 @@ ActiveRecord::Schema.define(version: 20141226080412) do t.string "import_status" t.float "repository_size", default: 0.0 t.integer "star_count", default: 0, null: false + t.string "import_type" + t.string "import_source" end add_index "projects", ["creator_id"], name: "index_projects_on_creator_id", using: :btree @@ -411,6 +413,7 @@ ActiveRecord::Schema.define(version: 20141226080412) do t.integer "notification_level", default: 1, null: false t.datetime "password_expires_at" t.integer "created_by_id" + t.datetime "last_credential_check_at" t.string "avatar" t.string "confirmation_token" t.datetime "confirmed_at" @@ -418,7 +421,7 @@ ActiveRecord::Schema.define(version: 20141226080412) do t.string "unconfirmed_email" t.boolean "hide_no_ssh_key", default: false t.string "website_url", default: "", null: false - t.datetime "last_credential_check_at" + t.string "github_access_token" end add_index "users", ["admin"], name: "index_users_on_admin", using: :btree diff --git a/lib/gitlab/github/client.rb b/lib/gitlab/github/client.rb new file mode 100644 index 00000000000..c6935a0b0ba --- /dev/null +++ b/lib/gitlab/github/client.rb @@ -0,0 +1,29 @@ +module Gitlab + module Github + class Client + attr_reader :client + + def initialize + @client = ::OAuth2::Client.new( + config.app_id, + config.app_secret, + github_options + ) + end + + private + + def config + Gitlab.config.omniauth.providers.select{|provider| provider.name == "github"}.first + end + + def github_options + { + :site => 'https://api.github.com', + :authorize_url => 'https://github.com/login/oauth/authorize', + :token_url => 'https://github.com/login/oauth/access_token' + } + end + end + end +end diff --git a/lib/gitlab/github/importer.rb b/lib/gitlab/github/importer.rb new file mode 100644 index 00000000000..c72a1c25e9e --- /dev/null +++ b/lib/gitlab/github/importer.rb @@ -0,0 +1,48 @@ +module Gitlab + module Github + class Importer + attr_reader :project + + def initialize(project) + @project = project + end + + def execute + client = octo_client(project.creator.github_access_token) + + #Issues && Comments + client.list_issues(project.import_source, state: :all).each do |issue| + if issue.pull_request.nil? + body = "*Created by: #{issue.user.login}*\n\n#{issue.body}" + + if issue.comments > 0 + body += "\n\n\n**Imported comments:**\n" + client.issue_comments(project.import_source, issue.number).each do |c| + body += "\n\n*By #{c.user.login} on #{c.created_at}*\n\n#{c.body}" + end + end + + project.issues.create!( + description: body, + title: issue.title, + state: issue.state == 'closed' ? 'closed' : 'opened', + author_id: gl_user_id(project, issue.user.id) + ) + end + end + end + + private + + def octo_client(access_token) + ::Octokit.auto_paginate = true + ::Octokit::Client.new(:access_token => access_token) + end + + def gl_user_id(project, github_id) + user = User.joins(:identities).find_by("identities.extern_uid = ?", github_id.to_s) + (user && user.id) || project.creator_id + end + end + end +end diff --git a/lib/gitlab/github/project_creator.rb b/lib/gitlab/github/project_creator.rb new file mode 100644 index 00000000000..682ef389e44 --- /dev/null +++ b/lib/gitlab/github/project_creator.rb @@ -0,0 +1,37 @@ +module Gitlab + module Github + class ProjectCreator + attr_reader :repo, :namespace, :current_user + + def initialize(repo, namespace, current_user) + @repo = repo + @namespace = namespace + @current_user = current_user + end + + def execute + @project = Project.new( + name: repo.name, + path: repo.name, + description: repo.description, + namespace: namespace, + creator: current_user, + visibility_level: repo.private ? Gitlab::VisibilityLevel::PRIVATE : Gitlab::VisibilityLevel::PUBLIC, + import_type: "github", + import_source: repo.full_name, + import_url: repo.clone_url.sub("https://", "https://#{current_user.github_access_token}@") + ) + + if @project.save! + @project.reload + + if @project.import_failed? + @project.import_retry + else + @project.import_start + end + end + end + end + end +end diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index c4d0d85b7f5..cf6e260f257 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -11,7 +11,7 @@ module Gitlab end def project_name_regex - /\A[a-zA-Z0-9_][a-zA-Z0-9_\-\. ]*\z/ + /\A[a-zA-Z0-9_.][a-zA-Z0-9_\-\. ]*\z/ end def project_regex_message diff --git a/spec/controllers/github_imports_controller_spec.rb b/spec/controllers/github_imports_controller_spec.rb new file mode 100644 index 00000000000..f1d2df8411a --- /dev/null +++ b/spec/controllers/github_imports_controller_spec.rb @@ -0,0 +1,64 @@ +require 'spec_helper' + +describe GithubImportsController do + let(:user) { create(:user, github_access_token: 'asd123') } + + before do + sign_in(user) + end + + describe "GET callback" do + it "updates access token" do + token = "asdasd12345" + Gitlab::Github::Client.any_instance.stub_chain(:client, :auth_code, :get_token, :token).and_return(token) + + get :callback + + user.reload.github_access_token.should == token + controller.should redirect_to(status_github_import_url) + end + end + + describe "GET status" do + before do + @repo = OpenStruct.new(login: 'vim', full_name: 'asd/vim') + end + + it "assigns variables" do + @project = create(:project, import_type: 'github', creator_id: user.id) + controller.stub_chain(:octo_client, :repos).and_return([@repo]) + controller.stub_chain(:octo_client, :orgs).and_return([]) + + get :status + + expect(assigns(:already_added_projects)).to eq([@project]) + expect(assigns(:repos)).to eq([@repo]) + end + + it "does not show already added project" do + @project = create(:project, import_type: 'github', creator_id: user.id, import_source: 'asd/vim') + controller.stub_chain(:octo_client, :repos).and_return([@repo]) + controller.stub_chain(:octo_client, :orgs).and_return([]) + + get :status + + expect(assigns(:already_added_projects)).to eq([@project]) + expect(assigns(:repos)).to eq([]) + end + end + + describe "POST create" do + before do + @repo = OpenStruct.new(login: 'vim', full_name: 'asd/vim', owner: OpenStruct.new(login: "john")) + end + + it "takes already existing namespace" do + namespace = create(:namespace, name: "john", owner: user) + Gitlab::Github::ProjectCreator.should_receive(:new).with(@repo, namespace, user). + and_return(double(execute: true)) + controller.stub_chain(:octo_client, :repo).and_return(@repo) + + post :create, format: :js + end + end +end diff --git a/spec/helpers/projects_helper_spec.rb b/spec/helpers/projects_helper_spec.rb index 114058e3095..2146b0b1383 100644 --- a/spec/helpers/projects_helper_spec.rb +++ b/spec/helpers/projects_helper_spec.rb @@ -20,4 +20,13 @@ describe ProjectsHelper do "" end end + + describe "#project_status_css_class" do + it "returns appropriate class" do + project_status_css_class("started").should == "active" + project_status_css_class("failed").should == "danger" + project_status_css_class("finished").should == "success" + end + + end end diff --git a/spec/lib/gitlab/github/project_creator.rb b/spec/lib/gitlab/github/project_creator.rb new file mode 100644 index 00000000000..0bade5619a5 --- /dev/null +++ b/spec/lib/gitlab/github/project_creator.rb @@ -0,0 +1,25 @@ +require 'spec_helper' + +describe Gitlab::Github::ProjectCreator do + let(:user) { create(:user, github_access_token: "asdffg") } + let(:repo) { OpenStruct.new( + login: 'vim', + name: 'vim', + private: true, + full_name: 'asd/vim', + clone_url: "https://gitlab.com/asd/vim.git", + owner: OpenStruct.new(login: "john")) + } + let(:namespace){ create(:namespace) } + + it 'creates project' do + Project.any_instance.stub(:add_import_job) + + project_creator = Gitlab::Github::ProjectCreator.new(repo, namespace, user) + project_creator.execute + project = Project.last + + project.import_url.should == "https://asdffg@gitlab.com/asd/vim.git" + project.visibility_level.should == Gitlab::VisibilityLevel::PRIVATE + end +end