From ad6d6232342558705c54ba70a94f9d7ddbd00f8c Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Tue, 17 Feb 2015 16:59:50 +0100 Subject: [PATCH] Add Bitbucket importer. --- CHANGELOG | 2 + Gemfile | 1 + Gemfile.lock | 5 + .../images/authbuttons/bitbucket_32.png | Bin 0 -> 2713 bytes .../images/authbuttons/bitbucket_64.png | Bin 0 -> 2163 bytes .../import/bitbucket_controller.rb | 74 +++++++++++++++ app/helpers/oauth_helper.rb | 4 +- app/helpers/projects_helper.rb | 4 + app/models/project.rb | 2 +- app/models/user.rb | 1 + app/views/import/base/create.js.haml | 7 ++ app/views/import/bitbucket/status.html.haml | 44 +++++++++ app/views/import/github/status.html.haml | 2 +- app/views/import/gitlab/status.html.haml | 2 +- .../_bitbucket_import_modal.html.haml | 9 ++ app/views/projects/new.html.haml | 13 +++ app/workers/repository_import_worker.rb | 2 + config/gitlab.yml.example | 18 ++-- config/routes.rb | 5 + ...tbucket_access_token_and_secret_to_user.rb | 6 ++ db/schema.rb | 40 ++++---- doc/integration/bitbucket.m | 1 + lib/gitlab/bitbucket_import/client.rb | 88 ++++++++++++++++++ lib/gitlab/bitbucket_import/importer.rb | 52 +++++++++++ lib/gitlab/bitbucket_import/key_adder.rb | 22 +++++ .../bitbucket_import/project_creator.rb | 39 ++++++++ lib/gitlab/github_import/client.rb | 6 +- lib/gitlab/github_import/importer.rb | 2 +- lib/gitlab/gitlab_import/client.rb | 10 +- lib/gitlab/gitlab_import/importer.rb | 2 +- lib/gitlab/import_formatter.rb | 2 +- .../import/bitbucket_controller_spec.rb | 77 +++++++++++++++ .../bitbucket_import/project_creator_spec.rb | 22 +++++ .../project_creator_spec.rb} | 7 +- ...ect_creator.rb => project_creator_spec.rb} | 5 +- 35 files changed, 522 insertions(+), 54 deletions(-) create mode 100644 app/assets/images/authbuttons/bitbucket_32.png create mode 100644 app/assets/images/authbuttons/bitbucket_64.png create mode 100644 app/controllers/import/bitbucket_controller.rb create mode 100644 app/views/import/bitbucket/status.html.haml create mode 100644 app/views/projects/_bitbucket_import_modal.html.haml create mode 100644 db/migrate/20150217123345_add_bitbucket_access_token_and_secret_to_user.rb create mode 100644 doc/integration/bitbucket.m create mode 100644 lib/gitlab/bitbucket_import/client.rb create mode 100644 lib/gitlab/bitbucket_import/importer.rb create mode 100644 lib/gitlab/bitbucket_import/key_adder.rb create mode 100644 lib/gitlab/bitbucket_import/project_creator.rb create mode 100644 spec/controllers/import/bitbucket_controller_spec.rb create mode 100644 spec/lib/gitlab/bitbucket_import/project_creator_spec.rb rename spec/lib/gitlab/{github/project_creator.rb => github_import/project_creator_spec.rb} (77%) rename spec/lib/gitlab/gitlab_import/{project_creator.rb => project_creator_spec.rb} (91%) diff --git a/CHANGELOG b/CHANGELOG index d879ee85728..5a3a2ca2e24 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -72,6 +72,8 @@ v 7.8.0 - Improve database performance for GitLab - Add Asana service (Jeremy Benoist) - Improve project web hooks with extra data + - Add Bitbucket omniauth provider. + - Add Bitbucket importer. v 7.7.2 - Update GitLab Shell to version 2.4.2 that fixes a bug when developers can push to protected branch diff --git a/Gemfile b/Gemfile index 093f7bacc06..ad01b2b43e0 100644 --- a/Gemfile +++ b/Gemfile @@ -30,6 +30,7 @@ gem 'omniauth-github' gem 'omniauth-shibboleth' gem 'omniauth-kerberos' gem 'omniauth-gitlab' +gem 'omniauth-bitbucket' gem 'doorkeeper', '2.1.0' gem "rack-oauth2", "~> 1.0.5" diff --git a/Gemfile.lock b/Gemfile.lock index 4bc47836e71..226591a50af 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -338,6 +338,10 @@ GEM omniauth (1.1.4) hashie (>= 1.2, < 3) rack + omniauth-bitbucket (0.0.2) + multi_json (~> 1.7) + omniauth (~> 1.1) + omniauth-oauth (~> 1.0) omniauth-github (1.1.1) omniauth (~> 1.0) omniauth-oauth2 (~> 1.1) @@ -701,6 +705,7 @@ DEPENDENCIES nprogress-rails octokit (= 3.7.0) omniauth (~> 1.1.3) + omniauth-bitbucket omniauth-github omniauth-gitlab omniauth-google-oauth2 diff --git a/app/assets/images/authbuttons/bitbucket_32.png b/app/assets/images/authbuttons/bitbucket_32.png new file mode 100644 index 0000000000000000000000000000000000000000..27702eb973d1c3c0b9a39eee082ca72d38339cc2 GIT binary patch literal 2713 zcmZ`*3p~^78y~tb)c!~k#|$NxjV;v7!WfgG5i4^UCKKD*9CHv4hY%>T%6zJQtvd&<}w?H)3 zN>;-BU|cc-W(YNa{Yb>7`hQ>E|ENTkpUNm)>Q7`M%U9WQF}|)5C9EhCPxqtxhgy-@ z6g1qx&`93^u5W1KVu(N*z>&LmqhM<|-!oiRW0({&o6dBh(}U1XR4yebSTqGApm3<+ zSM6%XDzPXJq$86`5f)ffMzoOrf7n_+h91mdQdlez-)M!u8e7XJP+0UJj>!E%8kPMm z#2R?5>d<#P+62B5W;M80K>U$lHMmM3+6d%wrHEG4UC7Pe zQbfrtOIAT^*~?)L{VdEX!-^0E6P@K%XZ!2LSw@972MrYN=)boR4V?6NKMeqEIEAya za3M;iyHSH&tW}m;!fReeSH{}}mI6~jcys)iB0=5BGF}sl`3n+YZ;O$J@IZ?nes^n& zVMxiPE5+>C5hK?=kso^kSgz2{+>*@LD!t`J?5}4`hUotDW^;O+)GEYu?$U29S8D?< z&^S^XE;#2|fhsd3Vs*^%8y;{|c`}5CvW+J=>n%AJr!;a*o!-?%&EW@RNc8OHb%h>o zF-;RBIoOUa75KSzwkaCpU9?ze^Y-rg-dD zdg^!*2o}?-_JX99D2`I3ROIS!vhg)dzustZps>=aIeNjk7FO-#(sOdosaH#d^M@i3 z%(x?7xusy-$fcl_oAgLhtDb*Wt+A%^eTUrftQ1d_8z?}sF(-y`rY5kiS#3V_Wht~Z zW^^0L$cr;%)yi4Wj*aWNE2sO5SG3vem(qeBLtqA`$=1i=pDd!g*M;h$Ka!2Z1{L&P z$u@NZf_5eq0f1l&S{OV&mPl?C(mI7m8cL$%eTDg+^U9PnzA3Ov%|6d%0Te zS`g9Ebc(KeWu~qhn80iD`wLe%tnqNS4AuJ37Q^mnLuQ@?+4*KJ3&7V?|8P_DC_b$< z)l}<&HKw6rB+^z7o3{z8gEn~)+z)CXDQ-szq7~_J4>^Lc6gxr8-#mEC)9O*nUX|p# zwgm!cSI^}R}dZwyi z#iBa|exJX#W= zLLDN=Kk`Mj1-!TVKv|et&L`n#Zn}Gx#C>?x3=Az0H(Q^Gwah`NHwnB5Q#a~Huv|`u zo=y!vPMb(PGn9>V>bS{&iFvSV|CYhxUF(vNBkylbFlzz&mjxaW{;Ai+_+#<+x*Nm0 zb95wMS+D2c2;irTLG8T`B+0*Y=qPQRQcQgt+~X+~*gpo;Ucw(fTO-F7x9ESfSzpn4 zuVdVBXC6#9c9(YL`OSHFsKk$JY9IL3bz!MWY=XoF^(CCcN0_I=@Gf_-u4~C1jW(~8ArF3J!^MM7`oRR z?RXrp&BGCP9J5u=V*YK!vCxHVy|T=)zK%DSPWwo=ayN5vqvzYA%8pCl&i(Tw;%!4Y zO(RNHZLr(*T#FZ%-HE@(ypyuW`WCf9{N2@)xgTc(5OTQjLU<8u-9Gb#A z4a$S!Qja;_`E37OgJ@^{=J0>gyas2C&zVv+s>GpoEssJ9AMEsl3-Xw03~Y)D&Oe~Z zLAt9`tN}|=9jv$`-&j?CQx7rH3v?-#X`iBw_AZpEbAPke0*>q0K3in!#2R!oI4ck` zytT2Px~ex*`oQCIL1<6NuUj4HxU;>92OmdrO&n++RkFo9@oJ^y1-r-YV?7@a{1Q+f zDSMkuzWLUw)J*Auw;Ks5i4)@nR46UA2 zmwM&%krW?sUNxhQeWFV(Zm8x&$|(57B*?obtFGUmCSPH|eB1aZ-I-RD%%KGDB5+zJ z@3l-Df*N8T1$_f&iZy>$j_z@Upo;^dL5P;d$2s(SrrMKzma{Ei%zdX$N7R5c?nwXWa;9p*_m)j*51XX05;U$nbepp!G-TG;r^8FVk9I3cu(*80QB)VcObzvy~ zft2JC1X3qrqa`mXGxAE6qZ1gHe-wRu(deHgt{}OJ`kdWr-RvBqd?~f>w(eZo-=6l$ gqZTZLJY{B5E#UAGZ=PVPRP?`yvv#m5vGk7mFT{djfB*mh literal 0 HcmV?d00001 diff --git a/app/assets/images/authbuttons/bitbucket_64.png b/app/assets/images/authbuttons/bitbucket_64.png new file mode 100644 index 0000000000000000000000000000000000000000..4b90a57bc7de93163275f99ea2971e2740af98af GIT binary patch literal 2163 zcmV-(2#oiMP)2uhW|vhI6clA;HOc~sCx{2egDXMtKom7mKv^T8ct;fPlX#%Ui}gx8f}-Mq z8i53&iTVRjz=KFcSXNM@D}srF8g{ySd}H3pR^Ig1Oi%ahB0rw&W_G&jtEa!JuIj3F z^7%Yyqwm?bInLn?j(6--#~D1u#044R2)luLa0a**G=ha-71#u}0UzW`Amn$q@!C}^ zYc{xsb@XMOj*ua){w;gl<&IN7-g#rmivQY~^Mn6>mLkB7X!IUn2)G{11FIQuzJvIa zb-uwEhJdaSGTbBrwBcRBncxMm-V$5@a_!(71cG0{a(!T_cO7^RoCbDR5pe~0Mh4V@ zyI4U~s%itn3f}pU^%SdfEw}^J8X4e~D!^j~RN=xIV6)s^pFcxOgSS)Ir(iyK7R&@U zf@xqfm;lCui@+sd3b+c~0%n1izyh!gYy>+ZvyCxGYI5QLI4}i z!iiM)3K(9bu^z}qkP)(ij^;{tj!EySgorMahk!K@(2D}RNCE5uFG4^b0$MHoFaVe7 zsA~l70hc7s0OA>Sc=Da;Ahcv@D+J^r;AwUN&o%}$LqGrlL3;wo$cVd~u{1nQ5t<>V zNERpo1OyQ9GX!*J7jWAXu$~!U6Hs^--wgT`NqtRs&xUpjx#P;doCjT&b2WoT{#MvMFE*80a*%|Dgt65laicFd&5$%C%(@3tYs9>fCdW4+A?4y1+b!u z5egNphXRJ$GQgvNdIO%xF90s;aG_!a`HD8MbY z7FbLH{w^TEr+`J)wE)RbKqCd@b^!r73V56XvNi!(3YgXb84wDU&Vb1xz?uQ0DmV!G z#Qu5ugWw1X$k+sU6mSRx{9_i70zpKB{~*531JZ0f*bV`8G6q-&!5RoyD}g6YL$a7l z>A|3m4CZGaim2l3c#`nWA1l(7SfD8pRBoL4T1En!H^Y6YT;A`ZXqk*45zyuKxs|yTa zr6hL$8x36cYnCTlZMw%tX~tM^wv-w7)!g7D0&2mQIQFHFKL8xrEqc+i0D4NE0uFdo z%8YXZeg*qcfE!1EO95H%wG^?3$+f{h94Uj@9_0{9p# z+<|%0r{V^2lccmm>cKd0E;yv{xpbM^<2GJmeU;^Kd%QdW_sGZ{w-5dWPBbKV#1TAF zX#6nnx4cd-l^n0RBMku;h}ClD11(^L>?Bu#YEmV~aYliyQbsu$FgA$*m+9C8d|xn| zU`?=tgW>^}#ZVvc5NIu@@e<>D(2Z?y%RMFL8Qag>l1d<(**C8S=YX;NeJ=P-v!_&~ zoBnBZJ1^NWITpYI*V#5D<7}oX3CqpN0jU!t_9OxT`zO0YBU4n*4~4Nr06iiXvkW~4 zw1d0Rdr8d61*lDM4Ac0pi2mo>grv0LswdK3VN4QhiURbF4H*D#3l(IC-CV^El=B%ZZR5Cy%EHMV5Ku&7k3kJ2*&5D@|U@co?6_fFw(WGy(AWqzjh5i_QTB<55nLX%`_t8aAj z!fy^Hoh=4n zkPLvSjsZPEOR>d@3K4>04Hd!|i>;r)Ua5}(#HH1xZE4u~CqyKcPvo$6DgtyMd|m}~ zA}V0EblG$SNTJhqVgXY+Lcqx~2J8%kzT(Ii9Kvd)ZB3oHV}T0(4N^hf~*P zThOszblPQZ;9c(_v2>^at++clAIxprM@Wn|dJgY+H;th)1laz&d~Yxu%m8nKFTs`= puf?};oj;Fd45Q%4{|-+g;9pZ(FBz?c<5BAccess denied! Please verify you can add deploy keys to this repository.

"") - else :plain job = $("tr#repo_#{@repo_id}") job.attr("id", "project_#{@project.id}") + target_field = job.find(".import-target") + target_field.empty() + target_field.append('#{link_to @project.path_with_namespace, @project}') $("table.import-jobs tbody").prepend(job) job.addClass("active").find(".import-actions").html(" started") diff --git a/app/views/import/bitbucket/status.html.haml b/app/views/import/bitbucket/status.html.haml new file mode 100644 index 00000000000..7a2613e4b03 --- /dev/null +++ b/app/views/import/bitbucket/status.html.haml @@ -0,0 +1,44 @@ +%h3.page-title + %i.fa.fa-bitbucket + Import repositories from Bitbucket + +%p.light + Select projects you want to import. +%hr +%p + = button_tag 'Import all projects', class: "btn btn-success js-import-all" + +%table.table.import-jobs + %thead + %tr + %th From Bitbucket + %th To GitLab + %th Status + %tbody + - @already_added_projects.each do |project| + %tr{id: "project_#{project.id}", class: "#{project_status_css_class(project.import_status)}"} + %td= project.import_source + %td + %strong= link_to project.path_with_namespace, project + %td.job-status + - if project.import_status == 'finished' + %span.cgreen + %i.fa.fa-check + done + - elsif project.import_status == 'started' + %i.fa.fa-spinner.fa-spin + started + - else + = project.human_import_status_name + + - @repos.each do |repo| + %tr{id: "repo_#{repo["owner"]}___#{repo["slug"]}"} + %td= "#{repo["owner"]}/#{repo["slug"]}" + %td.import-target + = "#{repo["owner"]}/#{repo["slug"]}" + %td.import-actions.job-status + = button_tag "Import", class: "btn js-add-to-import" + +:coffeescript + $ -> + new ImporterStatus("#{jobs_import_bitbucket_path}", "#{import_bitbucket_path}") diff --git a/app/views/import/github/status.html.haml b/app/views/import/github/status.html.haml index 84d9903fe15..b1538b1a41e 100644 --- a/app/views/import/github/status.html.haml +++ b/app/views/import/github/status.html.haml @@ -1,6 +1,6 @@ %h3.page-title %i.fa.fa-github - Import repositories from GitHub.com + Import repositories from GitHub %p.light Select projects you want to import. diff --git a/app/views/import/gitlab/status.html.haml b/app/views/import/gitlab/status.html.haml index d1e48dfad20..43db102994f 100644 --- a/app/views/import/gitlab/status.html.haml +++ b/app/views/import/gitlab/status.html.haml @@ -1,5 +1,5 @@ %h3.page-title - %i.fa.fa-github + %i.fa.fa-heart Import repositories from GitLab.com %p.light diff --git a/app/views/projects/_bitbucket_import_modal.html.haml b/app/views/projects/_bitbucket_import_modal.html.haml new file mode 100644 index 00000000000..dd7aacc7e6e --- /dev/null +++ b/app/views/projects/_bitbucket_import_modal.html.haml @@ -0,0 +1,9 @@ +%div#bitbucket_import_modal.modal.hide + .modal-dialog + .modal-content + .modal-header + %a.close{href: "#", "data-dismiss" => "modal"} × + %h3 GitHub OAuth import + .modal-body + You need to setup integration with Bitbucket first. + = link_to 'How to setup integration with Bitbucket', 'https://gitlab.com/gitlab-org/gitlab-ce/blob/master/doc/integration/bitbucket.md' \ No newline at end of file diff --git a/app/views/projects/new.html.haml b/app/views/projects/new.html.haml index 5216f308110..875c092fd1a 100644 --- a/app/views/projects/new.html.haml +++ b/app/views/projects/new.html.haml @@ -53,6 +53,19 @@ Import projects from GitHub = render 'github_import_modal' + .project-import.form-group + .col-sm-2 + .col-sm-10 + - if bitbucket_import_enabled? + = link_to status_import_bitbucket_path do + %i.fa.fa-bitbucket + Import projects from Bitbucket + - else + = link_to '#', class: 'how_to_import_link light' do + %i.fa.fa-bitbucket + Import projects from Bitbucket + = render 'bitbucket_import_modal' + - unless request.host == 'gitlab.com' .project-import.form-group .col-sm-2 diff --git a/app/workers/repository_import_worker.rb b/app/workers/repository_import_worker.rb index 5f9970d3795..d7e759fb470 100644 --- a/app/workers/repository_import_worker.rb +++ b/app/workers/repository_import_worker.rb @@ -14,6 +14,8 @@ class RepositoryImportWorker Gitlab::GithubImport::Importer.new(project).execute elsif project.import_type == 'gitlab' Gitlab::GitlabImport::Importer.new(project).execute + elsif project.import_type == 'bitbucket' + Gitlab::BitbucketImport::Importer.new(project).execute else true end diff --git a/config/gitlab.yml.example b/config/gitlab.yml.example index 044b1f66b25..6dff07cf9df 100644 --- a/config/gitlab.yml.example +++ b/config/gitlab.yml.example @@ -207,17 +207,19 @@ production: &base # arguments, followed by optional 'args' which can be either a hash or an array. # Documentation for this is available at http://doc.gitlab.com/ce/integration/omniauth.html providers: - # - { name: 'google_oauth2', app_id: 'YOUR APP ID', - # app_secret: 'YOUR APP SECRET', + # - { name: 'google_oauth2', app_id: 'YOUR_APP_ID', + # app_secret: 'YOUR_APP_SECRET', # args: { access_type: 'offline', approval_prompt: '' } } - # - { name: 'twitter', app_id: 'YOUR APP ID', - # app_secret: 'YOUR APP SECRET'} - # - { name: 'github', app_id: 'YOUR APP ID', - # app_secret: 'YOUR APP SECRET', + # - { name: 'twitter', app_id: 'YOUR_APP_ID', + # app_secret: 'YOUR_APP_SECRET'} + # - { name: 'github', app_id: 'YOUR_APP_ID', + # app_secret: 'YOUR_APP_SECRET', # args: { scope: 'user:email' } } - # - { name: 'gitlab', app_id: 'YOUR APP ID', - # app_secret: 'YOUR APP SECRET', + # - { name: 'gitlab', app_id: 'YOUR_APP_ID', + # app_secret: 'YOUR_APP_SECRET', # args: { scope: 'api' } } + # - { name: 'bitbucket', app_id: 'YOUR_APP_ID', + # app_secret: 'YOUR_APP_SECRET'} diff --git a/config/routes.rb b/config/routes.rb index ecd439aecea..57964bdc3b7 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -68,6 +68,11 @@ Gitlab::Application.routes.draw do get :jobs end + resource :bitbucket, only: [:create, :new], controller: :bitbucket do + get :status + get :callback + get :jobs + end resource :gitorious, only: [:create, :new], controller: :gitorious do get :status get :callback diff --git a/db/migrate/20150217123345_add_bitbucket_access_token_and_secret_to_user.rb b/db/migrate/20150217123345_add_bitbucket_access_token_and_secret_to_user.rb new file mode 100644 index 00000000000..23ac1b399ec --- /dev/null +++ b/db/migrate/20150217123345_add_bitbucket_access_token_and_secret_to_user.rb @@ -0,0 +1,6 @@ +class AddBitbucketAccessTokenAndSecretToUser < ActiveRecord::Migration + def change + add_column :users, :bitbucket_access_token, :string + add_column :users, :bitbucket_access_token_secret, :string + end +end diff --git a/db/schema.rb b/db/schema.rb index e11a068c9c5..8069d95c698 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -11,7 +11,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20150213121042) do +ActiveRecord::Schema.define(version: 20150217123345) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -410,12 +410,12 @@ ActiveRecord::Schema.define(version: 20150213121042) do end create_table "users", force: true do |t| - t.string "email", default: "", null: false - t.string "encrypted_password", default: "", null: false + t.string "email", default: "", null: false + t.string "encrypted_password", default: "", null: false t.string "reset_password_token" t.datetime "reset_password_sent_at" t.datetime "remember_created_at" - t.integer "sign_in_count", default: 0 + t.integer "sign_in_count", default: 0 t.datetime "current_sign_in_at" t.datetime "last_sign_in_at" t.string "current_sign_in_ip" @@ -423,22 +423,22 @@ ActiveRecord::Schema.define(version: 20150213121042) do t.datetime "created_at" t.datetime "updated_at" t.string "name" - t.boolean "admin", default: false, null: false - t.integer "projects_limit", default: 10 - t.string "skype", default: "", null: false - t.string "linkedin", default: "", null: false - t.string "twitter", default: "", null: false + t.boolean "admin", default: false, null: false + t.integer "projects_limit", default: 10 + t.string "skype", default: "", null: false + t.string "linkedin", default: "", null: false + t.string "twitter", default: "", null: false t.string "authentication_token" - t.integer "theme_id", default: 1, null: false + t.integer "theme_id", default: 1, null: false t.string "bio" - t.integer "failed_attempts", default: 0 + t.integer "failed_attempts", default: 0 t.datetime "locked_at" t.string "username" - t.boolean "can_create_group", default: true, null: false - t.boolean "can_create_team", default: true, null: false + t.boolean "can_create_group", default: true, null: false + t.boolean "can_create_team", default: true, null: false t.string "state" - t.integer "color_scheme_id", default: 1, null: false - t.integer "notification_level", default: 1, null: false + t.integer "color_scheme_id", default: 1, null: false + t.integer "notification_level", default: 1, null: false t.datetime "password_expires_at" t.integer "created_by_id" t.datetime "last_credential_check_at" @@ -447,13 +447,15 @@ ActiveRecord::Schema.define(version: 20150213121042) do t.datetime "confirmed_at" t.datetime "confirmation_sent_at" t.string "unconfirmed_email" - t.boolean "hide_no_ssh_key", default: false - t.string "website_url", default: "", null: false + t.boolean "hide_no_ssh_key", default: false + t.string "website_url", default: "", null: false t.string "github_access_token" t.string "gitlab_access_token" t.string "notification_email" - t.boolean "hide_no_password", default: false - t.boolean "password_automatically_set", default: false + t.boolean "hide_no_password", default: false + t.boolean "password_automatically_set", default: false + t.string "bitbucket_access_token" + t.string "bitbucket_access_token_secret" end add_index "users", ["admin"], name: "index_users_on_admin", using: :btree diff --git a/doc/integration/bitbucket.m b/doc/integration/bitbucket.m new file mode 100644 index 00000000000..30404ce4c54 --- /dev/null +++ b/doc/integration/bitbucket.m @@ -0,0 +1 @@ +TODO \ No newline at end of file diff --git a/lib/gitlab/bitbucket_import/client.rb b/lib/gitlab/bitbucket_import/client.rb new file mode 100644 index 00000000000..3d2ef78ee7d --- /dev/null +++ b/lib/gitlab/bitbucket_import/client.rb @@ -0,0 +1,88 @@ +module Gitlab + module BitbucketImport + class Client + attr_reader :consumer, :api + + def initialize(access_token = nil, access_token_secret = nil) + @consumer = ::OAuth::Consumer.new( + config.app_id, + config.app_secret, + bitbucket_options + ) + + if access_token && access_token_secret + @api = ::OAuth::AccessToken.new(@consumer, access_token, access_token_secret) + end + end + + def request_token(redirect_uri) + request_token = consumer.get_request_token(oauth_callback: redirect_uri) + + { + oauth_token: request_token.token, + oauth_token_secret: request_token.secret, + oauth_callback_confirmed: request_token.callback_confirmed?.to_s + } + end + + def authorize_url(request_token, redirect_uri) + request_token = ::OAuth::RequestToken.from_hash(consumer, request_token) if request_token.is_a?(Hash) + + if request_token.callback_confirmed? + request_token.authorize_url + else + request_token.authorize_url(oauth_callback: redirect_uri) + end + end + + def get_token(request_token, oauth_verifier, redirect_uri) + request_token = ::OAuth::RequestToken.from_hash(consumer, request_token) if request_token.is_a?(Hash) + + if request_token.callback_confirmed? + request_token.get_access_token(oauth_verifier: oauth_verifier) + else + request_token.get_access_token(oauth_callback: redirect_uri) + end + end + + def user + JSON.parse(api.get("/api/1.0/user").body) + end + + def issues(project_identifier) + JSON.parse(api.get("/api/1.0/repositories/#{project_identifier}/issues").body) + end + + def issue_comments(project_identifier, issue_id) + JSON.parse(api.get("/api/1.0/repositories/#{project_identifier}/issues/#{issue_id}/comments").body) + end + + def project(project_identifier) + JSON.parse(api.get("/api/1.0/repositories/#{project_identifier}").body) + end + + def deploy_key(project_identifier) + JSON.parse(api.get("/api/1.0/repositories/#{project_identifier}/deploy-keys").body).find { |key| key["label"] =~ /GitLab/ } + end + + def add_deploy_key(project_identifier, key) + JSON.parse(api.post("/api/1.0/repositories/#{project_identifier}/deploy-keys", key: key, label: "GitLab import key").body) + end + + def projects + JSON.parse(api.get("/api/1.0/user/repositories").body). + select { |repo| repo["scm"] == "git" } + end + + private + + def config + Gitlab.config.omniauth.providers.find { |provider| provider.name == "bitbucket"} + end + + def bitbucket_options + OmniAuth::Strategies::Bitbucket.default_options[:client_options] + end + end + end +end diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb new file mode 100644 index 00000000000..42c93707caa --- /dev/null +++ b/lib/gitlab/bitbucket_import/importer.rb @@ -0,0 +1,52 @@ +module Gitlab + module BitbucketImport + class Importer + attr_reader :project, :client + + def initialize(project) + @project = project + @client = Client.new(project.creator.bitbucket_access_token, project.creator.bitbucket_access_token_secret) + @formatter = Gitlab::ImportFormatter.new + end + + def execute + project_identifier = project.import_source + + return true unless client.project(project_identifier)["has_issues"] + + #Issues && Comments + issues = client.issues(project_identifier) + + issues["issues"].each do |issue| + body = @formatter.author_line(issue["reported_by"]["username"], issue["content"]) + + comments = client.issue_comments(project_identifier, issue["local_id"]) + + if comments.any? + body += @formatter.comments_header + end + + comments.each do |comment| + body += @formatter.comment(comment["author_info"]["username"], comment["utc_created_on"], comment["content"]) + end + + project.issues.create!( + description: body, + title: issue["title"], + state: %w(resolved invalid duplicate wontfix).include?(issue["status"]) ? 'closed' : 'opened', + author_id: gl_user_id(project, issue["reported_by"]["username"]) + ) + end + + true + end + + private + + def gl_user_id(project, bitbucket_id) + user = User.joins(:identities).find_by("identities.extern_uid = ? AND identities.provider = 'bitbucket'", bitbucket_id.to_s) + (user && user.id) || project.creator_id + end + end + end +end diff --git a/lib/gitlab/bitbucket_import/key_adder.rb b/lib/gitlab/bitbucket_import/key_adder.rb new file mode 100644 index 00000000000..207811237ba --- /dev/null +++ b/lib/gitlab/bitbucket_import/key_adder.rb @@ -0,0 +1,22 @@ +module Gitlab + module BitbucketImport + class KeyAdder + attr_reader :repo, :current_user, :client + + def initialize(repo, current_user) + @repo, @current_user = repo, current_user + @client = Client.new(current_user.bitbucket_access_token, current_user.bitbucket_access_token_secret) + end + + def execute + project_identifier = "#{repo["owner"]}/#{repo["slug"]}" + return true if client.deploy_key(project_identifier) + + # TODO: Point to actual public key. + client.add_deploy_key(project_identifier, File.read("/Users/douwemaan/.ssh/id_rsa.pub")) + + true + end + end + end +end diff --git a/lib/gitlab/bitbucket_import/project_creator.rb b/lib/gitlab/bitbucket_import/project_creator.rb new file mode 100644 index 00000000000..db33af2c2da --- /dev/null +++ b/lib/gitlab/bitbucket_import/project_creator.rb @@ -0,0 +1,39 @@ +module Gitlab + module BitbucketImport + class ProjectCreator + attr_reader :repo, :namespace, :current_user + + def initialize(repo, namespace, current_user) + @repo = repo + @namespace = namespace + @current_user = current_user + end + + def execute + @project = Project.new( + name: repo["name"], + path: repo["slug"], + description: repo["description"], + namespace: namespace, + creator: current_user, + visibility_level: repo["is_private"] ? Gitlab::VisibilityLevel::PRIVATE : Gitlab::VisibilityLevel::PUBLIC, + import_type: "bitbucket", + import_source: "#{repo["owner"]}/#{repo["slug"]}", + import_url: "ssh://git@bitbucket.org/#{repo["owner"]}/#{repo["slug"]}.git" + ) + + if @project.save! + @project.reload + + if @project.import_failed? + @project.import_retry + else + @project.import_start + end + end + + @project + end + end + end +end diff --git a/lib/gitlab/github_import/client.rb b/lib/gitlab/github_import/client.rb index c9904fe8779..676d226bddd 100644 --- a/lib/gitlab/github_import/client.rb +++ b/lib/gitlab/github_import/client.rb @@ -46,11 +46,7 @@ module Gitlab end def github_options - { - site: 'https://api.github.com', - authorize_url: 'https://github.com/login/oauth/authorize', - token_url: 'https://github.com/login/oauth/access_token' - } + OmniAuth::Strategies::GitHub.default_options[:client_options] end end end diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb index bc2b645b2d9..23832b3233c 100644 --- a/lib/gitlab/github_import/importer.rb +++ b/lib/gitlab/github_import/importer.rb @@ -20,7 +20,7 @@ module Gitlab body += @formatter.comments_header client.issue_comments(project.import_source, issue.number).each do |c| - body += @formatter.comment_to_md(c.user.login, c.created_at, c.body) + body += @formatter.comment(c.user.login, c.created_at, c.body) end end diff --git a/lib/gitlab/gitlab_import/client.rb b/lib/gitlab/gitlab_import/client.rb index 2206b68da99..ecf4ff94e39 100644 --- a/lib/gitlab/gitlab_import/client.rb +++ b/lib/gitlab/gitlab_import/client.rb @@ -9,7 +9,7 @@ module Gitlab @client = ::OAuth2::Client.new( config.app_id, config.app_secret, - github_options + gitlab_options ) if access_token @@ -70,12 +70,8 @@ module Gitlab Gitlab.config.omniauth.providers.find{|provider| provider.name == "gitlab"} end - def github_options - { - site: 'https://gitlab.com/', - authorize_url: 'oauth/authorize', - token_url: 'oauth/token' - } + def gitlab_options + OmniAuth::Strategies::GitLab.default_options[:client_options] end end end diff --git a/lib/gitlab/gitlab_import/importer.rb b/lib/gitlab/gitlab_import/importer.rb index 5f9b14399a4..c5304a0699b 100644 --- a/lib/gitlab/gitlab_import/importer.rb +++ b/lib/gitlab/gitlab_import/importer.rb @@ -25,7 +25,7 @@ module Gitlab end comments.each do |comment| - body += @formatter.comment_to_md(comment["author"]["name"], comment["created_at"], comment["body"]) + body += @formatter.comment(comment["author"]["name"], comment["created_at"], comment["body"]) end project.issues.create!( diff --git a/lib/gitlab/import_formatter.rb b/lib/gitlab/import_formatter.rb index ebb4b87f7e3..72e041a90b1 100644 --- a/lib/gitlab/import_formatter.rb +++ b/lib/gitlab/import_formatter.rb @@ -1,6 +1,6 @@ module Gitlab class ImportFormatter - def comment_to_md(author, date, body) + def comment(author, date, body) "\n\n*By #{author} on #{date}*\n\n#{body}" end diff --git a/spec/controllers/import/bitbucket_controller_spec.rb b/spec/controllers/import/bitbucket_controller_spec.rb new file mode 100644 index 00000000000..84e37ae5607 --- /dev/null +++ b/spec/controllers/import/bitbucket_controller_spec.rb @@ -0,0 +1,77 @@ +require 'spec_helper' + +describe Import::BitbucketController do + let(:user) { create(:user, bitbucket_access_token: 'asd123', bitbucket_access_token_secret: "sekret") } + + before do + sign_in(user) + end + + describe "GET callback" do + before do + session[:oauth_request_token] = {} + end + + it "updates access token" do + token = "asdasd12345" + secret = "sekrettt" + access_token = double(token: token, secret: secret) + Gitlab::BitbucketImport::Client.any_instance.stub(:get_token).and_return(access_token) + Gitlab.config.omniauth.providers << OpenStruct.new(app_id: "asd123", app_secret: "asd123", name: "bitbucket") + + get :callback + + expect(user.reload.bitbucket_access_token).to eq(token) + expect(user.reload.bitbucket_access_token_secret).to eq(secret) + expect(controller).to redirect_to(status_import_bitbucket_url) + end + end + + describe "GET status" do + before do + @repo = OpenStruct.new(slug: 'vim', owner: 'asd') + end + + it "assigns variables" do + @project = create(:project, import_type: 'bitbucket', creator_id: user.id) + controller.stub_chain(:client, :projects).and_return([@repo]) + + get :status + + expect(assigns(:already_added_projects)).to eq([@project]) + expect(assigns(:repos)).to eq([@repo]) + end + + it "does not show already added project" do + @project = create(:project, import_type: 'bitbucket', creator_id: user.id, import_source: 'asd/vim') + controller.stub_chain(:client, :projects).and_return([@repo]) + + get :status + + expect(assigns(:already_added_projects)).to eq([@project]) + expect(assigns(:repos)).to eq([]) + end + end + + describe "POST create" do + before do + @repo = { + slug: 'vim', + owner: "john" + }.with_indifferent_access + end + + it "takes already existing namespace" do + namespace = create(:namespace, name: "john", owner: user) + expect(Gitlab::BitbucketImport::KeyAdder). + to receive(:new).with(@repo, user). + and_return(double(execute: true)) + expect(Gitlab::BitbucketImport::ProjectCreator). + to receive(:new).with(@repo, namespace, user). + and_return(double(execute: true)) + controller.stub_chain(:client, :project).and_return(@repo) + + post :create, format: :js + end + end +end diff --git a/spec/lib/gitlab/bitbucket_import/project_creator_spec.rb b/spec/lib/gitlab/bitbucket_import/project_creator_spec.rb new file mode 100644 index 00000000000..f5523105848 --- /dev/null +++ b/spec/lib/gitlab/bitbucket_import/project_creator_spec.rb @@ -0,0 +1,22 @@ +require 'spec_helper' + +describe Gitlab::BitbucketImport::ProjectCreator do + let(:user) { create(:user, bitbucket_access_token: "asdffg", bitbucket_access_token_secret: "sekret") } + let(:repo) { { + name: 'Vim', + slug: 'vim', + is_private: true, + owner: "asd"}.with_indifferent_access + } + let(:namespace){ create(:namespace) } + + it 'creates project' do + allow_any_instance_of(Project).to receive(:add_import_job) + + project_creator = Gitlab::BitbucketImport::ProjectCreator.new(repo, namespace, user) + project = project_creator.execute + + expect(project.import_url).to eq("ssh://git@bitbucket.org/asd/vim.git") + expect(project.visibility_level).to eq(Gitlab::VisibilityLevel::PRIVATE) + end +end diff --git a/spec/lib/gitlab/github/project_creator.rb b/spec/lib/gitlab/github_import/project_creator_spec.rb similarity index 77% rename from spec/lib/gitlab/github/project_creator.rb rename to spec/lib/gitlab/github_import/project_creator_spec.rb index 3686ddbf170..8d594a112d4 100644 --- a/spec/lib/gitlab/github/project_creator.rb +++ b/spec/lib/gitlab/github_import/project_creator_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -describe Gitlab::Github::ProjectCreator do +describe Gitlab::GithubImport::ProjectCreator do let(:user) { create(:user, github_access_token: "asdffg") } let(:repo) { OpenStruct.new( login: 'vim', @@ -15,9 +15,8 @@ describe Gitlab::Github::ProjectCreator do it 'creates project' do allow_any_instance_of(Project).to receive(:add_import_job) - project_creator = Gitlab::Github::ProjectCreator.new(repo, namespace, user) - project_creator.execute - project = Project.last + project_creator = Gitlab::GithubImport::ProjectCreator.new(repo, namespace, user) + project = project_creator.execute expect(project.import_url).to eq("https://asdffg@gitlab.com/asd/vim.git") expect(project.visibility_level).to eq(Gitlab::VisibilityLevel::PRIVATE) diff --git a/spec/lib/gitlab/gitlab_import/project_creator.rb b/spec/lib/gitlab/gitlab_import/project_creator_spec.rb similarity index 91% rename from spec/lib/gitlab/gitlab_import/project_creator.rb rename to spec/lib/gitlab/gitlab_import/project_creator_spec.rb index e5d917830b0..4c0d64ed138 100644 --- a/spec/lib/gitlab/gitlab_import/project_creator.rb +++ b/spec/lib/gitlab/gitlab_import/project_creator_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' describe Gitlab::GitlabImport::ProjectCreator do let(:user) { create(:user, gitlab_access_token: "asdffg") } - let(:repo) {{ + let(:repo) { { name: 'vim', path: 'vim', visibility_level: Gitlab::VisibilityLevel::PRIVATE, @@ -16,8 +16,7 @@ describe Gitlab::GitlabImport::ProjectCreator do allow_any_instance_of(Project).to receive(:add_import_job) project_creator = Gitlab::GitlabImport::ProjectCreator.new(repo, namespace, user) - project_creator.execute - project = Project.last + project = project_creator.execute expect(project.import_url).to eq("https://oauth2:asdffg@gitlab.com/asd/vim.git") expect(project.visibility_level).to eq(Gitlab::VisibilityLevel::PRIVATE)