From 5f715f1d32c6f5ce25b3721bde8f476173afadc8 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Thu, 23 Mar 2017 03:54:49 +0900 Subject: [PATCH] Add scheduled_trigger model. Add cron parser. Plus, specs. --- app/models/ci/scheduled_trigger.rb | 23 ++++ app/workers/scheduled_trigger_worker.rb | 18 +++ ...0322070910_create_ci_scheduled_triggers.rb | 45 ++++++ db/schema.rb | 24 +++- lib/ci/cron_parser.rb | 30 ++++ spec/factories/ci/scheduled_triggers.rb | 42 ++++++ spec/lib/ci/cron_parser_spec.rb | 128 ++++++++++++++++++ spec/models/ci/scheduled_trigger_spec.rb | 38 ++++++ spec/workers/scheduled_trigger_worker_spec.rb | 11 ++ 9 files changed, 356 insertions(+), 3 deletions(-) create mode 100644 app/models/ci/scheduled_trigger.rb create mode 100644 app/workers/scheduled_trigger_worker.rb create mode 100644 db/migrate/20170322070910_create_ci_scheduled_triggers.rb create mode 100644 lib/ci/cron_parser.rb create mode 100644 spec/factories/ci/scheduled_triggers.rb create mode 100644 spec/lib/ci/cron_parser_spec.rb create mode 100644 spec/models/ci/scheduled_trigger_spec.rb create mode 100644 spec/workers/scheduled_trigger_worker_spec.rb diff --git a/app/models/ci/scheduled_trigger.rb b/app/models/ci/scheduled_trigger.rb new file mode 100644 index 00000000000..5b1ff7bd7a4 --- /dev/null +++ b/app/models/ci/scheduled_trigger.rb @@ -0,0 +1,23 @@ +module Ci + class ScheduledTrigger < ActiveRecord::Base + extend Ci::Model + + acts_as_paranoid + + belongs_to :project + belongs_to :owner, class_name: "User" + + def schedule_next_run! + next_time = Ci::CronParser.new(cron, cron_time_zone).next_time_from_now + update(:next_run_at => next_time) if next_time.present? + end + + def valid_ref? + true #TODO: + end + + def update_last_run! + update(:last_run_at => Time.now) + end + end +end diff --git a/app/workers/scheduled_trigger_worker.rb b/app/workers/scheduled_trigger_worker.rb new file mode 100644 index 00000000000..7dc17aa4332 --- /dev/null +++ b/app/workers/scheduled_trigger_worker.rb @@ -0,0 +1,18 @@ +class ScheduledTriggerWorker + include Sidekiq::Worker + include CronjobQueue + + def perform + # TODO: Update next_run_at + + Ci::ScheduledTriggers.where("next_run_at < ?", Time.now).find_each do |trigger| + begin + Ci::CreateTriggerRequestService.new.execute(trigger.project, trigger, trigger.ref) + rescue => e + Rails.logger.error "#{trigger.id}: Failed to trigger job: #{e.message}" + ensure + trigger.schedule_next_run! + end + end + end +end diff --git a/db/migrate/20170322070910_create_ci_scheduled_triggers.rb b/db/migrate/20170322070910_create_ci_scheduled_triggers.rb new file mode 100644 index 00000000000..91e4b42d2af --- /dev/null +++ b/db/migrate/20170322070910_create_ci_scheduled_triggers.rb @@ -0,0 +1,45 @@ +# See http://doc.gitlab.com/ce/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class CreateCiScheduledTriggers < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + # Set this constant to true if this migration requires downtime. + DOWNTIME = false + + # When a migration requires downtime you **must** uncomment the following + # constant and define a short and easy to understand explanation as to why the + # migration requires downtime. + # DOWNTIME_REASON = '' + + # When using the methods "add_concurrent_index" or "add_column_with_default" + # you must disable the use of transactions as these methods can not run in an + # existing transaction. When using "add_concurrent_index" make sure that this + # method is the _only_ method called in the migration, any other changes + # should go in a separate migration. This ensures that upon failure _only_ the + # index creation fails and can be retried or reverted easily. + # + # To disable transactions uncomment the following line and remove these + # comments: + # disable_ddl_transaction! + + def change + create_table :ci_scheduled_triggers do |t| + t.integer "project_id" + t.datetime "deleted_at" + t.datetime "created_at" + t.datetime "updated_at" + t.integer "owner_id" + t.string "description" + t.string "cron" + t.string "cron_time_zone" + t.datetime "next_run_at" + t.datetime "last_run_at" + t.string "ref" + end + + add_index :ci_scheduled_triggers, ["next_run_at"], name: "index_ci_scheduled_triggers_on_next_run_at", using: :btree + add_index :ci_scheduled_triggers, ["project_id"], name: "index_ci_scheduled_triggers_on_project_id", using: :btree + add_foreign_key :ci_scheduled_triggers, :users, column: :owner_id, on_delete: :cascade + end +end diff --git a/db/schema.rb b/db/schema.rb index 582f68cbee7..a101ce280fe 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -61,7 +61,6 @@ ActiveRecord::Schema.define(version: 20170405080720) do t.boolean "shared_runners_enabled", default: true, null: false t.integer "max_artifacts_size", default: 100, null: false t.string "runners_registration_token" - t.integer "max_pages_size", default: 100, null: false t.boolean "require_two_factor_authentication", default: false t.integer "two_factor_grace_period", default: 48 t.boolean "metrics_enabled", default: false @@ -111,6 +110,7 @@ ActiveRecord::Schema.define(version: 20170405080720) do t.string "plantuml_url" t.boolean "plantuml_enabled" t.integer "terminal_max_session_time", default: 0, null: false + t.integer "max_pages_size", default: 100, null: false t.string "default_artifacts_expire_in", default: "0", null: false t.integer "unique_ips_limit_per_user" t.integer "unique_ips_limit_time_window" @@ -290,6 +290,23 @@ ActiveRecord::Schema.define(version: 20170405080720) do add_index "ci_runners", ["locked"], name: "index_ci_runners_on_locked", using: :btree add_index "ci_runners", ["token"], name: "index_ci_runners_on_token", using: :btree + create_table "ci_scheduled_triggers", force: :cascade do |t| + t.integer "project_id" + t.datetime "deleted_at" + t.datetime "created_at" + t.datetime "updated_at" + t.integer "owner_id" + t.string "description" + t.string "cron" + t.string "cron_time_zone" + t.datetime "next_run_at" + t.datetime "last_run_at" + t.string "ref" + end + + add_index "ci_scheduled_triggers", ["next_run_at"], name: "index_ci_scheduled_triggers_on_next_run_at", using: :btree + add_index "ci_scheduled_triggers", ["project_id"], name: "index_ci_scheduled_triggers_on_project_id", using: :btree + create_table "ci_trigger_requests", force: :cascade do |t| t.integer "trigger_id", null: false t.text "variables" @@ -689,8 +706,8 @@ ActiveRecord::Schema.define(version: 20170405080720) do t.integer "visibility_level", default: 20, null: false t.boolean "request_access_enabled", default: false, null: false t.datetime "deleted_at" - t.text "description_html" t.boolean "lfs_enabled" + t.text "description_html" t.integer "parent_id" end @@ -1242,8 +1259,8 @@ ActiveRecord::Schema.define(version: 20170405080720) do t.datetime "otp_grace_period_started_at" t.boolean "ldap_email", default: false, null: false t.boolean "external", default: false - t.string "incoming_email_token" t.string "organization" + t.string "incoming_email_token" t.boolean "authorized_projects_populated" t.boolean "ghost" t.boolean "notified_of_own_activity" @@ -1298,6 +1315,7 @@ ActiveRecord::Schema.define(version: 20170405080720) do add_foreign_key "boards", "projects" add_foreign_key "chat_teams", "namespaces", on_delete: :cascade + add_foreign_key "ci_scheduled_triggers", "users", column: "owner_id", on_delete: :cascade add_foreign_key "ci_triggers", "users", column: "owner_id", name: "fk_e8e10d1964", on_delete: :cascade add_foreign_key "issue_metrics", "issues", on_delete: :cascade add_foreign_key "label_priorities", "labels", on_delete: :cascade diff --git a/lib/ci/cron_parser.rb b/lib/ci/cron_parser.rb new file mode 100644 index 00000000000..163cfc86aa7 --- /dev/null +++ b/lib/ci/cron_parser.rb @@ -0,0 +1,30 @@ +require 'rufus-scheduler' # Included in sidekiq-cron + +module Ci + class CronParser + def initialize(cron, cron_time_zone = 'UTC') + @cron = cron + @cron_time_zone = cron_time_zone + end + + def next_time_from_now + cronLine = try_parse_cron + return nil unless cronLine.present? + cronLine.next_time + end + + def valid_syntax? + try_parse_cron.present? ? true : false + end + + private + + def try_parse_cron + begin + Rufus::Scheduler.parse("#{@cron} #{@cron_time_zone}") + rescue + nil + end + end + end +end diff --git a/spec/factories/ci/scheduled_triggers.rb b/spec/factories/ci/scheduled_triggers.rb new file mode 100644 index 00000000000..9d45f4b4962 --- /dev/null +++ b/spec/factories/ci/scheduled_triggers.rb @@ -0,0 +1,42 @@ +FactoryGirl.define do + factory :ci_scheduled_trigger, class: Ci::ScheduledTrigger do + project factory: :empty_project + owner factory: :user + ref 'master' + + trait :cron_nightly_build do + cron '0 1 * * *' + cron_time_zone 'Europe/Istanbul' + end + + trait :cron_weekly_build do + cron '0 1 * * 5' + cron_time_zone 'Europe/Istanbul' + end + + trait :cron_monthly_build do + cron '0 1 22 * *' + cron_time_zone 'Europe/Istanbul' + end + + trait :cron_every_5_minutes do + cron '*/5 * * * *' + cron_time_zone 'Europe/Istanbul' + end + + trait :cron_every_5_hours do + cron '* */5 * * *' + cron_time_zone 'Europe/Istanbul' + end + + trait :cron_every_5_days do + cron '* * */5 * *' + cron_time_zone 'Europe/Istanbul' + end + + trait :cron_every_5_months do + cron '* * * */5 *' + cron_time_zone 'Europe/Istanbul' + end + end +end diff --git a/spec/lib/ci/cron_parser_spec.rb b/spec/lib/ci/cron_parser_spec.rb new file mode 100644 index 00000000000..58eb26c9421 --- /dev/null +++ b/spec/lib/ci/cron_parser_spec.rb @@ -0,0 +1,128 @@ +require 'spec_helper' + +module Ci + describe CronParser, lib: true do + describe '#next_time_from_now' do + subject { described_class.new(cron, cron_time_zone).next_time_from_now } + + context 'when cron and cron_time_zone are valid' do + context 'at 00:00, 00:10, 00:20, 00:30, 00:40, 00:50' do + let(:cron) { '*/10 * * * *' } + let(:cron_time_zone) { 'US/Pacific' } + + it 'returns next time from now' do + time = Time.now.in_time_zone(cron_time_zone) + time = time + 10.minutes + time = time.change(sec: 0, min: time.min-time.min%10) + is_expected.to eq(time) + end + end + + context 'at 10:00, 20:00' do + let(:cron) { '0 */10 * * *' } + let(:cron_time_zone) { 'US/Pacific' } + + it 'returns next time from now' do + time = Time.now.in_time_zone(cron_time_zone) + time = time + 10.hours + time = time.change(sec: 0, min: 0, hour: time.hour-time.hour%10) + is_expected.to eq(time) + end + end + + context 'when cron is every 10 days' do + let(:cron) { '0 0 */10 * *' } + let(:cron_time_zone) { 'US/Pacific' } + + it 'returns next time from now' do + time = Time.now.in_time_zone(cron_time_zone) + time = time + 10.days + time = time.change(sec: 0, min: 0, hour: 0, day: time.day-time.day%10) + is_expected.to eq(time) + end + end + + context 'when cron is every week 2:00 AM' do + let(:cron) { '0 2 * * *' } + let(:cron_time_zone) { 'US/Pacific' } + + it 'returns next time from now' do + time = Time.now.in_time_zone(cron_time_zone) + is_expected.to eq(time.change(sec: 0, min: 0, hour: 2, day: time.day+1)) + end + end + + context 'when cron_time_zone is US/Pacific' do + let(:cron) { '0 1 * * *' } + let(:cron_time_zone) { 'US/Pacific' } + + it 'returns next time from now' do + time = Time.now.in_time_zone(cron_time_zone) + is_expected.to eq(time.change(sec: 0, min: 0, hour: 1, day: time.day+1)) + end + end + + context 'when cron_time_zone is Europe/London' do + let(:cron) { '0 1 * * *' } + let(:cron_time_zone) { 'Europe/London' } + + it 'returns next time from now' do + time = Time.now.in_time_zone(cron_time_zone) + is_expected.to eq(time.change(sec: 0, min: 0, hour: 1, day: time.day+1)) + end + end + + context 'when cron_time_zone is Asia/Tokyo' do + let(:cron) { '0 1 * * *' } + let(:cron_time_zone) { 'Asia/Tokyo' } + + it 'returns next time from now' do + time = Time.now.in_time_zone(cron_time_zone) + is_expected.to eq(time.change(sec: 0, min: 0, hour: 1, day: time.day+1)) + end + end + end + + context 'when cron is given and cron_time_zone is not given' do + let(:cron) { '0 1 * * *' } + + it 'returns next time from now in utc' do + obj = described_class.new(cron).next_time_from_now + time = Time.now.in_time_zone('UTC') + expect(obj).to eq(time.change(sec: 0, min: 0, hour: 1, day: time.day+1)) + end + end + + context 'when cron and cron_time_zone are invalid' do + let(:cron) { 'hack' } + let(:cron_time_zone) { 'hack' } + + it 'returns nil' do + is_expected.to be_nil + end + end + end + + describe '#valid_syntax?' do + subject { described_class.new(cron, cron_time_zone).valid_syntax? } + + context 'when cron and cron_time_zone are valid' do + let(:cron) { '* * * * *' } + let(:cron_time_zone) { 'Europe/Istanbul' } + + it 'returns true' do + is_expected.to eq(true) + end + end + + context 'when cron and cron_time_zone are invalid' do + let(:cron) { 'hack' } + let(:cron_time_zone) { 'hack' } + + it 'returns false' do + is_expected.to eq(false) + end + end + end + end +end diff --git a/spec/models/ci/scheduled_trigger_spec.rb b/spec/models/ci/scheduled_trigger_spec.rb new file mode 100644 index 00000000000..68ba9c379b8 --- /dev/null +++ b/spec/models/ci/scheduled_trigger_spec.rb @@ -0,0 +1,38 @@ +require 'spec_helper' +require 'rufus-scheduler' # Included in sidekiq-cron + +describe Ci::ScheduledTrigger, models: true do + + describe 'associations' do + it { is_expected.to belong_to(:project) } + it { is_expected.to belong_to(:owner) } + end + + describe '#schedule_next_run!' do + context 'when cron and cron_time_zone are vaild' do + context 'when nightly build' do + it 'schedules next run' do + scheduled_trigger = create(:ci_scheduled_trigger, :cron_nightly_build) + scheduled_trigger.schedule_next_run! + puts "scheduled_trigger: #{scheduled_trigger.inspect}" + + expect(scheduled_trigger.cron).to be_nil + end + end + + context 'when weekly build' do + + end + + context 'when monthly build' do + + end + end + + context 'when cron and cron_time_zone are invaild' do + it 'schedules nothing' do + + end + end + end +end diff --git a/spec/workers/scheduled_trigger_worker_spec.rb b/spec/workers/scheduled_trigger_worker_spec.rb new file mode 100644 index 00000000000..c17536720a4 --- /dev/null +++ b/spec/workers/scheduled_trigger_worker_spec.rb @@ -0,0 +1,11 @@ +require 'spec_helper' + +describe ScheduledTriggerWorker do + subject { described_class.new.perform } + + context '#perform' do # TODO: + it 'does' do + is_expected.to be_nil + end + end +end