Yorick Peterse 237c8f66e6 Precalculate trending projects ... This commit introduces a Sidekiq worker that precalculates the list of trending projects on a daily basis. The resulting set is stored in a database table that is then queried by Project.trending. This setup means that Unicorn workers no longer _may_ have to calculate the list of trending projects. Furthermore it supports filtering without any complex caching mechanisms. The data in the "trending_projects" table is inserted in the same order as the project ranking. This means that getting the projects in the correct order is simply a matter of: SELECT projects.* FROM projects INNER JOIN trending_projects ON trending_projects.project_id = projects.id ORDER BY trending_projects.id ASC; Such a query will only take a few milliseconds at most (as measured on GitLab.com), opposed to a few seconds for the query used for calculating the project ranks. The migration in this commit does not require downtime and takes care of populating an initial list of trending projects.		2016-10-10 12:27:08 +02:00
..
20121220064453_init_schema.rb
20130102143055_rename_owner_to_creator_for_project.rb
20130110172407_add_public_to_project.rb
20130123114545_add_issues_tracker_to_project.rb
20130125090214_add_user_permissions.rb
20130131070232_remove_private_flag_from_project.rb
20130206084024_add_description_to_namsespace.rb
20130207104426_add_description_to_teams.rb
20130211085435_add_issues_tracker_id_to_project.rb
20130214154045_rename_state_to_merge_status_in_milestone.rb
20130218140952_add_state_to_issue.rb
20130218141038_add_state_to_merge_request.rb
20130218141117_add_state_to_milestone.rb
20130218141258_convert_closed_to_state_in_issue.rb
20130218141327_convert_closed_to_state_in_merge_request.rb
20130218141344_convert_closed_to_state_in_milestone.rb
20130218141444_remove_merged_from_merge_request.rb
20130218141507_remove_closed_from_issue.rb
20130218141536_remove_closed_from_merge_request.rb
20130218141554_remove_closed_from_milestone.rb
20130220124204_add_new_merge_status_to_merge_request.rb
20130220125544_convert_merge_status_in_merge_request.rb
20130220125545_remove_merge_status_from_merge_request.rb
20130220133245_rename_new_merge_status_to_merge_status_in_milestone.rb
20130304104623_add_state_to_user.rb
20130304104740_convert_blocked_to_state.rb
20130304105317_remove_blocked_from_user.rb
20130315124931_user_color_scheme.rb
20130318212250_add_snippets_to_features.rb
20130319214458_create_forked_project_links.rb
20130323174317_add_private_to_snippets.rb
20130324151736_add_type_to_snippets.rb
20130324172327_change_project_id_to_null_in_snipepts.rb
20130324203535_add_type_value_for_snippets.rb
20130325173941_add_notification_level_to_user.rb
20130326142630_add_index_to_users_authentication_token.rb
20130403003950_add_last_activity_column_into_project.rb
20130404164628_add_notification_level_to_user_project.rb
20130410175022_remove_wiki_table.rb
20130419190306_allow_merges_for_forks.rb
20130506085413_add_type_to_key.rb
20130506090604_create_deploy_keys_projects.rb
20130506095501_remove_project_id_from_key.rb
20130522141856_add_more_fields_to_service.rb
20130528184641_add_system_to_notes.rb
20130611210815_increase_snippet_text_column_size.rb
20130613165816_add_password_expires_at_to_users.rb
20130613173246_add_created_by_id_to_user.rb
20130614132337_add_improted_to_project.rb
20130617095603_create_users_groups.rb
20130621195223_add_notification_level_to_user_group.rb
20130622115340_add_more_db_index.rb
20130624162710_add_fingerprint_to_key.rb
20130711063759_create_project_group_links.rb
20130804151314_add_st_diff_to_note.rb
20130809124851_add_permission_check_to_user.rb
20130812143708_add_import_url_to_project.rb
20130819182730_add_internal_ids_to_issues_and_mr.rb
20130820102832_add_access_to_project_group_link.rb
20130821090530_remove_deprecated_tables.rb
20130821090531_add_internal_ids_to_milestones.rb
20130909132950_add_description_to_merge_request.rb
20130926081215_change_owner_id_for_group.rb
20131005191208_add_avatar_to_users.rb
20131009115346_add_confirmable_to_users.rb
20131106151520_remove_default_branch.rb
20131112114325_create_broadcast_messages.rb
20131112220935_add_visibility_level_to_projects.rb
20131129154016_add_archived_to_projects.rb
20131130165425_add_color_and_font_to_broadcast_messages.rb
20131202192556_add_event_fields_for_web_hook.rb
20131214224427_add_hide_no_ssh_key_to_users.rb
20131217102743_add_recipients_to_service.rb
20140116231608_add_website_url_to_users.rb
20140122112253_create_merge_request_diffs.rb
20140122114406_migrate_mr_diffs.rb
20140122122549_remove_m_rdiff_fields.rb
20140125162722_add_avatar_to_projects.rb
20140127170938_add_group_avatars.rb
20140209025651_create_emails.rb
20140214102325_add_api_key_to_services.rb
20140304005354_add_index_merge_request_diffs_on_merge_request_id.rb
20140305193308_add_tag_push_hooks_to_project_hook.rb
20140312145357_add_import_status_to_project.rb
20140313092127_migrate_already_imported_projects.rb
20140407135544_fix_namespaces.rb
20140414131055_change_state_to_allow_empty_merge_request_diffs.rb
20140415124820_limits_to_mysql.rb
20140416074002_add_index_on_iid.rb
20140416185734_index_on_current_sign_in_at.rb
20140428105831_add_notes_index_updated_at.rb
20140502115131_add_repo_size_to_db.rb
20140502125220_migrate_repo_size.rb
20140611135229_add_position_to_merge_request.rb
20140625115202_create_users_star_projects.rb
20140729134820_create_labels.rb
20140729140420_create_label_links.rb
20140729145339_migrate_project_tags.rb
20140729152420_migrate_taggable_labels.rb
20140730111702_add_index_to_labels.rb
20140903115954_migrate_to_new_shell.rb
20140907220153_serialize_service_properties.rb
20140914113604_add_members_table.rb
20140914145549_migrate_to_new_members_model.rb
20140914173417_remove_old_member_tables.rb
20141006143943_move_slack_service_to_webhook.rb
20141007100818_add_visibility_level_to_snippet.rb
20141118150935_add_audit_event.rb
20141121133009_add_timestamps_to_members.rb
20141121161704_add_identity_table.rb
20141205134006_add_locked_at_to_merge_request.rb
20141216155758_create_doorkeeper_tables.rb
20141217125223_add_owner_to_application.rb
20141223135007_add_import_data_to_project_table.rb
20141226080412_add_developers_can_push_to_protected_branches.rb
20150108073740_create_application_settings.rb
20150116234544_add_home_page_url_for_application_settings.rb
20150116234545_add_gitlab_access_token_to_user.rb
20150125163100_add_default_branch_protection_setting.rb
20150205211843_add_timestamps_to_identities.rb
20150206181414_add_index_to_created_at.rb
20150206222854_add_notification_email_to_user.rb
20150209222013_add_missing_index.rb
20150211172122_add_template_to_service.rb
20150211174341_allow_null_in_services_project_id.rb
20150213104043_add_twitter_sharing_enabled_to_application_settings.rb
20150213114800_add_hide_no_password_to_user.rb
20150213121042_add_password_automatically_set_to_user.rb
20150217123345_add_bitbucket_access_token_and_secret_to_user.rb
20150219004514_add_events_to_services.rb
20150223022001_set_missing_last_activity_at.rb
20150225065047_add_note_events_to_services.rb
20150301014758_add_restricted_visibility_levels_to_application_settings.rb
20150306023106_fix_namespace_duplication.rb
20150306023112_add_unique_index_to_namespace.rb
20150310194358_add_version_check_to_application_settings.rb
20150313012111_create_subscriptions_table.rb
20150320234437_add_location_to_user.rb
20150324155957_set_incorrect_assignee_id_to_null.rb
20150327122227_add_public_to_key.rb
20150327150017_add_import_data_to_project.rb
20150327223628_add_devise_two_factor_to_users.rb
20150328132231_add_max_attachment_size_to_application_settings.rb
20150331183602_add_devise_two_factor_backupable_to_users.rb
20150406133311_add_invite_data_to_member.rb
20150411000035_fix_identities.rb
20150411180045_rename_buildbox_service.rb
20150413192223_add_public_email_to_users.rb
20150417121913_create_project_import_data.rb
20150417122318_remove_import_data_from_project.rb
20150421120000_remove_periods_at_ends_of_usernames.rb
20150423033240_add_default_project_visibililty_to_application_settings.rb
20150425164646_gitlab_change_collation_for_tag_names.acts_as_taggable_on_engine.rb
20150425164647_remove_duplicate_tags.rb
20150425164648_add_missing_unique_indices.acts_as_taggable_on_engine.rb
20150425164649_add_taggings_counter_cache_to_tags.acts_as_taggable_on_engine.rb
20150425164650_add_missing_taggable_index.acts_as_taggable_on_engine.rb
20150425164651_change_collation_for_tag_names.acts_as_taggable_on_engine.rb
20150425173433_add_default_snippet_visibility_to_app_settings.rb
20150429002313_remove_abandoned_group_members_records.rb
20150502064022_add_restricted_signup_domains_to_application_settings.rb
20150509180749_convert_legacy_reference_notes.rb
20150516060434_add_note_events_to_web_hooks.rb
20150529111607_add_user_oauth_applications_to_application_settings.rb
20150529150354_add_after_sign_out_path_for_application_settings.rb
20150609141121_add_session_expire_delay_for_application_settings.rb
20150610065936_add_dashboard_to_users.rb
20150620233230_add_default_otp_required_for_login_value.rb
20150713160110_add_project_view_to_users.rb
20150717130904_add_commits_count_to_project.rb
20150730122406_add_updated_by_to_issuables_and_notes.rb
20150806104937_create_abuse_reports.rb
20150812080800_add_settings_import_sources.rb
20150814065925_remove_oauth_tokens_from_users.rb
20150817163600_deduplicate_user_identities.rb
20150818213832_add_sent_notifications.rb
20150824002011_add_enable_ssl_verification.rb
20150826001931_add_ci_tables.rb
20150902001023_add_template_to_label.rb
20150914215247_add_ci_tags.rb
20150915001905_enable_ssl_verification_by_default.rb
20150916000405_enable_ssl_verification_for_web_hooks.rb
20150916114643_add_help_page_text_to_application_settings.rb
20150916145038_add_index_for_committed_at_and_id.rb
20150918084513_add_ci_enabled_to_application_settings.rb
20150918161719_remove_invalid_milestones_from_merge_requests.rb
20150920010715_add_consumed_timestep_to_users.rb
20150920161119_add_line_code_to_sent_notification.rb
20150924125150_add_project_id_to_ci_commit.rb
20150924125436_migrate_project_id_for_ci_commits.rb
20150930001110_merge_request_error_field.rb
20150930095736_add_null_to_name_for_ci_projects.rb
20150930110012_add_group_share_lock.rb
20151002112914_add_stage_idx_to_builds.rb
20151002121400_add_index_for_builds.rb
20151002122929_add_ref_and_tag_to_builds.rb
20151002122943_migrate_ref_and_tag_to_build.rb
20151005075649_add_user_id_to_build.rb
20151005150751_add_layout_option_for_users.rb
20151005162154_remove_ci_enabled_from_application_settings.rb
20151007120511_namespaces_projects_path_lower_indexes.rb
20151008110232_add_users_lower_username_email_indexes.rb
20151008123042_add_type_and_description_to_builds.rb
20151008130321_migrate_name_to_description_for_builds.rb
20151008143519_add_admin_notification_email_setting.rb
20151012173029_set_jira_service_api_url.rb
20151013092124_add_artifacts_file_to_builds.rb
20151016131433_add_ci_projects_gl_project_id_index.rb
20151016195451_add_ci_builds_and_projects_indexes.rb
20151016195706_add_notes_line_code_index.rb
20151019111551_fix_build_tags.rb
20151019111703_fail_build_without_names.rb
20151020145526_add_services_template_index.rb
20151020173516_ci_limits_to_mysql.rb
20151020173906_add_ci_builds_index_for_status.rb
20151023112551_fail_build_with_empty_name.rb
20151023144219_remove_satellites.rb
20151026182941_add_project_path_index.rb
20151028152939_add_merge_when_build_succeeds_to_merge_request.rb
20151103001141_add_public_to_group.rb
20151103133339_add_shared_runners_setting.rb
20151103134857_create_lfs_objects.rb
20151103134958_create_lfs_objects_projects.rb
20151104105513_add_file_to_lfs_objects.rb
20151105094515_create_releases.rb
20151106000015_add_is_award_to_notes.rb
20151109100728_add_max_artifacts_size_to_application_settings.rb
20151109134526_add_issues_state_index.rb
20151109134916_add_projects_visibility_level_index.rb
20151110125604_add_import_error_to_project.rb
20151114113410_add_index_for_lfs_oid_and_size.rb
20151116144118_add_unique_for_lfs_oid_index.rb
20151118162244_add_projects_public_index.rb
20151201203948_raise_hook_url_limit.rb
20151203162133_add_hide_project_limit_to_users.rb
20151203162134_add_build_events_to_services.rb
20151209144329_migrate_ci_web_hooks.rb
20151209145909_migrate_ci_emails.rb
20151210030143_add_unlock_token_to_user.rb
20151210072243_add_runners_registration_token_to_application_settings.rb
20151210125232_migrate_ci_slack_service.rb
20151210125927_migrate_ci_hip_chat_service.rb
20151210125928_add_ci_to_project.rb
20151210125929_add_project_id_to_ci.rb
20151210125930_migrate_ci_to_project.rb
20151210125931_add_index_to_ci_tables.rb
20151210125932_drop_null_for_ci_tables.rb
20151218154042_add_tfa_to_application_settings.rb
20151221234414_add_tfa_additional_fields.rb
20151224123230_rename_emojis.rb
20151228111122_remove_public_from_namespace.rb
20151228150906_influxdb_settings.rb
20151228175719_add_recaptcha_to_application_settings.rb
20151229102248_influxdb_udp_port_setting.rb
20151229112614_influxdb_remote_database_setting.rb
20151230132518_add_artifacts_metadata_to_ci_build.rb
20151231152326_add_akismet_to_application_settings.rb
20151231202530_remove_alert_type_from_broadcast_messages.rb
20160106162223_add_index_milestones_title.rb
20160106164438_remove_influxdb_credentials.rb
20160109054846_create_spam_logs.rb
20160113111034_add_metrics_sample_interval.rb
20160118155830_add_sentry_to_application_settings.rb
20160118232755_add_ip_blocking_settings_to_application_settings.rb
20160119111158_add_services_category.rb
20160119112418_add_services_default.rb
20160119145451_add_ldap_email_to_users.rb
20160120172143_add_base_commit_sha_to_merge_request_diffs.rb
20160121030729_add_email_author_in_body_to_application_settings.rb
20160122185421_add_pending_delete_to_project.rb
20160128212447_remove_ip_blocking_settings_from_application_settings.rb
20160128233227_change_lfs_objects_size_column.rb
20160129135155_remove_dot_atom_path_ending_of_projects.rb
20160129155512_add_merge_commit_sha_to_merge_requests.rb
20160202091601_add_erasable_to_ci_build.rb
20160202164642_add_allow_guest_to_access_builds_project.rb
20160204144558_add_real_size_to_merge_request_diffs.rb
20160209130428_add_index_to_snippet.rb
20160212123307_create_tasks.rb
20160217100506_add_description_to_label.rb
20160217174422_add_note_to_tasks.rb
20160220123949_rename_tasks_to_todos.rb
20160222153918_create_appearances_ce.rb
20160223192159_add_confidential_to_issues.rb
20160225090018_add_delete_at_to_issues.rb
20160225101956_add_delete_at_to_merge_requests.rb
20160226114608_add_trigram_indexes_for_searching.rb
20160227120001_add_event_field_for_web_hook.rb
20160227120047_add_event_to_services.rb
20160229193553_add_main_language_to_repository.rb
20160301124843_add_visibility_level_to_groups.rb
20160302151724_add_import_credentials_to_project_import_data.rb
20160302152808_remove_wrong_import_url_from_projects.rb
20160305220806_remove_expires_at_from_snippets.rb
20160307221555_disallow_blank_line_code_on_note.rb
20160308212903_add_default_group_visibility_to_application_settings.rb
20160309140734_fix_todos.rb
20160310124959_add_due_date_to_issues.rb
20160310185910_add_external_flag_to_users.rb
20160314094147_add_priority_to_label.rb
20160314114439_add_requested_at_to_members.rb
20160314143402_projects_add_pushes_since_gc.rb
20160315135439_project_add_repository_check.rb
20160316123110_ci_runners_token_index.rb
20160316192622_change_target_id_to_null_on_todos.rb
20160316204731_add_commit_id_to_todos.rb
20160317092222_add_moved_to_to_issue.rb
20160320204112_index_namespaces_on_visibility_level.rb
20160324020319_remove_todos_for_deleted_issues.rb
20160328112808_create_notification_settings.rb
20160328115649_migrate_new_notification_setting.rb
20160328121138_add_notification_setting_index.rb
20160329144452_add_index_on_pending_delete_projects.rb
20160331133914_remove_todos_for_deleted_merge_requests.rb
20160331223143_remove_twitter_sharing_enabled_from_application_settings.rb
20160407120251_add_images_enabled_for_project.rb
20160412140240_add_repository_checks_enabled_setting.rb
20160412173416_add_fields_to_ci_commit.rb
20160412173417_update_ci_commit.rb
20160412173418_add_ci_commit_indexes.rb
20160413115152_add_token_to_web_hooks.rb
20160415062917_create_personal_access_tokens.rb
20160415133440_add_shared_runners_text_to_application_settings.rb
20160416180807_add_award_emoji.rb
20160416182152_convert_award_note_to_emoji_award.rb
20160419120017_add_metrics_packet_size.rb
20160419122101_add_only_allow_merge_if_build_succeeds_to_projects.rb
20160421130527_disable_repository_checks.rb
20160425045124_create_u2f_registrations.rb
20160504091942_add_disabled_oauth_sign_in_sources_to_application_settings.rb
20160504112519_add_run_untagged_to_ci_runner.rb
20160508194200_remove_wall_enabled_from_projects.rb
20160508202603_add_head_commit_id_to_merge_request_diffs.rb
20160508215820_add_type_to_notes.rb
20160508215920_add_positions_to_diff_notes.rb
20160508221410_set_type_on_legacy_diff_notes.rb
20160509091049_add_locked_to_ci_runner.rb
20160509201028_add_health_check_access_token_to_application_settings.rb
20160516174813_add_send_user_confirmation_email_to_application_settings.rb
20160516224534_add_start_commit_id_to_merge_request_diffs.rb
20160518200441_add_artifacts_expire_date_to_ci_builds.rb
20160519203051_add_developers_can_merge_to_protected_branches.rb
20160522215720_add_note_type_and_position_to_sent_notification.rb
20160525205328_remove_main_language_from_projects.rb
20160527020117_remove_notification_settings_for_deleted_projects.rb
20160528043124_add_users_state_index.rb
20160530150109_add_container_registry_token_expire_delay_to_application_settings.rb
20160603075128_add_has_external_issue_tracker_to_projects.rb
20160603180330_remove_duplicated_notification_settings.rb
20160603182247_add_index_to_notification_settings.rb
20160608155312_add_after_sign_up_text_to_application_settings.rb
20160608195742_add_repository_storage_to_projects.rb
20160608211215_add_user_default_external_to_application_settings.rb
20160610140403_remove_notification_setting_not_null_constraints.rb
20160610194713_remove_deprecated_issues_tracker_columns_from_projects.rb
20160610201627_migrate_users_notification_level.rb
20160610204157_add_deployments.rb
20160610204158_add_environments.rb
20160610211845_add_environment_to_builds.rb
20160610301627_remove_notification_level_from_users.rb
20160614182521_add_repository_storage_to_application_settings.rb
20160615142710_add_index_on_requested_at_to_members.rb
20160615173316_add_enabled_git_access_protocols_to_application_settings.rb
20160615191922_set_missing_stage_on_ci_builds.rb
20160616084004_change_project_of_environment.rb
20160616102642_remove_duplicated_keys.rb
20160616103005_remove_keys_fingerprint_index_if_exists.rb
20160616103948_add_unique_index_to_keys_fingerprint.rb
20160617301627_add_events_to_notification_settings.rb
20160620115026_add_index_on_runners_locked.rb
20160628085157_add_artifacts_size_to_ci_builds.rb
20160629025435_add_column_in_progress_merge_commit_sha_to_merge_requests.rb
20160703180340_add_index_on_award_emoji_user_and_name.rb
20160705054938_add_protected_branches_push_access.rb
20160705054952_add_protected_branches_merge_access.rb
20160705055254_move_from_developers_can_merge_to_protected_branches_merge_access.rb
20160705055308_move_from_developers_can_push_to_protected_branches_push_access.rb
20160705055809_remove_developers_can_push_from_protected_branches.rb
20160705055813_remove_developers_can_merge_from_protected_branches.rb
20160705163108_remove_requesters_that_are_owners.rb
20160707104333_add_lock_to_issuables.rb
20160712171823_remove_award_emojis_with_no_user.rb
20160713205315_add_domain_blacklist_to_application_settings.rb
20160715132507_add_user_id_to_pipeline.rb
20160715134306_add_index_for_pipeline_user_id.rb
20160715154212_add_request_access_enabled_to_projects.rb
20160715204316_add_request_access_enabled_to_groups.rb
20160715230841_rename_application_settings_restricted_signup_domains.rb
20160716115710_add_when_and_yaml_variables_to_ci_builds.rb
20160716115711_add_queued_at_to_ci_builds.rb
20160718153603_add_has_external_wiki_to_projects.rb
20160721081015_drop_and_readd_has_external_wiki_in_projects.rb
20160722221922_nullify_blank_type_on_notes.rb
20160724205507_add_resolved_to_notes.rb
20160725083350_add_external_url_to_enviroments.rb
20160725104020_merge_request_diff_remove_uniq.rb
20160725104452_merge_request_diff_add_index.rb
20160727163552_create_user_agent_details.rb
20160727191041_create_boards.rb
20160727193336_create_lists.rb
20160728081025_add_pipeline_events_to_web_hooks.rb
20160728103734_add_pipeline_events_to_services.rb
20160729173930_remove_project_id_from_spam_logs.rb
20160801163421_add_expires_at_to_member.rb
20160801163709_add_submitted_as_ham_to_spam_logs.rb
20160802010328_remove_builds_enable_index_on_projects.rb
20160803161903_add_unique_index_to_lists_label_id.rb
20160804150737_add_timestamps_to_members_again.rb
20160805041956_add_deleted_at_to_namespaces.rb
20160808085531_add_token_to_build.rb
20160808085602_add_index_for_build_token.rb
20160810102349_remove_ci_runner_trigram_indexes.rb
20160810142633_remove_redundant_indexes.rb
20160816161312_add_column_name_to_u2f_registrations.rb
20160817133006_add_koding_to_application_settings.rb
20160817154936_add_discussion_ids_to_notes.rb
20160818205718_add_expires_at_to_project_group_links.rb
20160819221631_add_index_to_note_discussion_id.rb
20160819221833_reset_diff_note_discussion_id_because_it_was_calculated_wrongly.rb
20160823081327_change_merge_error_to_text.rb
20160823213309_add_lfs_enabled_to_projects.rb
20160824103857_drop_unused_ci_tables.rb
20160824124900_add_table_issue_metrics.rb
20160825052008_add_table_merge_request_metrics.rb
20160827011312_ensure_lock_version_has_no_default.rb
20160829114652_add_markdown_cache_columns.rb
20160830203109_add_confidential_issues_events_to_web_hooks.rb
20160830211132_add_confidential_issues_events_to_services.rb
20160830232601_change_lock_version_not_null.rb
20160831214002_create_project_features.rb
20160831214543_migrate_project_features.rb
20160831223750_remove_features_enabled_from_projects.rb
20160901141443_set_confidential_issues_events_on_webhooks.rb
20160901213340_add_lfs_enabled_to_namespaces.rb
20160902122721_drop_gitorious_field_from_application_settings.rb
20160907131111_add_environment_type_to_environments.rb
20160913162434_remove_projects_pushes_since_gc.rb
20160913212128_change_artifacts_size_column.rb
20160915042921_create_merge_requests_closing_issues.rb
20160920160832_add_index_to_labels_title.rb
20160926145521_add_organization_to_user.rb
20161007133303_precalculate_trending_projects.rb
limits_to_mysql.rb