Commit graph

7 commits

Author SHA1 Message Date
GitLab Bot
88a0824944 Add latest changes from gitlab-org/gitlab@master 2020-02-04 12:09:00 +00:00
Peter Leitzen
8136fac26c Prefer flat_map over map + flatten
Convert several occurrences of `map` + `flatten` to
`flat_map` where applicable.
2019-07-26 10:44:13 +00:00
Toon Claes
83767dd0b3 Cleanup dead code and comments 2019-01-11 15:36:57 +01:00
Toon Claes
082cc12225 Fix failing spec with orphaned namespace 2019-01-11 15:36:57 +01:00
Toon Claes
d3e028b85f Load all projects, namespaces, routes in 1 query
Avoid doing sequential database queries to load the namespaces and the
routes of projects and namespaces.

This results in the following query:

```sql
SELECT "projects"."id" AS t0_r0,
       "projects"."name" AS t0_r1,
       "projects"."path" AS t0_r2,
       "projects"."description" AS t0_r3,
       "projects"."created_at" AS t0_r4,
       "projects"."updated_at" AS t0_r5,
       "projects"."creator_id" AS t0_r6,
       "projects"."namespace_id" AS t0_r7,
       "projects"."last_activity_at" AS t0_r8,
       "projects"."import_url" AS t0_r9,
       "projects"."visibility_level" AS t0_r10,
       "projects"."archived" AS t0_r11,
       "projects"."avatar" AS t0_r12,
       "projects"."import_status" AS t0_r13,
       "projects"."star_count" AS t0_r14,
       "projects"."import_type" AS t0_r15,
       "projects"."import_source" AS t0_r16,
       "projects"."import_error" AS t0_r17,
       "projects"."ci_id" AS t0_r18,
       "projects"."shared_runners_enabled" AS t0_r19,
       "projects"."runners_token" AS t0_r20,
       "projects"."build_coverage_regex" AS t0_r21,
       "projects"."build_allow_git_fetch" AS t0_r22,
       "projects"."build_timeout" AS t0_r23,
       "projects"."pending_delete" AS t0_r24,
       "projects"."public_builds" AS t0_r25,
       "projects"."last_repository_check_failed" AS t0_r26,
       "projects"."last_repository_check_at" AS t0_r27,
       "projects"."container_registry_enabled" AS t0_r28,
       "projects"."only_allow_merge_if_pipeline_succeeds" AS t0_r29,
       "projects"."has_external_issue_tracker" AS t0_r30,
       "projects"."repository_storage" AS t0_r31,
       "projects"."request_access_enabled" AS t0_r32,
       "projects"."has_external_wiki" AS t0_r33,
       "projects"."ci_config_path" AS t0_r34,
       "projects"."lfs_enabled" AS t0_r35,
       "projects"."description_html" AS t0_r36,
       "projects"."only_allow_merge_if_all_discussions_are_resolved" AS t0_r37,
       "projects"."printing_merge_request_link_enabled" AS t0_r38,
       "projects"."auto_cancel_pending_pipelines" AS t0_r39,
       "projects"."import_jid" AS t0_r40,
       "projects"."cached_markdown_version" AS t0_r41,
       "projects"."delete_error" AS t0_r42,
       "projects"."last_repository_updated_at" AS t0_r43,
       "projects"."storage_version" AS t0_r44,
       "projects"."resolve_outdated_diff_discussions" AS t0_r45,
       "projects"."repository_read_only" AS t0_r46,
       "projects"."merge_requests_ff_only_enabled" AS t0_r47,
       "projects"."merge_requests_rebase_enabled" AS t0_r48,
       "projects"."jobs_cache_index" AS t0_r49,
       "projects"."pages_https_only" AS t0_r50,
       "projects"."remote_mirror_available_overridden" AS t0_r51,
       "projects"."pool_repository_id" AS t0_r52,
       "projects"."runners_token_encrypted" AS t0_r53,
       "projects"."bfg_object_map" AS t0_r54,
       "namespaces"."id" AS t1_r0,
       "namespaces"."name" AS t1_r1,
       "namespaces"."path" AS t1_r2,
       "namespaces"."owner_id" AS t1_r3,
       "namespaces"."created_at" AS t1_r4,
       "namespaces"."updated_at" AS t1_r5,
       "namespaces"."type" AS t1_r6,
       "namespaces"."description" AS t1_r7,
       "namespaces"."avatar" AS t1_r8,
       "namespaces"."share_with_group_lock" AS t1_r9,
       "namespaces"."visibility_level" AS t1_r10,
       "namespaces"."request_access_enabled" AS t1_r11,
       "namespaces"."description_html" AS t1_r12,
       "namespaces"."lfs_enabled" AS t1_r13,
       "namespaces"."parent_id" AS t1_r14,
       "namespaces"."require_two_factor_authentication" AS t1_r15,
       "namespaces"."two_factor_grace_period" AS t1_r16,
       "namespaces"."cached_markdown_version" AS t1_r17,
       "namespaces"."runners_token" AS t1_r18,
       "namespaces"."runners_token_encrypted" AS t1_r19,
       "routes"."id" AS t2_r0,
       "routes"."source_id" AS t2_r1,
       "routes"."source_type" AS t2_r2,
       "routes"."path" AS t2_r3,
       "routes"."created_at" AS t2_r4,
       "routes"."updated_at" AS t2_r5,
       "routes"."name" AS t2_r6,
       "routes_projects"."id" AS t3_r0,
       "routes_projects"."source_id" AS t3_r1,
       "routes_projects"."source_type" AS t3_r2,
       "routes_projects"."path" AS t3_r3,
       "routes_projects"."created_at" AS t3_r4,
       "routes_projects"."updated_at" AS t3_r5,
       "routes_projects"."name" AS t3_r6
FROM "projects"
LEFT OUTER JOIN "namespaces" ON "namespaces"."id" = "projects"."namespace_id"
LEFT OUTER JOIN "routes" ON "routes"."source_id" = "namespaces"."id"
AND "routes"."source_type" = $1
LEFT OUTER JOIN "routes" "routes_projects" ON "routes_projects"."source_id" = "projects"."id"
AND "routes_projects"."source_type" = $2
LEFT OUTER JOIN "project_repositories" ON "projects"."id" = "project_repositories"."project_id"
WHERE ("projects"."storage_version" IS NULL
       OR "projects"."storage_version" = 0)
  AND "project_repositories"."project_id" IS NULL
  AND ("projects"."id" BETWEEN $3 AND $4);
  -- [["source_type", "Namespace"],
  --  ["source_type", "Project"],
  --  ["id", 1],
  --  ["id", 4]]
```
2019-01-11 15:36:57 +01:00
Stan Hu
618875c58c Fix duplicate disk path in Backfill ProjectRepos
On GitLab.com, we saw numerous duplicate disk entry inserts because
the migration was not taking the routes table into account. We now
implement this in the migration to be consistent.

Closes https://gitlab.com/gitlab-org/gitlab-ce/issues/56061
2019-01-11 15:36:51 +01:00
Douglas Barbosa Alexandre
754f66113e
Backfill project_repositories for legacy storage projects
Adds a background migration that will ensure all projects that
are on legacy storage have a row in `project_repositories`.
2018-12-18 21:13:38 -02:00