From a3b472a12ec6e57dbe2f650419f8064864686eab Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Wed, 10 Aug 2022 11:51:13 +0200 Subject: [PATCH] Add ability to list imports of a channel sync --- .../my-video-channel-syncs.component.html | 9 +- .../my-video-channel-syncs.component.ts | 2 +- .../video-channel-sync-edit.component.ts | 2 +- .../my-video-imports.component.html | 11 +- .../my-video-imports.component.scss | 6 + .../my-video-imports.component.ts | 6 +- .../video-channel/video-channel.service.ts | 19 +- .../shared-main/video/video-import.service.ts | 15 +- server/controllers/api/users/me.ts | 11 +- server/controllers/api/video-channel.ts | 9 +- server/initializers/constants.ts | 2 +- ...5-video-channel-sync-import-foreign-key.ts | 32 ++++ .../handlers/video-channel-import.ts | 20 +- .../video-channel-sync-latest-scheduler.ts | 4 - server/lib/sync-channel.ts | 7 + server/lib/video-import.ts | 3 +- server/middlewares/validators/shared/index.ts | 1 + .../validators/shared/video-channel-syncs.ts | 24 +++ .../validators/videos/video-channel-sync.ts | 16 +- .../validators/videos/video-channels.ts | 14 +- .../validators/videos/video-imports.ts | 19 +- server/models/video/video-import.ts | 79 ++++++-- .../api/check-params/channel-import-videos.ts | 172 ++++++++++++++++++ server/tests/api/check-params/index.ts | 1 + .../tests/api/check-params/video-channels.ts | 113 +----------- .../tests/api/check-params/video-imports.ts | 9 + .../tests/api/videos/channel-import-videos.ts | 72 +++++++- .../tests/api/videos/video-channel-syncs.ts | 12 ++ server/tests/api/videos/video-imports.ts | 9 + shared/models/server/job.model.ts | 2 + shared/models/videos/import/index.ts | 1 + .../videos/import/video-import.model.ts | 5 + .../videos-import-in-channel-create.model.ts | 4 + shared/server-commands/server/server.ts | 3 +- .../videos/channels-command.ts | 10 +- .../server-commands/videos/imports-command.ts | 6 +- support/doc/api/openapi.yaml | 14 ++ 37 files changed, 565 insertions(+), 179 deletions(-) create mode 100644 server/initializers/migrations/0735-video-channel-sync-import-foreign-key.ts create mode 100644 server/middlewares/validators/shared/video-channel-syncs.ts create mode 100644 server/tests/api/check-params/channel-import-videos.ts create mode 100644 shared/models/videos/import/videos-import-in-channel-create.model.ts diff --git a/client/src/app/+my-library/my-video-channel-syncs/my-video-channel-syncs.component.html b/client/src/app/+my-library/my-video-channel-syncs/my-video-channel-syncs.component.html index 5141607b1..c2fed8112 100644 --- a/client/src/app/+my-library/my-video-channel-syncs/my-video-channel-syncs.component.html +++ b/client/src/app/+my-library/my-video-channel-syncs/my-video-channel-syncs.component.html @@ -30,12 +30,13 @@ - + External Channel Channel State Created Last synchronization at + @@ -78,6 +79,12 @@ {{ videoChannelSync.createdAt | date: 'short' }} {{ videoChannelSync.lastSyncAt | date: 'short' }} + + + + List imports + + diff --git a/client/src/app/+my-library/my-video-channel-syncs/my-video-channel-syncs.component.ts b/client/src/app/+my-library/my-video-channel-syncs/my-video-channel-syncs.component.ts index 81bdaf9f2..0c429e5dd 100644 --- a/client/src/app/+my-library/my-video-channel-syncs/my-video-channel-syncs.component.ts +++ b/client/src/app/+my-library/my-video-channel-syncs/my-video-channel-syncs.component.ts @@ -100,7 +100,7 @@ export class MyVideoChannelSyncsComponent extends RestTable implements OnInit { } fullySynchronize (videoChannelSync: VideoChannelSync) { - this.videoChannelService.importVideos(videoChannelSync.channel.name, videoChannelSync.externalChannelUrl) + this.videoChannelService.importVideos(videoChannelSync.channel.name, videoChannelSync.externalChannelUrl, videoChannelSync.id) .subscribe({ next: () => { this.notifier.success($localize`Full synchronization requested successfully for ${videoChannelSync.channel.displayName}.`) diff --git a/client/src/app/+my-library/my-video-channel-syncs/video-channel-sync-edit/video-channel-sync-edit.component.ts b/client/src/app/+my-library/my-video-channel-syncs/video-channel-sync-edit/video-channel-sync-edit.component.ts index 836582609..9ceb6dfd1 100644 --- a/client/src/app/+my-library/my-video-channel-syncs/video-channel-sync-edit/video-channel-sync-edit.component.ts +++ b/client/src/app/+my-library/my-video-channel-syncs/video-channel-sync-edit/video-channel-sync-edit.component.ts @@ -59,7 +59,7 @@ export class VideoChannelSyncEditComponent extends FormReactive implements OnIni this.videoChannelSyncService.createSync(videoChannelSyncCreate) .pipe(mergeMap(({ videoChannelSync }) => { return importExistingVideos - ? this.videoChannelService.importVideos(videoChannelSync.channel.name, videoChannelSync.externalChannelUrl) + ? this.videoChannelService.importVideos(videoChannelSync.channel.name, videoChannelSync.externalChannelUrl, videoChannelSync.id) : Promise.resolve(null) })) .subscribe({ diff --git a/client/src/app/+my-library/my-video-imports/my-video-imports.component.html b/client/src/app/+my-library/my-video-imports/my-video-imports.component.html index fb0f6f5a3..866cd1a72 100644 --- a/client/src/app/+my-library/my-video-imports/my-video-imports.component.html +++ b/client/src/app/+my-library/my-video-imports/my-video-imports.component.html @@ -3,9 +3,18 @@ My imports +
+ + + + + My synchronizations + +
+ { this.videoImports = resultList.data diff --git a/client/src/app/shared/shared-main/video-channel/video-channel.service.ts b/client/src/app/shared/shared-main/video-channel/video-channel.service.ts index fa97025ac..5e3985526 100644 --- a/client/src/app/shared/shared-main/video-channel/video-channel.service.ts +++ b/client/src/app/shared/shared-main/video-channel/video-channel.service.ts @@ -3,7 +3,14 @@ import { catchError, map, tap } from 'rxjs/operators' import { HttpClient, HttpParams } from '@angular/common/http' import { Injectable } from '@angular/core' import { ComponentPaginationLight, RestExtractor, RestService } from '@app/core' -import { ActorImage, ResultList, VideoChannel as VideoChannelServer, VideoChannelCreate, VideoChannelUpdate } from '@shared/models' +import { + ActorImage, + ResultList, + VideoChannel as VideoChannelServer, + VideoChannelCreate, + VideoChannelUpdate, + VideosImportInChannelCreate +} from '@shared/models' import { environment } from '../../../../environments/environment' import { Account } from '../account' import { AccountService } from '../account/account.service' @@ -96,9 +103,15 @@ export class VideoChannelService { .pipe(catchError(err => this.restExtractor.handleError(err))) } - importVideos (videoChannelName: string, externalChannelUrl: string) { + importVideos (videoChannelName: string, externalChannelUrl: string, syncId?: number) { const path = VideoChannelService.BASE_VIDEO_CHANNEL_URL + videoChannelName + '/import-videos' - return this.authHttp.post(path, { externalChannelUrl }) + + const body: VideosImportInChannelCreate = { + externalChannelUrl, + videoChannelSyncId: syncId + } + + return this.authHttp.post(path, body) .pipe(catchError(err => this.restExtractor.handleError(err))) } } diff --git a/client/src/app/shared/shared-main/video/video-import.service.ts b/client/src/app/shared/shared-main/video/video-import.service.ts index 0a610ab1f..f9720033a 100644 --- a/client/src/app/shared/shared-main/video/video-import.service.ts +++ b/client/src/app/shared/shared-main/video/video-import.service.ts @@ -43,10 +43,23 @@ export class VideoImportService { .pipe(catchError(res => this.restExtractor.handleError(res))) } - getMyVideoImports (pagination: RestPagination, sort: SortMeta): Observable> { + getMyVideoImports (pagination: RestPagination, sort: SortMeta, search?: string): Observable> { let params = new HttpParams() params = this.restService.addRestGetParams(params, pagination, sort) + if (search) { + const filters = this.restService.parseQueryStringFilter(search, { + videoChannelSyncId: { + prefix: 'videoChannelSyncId:' + }, + targetUrl: { + prefix: 'targetUrl:' + } + }) + + params = this.restService.addObjectParams(params, filters) + } + return this.authHttp .get>(UserService.BASE_USERS_URL + '/me/videos/imports', { params }) .pipe( diff --git a/server/controllers/api/users/me.ts b/server/controllers/api/users/me.ts index 595abcf95..00f580ee9 100644 --- a/server/controllers/api/users/me.ts +++ b/server/controllers/api/users/me.ts @@ -25,7 +25,13 @@ import { usersUpdateMeValidator, usersVideoRatingValidator } from '../../../middlewares' -import { deleteMeValidator, usersVideosValidator, videoImportsSortValidator, videosSortValidator } from '../../../middlewares/validators' +import { + deleteMeValidator, + getMyVideoImportsValidator, + usersVideosValidator, + videoImportsSortValidator, + videosSortValidator +} from '../../../middlewares/validators' import { updateAvatarValidator } from '../../../middlewares/validators/actor-image' import { AccountModel } from '../../../models/account/account' import { AccountVideoRateModel } from '../../../models/account/account-video-rate' @@ -60,6 +66,7 @@ meRouter.get('/me/videos/imports', videoImportsSortValidator, setDefaultSort, setDefaultPagination, + getMyVideoImportsValidator, asyncMiddleware(getUserVideoImports) ) @@ -138,7 +145,7 @@ async function getUserVideoImports (req: express.Request, res: express.Response) const resultList = await VideoImportModel.listUserVideoImportsForApi({ userId: user.id, - ...pick(req.query, [ 'targetUrl', 'start', 'count', 'sort' ]) + ...pick(req.query, [ 'targetUrl', 'start', 'count', 'sort', 'search', 'videoChannelSyncId' ]) }) return res.json(getFormattedObjects(resultList.data, resultList.total)) diff --git a/server/controllers/api/video-channel.ts b/server/controllers/api/video-channel.ts index 89c7181bd..94285a78d 100644 --- a/server/controllers/api/video-channel.ts +++ b/server/controllers/api/video-channel.ts @@ -6,7 +6,7 @@ import { ActorFollowModel } from '@server/models/actor/actor-follow' import { getServerActor } from '@server/models/application/application' import { guessAdditionalAttributesFromQuery } from '@server/models/video/formatter/video-format-utils' import { MChannelBannerAccountDefault } from '@server/types/models' -import { ActorImageType, HttpStatusCode, VideoChannelCreate, VideoChannelUpdate } from '@shared/models' +import { ActorImageType, HttpStatusCode, VideoChannelCreate, VideoChannelUpdate, VideosImportInChannelCreate } from '@shared/models' import { auditLoggerFactory, getAuditIdFromRes, VideoChannelAuditView } from '../../helpers/audit-logger' import { resetSequelizeInstance } from '../../helpers/database-utils' import { buildNSFWFilter, createReqFiles, getCountVideos, isUserAbleToSearchRemoteURI } from '../../helpers/express-utils' @@ -166,7 +166,7 @@ videoChannelRouter.get('/:nameWithHost/followers', videoChannelRouter.post('/:nameWithHost/import-videos', authenticate, asyncMiddleware(videoChannelsNameWithHostValidator), - videoChannelImportVideosValidator, + asyncMiddleware(videoChannelImportVideosValidator), ensureIsLocalChannel, ensureCanManageChannel, asyncMiddleware(ensureChannelOwnerCanUpload), @@ -418,13 +418,14 @@ async function listVideoChannelFollowers (req: express.Request, res: express.Res } async function importVideosInChannel (req: express.Request, res: express.Response) { - const { externalChannelUrl } = req.body + const { externalChannelUrl } = req.body as VideosImportInChannelCreate await JobQueue.Instance.createJob({ type: 'video-channel-import', payload: { externalChannelUrl, - videoChannelId: res.locals.videoChannel.id + videoChannelId: res.locals.videoChannel.id, + partOfChannelSyncId: res.locals.videoChannelSync?.id } }) diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index 697a64d42..c2289ef36 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -25,7 +25,7 @@ import { CONFIG, registerConfigChangedHandler } from './config' // --------------------------------------------------------------------------- -const LAST_MIGRATION_VERSION = 730 +const LAST_MIGRATION_VERSION = 735 // --------------------------------------------------------------------------- diff --git a/server/initializers/migrations/0735-video-channel-sync-import-foreign-key.ts b/server/initializers/migrations/0735-video-channel-sync-import-foreign-key.ts new file mode 100644 index 000000000..ffe0b11ab --- /dev/null +++ b/server/initializers/migrations/0735-video-channel-sync-import-foreign-key.ts @@ -0,0 +1,32 @@ +import * as Sequelize from 'sequelize' + +async function up (utils: { + transaction: Sequelize.Transaction + queryInterface: Sequelize.QueryInterface + sequelize: Sequelize.Sequelize + db: any +}): Promise { + await utils.queryInterface.addColumn('videoImport', 'videoChannelSyncId', { + type: Sequelize.INTEGER, + defaultValue: null, + allowNull: true, + references: { + model: 'videoChannelSync', + key: 'id' + }, + onUpdate: 'CASCADE', + onDelete: 'SET NULL' + }, { transaction: utils.transaction }) +} + +async function down (utils: { + queryInterface: Sequelize.QueryInterface + transaction: Sequelize.Transaction +}) { + await utils.queryInterface.dropTable('videoChannelSync', { transaction: utils.transaction }) +} + +export { + up, + down +} diff --git a/server/lib/job-queue/handlers/video-channel-import.ts b/server/lib/job-queue/handlers/video-channel-import.ts index 9bdb2d269..9aaad659e 100644 --- a/server/lib/job-queue/handlers/video-channel-import.ts +++ b/server/lib/job-queue/handlers/video-channel-import.ts @@ -3,6 +3,8 @@ import { logger } from '@server/helpers/logger' import { CONFIG } from '@server/initializers/config' import { synchronizeChannel } from '@server/lib/sync-channel' import { VideoChannelModel } from '@server/models/video/video-channel' +import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync' +import { MChannelSync } from '@server/types/models' import { VideoChannelImportPayload } from '@shared/models' export async function processVideoChannelImport (job: Job) { @@ -12,13 +14,20 @@ export async function processVideoChannelImport (job: Job) { // Channel import requires only http upload to be allowed if (!CONFIG.IMPORT.VIDEOS.HTTP.ENABLED) { - logger.error('Cannot import channel as the HTTP upload is disabled') - return + throw new Error('Cannot import channel as the HTTP upload is disabled') } if (!CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.ENABLED) { - logger.error('Cannot import channel as the synchronization is disabled') - return + throw new Error('Cannot import channel as the synchronization is disabled') + } + + let channelSync: MChannelSync + if (payload.partOfChannelSyncId) { + channelSync = await VideoChannelSyncModel.loadWithChannel(payload.partOfChannelSyncId) + + if (!channelSync) { + throw new Error('Unlnown channel sync specified in videos channel import') + } } const videoChannel = await VideoChannelModel.loadAndPopulateAccount(payload.videoChannelId) @@ -28,7 +37,8 @@ export async function processVideoChannelImport (job: Job) { await synchronizeChannel({ channel: videoChannel, - externalChannelUrl: payload.externalChannelUrl + externalChannelUrl: payload.externalChannelUrl, + channelSync }) } catch (err) { logger.error(`Failed to import channel ${videoChannel.name}`, { err }) diff --git a/server/lib/schedulers/video-channel-sync-latest-scheduler.ts b/server/lib/schedulers/video-channel-sync-latest-scheduler.ts index fd9a35299..491ddaa87 100644 --- a/server/lib/schedulers/video-channel-sync-latest-scheduler.ts +++ b/server/lib/schedulers/video-channel-sync-latest-scheduler.ts @@ -36,10 +36,6 @@ export class VideoChannelSyncLatestScheduler extends AbstractScheduler { const onlyAfter = sync.lastSyncAt || sync.createdAt - sync.state = VideoChannelSyncState.PROCESSING - sync.lastSyncAt = new Date() - await sync.save() - await synchronizeChannel({ channel, externalChannelUrl: sync.externalChannelUrl, diff --git a/server/lib/sync-channel.ts b/server/lib/sync-channel.ts index 50f80e6f9..eb5ca1703 100644 --- a/server/lib/sync-channel.ts +++ b/server/lib/sync-channel.ts @@ -18,6 +18,12 @@ export async function synchronizeChannel (options: { }) { const { channel, externalChannelUrl, videosCountLimit, onlyAfter, channelSync } = options + if (channelSync) { + channelSync.state = VideoChannelSyncState.PROCESSING + channelSync.lastSyncAt = new Date() + await channelSync.save() + } + const user = await UserModel.loadByChannelActorId(channel.actorId) const youtubeDL = new YoutubeDLWrapper( externalChannelUrl, @@ -70,6 +76,7 @@ export async function synchronizeChannel (options: { children.push(job) } + // Will update the channel sync status const parent: CreateJobArgument = { type: 'after-video-channel-import', payload: { diff --git a/server/lib/video-import.ts b/server/lib/video-import.ts index fb9306967..de95116aa 100644 --- a/server/lib/video-import.ts +++ b/server/lib/video-import.ts @@ -206,7 +206,8 @@ async function buildYoutubeDLImport (options: { videoImportAttributes: { targetUrl, state: VideoImportState.PENDING, - userId: user.id + userId: user.id, + videoChannelSyncId: channelSync?.id } }) diff --git a/server/middlewares/validators/shared/index.ts b/server/middlewares/validators/shared/index.ts index fa89d05f2..bbd03b248 100644 --- a/server/middlewares/validators/shared/index.ts +++ b/server/middlewares/validators/shared/index.ts @@ -4,6 +4,7 @@ export * from './utils' export * from './video-blacklists' export * from './video-captions' export * from './video-channels' +export * from './video-channel-syncs' export * from './video-comments' export * from './video-imports' export * from './video-ownerships' diff --git a/server/middlewares/validators/shared/video-channel-syncs.ts b/server/middlewares/validators/shared/video-channel-syncs.ts new file mode 100644 index 000000000..a6e51eb97 --- /dev/null +++ b/server/middlewares/validators/shared/video-channel-syncs.ts @@ -0,0 +1,24 @@ +import express from 'express' +import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync' +import { HttpStatusCode } from '@shared/models' + +async function doesVideoChannelSyncIdExist (id: number, res: express.Response) { + const sync = await VideoChannelSyncModel.loadWithChannel(+id) + + if (!sync) { + res.fail({ + status: HttpStatusCode.NOT_FOUND_404, + message: 'Video channel sync not found' + }) + return false + } + + res.locals.videoChannelSync = sync + return true +} + +// --------------------------------------------------------------------------- + +export { + doesVideoChannelSyncIdExist +} diff --git a/server/middlewares/validators/videos/video-channel-sync.ts b/server/middlewares/validators/videos/video-channel-sync.ts index b18498243..081f09bba 100644 --- a/server/middlewares/validators/videos/video-channel-sync.ts +++ b/server/middlewares/validators/videos/video-channel-sync.ts @@ -3,10 +3,10 @@ import { body, param } from 'express-validator' import { isUrlValid } from '@server/helpers/custom-validators/activitypub/misc' import { logger } from '@server/helpers/logger' import { CONFIG } from '@server/initializers/config' -import { VideoChannelModel } from '@server/models/video/video-channel' import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync' import { HttpStatusCode, VideoChannelSyncCreate } from '@shared/models' import { areValidationErrors, doesVideoChannelIdExist } from '../shared' +import { doesVideoChannelSyncIdExist } from '../shared/video-channel-syncs' export const ensureSyncIsEnabled = (req: express.Request, res: express.Response, next: express.NextFunction) => { if (!CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.ENABLED) { @@ -48,18 +48,8 @@ export const ensureSyncExists = [ async (req: express.Request, res: express.Response, next: express.NextFunction) => { if (areValidationErrors(req, res)) return - const syncId = parseInt(req.params.id, 10) - const sync = await VideoChannelSyncModel.loadWithChannel(syncId) - - if (!sync) { - return res.fail({ - status: HttpStatusCode.NOT_FOUND_404, - message: 'Synchronization not found' - }) - } - - res.locals.videoChannelSync = sync - res.locals.videoChannel = await VideoChannelModel.loadAndPopulateAccount(sync.videoChannelId) + if (!await doesVideoChannelSyncIdExist(+req.params.id, res)) return + if (!await doesVideoChannelIdExist(res.locals.videoChannelSync.videoChannelId, res)) return return next() } diff --git a/server/middlewares/validators/videos/video-channels.ts b/server/middlewares/validators/videos/video-channels.ts index 88f8b814d..d53c777fa 100644 --- a/server/middlewares/validators/videos/video-channels.ts +++ b/server/middlewares/validators/videos/video-channels.ts @@ -3,8 +3,9 @@ import { body, param, query } from 'express-validator' import { isUrlValid } from '@server/helpers/custom-validators/activitypub/misc' import { CONFIG } from '@server/initializers/config' import { MChannelAccountDefault } from '@server/types/models' +import { VideosImportInChannelCreate } from '@shared/models' import { HttpStatusCode } from '../../../../shared/models/http/http-error-codes' -import { isBooleanValid, toBooleanOrNull } from '../../../helpers/custom-validators/misc' +import { isBooleanValid, isIdValid, toBooleanOrNull } from '../../../helpers/custom-validators/misc' import { isVideoChannelDescriptionValid, isVideoChannelDisplayNameValid, @@ -15,6 +16,7 @@ import { logger } from '../../../helpers/logger' import { ActorModel } from '../../../models/actor/actor' import { VideoChannelModel } from '../../../models/video/video-channel' import { areValidationErrors, checkUserQuota, doesVideoChannelNameWithHostExist } from '../shared' +import { doesVideoChannelSyncIdExist } from '../shared/video-channel-syncs' export const videoChannelsAddValidator = [ body('name').custom(isVideoChannelUsernameValid).withMessage('Should have a valid channel name'), @@ -145,11 +147,17 @@ export const videoChannelsListValidator = [ export const videoChannelImportVideosValidator = [ body('externalChannelUrl').custom(isUrlValid).withMessage('Should have a valid channel url'), - (req: express.Request, res: express.Response, next: express.NextFunction) => { + body('videoChannelSyncId') + .optional() + .custom(isIdValid).withMessage('Should have a valid channel sync id'), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { logger.debug('Checking videoChannelImport parameters', { parameters: req.body }) if (areValidationErrors(req, res)) return + const body: VideosImportInChannelCreate = req.body + if (!CONFIG.IMPORT.VIDEOS.HTTP.ENABLED) { return res.fail({ status: HttpStatusCode.FORBIDDEN_403, @@ -157,6 +165,8 @@ export const videoChannelImportVideosValidator = [ }) } + if (body.videoChannelSyncId && !await doesVideoChannelSyncIdExist(body.videoChannelSyncId, res)) return + return next() } ] diff --git a/server/middlewares/validators/videos/video-imports.ts b/server/middlewares/validators/videos/video-imports.ts index 9c6d213c4..3115acb21 100644 --- a/server/middlewares/validators/videos/video-imports.ts +++ b/server/middlewares/validators/videos/video-imports.ts @@ -1,5 +1,5 @@ import express from 'express' -import { body, param } from 'express-validator' +import { body, param, query } from 'express-validator' import { isResolvingToUnicastOnly } from '@server/helpers/dns' import { isPreImportVideoAccepted } from '@server/lib/moderation' import { Hooks } from '@server/lib/plugins/hooks' @@ -92,6 +92,20 @@ const videoImportAddValidator = getCommonVideoEditAttributes().concat([ } ]) +const getMyVideoImportsValidator = [ + query('videoChannelSyncId') + .optional() + .custom(isIdValid).withMessage('Should have correct videoChannelSync id'), + + (req: express.Request, res: express.Response, next: express.NextFunction) => { + logger.debug('Checking getMyVideoImportsValidator parameters', { parameters: req.params }) + + if (areValidationErrors(req, res)) return + + return next() + } +] + const videoImportDeleteValidator = [ param('id') .custom(isIdValid).withMessage('Should have correct import id'), @@ -143,7 +157,8 @@ const videoImportCancelValidator = [ export { videoImportAddValidator, videoImportCancelValidator, - videoImportDeleteValidator + videoImportDeleteValidator, + getMyVideoImportsValidator } // --------------------------------------------------------------------------- diff --git a/server/models/video/video-import.ts b/server/models/video/video-import.ts index b8e941623..da6b92c7a 100644 --- a/server/models/video/video-import.ts +++ b/server/models/video/video-import.ts @@ -1,4 +1,4 @@ -import { Op, WhereOptions } from 'sequelize' +import { IncludeOptions, Op, WhereOptions } from 'sequelize' import { AfterUpdate, AllowNull, @@ -22,8 +22,17 @@ import { isVideoImportStateValid, isVideoImportTargetUrlValid } from '../../help import { isVideoMagnetUriValid } from '../../helpers/custom-validators/videos' import { CONSTRAINTS_FIELDS, VIDEO_IMPORT_STATES } from '../../initializers/constants' import { UserModel } from '../user/user' -import { getSort, throwIfNotValid } from '../utils' +import { getSort, searchAttribute, throwIfNotValid } from '../utils' import { ScopeNames as VideoModelScopeNames, VideoModel } from './video' +import { VideoChannelSyncModel } from './video-channel-sync' + +const defaultVideoScope = () => { + return VideoModel.scope([ + VideoModelScopeNames.WITH_ACCOUNT_DETAILS, + VideoModelScopeNames.WITH_TAGS, + VideoModelScopeNames.WITH_THUMBNAILS + ]) +} @DefaultScope(() => ({ include: [ @@ -32,11 +41,11 @@ import { ScopeNames as VideoModelScopeNames, VideoModel } from './video' required: true }, { - model: VideoModel.scope([ - VideoModelScopeNames.WITH_ACCOUNT_DETAILS, - VideoModelScopeNames.WITH_TAGS, - VideoModelScopeNames.WITH_THUMBNAILS - ]), + model: defaultVideoScope(), + required: false + }, + { + model: VideoChannelSyncModel.unscoped(), required: false } ] @@ -113,6 +122,18 @@ export class VideoImportModel extends Model VideoChannelSyncModel) + @Column + videoChannelSyncId: number + + @BelongsTo(() => VideoChannelSyncModel, { + foreignKey: { + allowNull: true + }, + onDelete: 'set null' + }) + VideoChannelSync: VideoChannelSyncModel + @AfterUpdate static deleteVideoIfFailed (instance: VideoImportModel, options) { if (instance.state === VideoImportState.FAILED) { @@ -132,23 +153,44 @@ export class VideoImportModel extends Model t.name) }) : undefined + const videoChannelSync = this.VideoChannelSync + ? { id: this.VideoChannelSync.id, externalChannelUrl: this.VideoChannelSync.externalChannelUrl } + : undefined + return { id: this.id, @@ -210,7 +256,8 @@ export class VideoImportModel extends Model !!i.videoChannelSync) + expect(importsWithSyncId).to.have.lengthOf(2) + + for (const videoImport of importsWithSyncId) { + expect(videoImport.videoChannelSync).to.exist + expect(videoImport.videoChannelSync.id).to.equal(server.store.videoChannelSync.id) + } + }) + + it('Should be able to filter imports by this sync id', async function () { + const { total, data } = await server.imports.getMyVideoImports({ videoChannelSyncId: server.store.videoChannelSync.id }) + + expect(total).to.equal(2) + expect(data).to.have.lengthOf(2) + + for (const videoImport of data) { + expect(videoImport.videoChannelSync).to.exist + expect(videoImport.videoChannelSync.id).to.equal(server.store.videoChannelSync.id) + } + }) + after(async function () { await server?.kill() }) diff --git a/server/tests/api/videos/video-channel-syncs.ts b/server/tests/api/videos/video-channel-syncs.ts index 229c01f68..835d3cb09 100644 --- a/server/tests/api/videos/video-channel-syncs.ts +++ b/server/tests/api/videos/video-channel-syncs.ts @@ -23,7 +23,10 @@ describe('Test channel synchronizations', function () { describe('Sync using ' + mode, function () { let server: PeerTubeServer let command: ChannelSyncsCommand + let startTestDate: Date + + let rootChannelSyncId: number const userInfo = { accessToken: '', username: 'user1', @@ -90,6 +93,7 @@ describe('Test channel synchronizations', function () { token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) + rootChannelSyncId = videoChannelSync.id // Ensure any missing video not already fetched will be considered as new await changeDateForSync(videoChannelSync.id, '1970-01-01') @@ -208,6 +212,14 @@ describe('Test channel synchronizations', function () { } }) + it('Should list imports of a channel synchronization', async function () { + const { total, data } = await server.imports.getMyVideoImports({ videoChannelSyncId: rootChannelSyncId }) + + expect(total).to.equal(1) + expect(data).to.have.lengthOf(1) + expect(data[0].video.name).to.equal('test') + }) + it('Should remove user\'s channel synchronizations', async function () { await command.delete({ channelSyncId: userInfo.syncId }) diff --git a/server/tests/api/videos/video-imports.ts b/server/tests/api/videos/video-imports.ts index a487062a2..f082d4bd7 100644 --- a/server/tests/api/videos/video-imports.ts +++ b/server/tests/api/videos/video-imports.ts @@ -228,6 +228,15 @@ describe('Test video imports', function () { expect(videoImports[0].targetUrl).to.equal(FIXTURE_URLS.youtube) }) + it('Should search in my imports', async function () { + const { total, data: videoImports } = await servers[0].imports.getMyVideoImports({ search: 'peertube2' }) + expect(total).to.equal(1) + expect(videoImports).to.have.lengthOf(1) + + expect(videoImports[0].magnetUri).to.equal(FIXTURE_URLS.magnet) + expect(videoImports[0].video.name).to.equal('super peertube2 video') + }) + it('Should have the video listed on the two instances', async function () { this.timeout(120_000) diff --git a/shared/models/server/job.model.ts b/shared/models/server/job.model.ts index ba1f83684..9c0b5ea56 100644 --- a/shared/models/server/job.model.ts +++ b/shared/models/server/job.model.ts @@ -236,6 +236,8 @@ export interface VideoStudioEditionPayload { export interface VideoChannelImportPayload { externalChannelUrl: string videoChannelId: number + + partOfChannelSyncId?: number } export interface AfterVideoChannelImportPayload { diff --git a/shared/models/videos/import/index.ts b/shared/models/videos/import/index.ts index 8884ee8f2..b38a67b5f 100644 --- a/shared/models/videos/import/index.ts +++ b/shared/models/videos/import/index.ts @@ -1,3 +1,4 @@ export * from './video-import-create.model' export * from './video-import-state.enum' export * from './video-import.model' +export * from './videos-import-in-channel-create.model' diff --git a/shared/models/videos/import/video-import.model.ts b/shared/models/videos/import/video-import.model.ts index 92856c70f..6aed7a91a 100644 --- a/shared/models/videos/import/video-import.model.ts +++ b/shared/models/videos/import/video-import.model.ts @@ -16,4 +16,9 @@ export interface VideoImport { error?: string video?: Video & { tags: string[] } + + videoChannelSync?: { + id: number + externalChannelUrl: string + } } diff --git a/shared/models/videos/import/videos-import-in-channel-create.model.ts b/shared/models/videos/import/videos-import-in-channel-create.model.ts new file mode 100644 index 000000000..fbfef63f8 --- /dev/null +++ b/shared/models/videos/import/videos-import-in-channel-create.model.ts @@ -0,0 +1,4 @@ +export interface VideosImportInChannelCreate { + externalChannelUrl: string + videoChannelSyncId?: number +} diff --git a/shared/server-commands/server/server.ts b/shared/server-commands/server/server.ts index 7acbc978f..c05d16ad2 100644 --- a/shared/server-commands/server/server.ts +++ b/shared/server-commands/server/server.ts @@ -2,7 +2,7 @@ import { ChildProcess, fork } from 'child_process' import { copy } from 'fs-extra' import { join } from 'path' import { parallelTests, randomInt, root } from '@shared/core-utils' -import { Video, VideoChannel, VideoCreateResult, VideoDetails } from '@shared/models' +import { Video, VideoChannel, VideoChannelSync, VideoCreateResult, VideoDetails } from '@shared/models' import { BulkCommand } from '../bulk' import { CLICommand } from '../cli' import { CustomPagesCommand } from '../custom-pages' @@ -80,6 +80,7 @@ export class PeerTubeServer { } channel?: VideoChannel + videoChannelSync?: Partial video?: Video videoCreated?: VideoCreateResult diff --git a/shared/server-commands/videos/channels-command.ts b/shared/server-commands/videos/channels-command.ts index a688a120f..385d0fe73 100644 --- a/shared/server-commands/videos/channels-command.ts +++ b/shared/server-commands/videos/channels-command.ts @@ -6,7 +6,8 @@ import { VideoChannel, VideoChannelCreate, VideoChannelCreateResult, - VideoChannelUpdate + VideoChannelUpdate, + VideosImportInChannelCreate } from '@shared/models' import { unwrapBody } from '../requests' import { AbstractCommand, OverrideCommandOptions } from '../shared' @@ -182,11 +183,10 @@ export class ChannelsCommand extends AbstractCommand { }) } - importVideos (options: OverrideCommandOptions & { + importVideos (options: OverrideCommandOptions & VideosImportInChannelCreate & { channelName: string - externalChannelUrl: string }) { - const { channelName, externalChannelUrl } = options + const { channelName, externalChannelUrl, videoChannelSyncId } = options const path = `/api/v1/video-channels/${channelName}/import-videos` @@ -194,7 +194,7 @@ export class ChannelsCommand extends AbstractCommand { ...options, path, - fields: { externalChannelUrl }, + fields: { externalChannelUrl, videoChannelSyncId }, implicitToken: true, defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 }) diff --git a/shared/server-commands/videos/imports-command.ts b/shared/server-commands/videos/imports-command.ts index c931ac481..07d810ec1 100644 --- a/shared/server-commands/videos/imports-command.ts +++ b/shared/server-commands/videos/imports-command.ts @@ -57,15 +57,17 @@ export class ImportsCommand extends AbstractCommand { getMyVideoImports (options: OverrideCommandOptions & { sort?: string targetUrl?: string + videoChannelSyncId?: number + search?: string } = {}) { - const { sort, targetUrl } = options + const { sort, targetUrl, videoChannelSyncId, search } = options const path = '/api/v1/users/me/videos/imports' return this.getRequestBody>({ ...options, path, - query: { sort, targetUrl }, + query: { sort, targetUrl, videoChannelSyncId, search }, implicitToken: true, defaultExpectedStatus: HttpStatusCode.OK_200 }) diff --git a/support/doc/api/openapi.yaml b/support/doc/api/openapi.yaml index ac8cde565..c4bc507fd 100644 --- a/support/doc/api/openapi.yaml +++ b/support/doc/api/openapi.yaml @@ -1187,6 +1187,20 @@ paths: - $ref: '#/components/parameters/start' - $ref: '#/components/parameters/count' - $ref: '#/components/parameters/sort' + - + name: targetUrl + in: query + required: false + description: Filter on import target URL + schema: + type: string + - + name: videoChannelSyncId + in: query + required: false + description: Filter on imports created by a specific channel synchronization + schema: + type: number responses: '200': description: successful operation