diff --git a/.travis.yml b/.travis.yml index ecb44c514..78e25cf45 100644 --- a/.travis.yml +++ b/.travis.yml @@ -36,8 +36,9 @@ before_script: matrix: include: - env: TEST_SUITE=misc - - env: TEST_SUITE=api-fast - - env: TEST_SUITE=api-slow + - env: TEST_SUITE=api-1 + - env: TEST_SUITE=api-2 + - env: TEST_SUITE=api-3 - env: TEST_SUITE=cli - env: TEST_SUITE=lint diff --git a/config/default.yaml b/config/default.yaml index 60da192b4..6a02f254d 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -57,6 +57,11 @@ storage: log: level: 'info' # debug/info/warning/error +search: + remote_uri: # Add ability to fetch remote videos/actors by their URI, that may not be federated with your instance + users: true + anonymous: false + cache: previews: size: 500 # Max number of previews you want to cache diff --git a/config/production.yaml.example b/config/production.yaml.example index 9e8b57829..272a3cb46 100644 --- a/config/production.yaml.example +++ b/config/production.yaml.example @@ -58,6 +58,10 @@ storage: log: level: 'info' # debug/info/warning/error +search: + remote_uri: # Add ability to search remote videos/actors by URI, that may not be federated with your instance + users: true + anonymous: false ############################################################################### # diff --git a/scripts/travis.sh b/scripts/travis.sh index 390500ed4..c2785ffa7 100755 --- a/scripts/travis.sh +++ b/scripts/travis.sh @@ -12,19 +12,22 @@ killall -q peertube || true if [ "$1" = "misc" ]; then npm run build mocha --timeout 5000 --exit --require ts-node/register/type-check --bail server/tests/client.ts server/tests/activitypub.ts \ - server/tests/feeds/feeds.ts + server/tests/feeds/index.ts elif [ "$1" = "api" ]; then npm run build:server mocha --timeout 5000 --exit --require ts-node/register/type-check --bail server/tests/api/index.ts elif [ "$1" = "cli" ]; then npm run build:server mocha --timeout 5000 --exit --require ts-node/register/type-check --bail server/tests/cli/index.ts -elif [ "$1" = "api-fast" ]; then +elif [ "$1" = "api-1" ]; then npm run build:server - mocha --timeout 5000 --exit --require ts-node/register/type-check --bail server/tests/api/index-fast.ts -elif [ "$1" = "api-slow" ]; then + mocha --timeout 5000 --exit --require ts-node/register/type-check --bail server/tests/api/index-1.ts +elif [ "$1" = "api-2" ]; then npm run build:server - mocha --timeout 5000 --exit --require ts-node/register/type-check --bail server/tests/api/index-slow.ts + mocha --timeout 5000 --exit --require ts-node/register/type-check --bail server/tests/api/index-2.ts +elif [ "$1" = "api-3" ]; then + npm run build:server + mocha --timeout 5000 --exit --require ts-node/register/type-check --bail server/tests/api/index-3.ts elif [ "$1" = "lint" ]; then ( cd client npm run lint diff --git a/server/controllers/api/search.ts b/server/controllers/api/search.ts index 9c2c7d6c1..d95e7cac9 100644 --- a/server/controllers/api/search.ts +++ b/server/controllers/api/search.ts @@ -13,8 +13,10 @@ import { videosSearchSortValidator } from '../../middlewares' import { VideosSearchQuery } from '../../../shared/models/search' -import { getOrCreateAccountAndVideoAndChannel } from '../../lib/activitypub' +import { getOrCreateVideoAndAccountAndChannel } from '../../lib/activitypub' import { logger } from '../../helpers/logger' +import { User } from '../../../shared/models/users' +import { CONFIG } from '../../initializers/constants' const searchRouter = express.Router() @@ -56,20 +58,30 @@ async function searchVideosDB (query: VideosSearchQuery, res: express.Response) async function searchVideoUrl (url: string, res: express.Response) { let video: VideoModel + const user: User = res.locals.oauth ? res.locals.oauth.token.User : undefined - try { - const syncParam = { - likes: false, - dislikes: false, - shares: false, - comments: false, - thumbnail: true + // Check if we can fetch a remote video with the URL + if ( + CONFIG.SEARCH.REMOTE_URI.ANONYMOUS === true || + (CONFIG.SEARCH.REMOTE_URI.USERS === true && user !== undefined) + ) { + try { + const syncParam = { + likes: false, + dislikes: false, + shares: false, + comments: false, + thumbnail: true, + refreshVideo: false + } + + const res = await getOrCreateVideoAndAccountAndChannel(url, syncParam) + video = res ? res.video : undefined + } catch (err) { + logger.info('Cannot search remote video %s.', url) } - - const res = await getOrCreateAccountAndVideoAndChannel(url, syncParam) - video = res ? res.video : undefined - } catch (err) { - logger.info('Cannot search remote video %s.', url) + } else { + video = await VideoModel.loadByUrlAndPopulateAccount(url) } return res.json({ diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index 99b10a7fc..cd709cd3f 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -181,6 +181,12 @@ const CONFIG = { LOG: { LEVEL: config.get('log.level') }, + SEARCH: { + REMOTE_URI: { + USERS: config.get('search.remote_uri.users'), + ANONYMOUS: config.get('search.remote_uri.anonymous') + } + }, ADMIN: { get EMAIL () { return config.get('admin.email') } }, @@ -462,7 +468,8 @@ const ACTIVITY_PUB = { MAGNET: [ 'application/x-bittorrent;x-scheme-handler/magnet' ] }, MAX_RECURSION_COMMENTS: 100, - ACTOR_REFRESH_INTERVAL: 3600 * 24 * 1000 // 1 day + ACTOR_REFRESH_INTERVAL: 3600 * 24 * 1000, // 1 day + VIDEO_REFRESH_INTERVAL: 3600 * 24 * 1000 // 1 day } const ACTIVITY_PUB_ACTOR_TYPES: { [ id: string ]: ActivityPubActorType } = { @@ -574,6 +581,7 @@ if (isTestInstance() === true) { ACTIVITY_PUB.COLLECTION_ITEMS_PER_PAGE = 2 ACTIVITY_PUB.ACTOR_REFRESH_INTERVAL = 10 * 1000 // 10 seconds + ACTIVITY_PUB.VIDEO_REFRESH_INTERVAL = 10 * 1000 // 10 seconds CONSTRAINTS_FIELDS.ACTORS.AVATAR.FILE_SIZE.max = 100 * 1024 // 100KB diff --git a/server/lib/activitypub/process/process-announce.ts b/server/lib/activitypub/process/process-announce.ts index b08156aa1..814556817 100644 --- a/server/lib/activitypub/process/process-announce.ts +++ b/server/lib/activitypub/process/process-announce.ts @@ -6,7 +6,7 @@ import { VideoModel } from '../../../models/video/video' import { VideoShareModel } from '../../../models/video/video-share' import { getOrCreateActorAndServerAndModel } from '../actor' import { forwardVideoRelatedActivity } from '../send/utils' -import { getOrCreateAccountAndVideoAndChannel } from '../videos' +import { getOrCreateVideoAndAccountAndChannel } from '../videos' async function processAnnounceActivity (activity: ActivityAnnounce) { const actorAnnouncer = await getOrCreateActorAndServerAndModel(activity.actor) @@ -25,7 +25,7 @@ export { async function processVideoShare (actorAnnouncer: ActorModel, activity: ActivityAnnounce) { const objectUri = typeof activity.object === 'string' ? activity.object : activity.object.id - const { video } = await getOrCreateAccountAndVideoAndChannel(objectUri) + const { video } = await getOrCreateVideoAndAccountAndChannel(objectUri) return sequelizeTypescript.transaction(async t => { // Add share entry diff --git a/server/lib/activitypub/process/process-create.ts b/server/lib/activitypub/process/process-create.ts index 9655d015f..e8f5ade06 100644 --- a/server/lib/activitypub/process/process-create.ts +++ b/server/lib/activitypub/process/process-create.ts @@ -10,7 +10,7 @@ import { VideoAbuseModel } from '../../../models/video/video-abuse' import { VideoCommentModel } from '../../../models/video/video-comment' import { getOrCreateActorAndServerAndModel } from '../actor' import { resolveThread } from '../video-comments' -import { getOrCreateAccountAndVideoAndChannel } from '../videos' +import { getOrCreateVideoAndAccountAndChannel } from '../videos' import { forwardActivity, forwardVideoRelatedActivity } from '../send/utils' async function processCreateActivity (activity: ActivityCreate) { @@ -45,7 +45,7 @@ export { async function processCreateVideo (activity: ActivityCreate) { const videoToCreateData = activity.object as VideoTorrentObject - const { video } = await getOrCreateAccountAndVideoAndChannel(videoToCreateData) + const { video } = await getOrCreateVideoAndAccountAndChannel(videoToCreateData) return video } @@ -56,7 +56,7 @@ async function processCreateDislike (byActor: ActorModel, activity: ActivityCrea if (!byAccount) throw new Error('Cannot create dislike with the non account actor ' + byActor.url) - const { video } = await getOrCreateAccountAndVideoAndChannel(dislike.object) + const { video } = await getOrCreateVideoAndAccountAndChannel(dislike.object) return sequelizeTypescript.transaction(async t => { const rate = { @@ -83,7 +83,7 @@ async function processCreateDislike (byActor: ActorModel, activity: ActivityCrea async function processCreateView (byActor: ActorModel, activity: ActivityCreate) { const view = activity.object as ViewObject - const { video } = await getOrCreateAccountAndVideoAndChannel(view.object) + const { video } = await getOrCreateVideoAndAccountAndChannel(view.object) const actor = await ActorModel.loadByUrl(view.actor) if (!actor) throw new Error('Unknown actor ' + view.actor) @@ -103,7 +103,7 @@ async function processCreateVideoAbuse (actor: ActorModel, videoAbuseToCreateDat const account = actor.Account if (!account) throw new Error('Cannot create dislike with the non account actor ' + actor.url) - const { video } = await getOrCreateAccountAndVideoAndChannel(videoAbuseToCreateData.object) + const { video } = await getOrCreateVideoAndAccountAndChannel(videoAbuseToCreateData.object) return sequelizeTypescript.transaction(async t => { const videoAbuseData = { diff --git a/server/lib/activitypub/process/process-like.ts b/server/lib/activitypub/process/process-like.ts index d0865b78c..9e1664fd8 100644 --- a/server/lib/activitypub/process/process-like.ts +++ b/server/lib/activitypub/process/process-like.ts @@ -5,7 +5,7 @@ import { AccountVideoRateModel } from '../../../models/account/account-video-rat import { ActorModel } from '../../../models/activitypub/actor' import { getOrCreateActorAndServerAndModel } from '../actor' import { forwardVideoRelatedActivity } from '../send/utils' -import { getOrCreateAccountAndVideoAndChannel } from '../videos' +import { getOrCreateVideoAndAccountAndChannel } from '../videos' async function processLikeActivity (activity: ActivityLike) { const actor = await getOrCreateActorAndServerAndModel(activity.actor) @@ -27,7 +27,7 @@ async function processLikeVideo (byActor: ActorModel, activity: ActivityLike) { const byAccount = byActor.Account if (!byAccount) throw new Error('Cannot create like with the non account actor ' + byActor.url) - const { video } = await getOrCreateAccountAndVideoAndChannel(videoUrl) + const { video } = await getOrCreateVideoAndAccountAndChannel(videoUrl) return sequelizeTypescript.transaction(async t => { const rate = { diff --git a/server/lib/activitypub/process/process-undo.ts b/server/lib/activitypub/process/process-undo.ts index b6de107ad..eab9e3d61 100644 --- a/server/lib/activitypub/process/process-undo.ts +++ b/server/lib/activitypub/process/process-undo.ts @@ -9,7 +9,7 @@ import { AccountVideoRateModel } from '../../../models/account/account-video-rat import { ActorModel } from '../../../models/activitypub/actor' import { ActorFollowModel } from '../../../models/activitypub/actor-follow' import { forwardVideoRelatedActivity } from '../send/utils' -import { getOrCreateAccountAndVideoAndChannel } from '../videos' +import { getOrCreateVideoAndAccountAndChannel } from '../videos' import { VideoShareModel } from '../../../models/video/video-share' async function processUndoActivity (activity: ActivityUndo) { @@ -43,7 +43,7 @@ export { async function processUndoLike (actorUrl: string, activity: ActivityUndo) { const likeActivity = activity.object as ActivityLike - const { video } = await getOrCreateAccountAndVideoAndChannel(likeActivity.object) + const { video } = await getOrCreateVideoAndAccountAndChannel(likeActivity.object) return sequelizeTypescript.transaction(async t => { const byAccount = await AccountModel.loadByUrl(actorUrl, t) @@ -67,7 +67,7 @@ async function processUndoLike (actorUrl: string, activity: ActivityUndo) { async function processUndoDislike (actorUrl: string, activity: ActivityUndo) { const dislike = activity.object.object as DislikeObject - const { video } = await getOrCreateAccountAndVideoAndChannel(dislike.object) + const { video } = await getOrCreateVideoAndAccountAndChannel(dislike.object) return sequelizeTypescript.transaction(async t => { const byAccount = await AccountModel.loadByUrl(actorUrl, t) diff --git a/server/lib/activitypub/process/process-update.ts b/server/lib/activitypub/process/process-update.ts index 11226e275..07a5ff92f 100644 --- a/server/lib/activitypub/process/process-update.ts +++ b/server/lib/activitypub/process/process-update.ts @@ -1,4 +1,3 @@ -import * as Bluebird from 'bluebird' import { ActivityUpdate, VideoTorrentObject } from '../../../../shared/models/activitypub' import { ActivityPubActor } from '../../../../shared/models/activitypub/activitypub-actor' import { resetSequelizeInstance, retryTransactionWrapper } from '../../../helpers/database-utils' @@ -6,19 +5,10 @@ import { logger } from '../../../helpers/logger' import { sequelizeTypescript } from '../../../initializers' import { AccountModel } from '../../../models/account/account' import { ActorModel } from '../../../models/activitypub/actor' -import { TagModel } from '../../../models/video/tag' import { VideoChannelModel } from '../../../models/video/video-channel' -import { VideoFileModel } from '../../../models/video/video-file' import { fetchAvatarIfExists, getOrCreateActorAndServerAndModel, updateActorAvatarInstance, updateActorInstance } from '../actor' -import { - generateThumbnailFromUrl, - getOrCreateAccountAndVideoAndChannel, - getOrCreateVideoChannel, - videoActivityObjectToDBAttributes, - videoFileActivityUrlToDBAttributes -} from '../videos' +import { getOrCreateVideoAndAccountAndChannel, getOrCreateVideoChannel, updateVideoFromAP } from '../videos' import { sanitizeAndCheckVideoTorrentObject } from '../../../helpers/custom-validators/activitypub/videos' -import { VideoCaptionModel } from '../../../models/video/video-caption' async function processUpdateActivity (activity: ActivityUpdate) { const actor = await getOrCreateActorAndServerAndModel(activity.actor) @@ -49,91 +39,10 @@ async function processUpdateVideo (actor: ActorModel, activity: ActivityUpdate) return undefined } - const res = await getOrCreateAccountAndVideoAndChannel(videoObject.id) + const { video } = await getOrCreateVideoAndAccountAndChannel(videoObject.id) + const channelActor = await getOrCreateVideoChannel(videoObject) - // Fetch video channel outside the transaction - const newVideoChannelActor = await getOrCreateVideoChannel(videoObject) - const newVideoChannel = newVideoChannelActor.VideoChannel - - logger.debug('Updating remote video "%s".', videoObject.uuid) - let videoInstance = res.video - let videoFieldsSave: any - - try { - await sequelizeTypescript.transaction(async t => { - const sequelizeOptions = { - transaction: t - } - - videoFieldsSave = videoInstance.toJSON() - - // Check actor has the right to update the video - const videoChannel = videoInstance.VideoChannel - if (videoChannel.Account.Actor.id !== actor.id) { - throw new Error('Account ' + actor.url + ' does not own video channel ' + videoChannel.Actor.url) - } - - const videoData = await videoActivityObjectToDBAttributes(newVideoChannel, videoObject, activity.to) - videoInstance.set('name', videoData.name) - videoInstance.set('uuid', videoData.uuid) - videoInstance.set('url', videoData.url) - videoInstance.set('category', videoData.category) - videoInstance.set('licence', videoData.licence) - videoInstance.set('language', videoData.language) - videoInstance.set('description', videoData.description) - videoInstance.set('support', videoData.support) - videoInstance.set('nsfw', videoData.nsfw) - videoInstance.set('commentsEnabled', videoData.commentsEnabled) - videoInstance.set('waitTranscoding', videoData.waitTranscoding) - videoInstance.set('state', videoData.state) - videoInstance.set('duration', videoData.duration) - videoInstance.set('createdAt', videoData.createdAt) - videoInstance.set('updatedAt', videoData.updatedAt) - videoInstance.set('views', videoData.views) - videoInstance.set('privacy', videoData.privacy) - videoInstance.set('channelId', videoData.channelId) - - await videoInstance.save(sequelizeOptions) - - // Don't block on request - generateThumbnailFromUrl(videoInstance, videoObject.icon) - .catch(err => logger.warn('Cannot generate thumbnail of %s.', videoObject.id, { err })) - - // Remove old video files - const videoFileDestroyTasks: Bluebird[] = [] - for (const videoFile of videoInstance.VideoFiles) { - videoFileDestroyTasks.push(videoFile.destroy(sequelizeOptions)) - } - await Promise.all(videoFileDestroyTasks) - - const videoFileAttributes = videoFileActivityUrlToDBAttributes(videoInstance, videoObject) - const tasks = videoFileAttributes.map(f => VideoFileModel.create(f, sequelizeOptions)) - await Promise.all(tasks) - - // Update Tags - const tags = videoObject.tag.map(tag => tag.name) - const tagInstances = await TagModel.findOrCreateTags(tags, t) - await videoInstance.$set('Tags', tagInstances, sequelizeOptions) - - // Update captions - await VideoCaptionModel.deleteAllCaptionsOfRemoteVideo(videoInstance.id, t) - - const videoCaptionsPromises = videoObject.subtitleLanguage.map(c => { - return VideoCaptionModel.insertOrReplaceLanguage(videoInstance.id, c.identifier, t) - }) - await Promise.all(videoCaptionsPromises) - }) - - logger.info('Remote video with uuid %s updated', videoObject.uuid) - } catch (err) { - if (videoInstance !== undefined && videoFieldsSave !== undefined) { - resetSequelizeInstance(videoInstance, videoFieldsSave) - } - - // This is just a debug because we will retry the insert - logger.debug('Cannot update the remote video.', { err }) - throw err - } + return updateVideoFromAP(video, videoObject, actor, channelActor, activity.to) } async function processUpdateActor (actor: ActorModel, activity: ActivityUpdate) { diff --git a/server/lib/activitypub/share.ts b/server/lib/activitypub/share.ts index 698414867..fe3d73e9b 100644 --- a/server/lib/activitypub/share.ts +++ b/server/lib/activitypub/share.ts @@ -6,6 +6,11 @@ import { VideoShareModel } from '../../models/video/video-share' import { sendUndoAnnounce, sendVideoAnnounce } from './send' import { getAnnounceActivityPubUrl } from './url' import { VideoChannelModel } from '../../models/video/video-channel' +import * as Bluebird from 'bluebird' +import { doRequest } from '../../helpers/requests' +import { getOrCreateActorAndServerAndModel } from './actor' +import { logger } from '../../helpers/logger' +import { CRAWL_REQUEST_CONCURRENCY } from '../../initializers' async function shareVideoByServerAndChannel (video: VideoModel, t: Transaction) { if (video.privacy === VideoPrivacy.PRIVATE) return undefined @@ -22,8 +27,41 @@ async function changeVideoChannelShare (video: VideoModel, oldVideoChannel: Vide await shareByVideoChannel(video, t) } +async function addVideoShares (shareUrls: string[], instance: VideoModel) { + await Bluebird.map(shareUrls, async shareUrl => { + try { + // Fetch url + const { body } = await doRequest({ + uri: shareUrl, + json: true, + activityPub: true + }) + if (!body || !body.actor) throw new Error('Body of body actor is invalid') + + const actorUrl = body.actor + const actor = await getOrCreateActorAndServerAndModel(actorUrl) + + const entry = { + actorId: actor.id, + videoId: instance.id, + url: shareUrl + } + + await VideoShareModel.findOrCreate({ + where: { + url: shareUrl + }, + defaults: entry + }) + } catch (err) { + logger.warn('Cannot add share %s.', shareUrl, { err }) + } + }, { concurrency: CRAWL_REQUEST_CONCURRENCY }) +} + export { changeVideoChannelShare, + addVideoShares, shareVideoByServerAndChannel } diff --git a/server/lib/activitypub/video-comments.ts b/server/lib/activitypub/video-comments.ts index 14c7fde69..beff557bc 100644 --- a/server/lib/activitypub/video-comments.ts +++ b/server/lib/activitypub/video-comments.ts @@ -7,7 +7,7 @@ import { ActorModel } from '../../models/activitypub/actor' import { VideoModel } from '../../models/video/video' import { VideoCommentModel } from '../../models/video/video-comment' import { getOrCreateActorAndServerAndModel } from './actor' -import { getOrCreateAccountAndVideoAndChannel } from './videos' +import { getOrCreateVideoAndAccountAndChannel } from './videos' import * as Bluebird from 'bluebird' async function videoCommentActivityObjectToDBAttributes (video: VideoModel, actor: ActorModel, comment: VideoCommentObject) { @@ -91,7 +91,7 @@ async function resolveThread (url: string, comments: VideoCommentModel[] = []) { try { // Maybe it's a reply to a video? - const { video } = await getOrCreateAccountAndVideoAndChannel(url) + const { video } = await getOrCreateVideoAndAccountAndChannel(url) if (comments.length !== 0) { const firstReply = comments[ comments.length - 1 ] diff --git a/server/lib/activitypub/video-rates.ts b/server/lib/activitypub/video-rates.ts index 19011b4ab..1619251c3 100644 --- a/server/lib/activitypub/video-rates.ts +++ b/server/lib/activitypub/video-rates.ts @@ -2,6 +2,45 @@ import { Transaction } from 'sequelize' import { AccountModel } from '../../models/account/account' import { VideoModel } from '../../models/video/video' import { sendCreateDislike, sendLike, sendUndoDislike, sendUndoLike } from './send' +import { VideoRateType } from '../../../shared/models/videos' +import * as Bluebird from 'bluebird' +import { getOrCreateActorAndServerAndModel } from './actor' +import { AccountVideoRateModel } from '../../models/account/account-video-rate' +import { logger } from '../../helpers/logger' +import { CRAWL_REQUEST_CONCURRENCY } from '../../initializers' + +async function createRates (actorUrls: string[], video: VideoModel, rate: VideoRateType) { + let rateCounts = 0 + + await Bluebird.map(actorUrls, async actorUrl => { + try { + const actor = await getOrCreateActorAndServerAndModel(actorUrl) + const [ , created ] = await AccountVideoRateModel + .findOrCreate({ + where: { + videoId: video.id, + accountId: actor.Account.id + }, + defaults: { + videoId: video.id, + accountId: actor.Account.id, + type: rate + } + }) + + if (created) rateCounts += 1 + } catch (err) { + logger.warn('Cannot add rate %s for actor %s.', rate, actorUrl, { err }) + } + }, { concurrency: CRAWL_REQUEST_CONCURRENCY }) + + logger.info('Adding %d %s to video %s.', rateCounts, rate, video.uuid) + + // This is "likes" and "dislikes" + if (rateCounts !== 0) await video.increment(rate + 's', { by: rateCounts }) + + return +} async function sendVideoRateChange (account: AccountModel, video: VideoModel, @@ -24,5 +63,6 @@ async function sendVideoRateChange (account: AccountModel, } export { + createRates, sendVideoRateChange } diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts index fac1d3fc7..388c31fe5 100644 --- a/server/lib/activitypub/videos.ts +++ b/server/lib/activitypub/videos.ts @@ -5,29 +5,30 @@ import { join } from 'path' import * as request from 'request' import { ActivityIconObject, VideoState } from '../../../shared/index' import { VideoTorrentObject } from '../../../shared/models/activitypub/objects' -import { VideoPrivacy, VideoRateType } from '../../../shared/models/videos' +import { VideoPrivacy } from '../../../shared/models/videos' import { sanitizeAndCheckVideoTorrentObject } from '../../helpers/custom-validators/activitypub/videos' import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos' -import { retryTransactionWrapper } from '../../helpers/database-utils' +import { resetSequelizeInstance, retryTransactionWrapper, updateInstanceWithAnother } from '../../helpers/database-utils' import { logger } from '../../helpers/logger' import { doRequest, doRequestAndSaveToFile } from '../../helpers/requests' -import { ACTIVITY_PUB, CONFIG, CRAWL_REQUEST_CONCURRENCY, REMOTE_SCHEME, sequelizeTypescript, VIDEO_MIMETYPE_EXT } from '../../initializers' -import { AccountVideoRateModel } from '../../models/account/account-video-rate' +import { ACTIVITY_PUB, CONFIG, REMOTE_SCHEME, sequelizeTypescript, VIDEO_MIMETYPE_EXT } from '../../initializers' import { ActorModel } from '../../models/activitypub/actor' import { TagModel } from '../../models/video/tag' import { VideoModel } from '../../models/video/video' import { VideoChannelModel } from '../../models/video/video-channel' import { VideoFileModel } from '../../models/video/video-file' -import { VideoShareModel } from '../../models/video/video-share' -import { getOrCreateActorAndServerAndModel } from './actor' +import { getOrCreateActorAndServerAndModel, updateActorAvatarInstance } from './actor' import { addVideoComments } from './video-comments' import { crawlCollectionPage } from './crawl' import { sendCreateVideo, sendUpdateVideo } from './send' -import { shareVideoByServerAndChannel } from './index' import { isArray } from '../../helpers/custom-validators/misc' import { VideoCaptionModel } from '../../models/video/video-caption' import { JobQueue } from '../job-queue' import { ActivitypubHttpFetcherPayload } from '../job-queue/handlers/activitypub-http-fetcher' +import { getUrlFromWebfinger } from '../../helpers/webfinger' +import { createRates } from './video-rates' +import { addVideoShares, shareVideoByServerAndChannel } from './share' +import { AccountModel } from '../../models/account/account' async function federateVideoIfNeeded (video: VideoModel, isNewVideo: boolean, transaction?: sequelize.Transaction) { // If the video is not private and published, we federate it @@ -180,15 +181,11 @@ function getOrCreateVideoChannel (videoObject: VideoTorrentObject) { return getOrCreateActorAndServerAndModel(channel.id) } -async function getOrCreateVideo (videoObject: VideoTorrentObject, channelActor: ActorModel, waitThumbnail = false) { +async function createVideo (videoObject: VideoTorrentObject, channelActor: ActorModel, waitThumbnail = false) { logger.debug('Adding remote video %s.', videoObject.id) const videoCreated: VideoModel = await sequelizeTypescript.transaction(async t => { - const sequelizeOptions = { - transaction: t - } - const videoFromDatabase = await VideoModel.loadByUUIDOrURLAndPopulateAccount(videoObject.uuid, videoObject.id, t) - if (videoFromDatabase) return videoFromDatabase + const sequelizeOptions = { transaction: t } const videoData = await videoActivityObjectToDBAttributes(channelActor.VideoChannel, videoObject, videoObject.to) const video = VideoModel.build(videoData) @@ -230,26 +227,32 @@ async function getOrCreateVideo (videoObject: VideoTorrentObject, channelActor: } type SyncParam = { - likes: boolean, - dislikes: boolean, - shares: boolean, - comments: boolean, + likes: boolean + dislikes: boolean + shares: boolean + comments: boolean thumbnail: boolean + refreshVideo: boolean } -async function getOrCreateAccountAndVideoAndChannel ( +async function getOrCreateVideoAndAccountAndChannel ( videoObject: VideoTorrentObject | string, - syncParam: SyncParam = { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true } + syncParam: SyncParam = { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true, refreshVideo: false } ) { const videoUrl = typeof videoObject === 'string' ? videoObject : videoObject.id - const videoFromDatabase = await VideoModel.loadByUrlAndPopulateAccount(videoUrl) - if (videoFromDatabase) return { video: videoFromDatabase } + let videoFromDatabase = await VideoModel.loadByUrlAndPopulateAccount(videoUrl) + if (videoFromDatabase) { + const p = retryTransactionWrapper(refreshVideoIfNeeded, videoFromDatabase) + if (syncParam.refreshVideo === true) videoFromDatabase = await p - const fetchedVideo = await fetchRemoteVideo(videoUrl) + return { video: videoFromDatabase } + } + + const { videoObject: fetchedVideo } = await fetchRemoteVideo(videoUrl) if (!fetchedVideo) throw new Error('Cannot fetch remote video with url: ' + videoUrl) const channelActor = await getOrCreateVideoChannel(fetchedVideo) - const video = await retryTransactionWrapper(getOrCreateVideo, fetchedVideo, channelActor, syncParam.thumbnail) + const video = await retryTransactionWrapper(createVideo, fetchedVideo, channelActor, syncParam.thumbnail) // Process outside the transaction because we could fetch remote data @@ -290,72 +293,7 @@ async function getOrCreateAccountAndVideoAndChannel ( return { video } } -async function createRates (actorUrls: string[], video: VideoModel, rate: VideoRateType) { - let rateCounts = 0 - - await Bluebird.map(actorUrls, async actorUrl => { - try { - const actor = await getOrCreateActorAndServerAndModel(actorUrl) - const [ , created ] = await AccountVideoRateModel - .findOrCreate({ - where: { - videoId: video.id, - accountId: actor.Account.id - }, - defaults: { - videoId: video.id, - accountId: actor.Account.id, - type: rate - } - }) - - if (created) rateCounts += 1 - } catch (err) { - logger.warn('Cannot add rate %s for actor %s.', rate, actorUrl, { err }) - } - }, { concurrency: CRAWL_REQUEST_CONCURRENCY }) - - logger.info('Adding %d %s to video %s.', rateCounts, rate, video.uuid) - - // This is "likes" and "dislikes" - if (rateCounts !== 0) await video.increment(rate + 's', { by: rateCounts }) - - return -} - -async function addVideoShares (shareUrls: string[], instance: VideoModel) { - await Bluebird.map(shareUrls, async shareUrl => { - try { - // Fetch url - const { body } = await doRequest({ - uri: shareUrl, - json: true, - activityPub: true - }) - if (!body || !body.actor) throw new Error('Body of body actor is invalid') - - const actorUrl = body.actor - const actor = await getOrCreateActorAndServerAndModel(actorUrl) - - const entry = { - actorId: actor.id, - videoId: instance.id, - url: shareUrl - } - - await VideoShareModel.findOrCreate({ - where: { - url: shareUrl - }, - defaults: entry - }) - } catch (err) { - logger.warn('Cannot add share %s.', shareUrl, { err }) - } - }, { concurrency: CRAWL_REQUEST_CONCURRENCY }) -} - -async function fetchRemoteVideo (videoUrl: string): Promise { +async function fetchRemoteVideo (videoUrl: string): Promise<{ response: request.RequestResponse, videoObject: VideoTorrentObject }> { const options = { uri: videoUrl, method: 'GET', @@ -365,26 +303,143 @@ async function fetchRemoteVideo (videoUrl: string): Promise logger.info('Fetching remote video %s.', videoUrl) - const { body } = await doRequest(options) + const { response, body } = await doRequest(options) if (sanitizeAndCheckVideoTorrentObject(body) === false) { logger.debug('Remote video JSON is not valid.', { body }) - return undefined + return { response, videoObject: undefined } } - return body + return { response, videoObject: body } +} + +async function refreshVideoIfNeeded (video: VideoModel): Promise { + if (!video.isOutdated()) return video + + try { + const { response, videoObject } = await fetchRemoteVideo(video.url) + if (response.statusCode === 404) { + // Video does not exist anymore + await video.destroy() + return undefined + } + + if (videoObject === undefined) { + logger.warn('Cannot refresh remote video: invalid body.') + return video + } + + const channelActor = await getOrCreateVideoChannel(videoObject) + const account = await AccountModel.load(channelActor.VideoChannel.accountId) + return updateVideoFromAP(video, videoObject, account.Actor, channelActor) + + } catch (err) { + logger.warn('Cannot refresh video.', { err }) + return video + } +} + +async function updateVideoFromAP ( + video: VideoModel, + videoObject: VideoTorrentObject, + accountActor: ActorModel, + channelActor: ActorModel, + overrideTo?: string[] +) { + logger.debug('Updating remote video "%s".', videoObject.uuid) + let videoFieldsSave: any + + try { + const updatedVideo: VideoModel = await sequelizeTypescript.transaction(async t => { + const sequelizeOptions = { + transaction: t + } + + videoFieldsSave = video.toJSON() + + // Check actor has the right to update the video + const videoChannel = video.VideoChannel + if (videoChannel.Account.Actor.id !== accountActor.id) { + throw new Error('Account ' + accountActor.url + ' does not own video channel ' + videoChannel.Actor.url) + } + + const to = overrideTo ? overrideTo : videoObject.to + const videoData = await videoActivityObjectToDBAttributes(channelActor.VideoChannel, videoObject, to) + video.set('name', videoData.name) + video.set('uuid', videoData.uuid) + video.set('url', videoData.url) + video.set('category', videoData.category) + video.set('licence', videoData.licence) + video.set('language', videoData.language) + video.set('description', videoData.description) + video.set('support', videoData.support) + video.set('nsfw', videoData.nsfw) + video.set('commentsEnabled', videoData.commentsEnabled) + video.set('waitTranscoding', videoData.waitTranscoding) + video.set('state', videoData.state) + video.set('duration', videoData.duration) + video.set('createdAt', videoData.createdAt) + video.set('publishedAt', videoData.publishedAt) + video.set('views', videoData.views) + video.set('privacy', videoData.privacy) + video.set('channelId', videoData.channelId) + + await video.save(sequelizeOptions) + + // Don't block on request + generateThumbnailFromUrl(video, videoObject.icon) + .catch(err => logger.warn('Cannot generate thumbnail of %s.', videoObject.id, { err })) + + // Remove old video files + const videoFileDestroyTasks: Bluebird[] = [] + for (const videoFile of video.VideoFiles) { + videoFileDestroyTasks.push(videoFile.destroy(sequelizeOptions)) + } + await Promise.all(videoFileDestroyTasks) + + const videoFileAttributes = videoFileActivityUrlToDBAttributes(video, videoObject) + const tasks = videoFileAttributes.map(f => VideoFileModel.create(f, sequelizeOptions)) + await Promise.all(tasks) + + // Update Tags + const tags = videoObject.tag.map(tag => tag.name) + const tagInstances = await TagModel.findOrCreateTags(tags, t) + await video.$set('Tags', tagInstances, sequelizeOptions) + + // Update captions + await VideoCaptionModel.deleteAllCaptionsOfRemoteVideo(video.id, t) + + const videoCaptionsPromises = videoObject.subtitleLanguage.map(c => { + return VideoCaptionModel.insertOrReplaceLanguage(video.id, c.identifier, t) + }) + await Promise.all(videoCaptionsPromises) + }) + + logger.info('Remote video with uuid %s updated', videoObject.uuid) + + return updatedVideo + } catch (err) { + if (video !== undefined && videoFieldsSave !== undefined) { + resetSequelizeInstance(video, videoFieldsSave) + } + + // This is just a debug because we will retry the insert + logger.debug('Cannot update the remote video.', { err }) + throw err + } } export { + updateVideoFromAP, federateVideoIfNeeded, fetchRemoteVideo, - getOrCreateAccountAndVideoAndChannel, + getOrCreateVideoAndAccountAndChannel, fetchRemoteVideoStaticFile, fetchRemoteVideoDescription, generateThumbnailFromUrl, videoActivityObjectToDBAttributes, videoFileActivityUrlToDBAttributes, - getOrCreateVideo, + createVideo, getOrCreateVideoChannel, addVideoShares, createRates diff --git a/server/models/video/video.ts b/server/models/video/video.ts index 25a1cd177..7acbc60f7 100644 --- a/server/models/video/video.ts +++ b/server/models/video/video.ts @@ -56,6 +56,7 @@ import { generateImageFromVideoFile, getVideoFileFPS, getVideoFileResolution, tr import { logger } from '../../helpers/logger' import { getServerActor } from '../../helpers/utils' import { + ACTIVITY_PUB, API_VERSION, CONFIG, CONSTRAINTS_FIELDS, @@ -1004,21 +1005,6 @@ export class VideoModel extends Model { return VideoModel.scope([ ScopeNames.WITH_ACCOUNT_DETAILS, ScopeNames.WITH_FILES ]).findOne(query) } - static loadByUUIDOrURLAndPopulateAccount (uuid: string, url: string, t?: Sequelize.Transaction) { - const query: IFindOptions = { - where: { - [Sequelize.Op.or]: [ - { uuid }, - { url } - ] - } - } - - if (t !== undefined) query.transaction = t - - return VideoModel.scope([ ScopeNames.WITH_ACCOUNT_DETAILS, ScopeNames.WITH_FILES ]).findOne(query) - } - static loadAndPopulateAccountAndServerAndTags (id: number) { const options = { order: [ [ 'Tags', 'name', 'ASC' ] ] @@ -1646,6 +1632,17 @@ export class VideoModel extends Model { return 'PT' + this.duration + 'S' } + isOutdated () { + if (this.isOwned()) return false + + const now = Date.now() + const createdAtTime = this.createdAt.getTime() + const updatedAtTime = this.updatedAt.getTime() + + return (now - createdAtTime) > ACTIVITY_PUB.VIDEO_REFRESH_INTERVAL && + (now - updatedAtTime) > ACTIVITY_PUB.VIDEO_REFRESH_INTERVAL + } + private getBaseUrls () { let baseUrlHttp let baseUrlWs diff --git a/server/tests/api/index-1.ts b/server/tests/api/index-1.ts new file mode 100644 index 000000000..80d752f42 --- /dev/null +++ b/server/tests/api/index-1.ts @@ -0,0 +1,2 @@ +import './check-params' +import './search' diff --git a/server/tests/api/index-2.ts b/server/tests/api/index-2.ts new file mode 100644 index 000000000..ed93faa91 --- /dev/null +++ b/server/tests/api/index-2.ts @@ -0,0 +1,2 @@ +import './server' +import './users' diff --git a/server/tests/api/index-3.ts b/server/tests/api/index-3.ts new file mode 100644 index 000000000..39823b82c --- /dev/null +++ b/server/tests/api/index-3.ts @@ -0,0 +1 @@ +import './videos' diff --git a/server/tests/api/index-fast.ts b/server/tests/api/index-fast.ts deleted file mode 100644 index 02ffdd4f1..000000000 --- a/server/tests/api/index-fast.ts +++ /dev/null @@ -1,18 +0,0 @@ -// Order of the tests we want to execute -import './server/stats' -import './check-params' -import './users/users' -import './videos/single-server' -import './videos/video-abuse' -import './videos/video-captions' -import './videos/video-blacklist' -import './videos/video-blacklist-management' -import './videos/video-description' -import './videos/video-nsfw' -import './videos/video-privacy' -import './videos/services' -import './server/email' -import './server/config' -import './server/reverse-proxy' -import './search/search-videos' -import './server/tracker' diff --git a/server/tests/api/index-slow.ts b/server/tests/api/index-slow.ts deleted file mode 100644 index e24a7b664..000000000 --- a/server/tests/api/index-slow.ts +++ /dev/null @@ -1,12 +0,0 @@ -// Order of the tests we want to execute -import './videos/video-channels' -import './videos/video-transcoder' -import './videos/multiple-servers' -import './server/follows' -import './server/jobs' -import './videos/video-comments' -import './users/users-multiple-servers' -import './users/user-subscriptions' -import './server/handle-down' -import './videos/video-schedule-update' -import './videos/video-imports' diff --git a/server/tests/api/index.ts b/server/tests/api/index.ts index 258502d26..2d996dbf9 100644 --- a/server/tests/api/index.ts +++ b/server/tests/api/index.ts @@ -1,3 +1,4 @@ // Order of the tests we want to execute -import './index-fast' -import './index-slow' +import './index-1' +import './index-2' +import './index-3' diff --git a/server/tests/api/search/index.ts b/server/tests/api/search/index.ts new file mode 100644 index 000000000..64b3d0910 --- /dev/null +++ b/server/tests/api/search/index.ts @@ -0,0 +1,2 @@ +import './search-activitypub-videos' +import './search-videos' diff --git a/server/tests/api/search/search-activitypub-videos.ts b/server/tests/api/search/search-activitypub-videos.ts new file mode 100644 index 000000000..6dc792696 --- /dev/null +++ b/server/tests/api/search/search-activitypub-videos.ts @@ -0,0 +1,161 @@ +/* tslint:disable:no-unused-expression */ + +import * as chai from 'chai' +import 'mocha' +import { + addVideoChannel, + flushAndRunMultipleServers, + flushTests, + getVideosList, + killallServers, + removeVideo, + searchVideoWithToken, + ServerInfo, + setAccessTokensToServers, + updateVideo, + uploadVideo, + wait, + searchVideo +} from '../../utils' +import { waitJobs } from '../../utils/server/jobs' +import { Video, VideoPrivacy } from '../../../../shared/models/videos' + +const expect = chai.expect + +describe('Test a ActivityPub videos search', function () { + let servers: ServerInfo[] + let videoServer1UUID: string + let videoServer2UUID: string + + before(async function () { + this.timeout(120000) + + await flushTests() + + servers = await flushAndRunMultipleServers(2) + + await setAccessTokensToServers(servers) + + { + const res = await uploadVideo(servers[ 0 ].url, servers[ 0 ].accessToken, { name: 'video 1 on server 1' }) + videoServer1UUID = res.body.video.uuid + } + + { + const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 1 on server 2' }) + videoServer2UUID = res.body.video.uuid + } + + await waitJobs(servers) + }) + + it('Should not find a remote video', async function () { + { + const res = await searchVideoWithToken(servers[ 0 ].url, 'http://localhost:9002/videos/watch/43', servers[ 0 ].accessToken) + + expect(res.body.total).to.equal(0) + expect(res.body.data).to.be.an('array') + expect(res.body.data).to.have.lengthOf(0) + } + + { + const res = await searchVideo(servers[0].url, 'http://localhost:9002/videos/watch/' + videoServer2UUID) + + expect(res.body.total).to.equal(0) + expect(res.body.data).to.be.an('array') + expect(res.body.data).to.have.lengthOf(0) + } + }) + + it('Should search a local video', async function () { + const res = await searchVideo(servers[0].url, 'http://localhost:9001/videos/watch/' + videoServer1UUID) + + expect(res.body.total).to.equal(1) + expect(res.body.data).to.be.an('array') + expect(res.body.data).to.have.lengthOf(1) + expect(res.body.data[0].name).to.equal('video 1 on server 1') + }) + + it('Should search a remote video', async function () { + const res = await searchVideoWithToken(servers[0].url, 'http://localhost:9002/videos/watch/' + videoServer2UUID, servers[0].accessToken) + + expect(res.body.total).to.equal(1) + expect(res.body.data).to.be.an('array') + expect(res.body.data).to.have.lengthOf(1) + expect(res.body.data[0].name).to.equal('video 1 on server 2') + }) + + it('Should not list this remote video', async function () { + const res = await getVideosList(servers[0].url) + expect(res.body.total).to.equal(1) + expect(res.body.data).to.have.lengthOf(1) + expect(res.body.data[0].name).to.equal('video 1 on server 1') + }) + + it('Should update video of server 2, and refresh it on server 1', async function () { + this.timeout(60000) + + const channelAttributes = { + name: 'super_channel', + displayName: 'super channel' + } + const resChannel = await addVideoChannel(servers[1].url, servers[1].accessToken, channelAttributes) + const videoChannelId = resChannel.body.videoChannel.id + + const attributes = { + name: 'updated', + tag: [ 'tag1', 'tag2' ], + privacy: VideoPrivacy.UNLISTED, + channelId: videoChannelId + } + await updateVideo(servers[1].url, servers[1].accessToken, videoServer2UUID, attributes) + + await waitJobs(servers) + // Expire video + await wait(10000) + + // Will run refresh async + await searchVideoWithToken(servers[0].url, 'http://localhost:9002/videos/watch/' + videoServer2UUID, servers[0].accessToken) + + // Wait refresh + await wait(5000) + + const res = await searchVideoWithToken(servers[0].url, 'http://localhost:9002/videos/watch/' + videoServer2UUID, servers[0].accessToken) + expect(res.body.total).to.equal(1) + expect(res.body.data).to.have.lengthOf(1) + + const video: Video = res.body.data[0] + expect(video.name).to.equal('updated') + expect(video.channel.name).to.equal('super_channel') + expect(video.privacy.id).to.equal(VideoPrivacy.UNLISTED) + }) + + it('Should delete video of server 2, and delete it on server 1', async function () { + this.timeout(60000) + + await removeVideo(servers[1].url, servers[1].accessToken, videoServer2UUID) + + await waitJobs(servers) + // Expire video + await wait(10000) + + // Will run refresh async + await searchVideoWithToken(servers[0].url, 'http://localhost:9002/videos/watch/' + videoServer2UUID, servers[0].accessToken) + + // Wait refresh + await wait(5000) + + const res = await searchVideoWithToken(servers[0].url, 'http://localhost:9002/videos/watch/' + videoServer2UUID, servers[0].accessToken) + expect(res.body.total).to.equal(0) + expect(res.body.data).to.have.lengthOf(0) + }) + + after(async function () { + killallServers(servers) + + // Keep the logs if the test failed + if (this['ok']) { + await flushTests() + } + }) +}) diff --git a/server/tests/api/server/index.ts b/server/tests/api/server/index.ts new file mode 100644 index 000000000..eeb8b7a28 --- /dev/null +++ b/server/tests/api/server/index.ts @@ -0,0 +1,8 @@ +import './config' +import './email' +import './follows' +import './handle-down' +import './jobs' +import './reverse-proxy' +import './stats' +import './tracker' diff --git a/server/tests/api/users/index.ts b/server/tests/api/users/index.ts new file mode 100644 index 000000000..4ce87fb91 --- /dev/null +++ b/server/tests/api/users/index.ts @@ -0,0 +1,3 @@ +import './user-subscriptions' +import './users' +import './users-multiple-servers' diff --git a/server/tests/api/videos/index.ts b/server/tests/api/videos/index.ts new file mode 100644 index 000000000..9f1230767 --- /dev/null +++ b/server/tests/api/videos/index.ts @@ -0,0 +1,15 @@ +import './multiple-servers' +import './services' +import './single-server' +import './video-abuse' +import './video-blacklist' +import './video-blacklist-management' +import './video-captions' +import './video-channels' +import './video-comme' +import './video-description' +import './video-impo' +import './video-nsfw' +import './video-privacy' +import './video-schedule-update' +import './video-transcoder' diff --git a/server/tests/feeds/index.ts b/server/tests/feeds/index.ts new file mode 100644 index 000000000..aa6236a91 --- /dev/null +++ b/server/tests/feeds/index.ts @@ -0,0 +1 @@ +import './feeds' diff --git a/server/tests/index.ts b/server/tests/index.ts index 755fb2604..e659fd3df 100644 --- a/server/tests/index.ts +++ b/server/tests/index.ts @@ -1,5 +1,6 @@ // Order of the tests we want to execute import './client' import './activitypub' -import './api/' +import './feeds/' import './cli/' +import './api/'