Add ability to accept or not remote redundancies

config/default.yaml

@@ -126,6 +126,14 @@ redundancy:
 #        strategy: 'recently-added' # Cache recently added videos
 #        min_views: 10 # Having at least x views
 
+# Other instances that duplicate your content
+remote_redundancy:
+  videos:
+    # 'nobody': Do not accept remote redundancies
+    # 'anybody': Accept remote redundancies from anybody
+    # 'followings': Accept redundancies from instance followings
+    accept_from: 'anybody'
+
 csp:
   enabled: false
   report_only: true # CSP directives are still being tested, so disable the report only mode at your own risk!

config/production.yaml.example

@@ -127,6 +127,14 @@ redundancy:
 #        strategy: 'recently-added' # Cache recently added videos
 #        min_views: 10 # Having at least x views
 
+# Other instances that duplicate your content
+remote_redundancy:
+  videos:
+    # 'nobody': Do not accept remote redundancies
+    # 'anybody': Accept remote redundancies from anybody
+    # 'followings': Accept redundancies from instance followings
+    accept_from: 'anybody'
+
 csp:
   enabled: false
   report_only: true # CSP directives are still being tested, so disable the report only mode at your own risk!

server/initializers/checker-after-init.ts

@@ -11,6 +11,7 @@ import { RecentlyAddedStrategy } from '../../shared/models/redundancy'
 import { isArray } from '../helpers/custom-validators/misc'
 import { uniq } from 'lodash'
 import { WEBSERVER } from './constants'
+import { VideoRedundancyConfigFilter } from '@shared/models/redundancy/video-redundancy-config-filter.type'
 
 async function checkActivityPubUrls () {
   const actor = await getServerActor()
@@ -87,6 +88,13 @@ function checkConfig () {
     return 'Videos redundancy should be an array (you must uncomment lines containing - too)'
   }
 
+  // Remote redundancies
+  const acceptFrom = CONFIG.REMOTE_REDUNDANCY.VIDEOS.ACCEPT_FROM
+  const acceptFromValues = new Set<VideoRedundancyConfigFilter>([ 'nobody', 'anybody', 'followings' ])
+  if (acceptFromValues.has(acceptFrom) === false) {
+    return 'remote_redundancy.videos.accept_from has an incorrect value'
+  }
+
   // Check storage directory locations
   if (isProdInstance()) {
     const configStorage = config.get('storage')

server/initializers/checker-before-init.ts

@@ -31,7 +31,8 @@ function checkMissedConfig () {
     'tracker.enabled', 'tracker.private', 'tracker.reject_too_many_announces',
     'history.videos.max_age', 'views.videos.remote.max_age',
     'rates_limit.login.window', 'rates_limit.login.max', 'rates_limit.ask_send_email.window', 'rates_limit.ask_send_email.max',
-    'theme.default'
+    'theme.default',
+    'remote_redundancy.videos.accept_from'
   ]
   const requiredAlternatives = [
     [ // set

server/initializers/config.ts

@@ -5,6 +5,7 @@ import { VideosRedundancyStrategy } from '../../shared/models'
 import { buildPath, parseBytes, parseDurationToMs, root } from '../helpers/core-utils'
 import { NSFWPolicyType } from '../../shared/models/videos/nsfw-policy.type'
 import * as bytes from 'bytes'
+import { VideoRedundancyConfigFilter } from '@shared/models/redundancy/video-redundancy-config-filter.type'
 
 // Use a variable to reload the configuration if we need
 let config: IConfig = require('config')
@@ -117,6 +118,11 @@ const CONFIG = {
       STRATEGIES: buildVideosRedundancy(config.get<any[]>('redundancy.videos.strategies'))
     }
   },
+  REMOTE_REDUNDANCY: {
+    VIDEOS: {
+      ACCEPT_FROM: config.get<VideoRedundancyConfigFilter>('remote_redundancy.videos.accept_from')
+    }
+  },
   CSP: {
     ENABLED: config.get<boolean>('csp.enabled'),
     REPORT_ONLY: config.get<boolean>('csp.report_only'),

server/lib/activitypub/process/process-create.ts

@@ -12,6 +12,7 @@ import { PlaylistObject } from '../../../../shared/models/activitypub/objects/pl
 import { createOrUpdateVideoPlaylist } from '../playlist'
 import { APProcessorOptions } from '../../../typings/activitypub-processor.model'
 import { MActorSignature, MCommentOwnerVideo, MVideoAccountLightBlacklistAllFiles } from '../../../typings/models'
+import { isRedundancyAccepted } from '@server/lib/redundancy'
 
 async function processCreateActivity (options: APProcessorOptions<ActivityCreate>) {
   const { activity, byActor } = options
@@ -60,6 +61,8 @@ async function processCreateVideo (activity: ActivityCreate, notify: boolean) {
 }
 
 async function processCreateCacheFile (activity: ActivityCreate, byActor: MActorSignature) {
+  if (await isRedundancyAccepted(activity, byActor) !== true) return
+
   const cacheFile = activity.object as CacheFileObject
 
   const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: cacheFile.object })

server/lib/activitypub/process/process-update.ts

@@ -16,6 +16,7 @@ import { PlaylistObject } from '../../../../shared/models/activitypub/objects/pl
 import { createOrUpdateVideoPlaylist } from '../playlist'
 import { APProcessorOptions } from '../../../typings/activitypub-processor.model'
 import { MActorSignature, MAccountIdActor } from '../../../typings/models'
+import { isRedundancyAccepted } from '@server/lib/redundancy'
 
 async function processUpdateActivity (options: APProcessorOptions<ActivityUpdate>) {
   const { activity, byActor } = options
@@ -78,6 +79,8 @@ async function processUpdateVideo (actor: MActorSignature, activity: ActivityUpd
 }
 
 async function processUpdateCacheFile (byActor: MActorSignature, activity: ActivityUpdate) {
+  if (await isRedundancyAccepted(activity, byActor) !== true) return
+
   const cacheFileObject = activity.object as CacheFileObject
 
   if (!isCacheFileObjectValid(cacheFileObject)) {

server/lib/redundancy.ts

@@ -2,7 +2,11 @@ import { VideoRedundancyModel } from '../models/redundancy/video-redundancy'
 import { sendUndoCacheFile } from './activitypub/send'
 import { Transaction } from 'sequelize'
 import { getServerActor } from '../helpers/utils'
-import { MVideoRedundancyVideo } from '@server/typings/models'
+import { MActorSignature, MVideoRedundancyVideo } from '@server/typings/models'
+import { CONFIG } from '@server/initializers/config'
+import { logger } from '@server/helpers/logger'
+import { ActorFollowModel } from '@server/models/activitypub/actor-follow'
+import { Activity } from '@shared/models'
 
 async function removeVideoRedundancy (videoRedundancy: MVideoRedundancyVideo, t?: Transaction) {
   const serverActor = await getServerActor()
@@ -21,9 +25,30 @@ async function removeRedundanciesOfServer (serverId: number) {
   }
 }
 
+async function isRedundancyAccepted (activity: Activity, byActor: MActorSignature) {
+  const configAcceptFrom = CONFIG.REMOTE_REDUNDANCY.VIDEOS.ACCEPT_FROM
+  if (configAcceptFrom === 'nobody') {
+    logger.info('Do not accept remote redundancy %s due instance accept policy.', activity.id)
+    return false
+  }
+
+  if (configAcceptFrom === 'followings') {
+    const serverActor = await getServerActor()
+    const allowed = await ActorFollowModel.isFollowedBy(byActor.id, serverActor.id)
+
+    if (allowed !== true) {
+      logger.info('Do not accept remote redundancy %s because actor %s is not followed by our instance.', activity.id, byActor.url)
+      return false
+    }
+  }
+
+  return true
+}
+
 // ---------------------------------------------------------------------------
 
 export {
+  isRedundancyAccepted,
   removeRedundanciesOfServer,
   removeVideoRedundancy
 }

server/models/activitypub/actor-follow.ts

@@ -36,6 +36,7 @@ import {
   MActorFollowSubscriptions
 } from '@server/typings/models'
 import { ActivityPubActorType } from '@shared/models'
+import { VideoModel } from '@server/models/video/video'
 
 @Table({
   tableName: 'actorFollow',
@@ -151,6 +152,18 @@ export class ActorFollowModel extends Model<ActorFollowModel> {
     if (numberOfActorFollowsRemoved) logger.info('Removed bad %d actor follows.', numberOfActorFollowsRemoved)
   }
 
+  static isFollowedBy (actorId: number, followerActorId: number) {
+    const query = 'SELECT 1 FROM "actorFollow" WHERE "actorId" = $followerActorId AND "targetActorId" = $actorId LIMIT 1'
+    const options = {
+      type: QueryTypes.SELECT as QueryTypes.SELECT,
+      bind: { actorId, followerActorId },
+      raw: true
+    }
+
+    return VideoModel.sequelize.query(query, options)
+                     .then(results => results.length === 1)
+  }
+
   static loadByActorAndTarget (actorId: number, targetActorId: number, t?: Transaction): Bluebird<MActorFollowActorsDefault> {
     const query = {
       where: {

server/tests/api/redundancy/index.ts

@@ -1,2 +1,3 @@
+import './redundancy-constraints'
 import './redundancy'
 import './manage-redundancy'

server/tests/api/redundancy/redundancy-constraints.ts

@@ -0,0 +1,200 @@
+/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
+
+import * as chai from 'chai'
+import 'mocha'
+import {
+  cleanupTests,
+  flushAndRunServer,
+  follow,
+  killallServers,
+  reRunServer,
+  ServerInfo,
+  setAccessTokensToServers,
+  uploadVideo,
+  waitUntilLog
+} from '../../../../shared/extra-utils'
+import { waitJobs } from '../../../../shared/extra-utils/server/jobs'
+import { listVideoRedundancies, updateRedundancy } from '@shared/extra-utils/server/redundancy'
+
+const expect = chai.expect
+
+describe('Test redundancy constraints', function () {
+  let remoteServer: ServerInfo
+  let localServer: ServerInfo
+  let servers: ServerInfo[]
+
+  async function getTotalRedundanciesLocalServer () {
+    const res = await listVideoRedundancies({
+      url: localServer.url,
+      accessToken: localServer.accessToken,
+      target: 'my-videos'
+    })
+
+    return res.body.total
+  }
+
+  async function getTotalRedundanciesRemoteServer () {
+    const res = await listVideoRedundancies({
+      url: remoteServer.url,
+      accessToken: remoteServer.accessToken,
+      target: 'remote-videos'
+    })
+
+    return res.body.total
+  }
+
+  before(async function () {
+    this.timeout(120000)
+
+    {
+      const config = {
+        redundancy: {
+          videos: {
+            check_interval: '1 second',
+            strategies: [
+              {
+                strategy: 'recently-added',
+                min_lifetime: '1 hour',
+                size: '100MB',
+                min_views: 0
+              }
+            ]
+          }
+        }
+      }
+      remoteServer = await flushAndRunServer(1, config)
+    }
+
+    {
+      const config = {
+        remote_redundancy: {
+          videos: {
+            accept_from: 'nobody'
+          }
+        }
+      }
+      localServer = await flushAndRunServer(2, config)
+    }
+
+    servers = [ remoteServer, localServer ]
+
+    // Get the access tokens
+    await setAccessTokensToServers(servers)
+
+    await uploadVideo(localServer.url, localServer.accessToken, { name: 'video 1 server 2' })
+
+    await waitJobs(servers)
+
+    // Server 1 and server 2 follow each other
+    await follow(remoteServer.url, [ localServer.url ], remoteServer.accessToken)
+    await waitJobs(servers)
+    await updateRedundancy(remoteServer.url, remoteServer.accessToken, localServer.host, true)
+
+    await waitJobs(servers)
+  })
+
+  it('Should have redundancy on server 1 but not on server 2 with a nobody filter', async function () {
+    this.timeout(120000)
+
+    await waitJobs(servers)
+    await waitUntilLog(remoteServer, 'Duplicated ', 5)
+    await waitJobs(servers)
+
+    {
+      const total = await getTotalRedundanciesRemoteServer()
+      expect(total).to.equal(1)
+    }
+
+    {
+      const total = await getTotalRedundanciesLocalServer()
+      expect(total).to.equal(0)
+    }
+  })
+
+  it('Should have redundancy on server 1 and on server 2 with an anybody filter', async function () {
+    this.timeout(120000)
+
+    const config = {
+      remote_redundancy: {
+        videos: {
+          accept_from: 'anybody'
+        }
+      }
+    }
+    await killallServers([ localServer ])
+    await reRunServer(localServer, config)
+
+    await uploadVideo(localServer.url, localServer.accessToken, { name: 'video 2 server 2' })
+
+    await waitJobs(servers)
+    await waitUntilLog(remoteServer, 'Duplicated ', 10)
+    await waitJobs(servers)
+
+    {
+      const total = await getTotalRedundanciesRemoteServer()
+      expect(total).to.equal(2)
+    }
+
+    {
+      const total = await getTotalRedundanciesLocalServer()
+      expect(total).to.equal(1)
+    }
+  })
+
+  it('Should have redundancy on server 1 but not on server 2 with a followings filter', async function () {
+    this.timeout(120000)
+
+    const config = {
+      remote_redundancy: {
+        videos: {
+          accept_from: 'followings'
+        }
+      }
+    }
+    await killallServers([ localServer ])
+    await reRunServer(localServer, config)
+
+    await uploadVideo(localServer.url, localServer.accessToken, { name: 'video 3 server 2' })
+
+    await waitJobs(servers)
+    await waitUntilLog(remoteServer, 'Duplicated ', 15)
+    await waitJobs(servers)
+
+    {
+      const total = await getTotalRedundanciesRemoteServer()
+      expect(total).to.equal(3)
+    }
+
+    {
+      const total = await getTotalRedundanciesLocalServer()
+      expect(total).to.equal(1)
+    }
+  })
+
+  it('Should have redundancy on server 1 and on server 2 with followings filter now server 2 follows server 1', async function () {
+    this.timeout(120000)
+
+    await follow(localServer.url, [ remoteServer.url ], localServer.accessToken)
+    await waitJobs(servers)
+
+    await uploadVideo(localServer.url, localServer.accessToken, { name: 'video 4 server 2' })
+
+    await waitJobs(servers)
+    await waitUntilLog(remoteServer, 'Duplicated ', 20)
+    await waitJobs(servers)
+
+    {
+      const total = await getTotalRedundanciesRemoteServer()
+      expect(total).to.equal(4)
+    }
+
+    {
+      const total = await getTotalRedundanciesLocalServer()
+      expect(total).to.equal(2)
+    }
+  })
+
+  after(async function () {
+    await cleanupTests(servers)
+  })
+})

shared/models/redundancy/video-redundancy-config-filter.type.ts

@@ -0,0 +1 @@
+export type VideoRedundancyConfigFilter = 'nobody' | 'anybody' | 'followings'