diff --git a/config/default.yaml b/config/default.yaml
index 615910478..51f3ad833 100644
--- a/config/default.yaml
+++ b/config/default.yaml
@@ -200,3 +200,8 @@ services:
     # If false, we use an image link card that will redirect on your PeerTube instance
     # Change it to "true", and then test on https://cards-dev.twitter.com/validator to see if you are whitelisted
     whitelisted: false
+
+followers:
+  instance:
+    # Allow or not other instances to follow yours
+    enabled: true
diff --git a/config/production.yaml.example b/config/production.yaml.example
index 5299484a5..a2811abd6 100644
--- a/config/production.yaml.example
+++ b/config/production.yaml.example
@@ -180,8 +180,8 @@ auto_blacklist:
   # New videos automatically blacklisted so moderators can review before publishing
   videos:
     of_users:
-      enabled: false 
- 
+      enabled: false
+
 # Instance settings
 instance:
   name: 'PeerTube'
@@ -217,3 +217,8 @@ services:
     # If false, we use an image link card that will redirect on your PeerTube instance
     # Test on https://cards-dev.twitter.com/validator to see if you are whitelisted
     whitelisted: false
+
+followers:
+  instance:
+    # Allow or not other instances to follow yours
+    enabled: true
diff --git a/server/controllers/api/config.ts b/server/controllers/api/config.ts
index bd0ba4f9d..f9bb0b947 100644
--- a/server/controllers/api/config.ts
+++ b/server/controllers/api/config.ts
@@ -279,6 +279,11 @@ function customConfig (): CustomConfig {
           enabled: CONFIG.AUTO_BLACKLIST.VIDEOS.OF_USERS.ENABLED
         }
       }
+    },
+    followers: {
+      instance: {
+        enabled: CONFIG.FOLLOWERS.INSTANCE.ENABLED
+      }
     }
   }
 }
diff --git a/server/controllers/api/server/follows.ts b/server/controllers/api/server/follows.ts
index c00069f93..87cf091cb 100644
--- a/server/controllers/api/server/follows.ts
+++ b/server/controllers/api/server/follows.ts
@@ -139,7 +139,7 @@ async function removeFollowing (req: express.Request, res: express.Response) {
 async function removeFollower (req: express.Request, res: express.Response) {
   const follow = res.locals.follow
 
-  await sendReject(follow)
+  await sendReject(follow.ActorFollower, follow.ActorFollowing)
 
   await follow.destroy()
 
diff --git a/server/initializers/checker-before-init.ts b/server/initializers/checker-before-init.ts
index e26f38564..a9896907d 100644
--- a/server/initializers/checker-before-init.ts
+++ b/server/initializers/checker-before-init.ts
@@ -24,7 +24,8 @@ function checkMissedConfig () {
     'trending.videos.interval_days',
     'instance.name', 'instance.short_description', 'instance.description', 'instance.terms', 'instance.default_client_route',
     'instance.is_nsfw', 'instance.default_nsfw_policy', 'instance.robots', 'instance.securitytxt',
-    'services.twitter.username', 'services.twitter.whitelisted'
+    'services.twitter.username', 'services.twitter.whitelisted',
+    'followers.instance.enabled'
   ]
   const requiredAlternatives = [
     [ // set
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index ac19231d0..43c5ec54c 100644
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -324,6 +324,11 @@ const CONFIG = {
       get USERNAME () { return config.get<string>('services.twitter.username') },
       get WHITELISTED () { return config.get<boolean>('services.twitter.whitelisted') }
     }
+  },
+  FOLLOWERS: {
+    INSTANCE: {
+      get ENABLED () { return config.get<boolean>('followers.instance.enabled') }
+    }
   }
 }
 
diff --git a/server/lib/activitypub/process/process-follow.ts b/server/lib/activitypub/process/process-follow.ts
index 0cd537187..cecf09b47 100644
--- a/server/lib/activitypub/process/process-follow.ts
+++ b/server/lib/activitypub/process/process-follow.ts
@@ -1,12 +1,13 @@
 import { ActivityFollow } from '../../../../shared/models/activitypub'
 import { retryTransactionWrapper } from '../../../helpers/database-utils'
 import { logger } from '../../../helpers/logger'
-import { sequelizeTypescript } from '../../../initializers'
+import { sequelizeTypescript, CONFIG } from '../../../initializers'
 import { ActorModel } from '../../../models/activitypub/actor'
 import { ActorFollowModel } from '../../../models/activitypub/actor-follow'
-import { sendAccept } from '../send'
+import { sendAccept, sendReject } from '../send'
 import { Notifier } from '../../notifier'
 import { getAPId } from '../../../helpers/activitypub'
+import { getServerActor } from '../../../helpers/utils'
 
 async function processFollowActivity (activity: ActivityFollow, byActor: ActorModel) {
   const activityObject = getAPId(activity.object)
@@ -29,6 +30,11 @@ async function processFollow (actor: ActorModel, targetActorURL: string) {
     if (!targetActor) throw new Error('Unknown actor')
     if (targetActor.isOwned() === false) throw new Error('This is not a local actor.')
 
+    const serverActor = await getServerActor()
+    if (targetActor.id === serverActor.id && CONFIG.FOLLOWERS.INSTANCE.ENABLED === false) {
+      return sendReject(actor, targetActor)
+    }
+
     const [ actorFollow, created ] = await ActorFollowModel.findOrCreate({
       where: {
         actorId: actor.id,
diff --git a/server/lib/activitypub/send/send-accept.ts b/server/lib/activitypub/send/send-accept.ts
index b6abde13d..388a9ed23 100644
--- a/server/lib/activitypub/send/send-accept.ts
+++ b/server/lib/activitypub/send/send-accept.ts
@@ -17,7 +17,7 @@ async function sendAccept (actorFollow: ActorFollowModel) {
 
   logger.info('Creating job to accept follower %s.', follower.url)
 
-  const followUrl = getActorFollowActivityPubUrl(actorFollow)
+  const followUrl = getActorFollowActivityPubUrl(follower, me)
   const followData = buildFollowActivity(followUrl, follower, me)
 
   const url = getActorFollowAcceptActivityPubUrl(actorFollow)
diff --git a/server/lib/activitypub/send/send-follow.ts b/server/lib/activitypub/send/send-follow.ts
index 170b46b48..2c3d02014 100644
--- a/server/lib/activitypub/send/send-follow.ts
+++ b/server/lib/activitypub/send/send-follow.ts
@@ -14,7 +14,7 @@ function sendFollow (actorFollow: ActorFollowModel) {
 
   logger.info('Creating job to send follow request to %s.', following.url)
 
-  const url = getActorFollowActivityPubUrl(actorFollow)
+  const url = getActorFollowActivityPubUrl(me, following)
   const data = buildFollowActivity(url, me, following)
 
   return unicastTo(data, me, following.inboxUrl)
diff --git a/server/lib/activitypub/send/send-reject.ts b/server/lib/activitypub/send/send-reject.ts
index db8c2d86d..bac7ff556 100644
--- a/server/lib/activitypub/send/send-reject.ts
+++ b/server/lib/activitypub/send/send-reject.ts
@@ -1,15 +1,11 @@
 import { ActivityFollow, ActivityReject } from '../../../../shared/models/activitypub'
 import { ActorModel } from '../../../models/activitypub/actor'
-import { ActorFollowModel } from '../../../models/activitypub/actor-follow'
-import { getActorFollowAcceptActivityPubUrl, getActorFollowActivityPubUrl } from '../url'
+import { getActorFollowActivityPubUrl, getActorFollowRejectActivityPubUrl } from '../url'
 import { unicastTo } from './utils'
 import { buildFollowActivity } from './send-follow'
 import { logger } from '../../../helpers/logger'
 
-async function sendReject (actorFollow: ActorFollowModel) {
-  const follower = actorFollow.ActorFollower
-  const me = actorFollow.ActorFollowing
-
+async function sendReject (follower: ActorModel, following: ActorModel) {
   if (!follower.serverId) { // This should never happen
     logger.warn('Do not sending reject to local follower.')
     return
@@ -17,13 +13,13 @@ async function sendReject (actorFollow: ActorFollowModel) {
 
   logger.info('Creating job to reject follower %s.', follower.url)
 
-  const followUrl = getActorFollowActivityPubUrl(actorFollow)
-  const followData = buildFollowActivity(followUrl, follower, me)
+  const followUrl = getActorFollowActivityPubUrl(follower, following)
+  const followData = buildFollowActivity(followUrl, follower, following)
 
-  const url = getActorFollowAcceptActivityPubUrl(actorFollow)
-  const data = buildRejectActivity(url, me, followData)
+  const url = getActorFollowRejectActivityPubUrl(follower, following)
+  const data = buildRejectActivity(url, following, followData)
 
-  return unicastTo(data, me, follower.inboxUrl)
+  return unicastTo(data, following, follower.inboxUrl)
 }
 
 // ---------------------------------------------------------------------------
diff --git a/server/lib/activitypub/send/send-undo.ts b/server/lib/activitypub/send/send-undo.ts
index ecbf605d6..8727a121e 100644
--- a/server/lib/activitypub/send/send-undo.ts
+++ b/server/lib/activitypub/send/send-undo.ts
@@ -31,7 +31,7 @@ async function sendUndoFollow (actorFollow: ActorFollowModel, t: Transaction) {
 
   logger.info('Creating job to send an unfollow request to %s.', following.url)
 
-  const followUrl = getActorFollowActivityPubUrl(actorFollow)
+  const followUrl = getActorFollowActivityPubUrl(me, following)
   const undoUrl = getUndoActivityPubUrl(followUrl)
 
   const followActivity = buildFollowActivity(followUrl, me, following)
diff --git a/server/lib/activitypub/url.ts b/server/lib/activitypub/url.ts
index 7c2ee5bc6..401b83fc2 100644
--- a/server/lib/activitypub/url.ts
+++ b/server/lib/activitypub/url.ts
@@ -74,11 +74,8 @@ function getVideoDislikesActivityPubUrl (video: VideoModel) {
   return video.url + '/dislikes'
 }
 
-function getActorFollowActivityPubUrl (actorFollow: ActorFollowModel) {
-  const me = actorFollow.ActorFollower
-  const following = actorFollow.ActorFollowing
-
-  return me.url + '/follows/' + following.id
+function getActorFollowActivityPubUrl (follower: ActorModel, following: ActorModel) {
+  return follower.url + '/follows/' + following.id
 }
 
 function getActorFollowAcceptActivityPubUrl (actorFollow: ActorFollowModel) {
@@ -88,6 +85,10 @@ function getActorFollowAcceptActivityPubUrl (actorFollow: ActorFollowModel) {
   return follower.url + '/accepts/follows/' + me.id
 }
 
+function getActorFollowRejectActivityPubUrl (follower: ActorModel, following: ActorModel) {
+  return follower.url + '/rejects/follows/' + following.id
+}
+
 function getVideoAnnounceActivityPubUrl (byActor: ActorModel, video: VideoModel) {
   return video.url + '/announces/' + byActor.id
 }
@@ -120,6 +121,7 @@ export {
   getVideoViewActivityPubUrl,
   getVideoLikeActivityPubUrl,
   getVideoDislikeActivityPubUrl,
+  getActorFollowRejectActivityPubUrl,
   getVideoCommentActivityPubUrl,
   getDeleteActivityPubUrl,
   getVideoSharesActivityPubUrl,
diff --git a/server/middlewares/validators/follows.ts b/server/middlewares/validators/follows.ts
index ef4151efe..38df39fda 100644
--- a/server/middlewares/validators/follows.ts
+++ b/server/middlewares/validators/follows.ts
@@ -9,7 +9,6 @@ import { ActorFollowModel } from '../../models/activitypub/actor-follow'
 import { areValidationErrors } from './utils'
 import { ActorModel } from '../../models/activitypub/actor'
 import { loadActorUrlOrGetFromWebfinger } from '../../helpers/webfinger'
-import { getOrCreateActorAndServerAndModel } from '../../lib/activitypub'
 import { isValidActorHandle } from '../../helpers/custom-validators/activitypub/actor'
 
 const followValidator = [
@@ -66,12 +65,16 @@ const removeFollowerValidator = [
 
     if (areValidationErrors(req, res)) return
 
-    const serverActor = await getServerActor()
+    let follow: ActorFollowModel
+    try {
+      const actorUrl = await loadActorUrlOrGetFromWebfinger(req.params.nameWithHost)
+      const actor = await ActorModel.loadByUrl(actorUrl)
 
-    const actorUrl = await loadActorUrlOrGetFromWebfinger(req.params.nameWithHost)
-    const actor = await ActorModel.loadByUrl(actorUrl)
-
-    const follow = await ActorFollowModel.loadByActorAndTarget(actor.id, serverActor.id)
+      const serverActor = await getServerActor()
+      follow = await ActorFollowModel.loadByActorAndTarget(actor.id, serverActor.id)
+    } catch (err) {
+      logger.warn('Cannot get actor from handle.', { handle: req.params.nameWithHost, err })
+    }
 
     if (!follow) {
       return res
diff --git a/server/tests/api/check-params/config.ts b/server/tests/api/check-params/config.ts
index 0b333e2f4..d117f26e6 100644
--- a/server/tests/api/check-params/config.ts
+++ b/server/tests/api/check-params/config.ts
@@ -87,6 +87,11 @@ describe('Test config API validators', function () {
           enabled: false
         }
       }
+    },
+    followers: {
+      instance: {
+        enabled: false
+      }
     }
   }
 
diff --git a/server/tests/api/check-params/follows.ts b/server/tests/api/check-params/follows.ts
index 2ad1575a3..67fa43778 100644
--- a/server/tests/api/check-params/follows.ts
+++ b/server/tests/api/check-params/follows.ts
@@ -144,6 +144,46 @@ describe('Test server follows API validators', function () {
       })
     })
 
+    describe('When removing a follower', function () {
+      const path = '/api/v1/server/followers'
+
+      it('Should fail with an invalid token', async function () {
+        await makeDeleteRequest({
+          url: server.url,
+          path: path + '/toto@localhost:9002',
+          token: 'fake_token',
+          statusCodeExpected: 401
+        })
+      })
+
+      it('Should fail if the user is not an administrator', async function () {
+        await makeDeleteRequest({
+          url: server.url,
+          path: path + '/toto@localhost:9002',
+          token: userAccessToken,
+          statusCodeExpected: 403
+        })
+      })
+
+      it('Should fail with an invalid follower', async function () {
+        await makeDeleteRequest({
+          url: server.url,
+          path: path + '/toto',
+          token: server.accessToken,
+          statusCodeExpected: 400
+        })
+      })
+
+      it('Should fail with an unknown follower', async function () {
+        await makeDeleteRequest({
+          url: server.url,
+          path: path + '/toto@localhost:9003',
+          token: server.accessToken,
+          statusCodeExpected: 404
+        })
+      })
+    })
+
     describe('When removing following', function () {
       const path = '/api/v1/server/following'
 
diff --git a/server/tests/api/server/config.ts b/server/tests/api/server/config.ts
index b9f05e952..cb2700f29 100644
--- a/server/tests/api/server/config.ts
+++ b/server/tests/api/server/config.ts
@@ -63,6 +63,8 @@ function checkInitialConfig (data: CustomConfig) {
   expect(data.import.videos.http.enabled).to.be.true
   expect(data.import.videos.torrent.enabled).to.be.true
   expect(data.autoBlacklist.videos.ofUsers.enabled).to.be.false
+
+  expect(data.followers.instance.enabled).to.be.true
 }
 
 function checkUpdatedConfig (data: CustomConfig) {
@@ -105,6 +107,8 @@ function checkUpdatedConfig (data: CustomConfig) {
   expect(data.import.videos.http.enabled).to.be.false
   expect(data.import.videos.torrent.enabled).to.be.false
   expect(data.autoBlacklist.videos.ofUsers.enabled).to.be.true
+
+  expect(data.followers.instance.enabled).to.be.false
 }
 
 describe('Test config', function () {
@@ -234,6 +238,11 @@ describe('Test config', function () {
             enabled: true
           }
         }
+      },
+      followers: {
+        instance: {
+          enabled: false
+        }
       }
     }
     await updateCustomConfig(server.url, server.accessToken, newCustomConfig)
diff --git a/server/tests/api/server/follows-moderation.ts b/server/tests/api/server/follows-moderation.ts
index b1cbfb62c..a360706f2 100644
--- a/server/tests/api/server/follows-moderation.ts
+++ b/server/tests/api/server/follows-moderation.ts
@@ -2,7 +2,13 @@
 
 import * as chai from 'chai'
 import 'mocha'
-import { flushAndRunMultipleServers, killallServers, ServerInfo, setAccessTokensToServers } from '../../../../shared/utils/index'
+import {
+  flushAndRunMultipleServers,
+  killallServers,
+  ServerInfo,
+  setAccessTokensToServers,
+  updateCustomSubConfig
+} from '../../../../shared/utils/index'
 import {
   follow,
   getFollowersListPaginationAndSort,
@@ -14,6 +20,38 @@ import { ActorFollow } from '../../../../shared/models/actors'
 
 const expect = chai.expect
 
+async function checkHasFollowers (servers: ServerInfo[]) {
+  {
+    const res = await getFollowingListPaginationAndSort(servers[0].url, 0, 5, 'createdAt')
+    expect(res.body.total).to.equal(1)
+
+    const follow = res.body.data[0] as ActorFollow
+    expect(follow.follower.url).to.equal('http://localhost:9001/accounts/peertube')
+    expect(follow.following.url).to.equal('http://localhost:9002/accounts/peertube')
+  }
+
+  {
+    const res = await getFollowersListPaginationAndSort(servers[1].url, 0, 5, 'createdAt')
+    expect(res.body.total).to.equal(1)
+
+    const follow = res.body.data[0] as ActorFollow
+    expect(follow.follower.url).to.equal('http://localhost:9001/accounts/peertube')
+    expect(follow.following.url).to.equal('http://localhost:9002/accounts/peertube')
+  }
+}
+
+async function checkNoFollowers (servers: ServerInfo[]) {
+  {
+    const res = await getFollowingListPaginationAndSort(servers[ 0 ].url, 0, 5, 'createdAt')
+    expect(res.body.total).to.equal(0)
+  }
+
+  {
+    const res = await getFollowersListPaginationAndSort(servers[ 1 ].url, 0, 5, 'createdAt')
+    expect(res.body.total).to.equal(0)
+  }
+}
+
 describe('Test follows moderation', function () {
   let servers: ServerInfo[] = []
 
@@ -35,23 +73,7 @@ describe('Test follows moderation', function () {
   })
 
   it('Should have correct follows', async function () {
-    {
-      const res = await getFollowingListPaginationAndSort(servers[0].url, 0, 5, 'createdAt')
-      expect(res.body.total).to.equal(1)
-
-      const follow = res.body.data[0] as ActorFollow
-      expect(follow.follower.url).to.equal('http://localhost:9001/accounts/peertube')
-      expect(follow.following.url).to.equal('http://localhost:9002/accounts/peertube')
-    }
-
-    {
-      const res = await getFollowersListPaginationAndSort(servers[1].url, 0, 5, 'createdAt')
-      expect(res.body.total).to.equal(1)
-
-      const follow = res.body.data[0] as ActorFollow
-      expect(follow.follower.url).to.equal('http://localhost:9001/accounts/peertube')
-      expect(follow.following.url).to.equal('http://localhost:9002/accounts/peertube')
-    }
+    await checkHasFollowers(servers)
   })
 
   it('Should remove follower on server 2', async function () {
@@ -61,15 +83,41 @@ describe('Test follows moderation', function () {
   })
 
   it('Should not not have follows anymore', async function () {
-    {
-      const res = await getFollowingListPaginationAndSort(servers[ 0 ].url, 0, 1, 'createdAt')
-      expect(res.body.total).to.equal(0)
+    await checkNoFollowers(servers)
+  })
+
+  it('Should disable followers on server 2', async function () {
+    const subConfig = {
+      followers: {
+        instance: {
+          enabled: false
+        }
+      }
     }
 
-    {
-      const res = await getFollowingListPaginationAndSort(servers[ 0 ].url, 0, 1, 'createdAt')
-      expect(res.body.total).to.equal(0)
+    await updateCustomSubConfig(servers[1].url, servers[1].accessToken, subConfig)
+
+    await follow(servers[0].url, [ servers[1].url ], servers[0].accessToken)
+    await waitJobs(servers)
+
+    await checkNoFollowers(servers)
+  })
+
+  it('Should re enable followers on server 2', async function () {
+    const subConfig = {
+      followers: {
+        instance: {
+          enabled: true
+        }
+      }
     }
+
+    await updateCustomSubConfig(servers[1].url, servers[1].accessToken, subConfig)
+
+    await follow(servers[0].url, [ servers[1].url ], servers[0].accessToken)
+    await waitJobs(servers)
+
+    await checkHasFollowers(servers)
   })
 
   after(async function () {
diff --git a/shared/models/server/custom-config.model.ts b/shared/models/server/custom-config.model.ts
index 1607b40a8..642ffea39 100644
--- a/shared/models/server/custom-config.model.ts
+++ b/shared/models/server/custom-config.model.ts
@@ -86,4 +86,10 @@ export interface CustomConfig {
     }
   }
 
+  followers: {
+    instance: {
+      enabled: boolean
+    }
+  }
+
 }
diff --git a/shared/utils/server/config.ts b/shared/utils/server/config.ts
index eaa493a93..21c689714 100644
--- a/shared/utils/server/config.ts
+++ b/shared/utils/server/config.ts
@@ -119,6 +119,11 @@ function updateCustomSubConfig (url: string, token: string, newConfig: any) {
           enabled: false
         }
       }
+    },
+    followers: {
+      instance: {
+        enabled: true
+      }
     }
   }