diff --git a/package.json b/package.json index 18bce1123..dc15405ce 100644 --- a/package.json +++ b/package.json @@ -99,6 +99,7 @@ "fs-extra": "^10.0.0", "got": "^11.8.2", "helmet": "^4.1.0", + "hpagent": "^0.1.2", "http-problem-details": "^0.1.5", "http-signature": "1.3.5", "ip-anonymize": "^0.1.0", @@ -199,6 +200,7 @@ "marked-man": "^0.7.0", "mocha": "^9.0.0", "nodemon": "^2.0.1", + "proxy": "^1.0.2", "socket.io-client": "^4.0.1", "source-map-support": "^0.5.0", "supertest": "^6.0.1", diff --git a/server/helpers/proxy.ts b/server/helpers/proxy.ts new file mode 100644 index 000000000..8b82ccae0 --- /dev/null +++ b/server/helpers/proxy.ts @@ -0,0 +1,14 @@ +function getProxy () { + return process.env.HTTPS_PROXY || + process.env.HTTP_PROXY || + undefined +} + +function isProxyEnabled () { + return !!getProxy() +} + +export { + getProxy, + isProxyEnabled +} diff --git a/server/helpers/requests.ts b/server/helpers/requests.ts index 36e69458e..e09e23086 100644 --- a/server/helpers/requests.ts +++ b/server/helpers/requests.ts @@ -1,19 +1,21 @@ import { createWriteStream, remove } from 'fs-extra' import got, { CancelableRequest, Options as GotOptions, RequestError } from 'got' +import { HttpProxyAgent, HttpsProxyAgent } from 'hpagent' import { join } from 'path' import { CONFIG } from '../initializers/config' import { ACTIVITY_PUB, PEERTUBE_VERSION, REQUEST_TIMEOUT, WEBSERVER } from '../initializers/constants' import { pipelinePromise } from './core-utils' import { processImage } from './image-utils' import { logger } from './logger' +import { getProxy, isProxyEnabled } from './proxy' + +const httpSignature = require('http-signature') export interface PeerTubeRequestError extends Error { statusCode?: number responseBody?: any } -const httpSignature = require('http-signature') - type PeerTubeRequestOptions = { activityPub?: boolean bodyKBLimit?: number // 1MB @@ -29,6 +31,8 @@ type PeerTubeRequestOptions = { } & Pick const peertubeGot = got.extend({ + ...getAgent(), + headers: { 'user-agent': getUserAgent() }, @@ -153,6 +157,30 @@ async function downloadImage (url: string, destDir: string, destName: string, si } } +function getAgent () { + if (!isProxyEnabled()) return {} + + const proxy = getProxy() + + logger.info('Using proxy %s.', proxy) + + const proxyAgentOptions = { + keepAlive: true, + keepAliveMsecs: 1000, + maxSockets: 256, + maxFreeSockets: 256, + scheduling: 'lifo' as 'lifo', + proxy + } + + return { + agent: { + http: new HttpProxyAgent(proxyAgentOptions), + https: new HttpsProxyAgent(proxyAgentOptions) + } + } +} + function getUserAgent () { return `PeerTube/${PEERTUBE_VERSION} (+${WEBSERVER.URL})` } diff --git a/server/tests/api/server/index.ts b/server/tests/api/server/index.ts index 56e6eb5da..b16a22ee7 100644 --- a/server/tests/api/server/index.ts +++ b/server/tests/api/server/index.ts @@ -15,3 +15,4 @@ import './stats' import './tracker' import './no-client' import './plugins' +import './proxy' diff --git a/server/tests/api/server/proxy.ts b/server/tests/api/server/proxy.ts new file mode 100644 index 000000000..d5042ef27 --- /dev/null +++ b/server/tests/api/server/proxy.ts @@ -0,0 +1,72 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import 'mocha' +import * as chai from 'chai' +import { cleanupTests, createMultipleServers, doubleFollow, PeerTubeServer, setAccessTokensToServers, waitJobs } from '@shared/extra-utils' +import { MockProxy } from '@shared/extra-utils/mock-servers/mock-proxy' + +const expect = chai.expect + +describe('Test proxy', function () { + let servers: PeerTubeServer[] = [] + let proxy: MockProxy + + const goodEnv = { HTTP_PROXY: '' } + const badEnv = { HTTP_PROXY: 'http://localhost:9000' } + + before(async function () { + this.timeout(120000) + + proxy = new MockProxy() + + const proxyPort = await proxy.initialize() + servers = await createMultipleServers(2) + + goodEnv.HTTP_PROXY = 'http://localhost:' + proxyPort + + await setAccessTokensToServers(servers) + await doubleFollow(servers[0], servers[1]) + }) + + it('Should succeed federation with the appropriate proxy config', async function () { + await servers[0].kill() + await servers[0].run({}, { env: goodEnv }) + + await servers[0].videos.quickUpload({ name: 'video 1' }) + + await waitJobs(servers) + + for (const server of servers) { + const { total, data } = await server.videos.list() + expect(total).to.equal(1) + expect(data).to.have.lengthOf(1) + } + }) + + it('Should fail federation with a wrong proxy config', async function () { + await servers[0].kill() + await servers[0].run({}, { env: badEnv }) + + await servers[0].videos.quickUpload({ name: 'video 2' }) + + await waitJobs(servers) + + { + const { total, data } = await servers[0].videos.list() + expect(total).to.equal(2) + expect(data).to.have.lengthOf(2) + } + + { + const { total, data } = await servers[1].videos.list() + expect(total).to.equal(1) + expect(data).to.have.lengthOf(1) + } + }) + + after(async function () { + proxy.terminate() + + await cleanupTests(servers) + }) +}) diff --git a/shared/extra-utils/mock-servers/mock-proxy.ts b/shared/extra-utils/mock-servers/mock-proxy.ts new file mode 100644 index 000000000..5365f87d1 --- /dev/null +++ b/shared/extra-utils/mock-servers/mock-proxy.ts @@ -0,0 +1,27 @@ + +import { createServer, Server } from 'http' +import * as proxy from 'proxy' +import { randomInt } from '@shared/core-utils' + +class MockProxy { + private server: Server + + initialize () { + return new Promise(res => { + const port = 42501 + randomInt(1, 100) + + this.server = proxy(createServer()) + this.server.listen(port, () => res(port)) + }) + } + + terminate () { + if (this.server) this.server.close() + } +} + +// --------------------------------------------------------------------------- + +export { + MockProxy +} diff --git a/yarn.lock b/yarn.lock index 0f4fe3938..1179f273b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2002,6 +2002,16 @@ argparse@^2.0.1: resolved "https://registry.yarnpkg.com/argparse/-/argparse-2.0.1.tgz#246f50f3ca78a3240f6c997e8a9bd1eac49e4b38" integrity sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q== +args@5.0.1: + version "5.0.1" + resolved "https://registry.yarnpkg.com/args/-/args-5.0.1.tgz#4bf298df90a4799a09521362c579278cc2fdd761" + integrity sha512-1kqmFCFsPffavQFGt8OxJdIcETti99kySRUPMpOhaGjL6mRJn8HFU1OxKY5bMqfZKUwTQc1mZkAjmGYaVOHFtQ== + dependencies: + camelcase "5.0.0" + chalk "2.4.2" + leven "2.1.0" + mri "1.1.4" + array-differ@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/array-differ/-/array-differ-3.0.0.tgz#3cbb3d0f316810eafcc47624734237d6aee4ae6b" @@ -2200,6 +2210,11 @@ basic-auth-connect@^1.0.0: resolved "https://registry.yarnpkg.com/basic-auth-connect/-/basic-auth-connect-1.0.0.tgz#fdb0b43962ca7b40456a7c2bb48fe173da2d2122" integrity sha1-/bC0OWLKe0BFanwrtI/hc9otISI= +basic-auth-parser@0.0.2: + version "0.0.2" + resolved "https://registry.yarnpkg.com/basic-auth-parser/-/basic-auth-parser-0.0.2.tgz#ce9e71a77f23c1279eecd2659b2a46244c156e41" + integrity sha1-zp5xp38jwSee7NJlmypGJEwVbkE= + basic-auth@2.0.1, basic-auth@~2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/basic-auth/-/basic-auth-2.0.1.tgz#b998279bf47ce38344b4f3cf916d4679bbf51e3a" @@ -2616,6 +2631,11 @@ camelcase-keys@^4.0.0: map-obj "^2.0.0" quick-lru "^1.0.0" +camelcase@5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-5.0.0.tgz#03295527d58bd3cd4aa75363f35b2e8d97be2f42" + integrity sha512-faqwZqnWxbxn+F1d399ygeamQNy3lPp/H9H6rNrqYh4FSVCtcY+3cub1MxA8o9mDd55mM8Aghuu/kuyYA6VTsA== + camelcase@^4.1.0: version "4.1.0" resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-4.1.0.tgz#d545635be1e33c542649c69173e5de6acfae34dd" @@ -2663,7 +2683,7 @@ chai@^4.1.1: pathval "^1.1.1" type-detect "^4.0.5" -chalk@^2.0.0, chalk@^2.4.2: +chalk@2.4.2, chalk@^2.0.0, chalk@^2.4.2: version "2.4.2" resolved "https://registry.yarnpkg.com/chalk/-/chalk-2.4.2.tgz#cd42541677a54333cf541a49108c1432b44c9424" integrity sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ== @@ -4692,6 +4712,11 @@ hosted-git-info@^2.1.4: resolved "https://registry.yarnpkg.com/hosted-git-info/-/hosted-git-info-2.8.9.tgz#dffc0bf9a21c02209090f2aa69429e1414daf3f9" integrity sha512-mxIDAb9Lsm6DoOJ7xH+5+X4y1LU/4Hi50L9C5sIswK3JzULS4bwk1FvjdBgvYR4bzT4tuUQiC15FE2f5HbLvYw== +hpagent@^0.1.2: + version "0.1.2" + resolved "https://registry.yarnpkg.com/hpagent/-/hpagent-0.1.2.tgz#cab39c66d4df2d4377dbd212295d878deb9bdaa9" + integrity sha512-ePqFXHtSQWAFXYmj+JtOTHr84iNrII4/QRlAAPPE+zqnKy4xJo7Ie1Y4kC7AdB+LxLxSTTzBMASsEcy0q8YyvQ== + html-to-text@8.0.0: version "8.0.0" resolved "https://registry.yarnpkg.com/html-to-text/-/html-to-text-8.0.0.tgz#5848681a5a38d657a7bb58cf5006d1c29fe64ce3" @@ -5548,6 +5573,11 @@ latest-version@^5.0.0: dependencies: package-json "^6.3.0" +leven@2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/leven/-/leven-2.1.0.tgz#c2e7a9f772094dee9d34202ae8acce4687875580" + integrity sha1-wuep93IJTe6dNCAq6KzORoeHVYA= + levn@^0.4.1: version "0.4.1" resolved "https://registry.yarnpkg.com/levn/-/levn-0.4.1.tgz#ae4562c007473b932a6200d403268dd2fffc6ade" @@ -6196,6 +6226,11 @@ mp4-stream@^3.0.0: queue-microtask "^1.2.2" readable-stream "^3.0.6" +mri@1.1.4: + version "1.1.4" + resolved "https://registry.yarnpkg.com/mri/-/mri-1.1.4.tgz#7cb1dd1b9b40905f1fac053abe25b6720f44744a" + integrity sha512-6y7IjGPm8AzlvoUrwAaw1tLnUBudaS3752vcd8JtrpGGQn+rXIe63LFVHm/YMwtqAuh+LJPCFdlLYPWM1nYn6w== + ms@2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/ms/-/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8" @@ -7131,6 +7166,15 @@ proxy-addr@~2.0.5: forwarded "0.2.0" ipaddr.js "1.9.1" +proxy@^1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/proxy/-/proxy-1.0.2.tgz#e0cfbe11c0a7a8b238fd2d7134de4e2867578e7f" + integrity sha512-KNac2ueWRpjbUh77OAFPZuNdfEqNynm9DD4xHT14CccGpW8wKZwEkN0yjlb7X9G9Z9F55N0Q+1z+WfgAhwYdzQ== + dependencies: + args "5.0.1" + basic-auth-parser "0.0.2" + debug "^4.1.1" + pseudomap@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/pseudomap/-/pseudomap-1.0.2.tgz#f052a28da70e618917ef0a8ac34c1ae5a68286b3"