forks
/
peertube
1
0
Fork 0
Code Releases Activity

Add sitemap

This commit is contained in:
Chocobozzz 2018-12-05 17:27:24 +01:00
parent 3b3b18203f
commit 2feebf3e6a
No known key found for this signature in database
GPG Key ID: 583A612D890159BE
11 changed files with 241 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
package.json
View File

@ -148,6 +148,7 @@
"sequelize": "4.41.2",
"sequelize-typescript": "0.6.6",
"sharp": "^0.21.0",
"sitemap": "^2.1.0",
"srt-to-vtt": "^1.1.2",
"summon-install": "^0.4.3",
"useragent": "^2.3.0",

1
scripts/clean/server/test.sh
View File

@ -18,6 +18,7 @@ removeFiles () {
dropRedis () {
redis-cli KEYS "bull-localhost:900$1*" | grep -v empty | xargs --no-run-if-empty redis-cli DEL
redis-cli KEYS "redis-localhost:900$1*" | grep -v empty | xargs --no-run-if-empty redis-cli DEL
}
for i in $(seq 1 6); do

3
server.ts
View File

@ -87,7 +87,7 @@ import {
servicesRouter,
webfingerRouter,
trackerRouter,
createWebsocketServer
createWebsocketServer, botsRouter
} from './server/controllers'
import { advertiseDoNotTrack } from './server/middlewares/dnt'
import { Redis } from './server/lib/redis'
@ -156,6 +156,7 @@ app.use('/', activityPubRouter)
app.use('/', feedsRouter)
app.use('/', webfingerRouter)
app.use('/', trackerRouter)
app.use('/', botsRouter)
// Static files
app.use('/', staticRouter)

101
server/controllers/bots.ts Normal file
View File

@ -0,0 +1,101 @@
import * as express from 'express'
import { asyncMiddleware } from '../middlewares'
import { CONFIG, ROUTE_CACHE_LIFETIME } from '../initializers'
import * as sitemapModule from 'sitemap'
import { logger } from '../helpers/logger'
import { VideoModel } from '../models/video/video'
import { VideoChannelModel } from '../models/video/video-channel'
import { AccountModel } from '../models/account/account'
import { cacheRoute } from '../middlewares/cache'
import { buildNSFWFilter } from '../helpers/express-utils'
import { truncate } from 'lodash'
const botsRouter = express.Router()
// Special route that add OpenGraph and oEmbed tags
// Do not use a template engine for a so little thing
botsRouter.use('/sitemap.xml',
asyncMiddleware(cacheRoute(ROUTE_CACHE_LIFETIME.SITEMAP)),
asyncMiddleware(getSitemap)
)
// ---------------------------------------------------------------------------
export {
botsRouter
}
// ---------------------------------------------------------------------------
async function getSitemap (req: express.Request, res: express.Response) {
let urls = getSitemapBasicUrls()
urls = urls.concat(await getSitemapLocalVideoUrls())
urls = urls.concat(await getSitemapVideoChannelUrls())
urls = urls.concat(await getSitemapAccountUrls())
const sitemap = sitemapModule.createSitemap({
hostname: CONFIG.WEBSERVER.URL,
urls: urls
})
sitemap.toXML((err, xml) => {
if (err) {
logger.error('Cannot generate sitemap.', { err })
return res.sendStatus(500)
}
res.header('Content-Type', 'application/xml')
res.send(xml)
})
}
async function getSitemapVideoChannelUrls () {
const rows = await VideoChannelModel.listLocalsForSitemap('createdAt')
return rows.map(channel => ({
url: CONFIG.WEBSERVER.URL + '/video-channels/' + channel.Actor.preferredUsername
}))
}
async function getSitemapAccountUrls () {
const rows = await AccountModel.listLocalsForSitemap('createdAt')
return rows.map(channel => ({
url: CONFIG.WEBSERVER.URL + '/accounts/' + channel.Actor.preferredUsername
}))
}
async function getSitemapLocalVideoUrls () {
const resultList = await VideoModel.listForApi({
start: 0,
count: undefined,
sort: 'createdAt',
includeLocalVideos: true,
nsfw: buildNSFWFilter(),
filter: 'local',
withFiles: false
})
return resultList.data.map(v => ({
url: CONFIG.WEBSERVER.URL + '/videos/watch/' + v.uuid,
video: [
{
title: v.name,
// Sitemap description should be < 2000 characters
description: truncate(v.description || v.name, { length: 2000, omission: '...' }),
player_loc: CONFIG.WEBSERVER.URL + '/videos/embed/' + v.uuid,
thumbnail_loc: v.getThumbnailStaticPath()
}
]
}))
}
function getSitemapBasicUrls () {
const paths = [
'/about/instance',
'/videos/local'
]
return paths.map(p => ({ url: CONFIG.WEBSERVER.URL + p }))
}

1
server/controllers/index.ts
View File

@ -6,3 +6,4 @@ export * from './services'
export * from './static'
export * from './webfinger'
export * from './tracker'
export * from './bots'

4
server/helpers/express-utils.ts
View File

@ -7,12 +7,12 @@ import { extname } from 'path'
import { isArray } from './custom-validators/misc'
import { UserModel } from '../models/account/user'
function buildNSFWFilter (res: express.Response, paramNSFW?: string) {
function buildNSFWFilter (res?: express.Response, paramNSFW?: string) {
if (paramNSFW === 'true') return true
if (paramNSFW === 'false') return false
if (paramNSFW === 'both') return undefined
if (res.locals.oauth) {
if (res && res.locals.oauth) {
const user: UserModel = res.locals.oauth.token.User
// User does not want NSFW videos

1
server/initializers/constants.ts
View File

@ -61,6 +61,7 @@ const OAUTH_LIFETIME = {
const ROUTE_CACHE_LIFETIME = {
FEEDS: '15 minutes',
ROBOTS: '2 hours',
SITEMAP: '1 day',
SECURITYTXT: '2 hours',
NODEINFO: '10 minutes',
DNT_POLICY: '1 week',

21
server/models/account/account.ts
View File

@ -241,6 +241,27 @@ export class AccountModel extends Model<AccountModel> {
})
}
static listLocalsForSitemap (sort: string) {
const query = {
attributes: [ ],
offset: 0,
order: getSort(sort),
include: [
{
attributes: [ 'preferredUsername', 'serverId' ],
model: ActorModel.unscoped(),
where: {
serverId: null
}
}
]
}
return AccountModel
.unscoped()
.findAll(query)
}
toFormattedJSON (): Account {
const actor = this.Actor.toFormattedJSON()
const account = {

21
server/models/video/video-channel.ts
View File

@ -233,6 +233,27 @@ export class VideoChannelModel extends Model<VideoChannelModel> {
})
}
static listLocalsForSitemap (sort: string) {
const query = {
attributes: [ ],
offset: 0,
order: getSort(sort),
include: [
{
attributes: [ 'preferredUsername', 'serverId' ],
model: ActorModel.unscoped(),
where: {
serverId: null
}
}
]
}
return VideoChannelModel
.unscoped()
.findAll(query)
}
static searchForApi (options: {
actorId: number
search: string

72
server/tests/misc-endpoints.ts
View File

@ -2,7 +2,18 @@
import 'mocha'
import * as chai from 'chai'
import { flushTests, killallServers, makeGetRequest, runServer, ServerInfo } from './utils'
import {
addVideoChannel,
createUser,
flushTests,
killallServers,
makeGetRequest,
runServer,
ServerInfo,
setAccessTokensToServers,
uploadVideo
} from './utils'
import { VideoPrivacy } from '../../shared/models/videos'
const expect = chai.expect
@ -15,6 +26,7 @@ describe('Test misc endpoints', function () {
await flushTests()
server = await runServer(1)
await setAccessTokensToServers([ server ])
})
describe('Test a well known endpoints', function () {
@ -93,6 +105,64 @@ describe('Test misc endpoints', function () {
})
})
describe('Test bots endpoints', function () {
it('Should get the empty sitemap', async function () {
const res = await makeGetRequest({
url: server.url,
path: '/sitemap.xml',
statusCodeExpected: 200
})
expect(res.text).to.contain('xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"')
expect(res.text).to.contain('<url><loc>http://localhost:9001/about/instance</loc></url>')
})
it('Should get the empty cached sitemap', async function () {
const res = await makeGetRequest({
url: server.url,
path: '/sitemap.xml',
statusCodeExpected: 200
})
expect(res.text).to.contain('xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"')
expect(res.text).to.contain('<url><loc>http://localhost:9001/about/instance</loc></url>')
})
it('Should add videos, channel and accounts and get sitemap', async function () {
this.timeout(35000)
await uploadVideo(server.url, server.accessToken, { name: 'video 1', nsfw: false })
await uploadVideo(server.url, server.accessToken, { name: 'video 2', nsfw: false })
await uploadVideo(server.url, server.accessToken, { name: 'video 3', privacy: VideoPrivacy.PRIVATE })
await addVideoChannel(server.url, server.accessToken, { name: 'channel1', displayName: 'channel 1' })
await addVideoChannel(server.url, server.accessToken, { name: 'channel2', displayName: 'channel 2' })
await createUser(server.url, server.accessToken, 'user1', 'password')
await createUser(server.url, server.accessToken, 'user2', 'password')
const res = await makeGetRequest({
url: server.url,
path: '/sitemap.xml?t=1', // avoid using cache
statusCodeExpected: 200
})
expect(res.text).to.contain('xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"')
expect(res.text).to.contain('<url><loc>http://localhost:9001/about/instance</loc></url>')
expect(res.text).to.contain('<video:title><![CDATA[video 1]]></video:title>')
expect(res.text).to.contain('<video:title><![CDATA[video 2]]></video:title>')
expect(res.text).to.not.contain('<video:title><![CDATA[video 3]]></video:title>')
expect(res.text).to.contain('<url><loc>http://localhost:9001/video-channels/channel1</loc></url>')
expect(res.text).to.contain('<url><loc>http://localhost:9001/video-channels/channel2</loc></url>')
expect(res.text).to.contain('<url><loc>http://localhost:9001/accounts/user1</loc></url>')
expect(res.text).to.contain('<url><loc>http://localhost:9001/accounts/user2</loc></url>')
})
})
after(async function () {
killallServers([ server ])
})

19
yarn.lock
View File

@ -7457,6 +7457,15 @@ simple-websocket@^7.0.1:
readable-stream "^2.0.5"
ws "^6.0.0"
sitemap@^2.1.0:
version "2.1.0"
resolved "https://registry.yarnpkg.com/sitemap/-/sitemap-2.1.0.tgz#1633cb88c196d755ad94becfb1c1bcacc6d3425a"
integrity sha512-AkfA7RDVCITQo+j5CpXsMJlZ/8ENO2NtgMHYIh+YMvex2Hao/oe3MQgNa03p0aWY6srCfUA1Q02OgiWCAiuccA==
dependencies:
lodash "^4.17.10"
url-join "^4.0.0"
xmlbuilder "^10.0.0"
slash@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/slash/-/slash-1.0.0.tgz#c41f2f6c39fc16d1cd17ad4b5d896114ae470d55"
@ -8592,6 +8601,11 @@ urix@^0.1.0:
resolved "https://registry.yarnpkg.com/urix/-/urix-0.1.0.tgz#da937f7a62e21fec1fd18d49b35c2935067a6c72"
integrity sha1-2pN/emLiH+wf0Y1Js1wpNQZ6bHI=
url-join@^4.0.0:
version "4.0.0"
resolved "https://registry.yarnpkg.com/url-join/-/url-join-4.0.0.tgz#4d3340e807d3773bda9991f8305acdcc2a665d2a"
integrity sha1-TTNA6AfTdzvamZH4MFrNzCpmXSo=
url-parse-lax@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/url-parse-lax/-/url-parse-lax-1.0.0.tgz#7af8f303645e9bd79a272e7a14ac68bc0609da73"
@ -9001,6 +9015,11 @@ xml@^1.0.1:
resolved "https://registry.yarnpkg.com/xml/-/xml-1.0.1.tgz#78ba72020029c5bc87b8a81a3cfcd74b4a2fc1e5"
integrity sha1-eLpyAgApxbyHuKgaPPzXS0ovweU=
xmlbuilder@^10.0.0:
version "10.1.1"
resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-10.1.1.tgz#8cae6688cc9b38d850b7c8d3c0a4161dcaf475b0"
integrity sha512-OyzrcFLL/nb6fMGHbiRDuPup9ljBycsdCypwuyg5AAHvyWzGfChJpCXMG88AGTIMFhGZ9RccFN1e6lhg3hkwKg==
xmlbuilder@~9.0.1:
version "9.0.7"
resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-9.0.7.tgz#132ee63d2ec5565c557e20f4c22df9aca686b10d"