diff --git a/.travis.yml b/.travis.yml index 9fcd50dee..76efedd50 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,8 +1,8 @@ language: node_js node_js: - - "4.6" - - "6.9" + - "4" + - "6" env: - CXX=g++-4.8 @@ -13,11 +13,12 @@ addons: - ubuntu-toolchain-r-test packages: - g++-4.8 + postgresql: "9.4" sudo: false services: - - mongodb + - postgresql before_install: if [[ `npm -v` != 3* ]]; then npm i -g npm@3; fi @@ -29,6 +30,13 @@ before_script: - cp ffmpeg-*-64bit-static/{ffmpeg,ffprobe,ffserver} $HOME/bin - export PATH=$HOME/bin:$PATH - export NODE_TEST_IMAGE=true + - psql -c 'create database peertube_test1;' -U postgres + - psql -c 'create database peertube_test2;' -U postgres + - psql -c 'create database peertube_test3;' -U postgres + - psql -c 'create database peertube_test4;' -U postgres + - psql -c 'create database peertube_test5;' -U postgres + - psql -c 'create database peertube_test6;' -U postgres + - psql -c "create user peertube with password 'peertube';" -U postgres after_failure: - cat test1/logs/all-logs.log diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index cc6864fac..ebcffd6cb 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -54,7 +54,16 @@ * A pod is a websocket tracker which is responsible for all the video uploaded in it * A pod has an administrator that can add/remove users, make friends and quit friends * A pod has different user accounts that can upload videos - * All pods have an index of all videos of the network (name, origin pod url, small description, uploader username, magnet Uri, thumbnail name, created date and the thumbnail file). For example, a test with 1000000 videos with alphanum characters and the following lengths: name = 50, author = 50, url = 25, description = 250, magnerUri = 200, thumbnail name = 50 has a mongodb size of ~ 4GB. To this, we add 1 000 000 thumbnails of 5-15 KB so 15GB maximum + * All pods have an index of all videos of the network (name, origin pod url, small description, uploader username, magnet Uri, thumbnail name, created date and the thumbnail file). For example, a test with 1000000 videos (3 tags each) with alphanum characters and the following lengths: name = 50, author = 50, podHost = 25, description = 250, videoExtension = 4, remoteId = 50, infoHash = 50 and tag = 10 has a PostgreSQL size of ~ 2GB with all the useful indexes. To this, we add 1 000 000 thumbnails of 5-15 KB so 15GB maximum + + table_name | row_estimate | index | toast | table + pod | 983416 | 140 MB | 83 MB | 57 MB + author | 1e+06 | 229 MB | 140 MB | 89 MB + tag | 2.96758e+06 | 309 MB | 182 MB | 127 MB + video | 1e+06 | 723 MB | 263 MB | 460 MB + video_tag | 3e+06 | 316 MB | 212 MB | 104 MB + + * After having uploaded a video, the server seeds it (WebSeed protocol), adds the meta data in its database and makes a secure request to all of its friends * If a user wants to watch a video, he asks its pod the magnetUri and the frontend adds the torrent (with WebTorrent), creates the HTML5 video tag and streams the file into it * A user watching a video seeds it too (BitTorrent) so another user who is watching the same video can get the data from the origin server and the user 1 (etc) diff --git a/README.md b/README.md index 1a9b03201..f9de61c72 100644 --- a/README.md +++ b/README.md @@ -121,7 +121,7 @@ Thanks to [WebTorrent](https://github.com/feross/webtorrent), we can make P2P (t * **NodeJS >= 4.x** * **npm >= 3.x** * OpenSSL (cli) - * MongoDB + * PostgreSQL * ffmpeg #### Debian @@ -131,7 +131,7 @@ Thanks to [WebTorrent](https://github.com/feross/webtorrent), we can make P2P (t * Run: # apt-get update - # apt-get install ffmpeg mongodb openssl + # apt-get install ffmpeg postgresql-9.4 openssl # npm install -g npm@3 #### Other distribution... (PR welcome) @@ -238,7 +238,7 @@ Here are some simple schemes: Join a network -Many networks

diff --git a/client/package.json b/client/package.json index c31f04f7b..e9f41959d 100644 --- a/client/package.json +++ b/client/package.json @@ -18,14 +18,14 @@ }, "license": "GPLv3", "dependencies": { - "@angular/common": "~2.3.0", - "@angular/compiler": "~2.3.0", - "@angular/core": "~2.3.0", - "@angular/forms": "~2.3.0", - "@angular/http": "~2.3.0", - "@angular/platform-browser": "~2.3.0", - "@angular/platform-browser-dynamic": "~2.3.0", - "@angular/router": "~3.3.0", + "@angular/common": "~2.4.1", + "@angular/compiler": "~2.4.1", + "@angular/core": "~2.4.1", + "@angular/forms": "~2.4.1", + "@angular/http": "~2.4.1", + "@angular/platform-browser": "~2.4.1", + "@angular/platform-browser-dynamic": "~2.4.1", + "@angular/router": "~3.4.1", "@angularclass/hmr": "^1.2.0", "@angularclass/hmr-loader": "^3.0.2", "@types/core-js": "^0.9.28", @@ -51,7 +51,7 @@ "ie-shim": "^0.1.0", "intl": "^1.2.4", "json-loader": "^0.5.4", - "ng2-bootstrap": "1.1.16", + "ng2-bootstrap": "1.1.16-10", "ng2-file-upload": "^1.1.0", "ng2-meta": "^2.0.0", "node-sass": "^3.10.0", diff --git a/client/src/app/admin/friends/friend-list/friend-list.component.html b/client/src/app/admin/friends/friend-list/friend-list.component.html index 4236fc5f6..06258f8c8 100644 --- a/client/src/app/admin/friends/friend-list/friend-list.component.html +++ b/client/src/app/admin/friends/friend-list/friend-list.component.html @@ -15,7 +15,7 @@ {{ friend.id }} {{ friend.host }} {{ friend.score }} - {{ friend.createdDate | date: 'medium' }} + {{ friend.createdAt | date: 'medium' }} diff --git a/client/src/app/admin/friends/friend-list/friend-list.component.ts b/client/src/app/admin/friends/friend-list/friend-list.component.ts index 88c4800ee..bec10162c 100644 --- a/client/src/app/admin/friends/friend-list/friend-list.component.ts +++ b/client/src/app/admin/friends/friend-list/friend-list.component.ts @@ -30,7 +30,7 @@ export class FriendListComponent implements OnInit { private getFriends() { this.friendService.getFriends().subscribe( - friends => this.friends = friends, + res => this.friends = res.friends, err => alert(err.text) ); diff --git a/client/src/app/admin/friends/shared/friend.model.ts b/client/src/app/admin/friends/shared/friend.model.ts index 3c23feebc..462cc82ed 100644 --- a/client/src/app/admin/friends/shared/friend.model.ts +++ b/client/src/app/admin/friends/shared/friend.model.ts @@ -2,5 +2,5 @@ export interface Friend { id: string; host: string; score: number; - createdDate: Date; + createdAt: Date; } diff --git a/client/src/app/admin/friends/shared/friend.service.ts b/client/src/app/admin/friends/shared/friend.service.ts index 8a1ba6b02..85ac04ba0 100644 --- a/client/src/app/admin/friends/shared/friend.service.ts +++ b/client/src/app/admin/friends/shared/friend.service.ts @@ -2,7 +2,7 @@ import { Injectable } from '@angular/core'; import { Observable } from 'rxjs/Observable'; import { Friend } from './friend.model'; -import { AuthHttp, RestExtractor } from '../../../shared'; +import { AuthHttp, RestExtractor, ResultList } from '../../../shared'; @Injectable() export class FriendService { @@ -13,11 +13,10 @@ export class FriendService { private restExtractor: RestExtractor ) {} - getFriends(): Observable { + getFriends() { return this.authHttp.get(FriendService.BASE_FRIEND_URL) - // Not implemented as a data list by the server yet - // .map(this.restExtractor.extractDataList) - .map((res) => res.json()) + .map(this.restExtractor.extractDataList) + .map(this.extractFriends) .catch((res) => this.restExtractor.handleError(res)); } @@ -36,4 +35,11 @@ export class FriendService { .map(res => res.status) .catch((res) => this.restExtractor.handleError(res)); } + + private extractFriends(result: ResultList) { + const friends: Friend[] = result.data; + const totalFriends = result.total; + + return { friends, totalFriends }; + } } diff --git a/client/src/app/admin/requests/request-stats/request-stats.component.html b/client/src/app/admin/requests/request-stats/request-stats.component.html index b5ac59a9a..6698eac48 100644 --- a/client/src/app/admin/requests/request-stats/request-stats.component.html +++ b/client/src/app/admin/requests/request-stats/request-stats.component.html @@ -18,6 +18,6 @@
Remaining requests: - {{ stats.requests.length }} + {{ stats.totalRequests }}
diff --git a/client/src/app/admin/requests/request-stats/request-stats.component.ts b/client/src/app/admin/requests/request-stats/request-stats.component.ts index d20b12199..9e2af219c 100644 --- a/client/src/app/admin/requests/request-stats/request-stats.component.ts +++ b/client/src/app/admin/requests/request-stats/request-stats.component.ts @@ -19,7 +19,7 @@ export class RequestStatsComponent implements OnInit, OnDestroy { } ngOnDestroy() { - if (this.stats.secondsInterval !== null) { + if (this.stats !== null && this.stats.secondsInterval !== null) { clearInterval(this.interval); } } diff --git a/client/src/app/admin/requests/shared/request-stats.model.ts b/client/src/app/admin/requests/shared/request-stats.model.ts index 766e80836..49ecbc79e 100644 --- a/client/src/app/admin/requests/shared/request-stats.model.ts +++ b/client/src/app/admin/requests/shared/request-stats.model.ts @@ -7,18 +7,18 @@ export class RequestStats { maxRequestsInParallel: number; milliSecondsInterval: number; remainingMilliSeconds: number; - requests: Request[]; + totalRequests: number; constructor(hash: { maxRequestsInParallel: number, milliSecondsInterval: number, remainingMilliSeconds: number, - requests: Request[]; + totalRequests: number; }) { this.maxRequestsInParallel = hash.maxRequestsInParallel; this.milliSecondsInterval = hash.milliSecondsInterval; this.remainingMilliSeconds = hash.remainingMilliSeconds; - this.requests = hash.requests; + this.totalRequests = hash.totalRequests; } get remainingSeconds() { diff --git a/client/src/app/admin/users/user-list/user-list.component.html b/client/src/app/admin/users/user-list/user-list.component.html index 328b1be77..36193d119 100644 --- a/client/src/app/admin/users/user-list/user-list.component.html +++ b/client/src/app/admin/users/user-list/user-list.component.html @@ -14,7 +14,7 @@ {{ user.id }} {{ user.username }} - {{ user.createdDate | date: 'medium' }} + {{ user.createdAt | date: 'medium' }} diff --git a/client/src/app/shared/auth/auth-user.model.ts b/client/src/app/shared/auth/auth-user.model.ts index bdd5ea5a9..f560351f4 100644 --- a/client/src/app/shared/auth/auth-user.model.ts +++ b/client/src/app/shared/auth/auth-user.model.ts @@ -7,9 +7,6 @@ export class AuthUser extends User { USERNAME: 'username' }; - id: string; - role: string; - username: string; tokens: Tokens; static load() { @@ -17,7 +14,7 @@ export class AuthUser extends User { if (usernameLocalStorage) { return new AuthUser( { - id: localStorage.getItem(this.KEYS.ID), + id: parseInt(localStorage.getItem(this.KEYS.ID)), username: localStorage.getItem(this.KEYS.USERNAME), role: localStorage.getItem(this.KEYS.ROLE) }, @@ -35,7 +32,7 @@ export class AuthUser extends User { Tokens.flush(); } - constructor(userHash: { id: string, username: string, role: string }, hashTokens: any) { + constructor(userHash: { id: number, username: string, role: string }, hashTokens: any) { super(userHash); this.tokens = new Tokens(hashTokens); } @@ -58,7 +55,7 @@ export class AuthUser extends User { } save() { - localStorage.setItem(AuthUser.KEYS.ID, this.id); + localStorage.setItem(AuthUser.KEYS.ID, this.id.toString()); localStorage.setItem(AuthUser.KEYS.USERNAME, this.username); localStorage.setItem(AuthUser.KEYS.ROLE, this.role); this.tokens.save(); diff --git a/client/src/app/shared/search/search-field.type.ts b/client/src/app/shared/search/search-field.type.ts index 5228ee68a..6be584ed1 100644 --- a/client/src/app/shared/search/search-field.type.ts +++ b/client/src/app/shared/search/search-field.type.ts @@ -1 +1 @@ -export type SearchField = "name" | "author" | "podUrl" | "magnetUri" | "tags"; +export type SearchField = "name" | "author" | "host" | "magnetUri" | "tags"; diff --git a/client/src/app/shared/search/search.component.ts b/client/src/app/shared/search/search.component.ts index b6237469b..9f7e156ec 100644 --- a/client/src/app/shared/search/search.component.ts +++ b/client/src/app/shared/search/search.component.ts @@ -14,8 +14,8 @@ export class SearchComponent implements OnInit { fieldChoices = { name: 'Name', author: 'Author', - podUrl: 'Pod Url', - magnetUri: 'Magnet Uri', + host: 'Pod Host', + magnetUri: 'Magnet URI', tags: 'Tags' }; searchCriterias: Search = { diff --git a/client/src/app/shared/shared.module.ts b/client/src/app/shared/shared.module.ts index 141922322..748c5d520 100644 --- a/client/src/app/shared/shared.module.ts +++ b/client/src/app/shared/shared.module.ts @@ -5,10 +5,10 @@ import { FormsModule, ReactiveFormsModule } from '@angular/forms'; import { RouterModule } from '@angular/router'; import { BytesPipe } from 'angular-pipes/src/math/bytes.pipe'; -import { DropdownModule } from 'ng2-bootstrap/components/dropdown'; -import { ProgressbarModule } from 'ng2-bootstrap/components/progressbar'; -import { PaginationModule } from 'ng2-bootstrap/components/pagination'; -import { ModalModule } from 'ng2-bootstrap/components/modal'; +import { DropdownModule } from 'ng2-bootstrap/dropdown'; +import { ProgressbarModule } from 'ng2-bootstrap/progressbar'; +import { PaginationModule } from 'ng2-bootstrap/pagination'; +import { ModalModule } from 'ng2-bootstrap/modal'; import { FileUploadModule } from 'ng2-file-upload/ng2-file-upload'; import { AUTH_HTTP_PROVIDERS } from './auth'; @@ -23,11 +23,12 @@ import { SearchComponent, SearchService } from './search'; HttpModule, RouterModule, - DropdownModule, - FileUploadModule, - ModalModule, - PaginationModule, - ProgressbarModule + DropdownModule.forRoot(), + ModalModule.forRoot(), + PaginationModule.forRoot(), + ProgressbarModule.forRoot(), + + FileUploadModule ], declarations: [ diff --git a/client/src/app/shared/users/user.model.ts b/client/src/app/shared/users/user.model.ts index 726495d11..52d89e004 100644 --- a/client/src/app/shared/users/user.model.ts +++ b/client/src/app/shared/users/user.model.ts @@ -1,16 +1,16 @@ export class User { - id: string; + id: number; username: string; role: string; - createdDate: Date; + createdAt: Date; - constructor(hash: { id: string, username: string, role: string, createdDate?: Date }) { + constructor(hash: { id: number, username: string, role: string, createdAt?: Date }) { this.id = hash.id; this.username = hash.username; this.role = hash.role; - if (hash.createdDate) { - this.createdDate = hash.createdDate; + if (hash.createdAt) { + this.createdAt = hash.createdAt; } } diff --git a/client/src/app/videos/shared/sort-field.type.ts b/client/src/app/videos/shared/sort-field.type.ts index 6e8cc7936..74908e344 100644 --- a/client/src/app/videos/shared/sort-field.type.ts +++ b/client/src/app/videos/shared/sort-field.type.ts @@ -1,3 +1,3 @@ export type SortField = "name" | "-name" | "duration" | "-duration" - | "createdDate" | "-createdDate"; + | "createdAt" | "-createdAt"; diff --git a/client/src/app/videos/shared/video.model.ts b/client/src/app/videos/shared/video.model.ts index b51a0e9de..fae001d78 100644 --- a/client/src/app/videos/shared/video.model.ts +++ b/client/src/app/videos/shared/video.model.ts @@ -1,7 +1,7 @@ export class Video { author: string; by: string; - createdDate: Date; + createdAt: Date; description: string; duration: string; id: string; @@ -27,7 +27,7 @@ export class Video { constructor(hash: { author: string, - createdDate: string, + createdAt: string, description: string, duration: number; id: string, @@ -39,7 +39,7 @@ export class Video { thumbnailPath: string }) { this.author = hash.author; - this.createdDate = new Date(hash.createdDate); + this.createdAt = new Date(hash.createdAt); this.description = hash.description; this.duration = Video.createDurationString(hash.duration); this.id = hash.id; diff --git a/client/src/app/videos/video-list/video-list.component.ts b/client/src/app/videos/video-list/video-list.component.ts index a8b92480b..6c42ba5be 100644 --- a/client/src/app/videos/video-list/video-list.component.ts +++ b/client/src/app/videos/video-list/video-list.component.ts @@ -145,7 +145,7 @@ export class VideoListComponent implements OnInit, OnDestroy { }; } - this.sort = routeParams['sort'] || '-createdDate'; + this.sort = routeParams['sort'] || '-createdAt'; if (routeParams['page'] !== undefined) { this.pagination.currentPage = parseInt(routeParams['page']); diff --git a/client/src/app/videos/video-list/video-miniature.component.html b/client/src/app/videos/video-list/video-miniature.component.html index 16513902b..f2f4a53a9 100644 --- a/client/src/app/videos/video-list/video-miniature.component.html +++ b/client/src/app/videos/video-list/video-miniature.component.html @@ -23,6 +23,6 @@ {{ video.by }} - {{ video.createdDate | date:'short' }} + {{ video.createdAt | date:'short' }} diff --git a/client/src/app/videos/video-list/video-miniature.component.scss b/client/src/app/videos/video-list/video-miniature.component.scss index 6b3fa3bf0..d70b1b50d 100644 --- a/client/src/app/videos/video-list/video-miniature.component.scss +++ b/client/src/app/videos/video-list/video-miniature.component.scss @@ -79,7 +79,7 @@ } } - .video-miniature-author, .video-miniature-created-date { + .video-miniature-author, .video-miniature-created-at { display: block; margin-left: 1px; font-size: 12px; diff --git a/client/src/app/videos/video-list/video-sort.component.ts b/client/src/app/videos/video-list/video-sort.component.ts index ca94b07c2..53951deb4 100644 --- a/client/src/app/videos/video-list/video-sort.component.ts +++ b/client/src/app/videos/video-list/video-sort.component.ts @@ -17,8 +17,8 @@ export class VideoSortComponent { '-name': 'Name - Desc', 'duration': 'Duration - Asc', '-duration': 'Duration - Desc', - 'createdDate': 'Created Date - Asc', - '-createdDate': 'Created Date - Desc' + 'createdAt': 'Created Date - Asc', + '-createdAt': 'Created Date - Desc' }; get choiceKeys() { diff --git a/client/src/app/videos/video-watch/video-magnet.component.ts b/client/src/app/videos/video-watch/video-magnet.component.ts index 2894e7df6..8bee848a4 100644 --- a/client/src/app/videos/video-watch/video-magnet.component.ts +++ b/client/src/app/videos/video-watch/video-magnet.component.ts @@ -1,6 +1,6 @@ import { Component, Input, ViewChild } from '@angular/core'; -import { ModalDirective } from 'ng2-bootstrap/components/modal'; +import { ModalDirective } from 'ng2-bootstrap/modal'; import { Video } from '../shared'; diff --git a/client/src/app/videos/video-watch/video-share.component.ts b/client/src/app/videos/video-watch/video-share.component.ts index 8e6de1294..0b85052cd 100644 --- a/client/src/app/videos/video-watch/video-share.component.ts +++ b/client/src/app/videos/video-watch/video-share.component.ts @@ -1,6 +1,6 @@ import { Component, Input, ViewChild } from '@angular/core'; -import { ModalDirective } from 'ng2-bootstrap/components/modal'; +import { ModalDirective } from 'ng2-bootstrap/modal'; import { Video } from '../shared'; diff --git a/client/src/app/videos/video-watch/video-watch.component.html b/client/src/app/videos/video-watch/video-watch.component.html index 0f0fa68cc..a726ef3ff 100644 --- a/client/src/app/videos/video-watch/video-watch.component.html +++ b/client/src/app/videos/video-watch/video-watch.component.html @@ -47,7 +47,7 @@ {{ video.by }} - on {{ video.createdDate | date:'short' }} + on {{ video.createdAt | date:'short' }} diff --git a/client/src/vendor.ts b/client/src/vendor.ts index 760fc8257..436c58f48 100644 --- a/client/src/vendor.ts +++ b/client/src/vendor.ts @@ -29,7 +29,7 @@ import 'angular-pipes/src/math/bytes.pipe'; import 'ng2-file-upload'; import 'video.js'; import 'ng2-meta'; -import 'ng2-bootstrap/components/pagination'; -import 'ng2-bootstrap/components/dropdown'; -import 'ng2-bootstrap/components/progressbar'; -import 'ng2-bootstrap/components/modal'; +import 'ng2-bootstrap/pagination'; +import 'ng2-bootstrap/dropdown'; +import 'ng2-bootstrap/progressbar'; +import 'ng2-bootstrap/modal'; diff --git a/config/default.yaml b/config/default.yaml index 90f4b9466..0939ae4ba 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -8,8 +8,10 @@ webserver: database: hostname: 'localhost' - port: 27017 - suffix: '-dev' + port: 5432 + suffix: '_dev' + username: 'peertube' + password: 'peertube' # From the project root directory storage: diff --git a/config/production.yaml.example b/config/production.yaml.example index 056ace434..005444e73 100644 --- a/config/production.yaml.example +++ b/config/production.yaml.example @@ -1,3 +1,6 @@ +listen: + port: 9000 + # Correspond to your reverse proxy "listen" configuration webserver: https: false @@ -5,4 +8,17 @@ webserver: port: 80 database: - suffix: '-prod' + hostname: 'localhost' + port: 5432 + suffix: '_prod' + username: peertube + password: peertube + +# From the project root directory +storage: + certs: 'certs/' + videos: 'videos/' + logs: 'logs/' + previews: 'previews/' + thumbnails: 'thumbnails/' + torrents: 'torrents/' diff --git a/config/test-1.yaml b/config/test-1.yaml index 6dcc7f294..b2a0a5422 100644 --- a/config/test-1.yaml +++ b/config/test-1.yaml @@ -6,7 +6,7 @@ webserver: port: 9001 database: - suffix: '-test1' + suffix: '_test1' # From the project root directory storage: diff --git a/config/test-2.yaml b/config/test-2.yaml index 209525963..7285f3394 100644 --- a/config/test-2.yaml +++ b/config/test-2.yaml @@ -6,7 +6,7 @@ webserver: port: 9002 database: - suffix: '-test2' + suffix: '_test2' # From the project root directory storage: diff --git a/config/test-3.yaml b/config/test-3.yaml index 15719d107..138a2cd53 100644 --- a/config/test-3.yaml +++ b/config/test-3.yaml @@ -6,7 +6,7 @@ webserver: port: 9003 database: - suffix: '-test3' + suffix: '_test3' # From the project root directory storage: diff --git a/config/test-4.yaml b/config/test-4.yaml index e6f34d013..7425f4af7 100644 --- a/config/test-4.yaml +++ b/config/test-4.yaml @@ -6,7 +6,7 @@ webserver: port: 9004 database: - suffix: '-test4' + suffix: '_test4' # From the project root directory storage: diff --git a/config/test-5.yaml b/config/test-5.yaml index fdeec76d4..1bf0de658 100644 --- a/config/test-5.yaml +++ b/config/test-5.yaml @@ -6,7 +6,7 @@ webserver: port: 9005 database: - suffix: '-test5' + suffix: '_test5' # From the project root directory storage: diff --git a/config/test-6.yaml b/config/test-6.yaml index 0c9630c89..7303a08fc 100644 --- a/config/test-6.yaml +++ b/config/test-6.yaml @@ -6,7 +6,7 @@ webserver: port: 9006 database: - suffix: '-test6' + suffix: '_test6' # From the project root directory storage: diff --git a/config/test.yaml b/config/test.yaml index 06705a987..493a23076 100644 --- a/config/test.yaml +++ b/config/test.yaml @@ -6,4 +6,4 @@ webserver: database: hostname: 'localhost' - port: 27017 + port: 5432 diff --git a/package.json b/package.json index 300af4867..554ad16df 100644 --- a/package.json +++ b/package.json @@ -56,17 +56,19 @@ "lodash": "^4.11.1", "magnet-uri": "^5.1.4", "mkdirp": "^0.5.1", - "mongoose": "^4.0.5", "morgan": "^1.5.3", "multer": "^1.1.0", "openssl-wrapper": "^0.3.4", "parse-torrent": "^5.8.0", "password-generator": "^2.0.2", + "pg": "^6.1.0", + "pg-hstore": "^2.3.2", "request": "^2.57.0", "request-replay": "^1.0.2", "rimraf": "^2.5.4", + "safe-buffer": "^5.0.1", "scripty": "^1.5.0", - "ursa": "^0.9.1", + "sequelize": "^3.27.0", "winston": "^2.1.1", "ws": "^1.1.1" }, diff --git a/scripts/clean/server/test.sh b/scripts/clean/server/test.sh index 927671dd4..35d3ad50f 100755 --- a/scripts/clean/server/test.sh +++ b/scripts/clean/server/test.sh @@ -1,6 +1,7 @@ #!/usr/bin/env sh for i in $(seq 1 6); do - printf "use peertube-test%s;\ndb.dropDatabase();" "$i" | mongo + dropdb "peertube_test$i" rm -rf "./test$i" + createdb "peertube_test$i" done diff --git a/scripts/danger/clean/cleaner.js b/scripts/danger/clean/cleaner.js index 10e91e700..1a1e3dee7 100644 --- a/scripts/danger/clean/cleaner.js +++ b/scripts/danger/clean/cleaner.js @@ -1,24 +1,25 @@ +const eachSeries = require('async/eachSeries') const rimraf = require('rimraf') -const mongoose = require('mongoose') -mongoose.Promise = global.Promise const constants = require('../../../server/initializers/constants') +const db = require('../../../server/initializers/database') -const mongodbUrl = 'mongodb://' + constants.CONFIG.DATABASE.HOSTNAME + ':' + constants.CONFIG.DATABASE.PORT + '/' + constants.CONFIG.DATABASE.DBNAME -mongoose.connect(mongodbUrl, function () { - console.info('Deleting MongoDB %s database.', constants.CONFIG.DATABASE.DBNAME) - mongoose.connection.dropDatabase(function () { - mongoose.connection.close() - }) -}) - -const STORAGE = constants.CONFIG.STORAGE -Object.keys(STORAGE).forEach(function (storage) { - const storageDir = STORAGE[storage] - - rimraf(storageDir, function (err) { +db.init(true, function () { + db.sequelize.drop().asCallback(function (err) { if (err) throw err - console.info('Deleting %s.', storageDir) + console.info('Tables of %s deleted.', db.sequelize.config.database) + + const STORAGE = constants.CONFIG.STORAGE + eachSeries(Object.keys(STORAGE), function (storage, callbackEach) { + const storageDir = STORAGE[storage] + + rimraf(storageDir, function (err) { + console.info('%s deleted.', storageDir) + return callbackEach(err) + }) + }, function () { + process.exit(0) + }) }) }) diff --git a/scripts/danger/clean/dev.sh b/scripts/danger/clean/dev.sh index 5f09565cf..f4a5d1e8a 100755 --- a/scripts/danger/clean/dev.sh +++ b/scripts/danger/clean/dev.sh @@ -1,6 +1,7 @@ #!/bin/bash -read -p "This will remove all directories and Mongo database. Are you sure? " -n 1 -r +read -p "This will remove all directories and SQL tables. Are you sure? (y/*) " -n 1 -r +echo if [[ "$REPLY" =~ ^[Yy]$ ]]; then NODE_ENV=test node "./scripts/danger/clean/cleaner" diff --git a/scripts/danger/clean/prod.sh b/scripts/danger/clean/prod.sh index e6c92671d..7ee13ba83 100755 --- a/scripts/danger/clean/prod.sh +++ b/scripts/danger/clean/prod.sh @@ -1,6 +1,7 @@ #!/bin/bash -read -p "This will remove all directories and Mongo database. Are you sure? " -n 1 -r +read -p "This will remove all directories and SQL tables. Are you sure? (y/*) " -n 1 -r +echo if [[ "$REPLY" =~ ^[Yy]$ ]]; then NODE_ENV=production node "./scripts/danger/clean/cleaner" diff --git a/scripts/mongo-to-postgre.js b/scripts/mongo-to-postgre.js new file mode 100755 index 000000000..4a581b46a --- /dev/null +++ b/scripts/mongo-to-postgre.js @@ -0,0 +1,244 @@ +#!/usr/bin/env node + +'use strict' + +// TODO: document this script + +const program = require('commander') +const eachSeries = require('async/eachSeries') +const series = require('async/series') +const waterfall = require('async/waterfall') +const fs = require('fs') +const path = require('path') +const MongoClient = require('mongodb').MongoClient + +const constants = require('../server/initializers/constants') + +program + .option('-mh, --mongo-host [host]', 'MongoDB host', 'localhost') + .option('-mp, --mongo-port [weight]', 'MongoDB port', '27017') + .option('-md, --mongo-database [dbname]', 'MongoDB database') + .parse(process.argv) + +if (!program.mongoDatabase) { + console.error('The mongodb database is mandatory.') + process.exit(-1) +} + +const mongoUrl = 'mongodb://' + program.mongoHost + ':' + program.mongoPort + '/' + program.mongoDatabase +const dbSequelize = require('../server/initializers/database') + +console.log('Connecting to ' + mongoUrl) +MongoClient.connect(mongoUrl, function (err, dbMongo) { + if (err) throw err + + console.log('Connected to ' + mongoUrl) + + const videoMongo = dbMongo.collection('videos') + const userMongo = dbMongo.collection('users') + const podMongo = dbMongo.collection('pods') + + podMongo.count(function (err, podsLength) { + if (err) throw err + + if (podsLength > 0) { + console.error('You need to quit friends first.') + process.exit(-1) + } + + console.log('Connecting to ' + dbSequelize.sequelize.config.database) + dbSequelize.init(true, function (err) { + if (err) throw err + + console.log('Connected to SQL database %s.', dbSequelize.sequelize.config.database) + + series([ + function (next) { + dbSequelize.sequelize.sync({ force: true }).asCallback(next) + }, + + function (next) { + migrateVideos(videoMongo, dbSequelize, next) + }, + + function (next) { + migrateUsers(userMongo, dbSequelize, next) + } + ], function (err) { + if (err) console.error(err) + + process.exit(0) + }) + }) + }) +}) + +// --------------------------------------------------------------------------- + +function migrateUsers (userMongo, dbSequelize, callback) { + userMongo.find().toArray(function (err, mongoUsers) { + if (err) return callback(err) + + eachSeries(mongoUsers, function (mongoUser, callbackEach) { + console.log('Migrating user %s', mongoUser.username) + + const userData = { + username: mongoUser.username, + password: mongoUser.password, + role: mongoUser.role + } + const options = { + hooks: false + } + + dbSequelize.User.create(userData, options).asCallback(callbackEach) + }, callback) + }) +} + +function migrateVideos (videoMongo, dbSequelize, finalCallback) { + videoMongo.find().toArray(function (err, mongoVideos) { + if (err) return finalCallback(err) + + eachSeries(mongoVideos, function (mongoVideo, callbackEach) { + console.log('Migrating video %s.', mongoVideo.name) + + waterfall([ + + function startTransaction (callback) { + dbSequelize.sequelize.transaction().asCallback(function (err, t) { + return callback(err, t) + }) + }, + + function findOrCreatePod (t, callback) { + if (mongoVideo.remoteId === null) return callback(null, t, null) + + const query = { + where: { + host: mongoVideo.podHost + }, + defaults: { + host: mongoVideo.podHost + }, + transaction: t + } + + dbSequelize.Pod.findOrCreate(query).asCallback(function (err, result) { + // [ instance, wasCreated ] + return callback(err, t, result[0]) + }) + }, + + function findOrCreateAuthor (t, pod, callback) { + const podId = pod ? pod.id : null + const username = mongoVideo.author + + const query = { + where: { + podId, + name: username + }, + defaults: { + podId, + name: username + }, + transaction: t + } + + dbSequelize.Author.findOrCreate(query).asCallback(function (err, result) { + // [ instance, wasCreated ] + return callback(err, t, result[0]) + }) + }, + + function findOrCreateTags (t, author, callback) { + const tags = mongoVideo.tags + const tagInstances = [] + + eachSeries(tags, function (tag, callbackEach) { + const query = { + where: { + name: tag + }, + defaults: { + name: tag + }, + transaction: t + } + + dbSequelize.Tag.findOrCreate(query).asCallback(function (err, res) { + if (err) return callbackEach(err) + + // res = [ tag, isCreated ] + const tag = res[0] + tagInstances.push(tag) + return callbackEach() + }) + }, function (err) { + return callback(err, t, author, tagInstances) + }) + }, + + function createVideoObject (t, author, tagInstances, callback) { + const videoData = { + name: mongoVideo.name, + remoteId: mongoVideo.remoteId, + extname: mongoVideo.extname, + infoHash: mongoVideo.magnet.infoHash, + description: mongoVideo.description, + authorId: author.id, + duration: mongoVideo.duration, + createdAt: mongoVideo.createdDate + } + + const video = dbSequelize.Video.build(videoData) + + return callback(null, t, tagInstances, video) + }, + + function moveVideoFile (t, tagInstances, video, callback) { + const basePath = constants.CONFIG.STORAGE.VIDEOS_DIR + const src = path.join(basePath, mongoVideo._id.toString()) + video.extname + const dst = path.join(basePath, video.id) + video.extname + fs.rename(src, dst, function (err) { + if (err) return callback(err) + + return callback(null, t, tagInstances, video) + }) + }, + + function insertVideoIntoDB (t, tagInstances, video, callback) { + const options = { + transaction: t + } + + video.save(options).asCallback(function (err, videoCreated) { + return callback(err, t, tagInstances, videoCreated) + }) + }, + + function associateTagsToVideo (t, tagInstances, video, callback) { + const options = { transaction: t } + + video.setTags(tagInstances, options).asCallback(function (err) { + return callback(err, t) + }) + } + + ], function (err, t) { + if (err) { + // Abort transaction? + if (t) t.rollback() + + return callbackEach(err) + } + + // Commit transaction + t.commit() + + return callbackEach() + }) + }, finalCallback) + }) +} diff --git a/scripts/play.sh b/scripts/play.sh index 33dc1a545..ab812025c 100755 --- a/scripts/play.sh +++ b/scripts/play.sh @@ -5,7 +5,9 @@ if [ ! -f server.js ]; then exit -1 fi -for i in 1 2 3; do +max=${1:-3} + +for i in $(seq 1 $max); do NODE_ENV=test NODE_APP_INSTANCE=$i node server.js & sleep 1 done diff --git a/scripts/update-host.js b/scripts/update-host.js index 8a17f2402..91051e68b 100755 --- a/scripts/update-host.js +++ b/scripts/update-host.js @@ -5,31 +5,24 @@ // TODO: document this script const fs = require('fs') -const mongoose = require('mongoose') const parseTorrent = require('parse-torrent') const constants = require('../server/initializers/constants') -const database = require('../server/initializers/database') - -database.connect() +const db = require('../server/initializers/database') const friends = require('../server/lib/friends') -const Video = mongoose.model('Video') -friends.hasFriends(function (err, hasFriends) { - if (err) throw err - - if (hasFriends === true) { - console.log('Cannot update host because you have friends!') - process.exit(-1) - } - - console.log('Updating videos host in database.') - Video.update({ }, { podHost: constants.CONFIG.WEBSERVER.HOST }, { multi: true }, function (err) { +db.init(true, function () { + friends.hasFriends(function (err, hasFriends) { if (err) throw err + if (hasFriends === true) { + console.log('Cannot update host because you have friends!') + process.exit(-1) + } + console.log('Updating torrent files.') - Video.find().lean().exec(function (err, videos) { + db.Video.list(function (err, videos) { if (err) throw err videos.forEach(function (video) { diff --git a/server.js b/server.js index 6eb022000..7503072af 100644 --- a/server.js +++ b/server.js @@ -17,10 +17,10 @@ const app = express() // ----------- Database ----------- const constants = require('./server/initializers/constants') -const database = require('./server/initializers/database') const logger = require('./server/helpers/logger') - -database.connect() +// Initialize database and models +const db = require('./server/initializers/database') +db.init() // ----------- Checker ----------- const checker = require('./server/initializers/checker') @@ -39,9 +39,7 @@ if (errorMessage !== null) { const customValidators = require('./server/helpers/custom-validators') const installer = require('./server/initializers/installer') const migrator = require('./server/initializers/migrator') -const mongoose = require('mongoose') const routes = require('./server/controllers') -const Request = mongoose.model('Request') // ----------- Command line ----------- @@ -59,7 +57,8 @@ app.use(expressValidator({ customValidators.misc, customValidators.pods, customValidators.users, - customValidators.videos + customValidators.videos, + customValidators.remote.videos ) })) @@ -130,7 +129,7 @@ installer.installApplication(function (err) { // ----------- Make the server listening ----------- server.listen(port, function () { // Activate the pool requests - Request.activate() + db.Request.activate() logger.info('Server listening on port %d', port) logger.info('Webserver: %s', constants.CONFIG.WEBSERVER.URL) diff --git a/server/controllers/api/clients.js b/server/controllers/api/clients.js index 7755f6c2b..cf83cb835 100644 --- a/server/controllers/api/clients.js +++ b/server/controllers/api/clients.js @@ -1,13 +1,11 @@ 'use strict' const express = require('express') -const mongoose = require('mongoose') const constants = require('../../initializers/constants') +const db = require('../../initializers/database') const logger = require('../../helpers/logger') -const Client = mongoose.model('OAuthClient') - const router = express.Router() router.get('/local', getLocalClient) @@ -27,12 +25,12 @@ function getLocalClient (req, res, next) { return res.type('json').status(403).end() } - Client.loadFirstClient(function (err, client) { + db.OAuthClient.loadFirstClient(function (err, client) { if (err) return next(err) if (!client) return next(new Error('No client available.')) res.json({ - client_id: client._id, + client_id: client.clientId, client_secret: client.clientSecret }) }) diff --git a/server/controllers/api/index.js b/server/controllers/api/index.js index 4cb65ed55..f13ff922c 100644 --- a/server/controllers/api/index.js +++ b/server/controllers/api/index.js @@ -2,6 +2,8 @@ const express = require('express') +const utils = require('../../helpers/utils') + const router = express.Router() const clientsController = require('./clients') @@ -18,7 +20,7 @@ router.use('/requests', requestsController) router.use('/users', usersController) router.use('/videos', videosController) router.use('/ping', pong) -router.use('/*', badRequest) +router.use('/*', utils.badRequest) // --------------------------------------------------------------------------- @@ -29,7 +31,3 @@ module.exports = router function pong (req, res, next) { return res.send('pong').status(200).end() } - -function badRequest (req, res, next) { - res.type('json').status(400).end() -} diff --git a/server/controllers/api/pods.js b/server/controllers/api/pods.js index 7857fcee0..38702face 100644 --- a/server/controllers/api/pods.js +++ b/server/controllers/api/pods.js @@ -1,10 +1,11 @@ 'use strict' const express = require('express') -const mongoose = require('mongoose') const waterfall = require('async/waterfall') +const db = require('../../initializers/database') const logger = require('../../helpers/logger') +const utils = require('../../helpers/utils') const friends = require('../../lib/friends') const middlewares = require('../../middlewares') const admin = middlewares.admin @@ -15,7 +16,6 @@ const validators = middlewares.validators.pods const signatureValidator = middlewares.validators.remote.signature const router = express.Router() -const Pod = mongoose.model('Pod') router.get('/', listPods) router.post('/', @@ -37,7 +37,7 @@ router.get('/quitfriends', ) // Post because this is a secured request router.post('/remove', - signatureValidator, + signatureValidator.signature, checkSignature, removePods ) @@ -53,15 +53,15 @@ function addPods (req, res, next) { waterfall([ function addPod (callback) { - const pod = new Pod(informations) - pod.save(function (err, podCreated) { + const pod = db.Pod.build(informations) + pod.save().asCallback(function (err, podCreated) { // Be sure about the number of parameters for the callback return callback(err, podCreated) }) }, function sendMyVideos (podCreated, callback) { - friends.sendOwnedVideosToPod(podCreated._id) + friends.sendOwnedVideosToPod(podCreated.id) callback(null) }, @@ -84,10 +84,10 @@ function addPods (req, res, next) { } function listPods (req, res, next) { - Pod.list(function (err, podsList) { + db.Pod.list(function (err, podsList) { if (err) return next(err) - res.json(getFormatedPods(podsList)) + res.json(utils.getFormatedObjects(podsList, podsList.length)) }) } @@ -111,11 +111,11 @@ function removePods (req, res, next) { waterfall([ function loadPod (callback) { - Pod.loadByHost(host, callback) + db.Pod.loadByHost(host, callback) }, - function removePod (pod, callback) { - pod.remove(callback) + function deletePod (pod, callback) { + pod.destroy().asCallback(callback) } ], function (err) { if (err) return next(err) @@ -131,15 +131,3 @@ function quitFriends (req, res, next) { res.type('json').status(204).end() }) } - -// --------------------------------------------------------------------------- - -function getFormatedPods (pods) { - const formatedPods = [] - - pods.forEach(function (pod) { - formatedPods.push(pod.toFormatedJSON()) - }) - - return formatedPods -} diff --git a/server/controllers/api/remote.js b/server/controllers/api/remote.js deleted file mode 100644 index f1046c534..000000000 --- a/server/controllers/api/remote.js +++ /dev/null @@ -1,86 +0,0 @@ -'use strict' - -const each = require('async/each') -const eachSeries = require('async/eachSeries') -const express = require('express') -const mongoose = require('mongoose') - -const middlewares = require('../../middlewares') -const secureMiddleware = middlewares.secure -const validators = middlewares.validators.remote -const logger = require('../../helpers/logger') - -const router = express.Router() -const Video = mongoose.model('Video') - -router.post('/videos', - validators.signature, - secureMiddleware.checkSignature, - validators.remoteVideos, - remoteVideos -) - -// --------------------------------------------------------------------------- - -module.exports = router - -// --------------------------------------------------------------------------- - -function remoteVideos (req, res, next) { - const requests = req.body.data - const fromHost = req.body.signature.host - - // We need to process in the same order to keep consistency - // TODO: optimization - eachSeries(requests, function (request, callbackEach) { - const videoData = request.data - - if (request.type === 'add') { - addRemoteVideo(videoData, fromHost, callbackEach) - } else if (request.type === 'remove') { - removeRemoteVideo(videoData, fromHost, callbackEach) - } else { - logger.error('Unkown remote request type %s.', request.type) - } - }, function (err) { - if (err) logger.error('Error managing remote videos.', { error: err }) - }) - - // We don't need to keep the other pod waiting - return res.type('json').status(204).end() -} - -function addRemoteVideo (videoToCreateData, fromHost, callback) { - logger.debug('Adding remote video "%s".', videoToCreateData.name) - - const video = new Video(videoToCreateData) - video.podHost = fromHost - Video.generateThumbnailFromBase64(video, videoToCreateData.thumbnailBase64, function (err) { - if (err) { - logger.error('Cannot generate thumbnail from base 64 data.', { error: err }) - return callback(err) - } - - video.save(callback) - }) -} - -function removeRemoteVideo (videoToRemoveData, fromHost, callback) { - // We need the list because we have to remove some other stuffs (thumbnail etc) - Video.listByHostAndRemoteId(fromHost, videoToRemoveData.remoteId, function (err, videosList) { - if (err) { - logger.error('Cannot list videos from host and magnets.', { error: err }) - return callback(err) - } - - if (videosList.length === 0) { - logger.error('No remote video was found for this pod.', { magnetUri: videoToRemoveData.magnetUri, podHost: fromHost }) - } - - each(videosList, function (video, callbackEach) { - logger.debug('Removing remote video %s.', video.magnetUri) - - video.remove(callbackEach) - }, callback) - }) -} diff --git a/server/controllers/api/remote/index.js b/server/controllers/api/remote/index.js new file mode 100644 index 000000000..2947632d5 --- /dev/null +++ b/server/controllers/api/remote/index.js @@ -0,0 +1,16 @@ +'use strict' + +const express = require('express') + +const utils = require('../../../helpers/utils') + +const router = express.Router() + +const videosRemoteController = require('./videos') + +router.use('/videos', videosRemoteController) +router.use('/*', utils.badRequest) + +// --------------------------------------------------------------------------- + +module.exports = router diff --git a/server/controllers/api/remote/videos.js b/server/controllers/api/remote/videos.js new file mode 100644 index 000000000..c45a86dbb --- /dev/null +++ b/server/controllers/api/remote/videos.js @@ -0,0 +1,328 @@ +'use strict' + +const eachSeries = require('async/eachSeries') +const express = require('express') +const waterfall = require('async/waterfall') + +const db = require('../../../initializers/database') +const middlewares = require('../../../middlewares') +const secureMiddleware = middlewares.secure +const videosValidators = middlewares.validators.remote.videos +const signatureValidators = middlewares.validators.remote.signature +const logger = require('../../../helpers/logger') +const utils = require('../../../helpers/utils') + +const router = express.Router() + +router.post('/', + signatureValidators.signature, + secureMiddleware.checkSignature, + videosValidators.remoteVideos, + remoteVideos +) + +// --------------------------------------------------------------------------- + +module.exports = router + +// --------------------------------------------------------------------------- + +function remoteVideos (req, res, next) { + const requests = req.body.data + const fromPod = res.locals.secure.pod + + // We need to process in the same order to keep consistency + // TODO: optimization + eachSeries(requests, function (request, callbackEach) { + const data = request.data + + switch (request.type) { + case 'add': + addRemoteVideoRetryWrapper(data, fromPod, callbackEach) + break + + case 'update': + updateRemoteVideoRetryWrapper(data, fromPod, callbackEach) + break + + case 'remove': + removeRemoteVideo(data, fromPod, callbackEach) + break + + case 'report-abuse': + reportAbuseRemoteVideo(data, fromPod, callbackEach) + break + + default: + logger.error('Unkown remote request type %s.', request.type) + } + }, function (err) { + if (err) logger.error('Error managing remote videos.', { error: err }) + }) + + // We don't need to keep the other pod waiting + return res.type('json').status(204).end() +} + +// Handle retries on fail +function addRemoteVideoRetryWrapper (videoToCreateData, fromPod, finalCallback) { + utils.transactionRetryer( + function (callback) { + return addRemoteVideo(videoToCreateData, fromPod, callback) + }, + function (err) { + if (err) { + logger.error('Cannot insert the remote video with many retries.', { error: err }) + } + + // Do not return the error, continue the process + return finalCallback(null) + } + ) +} + +function addRemoteVideo (videoToCreateData, fromPod, finalCallback) { + logger.debug('Adding remote video "%s".', videoToCreateData.remoteId) + + waterfall([ + + function startTransaction (callback) { + db.sequelize.transaction({ isolationLevel: 'SERIALIZABLE' }).asCallback(function (err, t) { + return callback(err, t) + }) + }, + + function findOrCreateAuthor (t, callback) { + const name = videoToCreateData.author + const podId = fromPod.id + // This author is from another pod so we do not associate a user + const userId = null + + db.Author.findOrCreateAuthor(name, podId, userId, t, function (err, authorInstance) { + return callback(err, t, authorInstance) + }) + }, + + function findOrCreateTags (t, author, callback) { + const tags = videoToCreateData.tags + + db.Tag.findOrCreateTags(tags, t, function (err, tagInstances) { + return callback(err, t, author, tagInstances) + }) + }, + + function createVideoObject (t, author, tagInstances, callback) { + const videoData = { + name: videoToCreateData.name, + remoteId: videoToCreateData.remoteId, + extname: videoToCreateData.extname, + infoHash: videoToCreateData.infoHash, + description: videoToCreateData.description, + authorId: author.id, + duration: videoToCreateData.duration, + createdAt: videoToCreateData.createdAt, + // FIXME: updatedAt does not seems to be considered by Sequelize + updatedAt: videoToCreateData.updatedAt + } + + const video = db.Video.build(videoData) + + return callback(null, t, tagInstances, video) + }, + + function generateThumbnail (t, tagInstances, video, callback) { + db.Video.generateThumbnailFromData(video, videoToCreateData.thumbnailData, function (err) { + if (err) { + logger.error('Cannot generate thumbnail from data.', { error: err }) + return callback(err) + } + + return callback(err, t, tagInstances, video) + }) + }, + + function insertVideoIntoDB (t, tagInstances, video, callback) { + const options = { + transaction: t + } + + video.save(options).asCallback(function (err, videoCreated) { + return callback(err, t, tagInstances, videoCreated) + }) + }, + + function associateTagsToVideo (t, tagInstances, video, callback) { + const options = { transaction: t } + + video.setTags(tagInstances, options).asCallback(function (err) { + return callback(err, t) + }) + } + + ], function (err, t) { + if (err) { + // This is just a debug because we will retry the insert + logger.debug('Cannot insert the remote video.', { error: err }) + + // Abort transaction? + if (t) t.rollback() + + return finalCallback(err) + } + + // Commit transaction + t.commit().asCallback(function (err) { + if (err) return finalCallback(err) + + logger.info('Remote video %s inserted.', videoToCreateData.name) + return finalCallback(null) + }) + }) +} + +// Handle retries on fail +function updateRemoteVideoRetryWrapper (videoAttributesToUpdate, fromPod, finalCallback) { + utils.transactionRetryer( + function (callback) { + return updateRemoteVideo(videoAttributesToUpdate, fromPod, callback) + }, + function (err) { + if (err) { + logger.error('Cannot update the remote video with many retries.', { error: err }) + } + + // Do not return the error, continue the process + return finalCallback(null) + } + ) +} + +function updateRemoteVideo (videoAttributesToUpdate, fromPod, finalCallback) { + logger.debug('Updating remote video "%s".', videoAttributesToUpdate.remoteId) + + waterfall([ + + function startTransaction (callback) { + db.sequelize.transaction({ isolationLevel: 'SERIALIZABLE' }).asCallback(function (err, t) { + return callback(err, t) + }) + }, + + function findVideo (t, callback) { + fetchVideo(fromPod.host, videoAttributesToUpdate.remoteId, function (err, videoInstance) { + return callback(err, t, videoInstance) + }) + }, + + function findOrCreateTags (t, videoInstance, callback) { + const tags = videoAttributesToUpdate.tags + + db.Tag.findOrCreateTags(tags, t, function (err, tagInstances) { + return callback(err, t, videoInstance, tagInstances) + }) + }, + + function updateVideoIntoDB (t, videoInstance, tagInstances, callback) { + const options = { transaction: t } + + videoInstance.set('name', videoAttributesToUpdate.name) + videoInstance.set('description', videoAttributesToUpdate.description) + videoInstance.set('infoHash', videoAttributesToUpdate.infoHash) + videoInstance.set('duration', videoAttributesToUpdate.duration) + videoInstance.set('createdAt', videoAttributesToUpdate.createdAt) + videoInstance.set('updatedAt', videoAttributesToUpdate.updatedAt) + videoInstance.set('extname', videoAttributesToUpdate.extname) + + videoInstance.save(options).asCallback(function (err) { + return callback(err, t, videoInstance, tagInstances) + }) + }, + + function associateTagsToVideo (t, videoInstance, tagInstances, callback) { + const options = { transaction: t } + + videoInstance.setTags(tagInstances, options).asCallback(function (err) { + return callback(err, t) + }) + } + + ], function (err, t) { + if (err) { + // This is just a debug because we will retry the insert + logger.debug('Cannot update the remote video.', { error: err }) + + // Abort transaction? + if (t) t.rollback() + + return finalCallback(err) + } + + // Commit transaction + t.commit().asCallback(function (err) { + if (err) return finalCallback(err) + + logger.info('Remote video %s updated', videoAttributesToUpdate.name) + return finalCallback(null) + }) + }) +} + +function removeRemoteVideo (videoToRemoveData, fromPod, callback) { + // We need the instance because we have to remove some other stuffs (thumbnail etc) + fetchVideo(fromPod.host, videoToRemoveData.remoteId, function (err, video) { + // Do not return the error, continue the process + if (err) return callback(null) + + logger.debug('Removing remote video %s.', video.remoteId) + video.destroy().asCallback(function (err) { + // Do not return the error, continue the process + if (err) { + logger.error('Cannot remove remote video with id %s.', videoToRemoveData.remoteId, { error: err }) + } + + return callback(null) + }) + }) +} + +function reportAbuseRemoteVideo (reportData, fromPod, callback) { + db.Video.load(reportData.videoRemoteId, function (err, video) { + if (err || !video) { + if (!err) err = new Error('video not found') + + logger.error('Cannot load video from id.', { error: err, id: reportData.videoRemoteId }) + // Do not return the error, continue the process + return callback(null) + } + + logger.debug('Reporting remote abuse for video %s.', video.id) + + const videoAbuseData = { + reporterUsername: reportData.reporterUsername, + reason: reportData.reportReason, + reporterPodId: fromPod.id, + videoId: video.id + } + + db.VideoAbuse.create(videoAbuseData).asCallback(function (err) { + if (err) { + logger.error('Cannot create remote abuse video.', { error: err }) + } + + return callback(null) + }) + }) +} + +function fetchVideo (podHost, remoteId, callback) { + db.Video.loadByHostAndRemoteId(podHost, remoteId, function (err, video) { + if (err || !video) { + if (!err) err = new Error('video not found') + + logger.error('Cannot load video from host and remote id.', { error: err, podHost, remoteId }) + return callback(err) + } + + return callback(null, video) + }) +} diff --git a/server/controllers/api/requests.js b/server/controllers/api/requests.js index 52aad6997..1f9193fc8 100644 --- a/server/controllers/api/requests.js +++ b/server/controllers/api/requests.js @@ -1,15 +1,13 @@ 'use strict' const express = require('express') -const mongoose = require('mongoose') const constants = require('../../initializers/constants') +const db = require('../../initializers/database') const middlewares = require('../../middlewares') const admin = middlewares.admin const oAuth = middlewares.oauth -const Request = mongoose.model('Request') - const router = express.Router() router.get('/stats', @@ -25,13 +23,13 @@ module.exports = router // --------------------------------------------------------------------------- function getStatsRequests (req, res, next) { - Request.list(function (err, requests) { + db.Request.countTotalRequests(function (err, totalRequests) { if (err) return next(err) return res.json({ - requests: requests, + totalRequests: totalRequests, maxRequestsInParallel: constants.REQUESTS_IN_PARALLEL, - remainingMilliSeconds: Request.remainingMilliSeconds(), + remainingMilliSeconds: db.Request.remainingMilliSeconds(), milliSecondsInterval: constants.REQUESTS_INTERVAL }) }) diff --git a/server/controllers/api/users.js b/server/controllers/api/users.js index b4d687312..6cd0e84f7 100644 --- a/server/controllers/api/users.js +++ b/server/controllers/api/users.js @@ -1,13 +1,12 @@ 'use strict' -const each = require('async/each') const express = require('express') -const mongoose = require('mongoose') const waterfall = require('async/waterfall') const constants = require('../../initializers/constants') -const friends = require('../../lib/friends') +const db = require('../../initializers/database') const logger = require('../../helpers/logger') +const utils = require('../../helpers/utils') const middlewares = require('../../middlewares') const admin = middlewares.admin const oAuth = middlewares.oauth @@ -17,9 +16,6 @@ const validatorsPagination = middlewares.validators.pagination const validatorsSort = middlewares.validators.sort const validatorsUsers = middlewares.validators.users -const User = mongoose.model('User') -const Video = mongoose.model('Video') - const router = express.Router() router.get('/me', oAuth.authenticate, getUserInformation) @@ -62,13 +58,13 @@ module.exports = router // --------------------------------------------------------------------------- function createUser (req, res, next) { - const user = new User({ + const user = db.User.build({ username: req.body.username, password: req.body.password, role: constants.USER_ROLES.USER }) - user.save(function (err, createdUser) { + user.save().asCallback(function (err, createdUser) { if (err) return next(err) return res.type('json').status(204).end() @@ -76,7 +72,7 @@ function createUser (req, res, next) { } function getUserInformation (req, res, next) { - User.loadByUsername(res.locals.oauth.token.user.username, function (err, user) { + db.User.loadByUsername(res.locals.oauth.token.user.username, function (err, user) { if (err) return next(err) return res.json(user.toFormatedJSON()) @@ -84,48 +80,21 @@ function getUserInformation (req, res, next) { } function listUsers (req, res, next) { - User.listForApi(req.query.start, req.query.count, req.query.sort, function (err, usersList, usersTotal) { + db.User.listForApi(req.query.start, req.query.count, req.query.sort, function (err, usersList, usersTotal) { if (err) return next(err) - res.json(getFormatedUsers(usersList, usersTotal)) + res.json(utils.getFormatedObjects(usersList, usersTotal)) }) } function removeUser (req, res, next) { waterfall([ - function getUser (callback) { - User.loadById(req.params.id, callback) + function loadUser (callback) { + db.User.loadById(req.params.id, callback) }, - function getVideos (user, callback) { - Video.listOwnedByAuthor(user.username, function (err, videos) { - return callback(err, user, videos) - }) - }, - - function removeVideosFromDB (user, videos, callback) { - each(videos, function (video, callbackEach) { - video.remove(callbackEach) - }, function (err) { - return callback(err, user, videos) - }) - }, - - function sendInformationToFriends (user, videos, callback) { - videos.forEach(function (video) { - const params = { - name: video.name, - magnetUri: video.magnetUri - } - - friends.removeVideoToFriends(params) - }) - - return callback(null, user) - }, - - function removeUserFromDB (user, callback) { - user.remove(callback) + function deleteUser (user, callback) { + user.destroy().asCallback(callback) } ], function andFinally (err) { if (err) { @@ -138,11 +107,11 @@ function removeUser (req, res, next) { } function updateUser (req, res, next) { - User.loadByUsername(res.locals.oauth.token.user.username, function (err, user) { + db.User.loadByUsername(res.locals.oauth.token.user.username, function (err, user) { if (err) return next(err) user.password = req.body.password - user.save(function (err) { + user.save().asCallback(function (err) { if (err) return next(err) return res.sendStatus(204) @@ -153,18 +122,3 @@ function updateUser (req, res, next) { function success (req, res, next) { res.end() } - -// --------------------------------------------------------------------------- - -function getFormatedUsers (users, usersTotal) { - const formatedUsers = [] - - users.forEach(function (user) { - formatedUsers.push(user.toFormatedJSON()) - }) - - return { - total: usersTotal, - data: formatedUsers - } -} diff --git a/server/controllers/api/videos.js b/server/controllers/api/videos.js index daf452573..2c4af520e 100644 --- a/server/controllers/api/videos.js +++ b/server/controllers/api/videos.js @@ -2,15 +2,16 @@ const express = require('express') const fs = require('fs') -const mongoose = require('mongoose') const multer = require('multer') const path = require('path') const waterfall = require('async/waterfall') const constants = require('../../initializers/constants') +const db = require('../../initializers/database') const logger = require('../../helpers/logger') const friends = require('../../lib/friends') const middlewares = require('../../middlewares') +const admin = middlewares.admin const oAuth = middlewares.oauth const pagination = middlewares.pagination const validators = middlewares.validators @@ -22,7 +23,6 @@ const sort = middlewares.sort const utils = require('../../helpers/utils') const router = express.Router() -const Video = mongoose.model('Video') // multer configuration const storage = multer.diskStorage({ @@ -44,6 +44,21 @@ const storage = multer.diskStorage({ const reqFiles = multer({ storage: storage }).fields([{ name: 'videofile', maxCount: 1 }]) +router.get('/abuse', + oAuth.authenticate, + admin.ensureIsAdmin, + validatorsPagination.pagination, + validatorsSort.videoAbusesSort, + sort.setVideoAbusesSort, + pagination.setPagination, + listVideoAbuses +) +router.post('/:id/abuse', + oAuth.authenticate, + validatorsVideos.videoAbuseReport, + reportVideoAbuseRetryWrapper +) + router.get('/', validatorsPagination.pagination, validatorsSort.videosSort, @@ -51,11 +66,17 @@ router.get('/', pagination.setPagination, listVideos ) +router.put('/:id', + oAuth.authenticate, + reqFiles, + validatorsVideos.videosUpdate, + updateVideoRetryWrapper +) router.post('/', oAuth.authenticate, reqFiles, validatorsVideos.videosAdd, - addVideo + addVideoRetryWrapper ) router.get('/:id', validatorsVideos.videosGet, @@ -82,117 +103,264 @@ module.exports = router // --------------------------------------------------------------------------- -function addVideo (req, res, next) { - const videoFile = req.files.videofile[0] +// Wrapper to video add that retry the function if there is a database error +// We need this because we run the transaction in SERIALIZABLE isolation that can fail +function addVideoRetryWrapper (req, res, next) { + utils.transactionRetryer( + function (callback) { + return addVideo(req, res, req.files.videofile[0], callback) + }, + function (err) { + if (err) { + logger.error('Cannot insert the video with many retries.', { error: err }) + return next(err) + } + + // TODO : include Location of the new video -> 201 + return res.type('json').status(204).end() + } + ) +} + +function addVideo (req, res, videoFile, callback) { const videoInfos = req.body waterfall([ - function createVideoObject (callback) { - const id = mongoose.Types.ObjectId() + function startTransaction (callbackWaterfall) { + db.sequelize.transaction({ isolationLevel: 'SERIALIZABLE' }).asCallback(function (err, t) { + return callbackWaterfall(err, t) + }) + }, + + function findOrCreateAuthor (t, callbackWaterfall) { + const user = res.locals.oauth.token.User + + const name = user.username + // null because it is OUR pod + const podId = null + const userId = user.id + + db.Author.findOrCreateAuthor(name, podId, userId, t, function (err, authorInstance) { + return callbackWaterfall(err, t, authorInstance) + }) + }, + + function findOrCreateTags (t, author, callbackWaterfall) { + const tags = videoInfos.tags + + db.Tag.findOrCreateTags(tags, t, function (err, tagInstances) { + return callbackWaterfall(err, t, author, tagInstances) + }) + }, + + function createVideoObject (t, author, tagInstances, callbackWaterfall) { const videoData = { - _id: id, name: videoInfos.name, remoteId: null, extname: path.extname(videoFile.filename), description: videoInfos.description, - author: res.locals.oauth.token.user.username, duration: videoFile.duration, - tags: videoInfos.tags + authorId: author.id } - const video = new Video(videoData) + const video = db.Video.build(videoData) - return callback(null, video) + return callbackWaterfall(null, t, author, tagInstances, video) }, - // Set the videoname the same as the MongoDB id - function renameVideoFile (video, callback) { + // Set the videoname the same as the id + function renameVideoFile (t, author, tagInstances, video, callbackWaterfall) { const videoDir = constants.CONFIG.STORAGE.VIDEOS_DIR const source = path.join(videoDir, videoFile.filename) const destination = path.join(videoDir, video.getVideoFilename()) fs.rename(source, destination, function (err) { - return callback(err, video) + if (err) return callbackWaterfall(err) + + // This is important in case if there is another attempt + videoFile.filename = video.getVideoFilename() + return callbackWaterfall(null, t, author, tagInstances, video) }) }, - function insertIntoDB (video, callback) { - video.save(function (err, video) { - // Assert there are only one argument sent to the next function (video) - return callback(err, video) + function insertVideoIntoDB (t, author, tagInstances, video, callbackWaterfall) { + const options = { transaction: t } + + // Add tags association + video.save(options).asCallback(function (err, videoCreated) { + if (err) return callbackWaterfall(err) + + // Do not forget to add Author informations to the created video + videoCreated.Author = author + + return callbackWaterfall(err, t, tagInstances, videoCreated) }) }, - function sendToFriends (video, callback) { - video.toRemoteJSON(function (err, remoteVideo) { - if (err) return callback(err) + function associateTagsToVideo (t, tagInstances, video, callbackWaterfall) { + const options = { transaction: t } + + video.setTags(tagInstances, options).asCallback(function (err) { + video.Tags = tagInstances + + return callbackWaterfall(err, t, video) + }) + }, + + function sendToFriends (t, video, callbackWaterfall) { + video.toAddRemoteJSON(function (err, remoteVideo) { + if (err) return callbackWaterfall(err) // Now we'll add the video's meta data to our friends - friends.addVideoToFriends(remoteVideo) - - return callback(null) + friends.addVideoToFriends(remoteVideo, t, function (err) { + return callbackWaterfall(err, t) + }) }) } - ], function andFinally (err) { + ], function andFinally (err, t) { if (err) { - logger.error('Cannot insert the video.') - return next(err) + // This is just a debug because we will retry the insert + logger.debug('Cannot insert the video.', { error: err }) + + // Abort transaction? + if (t) t.rollback() + + return callback(err) } - // TODO : include Location of the new video -> 201 - return res.type('json').status(204).end() + // Commit transaction + t.commit().asCallback(function (err) { + if (err) return callback(err) + + logger.info('Video with name %s created.', videoInfos.name) + return callback(null) + }) + }) +} + +function updateVideoRetryWrapper (req, res, next) { + utils.transactionRetryer( + function (callback) { + return updateVideo(req, res, callback) + }, + function (err) { + if (err) { + logger.error('Cannot update the video with many retries.', { error: err }) + return next(err) + } + + // TODO : include Location of the new video -> 201 + return res.type('json').status(204).end() + } + ) +} + +function updateVideo (req, res, finalCallback) { + const videoInstance = res.locals.video + const videoFieldsSave = videoInstance.toJSON() + const videoInfosToUpdate = req.body + + waterfall([ + + function startTransaction (callback) { + db.sequelize.transaction({ isolationLevel: 'SERIALIZABLE' }).asCallback(function (err, t) { + return callback(err, t) + }) + }, + + function findOrCreateTags (t, callback) { + if (videoInfosToUpdate.tags) { + db.Tag.findOrCreateTags(videoInfosToUpdate.tags, t, function (err, tagInstances) { + return callback(err, t, tagInstances) + }) + } else { + return callback(null, t, null) + } + }, + + function updateVideoIntoDB (t, tagInstances, callback) { + const options = { + transaction: t + } + + if (videoInfosToUpdate.name) videoInstance.set('name', videoInfosToUpdate.name) + if (videoInfosToUpdate.description) videoInstance.set('description', videoInfosToUpdate.description) + + videoInstance.save(options).asCallback(function (err) { + return callback(err, t, tagInstances) + }) + }, + + function associateTagsToVideo (t, tagInstances, callback) { + if (tagInstances) { + const options = { transaction: t } + + videoInstance.setTags(tagInstances, options).asCallback(function (err) { + videoInstance.Tags = tagInstances + + return callback(err, t) + }) + } else { + return callback(null, t) + } + }, + + function sendToFriends (t, callback) { + const json = videoInstance.toUpdateRemoteJSON() + + // Now we'll update the video's meta data to our friends + friends.updateVideoToFriends(json, t, function (err) { + return callback(err, t) + }) + } + + ], function andFinally (err, t) { + if (err) { + logger.debug('Cannot update the video.', { error: err }) + + // Abort transaction? + if (t) t.rollback() + + // Force fields we want to update + // If the transaction is retried, sequelize will think the object has not changed + // So it will skip the SQL request, even if the last one was ROLLBACKed! + Object.keys(videoFieldsSave).forEach(function (key) { + const value = videoFieldsSave[key] + videoInstance.set(key, value) + }) + + return finalCallback(err) + } + + // Commit transaction + t.commit().asCallback(function (err) { + if (err) return finalCallback(err) + + logger.info('Video with name %s updated.', videoInfosToUpdate.name) + return finalCallback(null) + }) }) } function getVideo (req, res, next) { - Video.load(req.params.id, function (err, video) { - if (err) return next(err) - - if (!video) { - return res.type('json').status(204).end() - } - - res.json(video.toFormatedJSON()) - }) + const videoInstance = res.locals.video + res.json(videoInstance.toFormatedJSON()) } function listVideos (req, res, next) { - Video.listForApi(req.query.start, req.query.count, req.query.sort, function (err, videosList, videosTotal) { + db.Video.listForApi(req.query.start, req.query.count, req.query.sort, function (err, videosList, videosTotal) { if (err) return next(err) - res.json(getFormatedVideos(videosList, videosTotal)) + res.json(utils.getFormatedObjects(videosList, videosTotal)) }) } function removeVideo (req, res, next) { - const videoId = req.params.id + const videoInstance = res.locals.video - waterfall([ - function getVideo (callback) { - Video.load(videoId, callback) - }, - - function removeFromDB (video, callback) { - video.remove(function (err) { - if (err) return callback(err) - - return callback(null, video) - }) - }, - - function sendInformationToFriends (video, callback) { - const params = { - name: video.name, - remoteId: video._id - } - - friends.removeVideoToFriends(params) - - return callback(null) - } - ], function andFinally (err) { + videoInstance.destroy().asCallback(function (err) { if (err) { logger.error('Errors when removed the video.', { error: err }) return next(err) @@ -203,25 +371,97 @@ function removeVideo (req, res, next) { } function searchVideos (req, res, next) { - Video.search(req.params.value, req.query.field, req.query.start, req.query.count, req.query.sort, - function (err, videosList, videosTotal) { + db.Video.searchAndPopulateAuthorAndPodAndTags( + req.params.value, req.query.field, req.query.start, req.query.count, req.query.sort, + function (err, videosList, videosTotal) { + if (err) return next(err) + + res.json(utils.getFormatedObjects(videosList, videosTotal)) + } + ) +} + +function listVideoAbuses (req, res, next) { + db.VideoAbuse.listForApi(req.query.start, req.query.count, req.query.sort, function (err, abusesList, abusesTotal) { if (err) return next(err) - res.json(getFormatedVideos(videosList, videosTotal)) + res.json(utils.getFormatedObjects(abusesList, abusesTotal)) }) } -// --------------------------------------------------------------------------- +function reportVideoAbuseRetryWrapper (req, res, next) { + utils.transactionRetryer( + function (callback) { + return reportVideoAbuse(req, res, callback) + }, + function (err) { + if (err) { + logger.error('Cannot report abuse to the video with many retries.', { error: err }) + return next(err) + } -function getFormatedVideos (videos, videosTotal) { - const formatedVideos = [] + return res.type('json').status(204).end() + } + ) +} - videos.forEach(function (video) { - formatedVideos.push(video.toFormatedJSON()) - }) +function reportVideoAbuse (req, res, finalCallback) { + const videoInstance = res.locals.video + const reporterUsername = res.locals.oauth.token.User.username - return { - total: videosTotal, - data: formatedVideos + const abuse = { + reporterUsername, + reason: req.body.reason, + videoId: videoInstance.id, + reporterPodId: null // This is our pod that reported this abuse } + + waterfall([ + + function startTransaction (callback) { + db.sequelize.transaction().asCallback(function (err, t) { + return callback(err, t) + }) + }, + + function createAbuse (t, callback) { + db.VideoAbuse.create(abuse).asCallback(function (err, abuse) { + return callback(err, t, abuse) + }) + }, + + function sendToFriendsIfNeeded (t, abuse, callback) { + // We send the information to the destination pod + if (videoInstance.isOwned() === false) { + const reportData = { + reporterUsername, + reportReason: abuse.reason, + videoRemoteId: videoInstance.remoteId + } + + friends.reportAbuseVideoToFriend(reportData, videoInstance) + } + + return callback(null, t) + } + + ], function andFinally (err, t) { + if (err) { + logger.debug('Cannot update the video.', { error: err }) + + // Abort transaction? + if (t) t.rollback() + + return finalCallback(err) + } + + // Commit transaction + t.commit().asCallback(function (err) { + if (err) return finalCallback(err) + + logger.info('Abuse report for video %s created.', videoInstance.name) + return finalCallback(null) + }) + }) } + diff --git a/server/controllers/client.js b/server/controllers/client.js index 572db6133..8c242af07 100644 --- a/server/controllers/client.js +++ b/server/controllers/client.js @@ -3,13 +3,12 @@ const parallel = require('async/parallel') const express = require('express') const fs = require('fs') -const mongoose = require('mongoose') const path = require('path') const validator = require('express-validator').validator const constants = require('../initializers/constants') +const db = require('../initializers/database') -const Video = mongoose.model('Video') const router = express.Router() const opengraphComment = '' @@ -45,14 +44,14 @@ function addOpenGraphTags (htmlStringPage, video) { if (video.isOwned()) { basePreviewUrlHttp = constants.CONFIG.WEBSERVER.URL } else { - basePreviewUrlHttp = constants.REMOTE_SCHEME.HTTP + '://' + video.podHost + basePreviewUrlHttp = constants.REMOTE_SCHEME.HTTP + '://' + video.Author.Pod.host } // We fetch the remote preview (bigger than the thumbnail) // This should not overhead the remote server since social websites put in a cache the OpenGraph tags // We can't use the thumbnail because these social websites want bigger images (> 200x200 for Facebook for example) const previewUrl = basePreviewUrlHttp + constants.STATIC_PATHS.PREVIEWS + video.getPreviewName() - const videoUrl = constants.CONFIG.WEBSERVER.URL + '/videos/watch/' + video._id + const videoUrl = constants.CONFIG.WEBSERVER.URL + '/videos/watch/' + video.id const metaTags = { 'og:type': 'video', @@ -86,7 +85,7 @@ function generateWatchHtmlPage (req, res, next) { const videoId = req.params.id // Let Angular application handle errors - if (!validator.isMongoId(videoId)) return res.sendFile(indexPath) + if (!validator.isUUID(videoId, 4)) return res.sendFile(indexPath) parallel({ file: function (callback) { @@ -94,7 +93,7 @@ function generateWatchHtmlPage (req, res, next) { }, video: function (callback) { - Video.load(videoId, callback) + db.Video.loadAndPopulateAuthorAndPodAndTags(videoId, callback) } }, function (err, results) { if (err) return next(err) diff --git a/server/helpers/custom-validators/index.js b/server/helpers/custom-validators/index.js index 96b5b20b9..9383e0304 100644 --- a/server/helpers/custom-validators/index.js +++ b/server/helpers/custom-validators/index.js @@ -2,12 +2,14 @@ const miscValidators = require('./misc') const podsValidators = require('./pods') +const remoteValidators = require('./remote') const usersValidators = require('./users') const videosValidators = require('./videos') const validators = { misc: miscValidators, pods: podsValidators, + remote: remoteValidators, users: usersValidators, videos: videosValidators } diff --git a/server/helpers/custom-validators/pods.js b/server/helpers/custom-validators/pods.js index 0154a2424..8bb3733ff 100644 --- a/server/helpers/custom-validators/pods.js +++ b/server/helpers/custom-validators/pods.js @@ -5,14 +5,19 @@ const validator = require('express-validator').validator const miscValidators = require('./misc') const podsValidators = { - isEachUniqueHostValid + isEachUniqueHostValid, + isHostValid +} + +function isHostValid (host) { + return validator.isURL(host) && host.split('://').length === 1 } function isEachUniqueHostValid (hosts) { return miscValidators.isArray(hosts) && hosts.length !== 0 && hosts.every(function (host) { - return validator.isURL(host) && host.split('://').length === 1 && hosts.indexOf(host) === hosts.lastIndexOf(host) + return isHostValid(host) && hosts.indexOf(host) === hosts.lastIndexOf(host) }) } diff --git a/server/helpers/custom-validators/remote/index.js b/server/helpers/custom-validators/remote/index.js new file mode 100644 index 000000000..1939a95f4 --- /dev/null +++ b/server/helpers/custom-validators/remote/index.js @@ -0,0 +1,11 @@ +'use strict' + +const remoteVideosValidators = require('./videos') + +const validators = { + videos: remoteVideosValidators +} + +// --------------------------------------------------------------------------- + +module.exports = validators diff --git a/server/helpers/custom-validators/remote/videos.js b/server/helpers/custom-validators/remote/videos.js new file mode 100644 index 000000000..7c27b9dbb --- /dev/null +++ b/server/helpers/custom-validators/remote/videos.js @@ -0,0 +1,73 @@ +'use strict' + +const videosValidators = require('../videos') +const miscValidators = require('../misc') + +const remoteVideosValidators = { + isEachRemoteRequestVideosValid +} + +function isEachRemoteRequestVideosValid (requests) { + return miscValidators.isArray(requests) && + requests.every(function (request) { + const video = request.data + return ( + isRequestTypeAddValid(request.type) && + videosValidators.isVideoAuthorValid(video.author) && + videosValidators.isVideoDateValid(video.createdAt) && + videosValidators.isVideoDateValid(video.updatedAt) && + videosValidators.isVideoDescriptionValid(video.description) && + videosValidators.isVideoDurationValid(video.duration) && + videosValidators.isVideoInfoHashValid(video.infoHash) && + videosValidators.isVideoNameValid(video.name) && + videosValidators.isVideoTagsValid(video.tags) && + videosValidators.isVideoThumbnailDataValid(video.thumbnailData) && + videosValidators.isVideoRemoteIdValid(video.remoteId) && + videosValidators.isVideoExtnameValid(video.extname) + ) || + ( + isRequestTypeUpdateValid(request.type) && + videosValidators.isVideoDateValid(video.createdAt) && + videosValidators.isVideoDateValid(video.updatedAt) && + videosValidators.isVideoDescriptionValid(video.description) && + videosValidators.isVideoDurationValid(video.duration) && + videosValidators.isVideoInfoHashValid(video.infoHash) && + videosValidators.isVideoNameValid(video.name) && + videosValidators.isVideoTagsValid(video.tags) && + videosValidators.isVideoRemoteIdValid(video.remoteId) && + videosValidators.isVideoExtnameValid(video.extname) + ) || + ( + isRequestTypeRemoveValid(request.type) && + videosValidators.isVideoRemoteIdValid(video.remoteId) + ) || + ( + isRequestTypeReportAbuseValid(request.type) && + videosValidators.isVideoRemoteIdValid(request.data.videoRemoteId) && + videosValidators.isVideoAbuseReasonValid(request.data.reportReason) && + videosValidators.isVideoAbuseReporterUsernameValid(request.data.reporterUsername) + ) + }) +} + +// --------------------------------------------------------------------------- + +module.exports = remoteVideosValidators + +// --------------------------------------------------------------------------- + +function isRequestTypeAddValid (value) { + return value === 'add' +} + +function isRequestTypeUpdateValid (value) { + return value === 'update' +} + +function isRequestTypeRemoveValid (value) { + return value === 'remove' +} + +function isRequestTypeReportAbuseValid (value) { + return value === 'report-abuse' +} diff --git a/server/helpers/custom-validators/videos.js b/server/helpers/custom-validators/videos.js index 1a7753265..7f727854d 100644 --- a/server/helpers/custom-validators/videos.js +++ b/server/helpers/custom-validators/videos.js @@ -6,43 +6,22 @@ const constants = require('../../initializers/constants') const usersValidators = require('./users') const miscValidators = require('./misc') const VIDEOS_CONSTRAINTS_FIELDS = constants.CONSTRAINTS_FIELDS.VIDEOS +const VIDEO_ABUSES_CONSTRAINTS_FIELDS = constants.CONSTRAINTS_FIELDS.VIDEO_ABUSES const videosValidators = { - isEachRemoteVideosValid, isVideoAuthorValid, isVideoDateValid, isVideoDescriptionValid, isVideoDurationValid, - isVideoMagnetValid, + isVideoInfoHashValid, isVideoNameValid, - isVideoPodHostValid, isVideoTagsValid, isVideoThumbnailValid, - isVideoThumbnail64Valid -} - -function isEachRemoteVideosValid (requests) { - return miscValidators.isArray(requests) && - requests.every(function (request) { - const video = request.data - return ( - isRequestTypeAddValid(request.type) && - isVideoAuthorValid(video.author) && - isVideoDateValid(video.createdDate) && - isVideoDescriptionValid(video.description) && - isVideoDurationValid(video.duration) && - isVideoMagnetValid(video.magnet) && - isVideoNameValid(video.name) && - isVideoTagsValid(video.tags) && - isVideoThumbnail64Valid(video.thumbnailBase64) && - isVideoRemoteIdValid(video.remoteId) - ) || - ( - isRequestTypeRemoveValid(request.type) && - isVideoNameValid(video.name) && - isVideoRemoteIdValid(video.remoteId) - ) - }) + isVideoThumbnailDataValid, + isVideoExtnameValid, + isVideoRemoteIdValid, + isVideoAbuseReasonValid, + isVideoAbuseReporterUsernameValid } function isVideoAuthorValid (value) { @@ -61,19 +40,18 @@ function isVideoDurationValid (value) { return validator.isInt(value + '', VIDEOS_CONSTRAINTS_FIELDS.DURATION) } -function isVideoMagnetValid (value) { - return validator.isLength(value.infoHash, VIDEOS_CONSTRAINTS_FIELDS.MAGNET.INFO_HASH) +function isVideoExtnameValid (value) { + return VIDEOS_CONSTRAINTS_FIELDS.EXTNAME.indexOf(value) !== -1 +} + +function isVideoInfoHashValid (value) { + return validator.isLength(value, VIDEOS_CONSTRAINTS_FIELDS.INFO_HASH) } function isVideoNameValid (value) { return validator.isLength(value, VIDEOS_CONSTRAINTS_FIELDS.NAME) } -function isVideoPodHostValid (value) { - // TODO: set options (TLD...) - return validator.isURL(value) -} - function isVideoTagsValid (tags) { return miscValidators.isArray(tags) && validator.isInt(tags.length, VIDEOS_CONSTRAINTS_FIELDS.TAGS) && @@ -87,25 +65,22 @@ function isVideoThumbnailValid (value) { return validator.isLength(value, VIDEOS_CONSTRAINTS_FIELDS.THUMBNAIL) } -function isVideoThumbnail64Valid (value) { - return validator.isBase64(value) && - validator.isByteLength(value, VIDEOS_CONSTRAINTS_FIELDS.THUMBNAIL64) +function isVideoThumbnailDataValid (value) { + return validator.isByteLength(value, VIDEOS_CONSTRAINTS_FIELDS.THUMBNAIL_DATA) } function isVideoRemoteIdValid (value) { - return validator.isMongoId(value) + return validator.isUUID(value, 4) +} + +function isVideoAbuseReasonValid (value) { + return validator.isLength(value, VIDEO_ABUSES_CONSTRAINTS_FIELDS.REASON) +} + +function isVideoAbuseReporterUsernameValid (value) { + return usersValidators.isUserUsernameValid(value) } // --------------------------------------------------------------------------- module.exports = videosValidators - -// --------------------------------------------------------------------------- - -function isRequestTypeAddValid (value) { - return value === 'add' -} - -function isRequestTypeRemoveValid (value) { - return value === 'remove' -} diff --git a/server/helpers/logger.js b/server/helpers/logger.js index fcc1789fd..281acedb8 100644 --- a/server/helpers/logger.js +++ b/server/helpers/logger.js @@ -22,7 +22,8 @@ const logger = new winston.Logger({ json: true, maxsize: 5242880, maxFiles: 5, - colorize: false + colorize: false, + prettyPrint: true }), new winston.transports.Console({ level: 'debug', @@ -30,7 +31,8 @@ const logger = new winston.Logger({ handleExceptions: true, humanReadableUnhandledException: true, json: false, - colorize: true + colorize: true, + prettyPrint: true }) ], exitOnError: true diff --git a/server/helpers/peertube-crypto.js b/server/helpers/peertube-crypto.js index 2e07df00e..0f1e02ad6 100644 --- a/server/helpers/peertube-crypto.js +++ b/server/helpers/peertube-crypto.js @@ -1,16 +1,13 @@ 'use strict' -const bcrypt = require('bcrypt') const crypto = require('crypto') +const bcrypt = require('bcrypt') const fs = require('fs') const openssl = require('openssl-wrapper') -const ursa = require('ursa') const constants = require('../initializers/constants') const logger = require('./logger') -const algorithm = 'aes-256-ctr' - const peertubeCrypto = { checkSignature, comparePassword, @@ -19,12 +16,51 @@ const peertubeCrypto = { sign } -function checkSignature (publicKey, rawData, hexSignature) { - const crt = ursa.createPublicKey(publicKey) - const isValid = crt.hashAndVerify('sha256', new Buffer(rawData).toString('hex'), hexSignature, 'hex') +function checkSignature (publicKey, data, hexSignature) { + const verify = crypto.createVerify(constants.SIGNATURE_ALGORITHM) + + let dataString + if (typeof data === 'string') { + dataString = data + } else { + try { + dataString = JSON.stringify(data) + } catch (err) { + logger.error('Cannot check signature.', { error: err }) + return false + } + } + + verify.update(dataString, 'utf8') + + const isValid = verify.verify(publicKey, hexSignature, constants.SIGNATURE_ENCODING) return isValid } +function sign (data) { + const sign = crypto.createSign(constants.SIGNATURE_ALGORITHM) + + let dataString + if (typeof data === 'string') { + dataString = data + } else { + try { + dataString = JSON.stringify(data) + } catch (err) { + logger.error('Cannot sign data.', { error: err }) + return '' + } + } + + sign.update(dataString, 'utf8') + + // TODO: make async + const myKey = fs.readFileSync(constants.CONFIG.STORAGE.CERT_DIR + 'peertube.key.pem') + const signature = sign.sign(myKey, constants.SIGNATURE_ENCODING) + + return signature +} + function comparePassword (plainPassword, hashPassword, callback) { bcrypt.compare(plainPassword, hashPassword, function (err, isPasswordMatch) { if (err) return callback(err) @@ -55,13 +91,6 @@ function cryptPassword (password, callback) { }) } -function sign (data) { - const myKey = ursa.createPrivateKey(fs.readFileSync(constants.CONFIG.STORAGE.CERT_DIR + 'peertube.key.pem')) - const signature = myKey.hashAndSign('sha256', data, 'utf8', 'hex') - - return signature -} - // --------------------------------------------------------------------------- module.exports = peertubeCrypto @@ -113,11 +142,3 @@ function createCerts (callback) { }) }) } - -function generatePassword (callback) { - crypto.randomBytes(32, function (err, buf) { - if (err) return callback(err) - - callback(null, buf.toString('utf8')) - }) -} diff --git a/server/helpers/requests.js b/server/helpers/requests.js index b0cda09fe..095b95e1c 100644 --- a/server/helpers/requests.js +++ b/server/helpers/requests.js @@ -28,31 +28,37 @@ function makeSecureRequest (params, callback) { url: constants.REMOTE_SCHEME.HTTP + '://' + params.toPod.host + params.path } - // Add data with POST requst ? - if (params.method === 'POST') { - requestParams.json = {} - - // Add signature if it is specified in the params - if (params.sign === true) { - const host = constants.CONFIG.WEBSERVER.HOST - - requestParams.json.signature = { - host, - signature: peertubeCrypto.sign(host) - } - } - - // If there are data informations - if (params.data) { - requestParams.json.data = params.data - request.post(requestParams, callback) - } else { - // No data - request.post(requestParams, callback) - } - } else { - request.get(requestParams, callback) + if (params.method !== 'POST') { + return callback(new Error('Cannot make a secure request with a non POST method.')) } + + requestParams.json = {} + + // Add signature if it is specified in the params + if (params.sign === true) { + const host = constants.CONFIG.WEBSERVER.HOST + + let dataToSign + if (params.data) { + dataToSign = dataToSign = params.data + } else { + // We do not have data to sign so we just take our host + // It is not ideal but the connection should be in HTTPS + dataToSign = host + } + + requestParams.json.signature = { + host, // Which host we pretend to be + signature: peertubeCrypto.sign(dataToSign) + } + } + + // If there are data informations + if (params.data) { + requestParams.json.data = params.data + } + + request.post(requestParams, callback) } // --------------------------------------------------------------------------- diff --git a/server/helpers/utils.js b/server/helpers/utils.js index 9f27671b6..a902850cd 100644 --- a/server/helpers/utils.js +++ b/server/helpers/utils.js @@ -1,13 +1,21 @@ 'use strict' const crypto = require('crypto') +const retry = require('async/retry') const logger = require('./logger') const utils = { + badRequest, cleanForExit, generateRandomString, - isTestInstance + isTestInstance, + getFormatedObjects, + transactionRetryer +} + +function badRequest (req, res, next) { + res.type('json').status(400).end() } function generateRandomString (size, callback) { @@ -27,6 +35,31 @@ function isTestInstance () { return (process.env.NODE_ENV === 'test') } +function getFormatedObjects (objects, objectsTotal) { + const formatedObjects = [] + + objects.forEach(function (object) { + formatedObjects.push(object.toFormatedJSON()) + }) + + return { + total: objectsTotal, + data: formatedObjects + } +} + +function transactionRetryer (func, callback) { + retry({ + times: 5, + + errorFilter: function (err) { + const willRetry = (err.name === 'SequelizeDatabaseError') + logger.debug('Maybe retrying the transaction function.', { willRetry }) + return willRetry + } + }, func, callback) +} + // --------------------------------------------------------------------------- module.exports = utils diff --git a/server/initializers/checker.js b/server/initializers/checker.js index aea013fa9..6471bb4f1 100644 --- a/server/initializers/checker.js +++ b/server/initializers/checker.js @@ -1,10 +1,8 @@ 'use strict' const config = require('config') -const mongoose = require('mongoose') -const Client = mongoose.model('OAuthClient') -const User = mongoose.model('User') +const db = require('./database') const checker = { checkConfig, @@ -29,7 +27,7 @@ function checkConfig () { function checkMissedConfig () { const required = [ 'listen.port', 'webserver.https', 'webserver.hostname', 'webserver.port', - 'database.hostname', 'database.port', 'database.suffix', + 'database.hostname', 'database.port', 'database.suffix', 'database.username', 'database.password', 'storage.certs', 'storage.videos', 'storage.logs', 'storage.thumbnails', 'storage.previews' ] const miss = [] @@ -44,15 +42,15 @@ function checkMissedConfig () { } function clientsExist (callback) { - Client.list(function (err, clients) { + db.OAuthClient.countTotal(function (err, totalClients) { if (err) return callback(err) - return callback(null, clients.length !== 0) + return callback(null, totalClients !== 0) }) } function usersExist (callback) { - User.countTotal(function (err, totalUsers) { + db.User.countTotal(function (err, totalUsers) { if (err) return callback(err) return callback(null, totalUsers !== 0) diff --git a/server/initializers/constants.js b/server/initializers/constants.js index 3ddf87454..97e3c5296 100644 --- a/server/initializers/constants.js +++ b/server/initializers/constants.js @@ -1,7 +1,6 @@ 'use strict' const config = require('config') -const maxBy = require('lodash/maxBy') const path = require('path') // --------------------------------------------------------------------------- @@ -14,13 +13,14 @@ const PAGINATION_COUNT_DEFAULT = 15 // Sortable columns per schema const SEARCHABLE_COLUMNS = { - VIDEOS: [ 'name', 'magnetUri', 'podHost', 'author', 'tags' ] + VIDEOS: [ 'name', 'magnetUri', 'host', 'author', 'tags' ] } // Sortable columns per schema const SORTABLE_COLUMNS = { - USERS: [ 'username', '-username', 'createdDate', '-createdDate' ], - VIDEOS: [ 'name', '-name', 'duration', '-duration', 'createdDate', '-createdDate' ] + USERS: [ 'username', '-username', 'createdAt', '-createdAt' ], + VIDEO_ABUSES: [ 'createdAt', '-createdAt' ], + VIDEOS: [ 'name', '-name', 'duration', '-duration', 'createdAt', '-createdAt' ] } const OAUTH_LIFETIME = { @@ -37,7 +37,9 @@ const CONFIG = { DATABASE: { DBNAME: 'peertube' + config.get('database.suffix'), HOSTNAME: config.get('database.hostname'), - PORT: config.get('database.port') + PORT: config.get('database.port'), + USERNAME: config.get('database.username'), + PASSWORD: config.get('database.password') }, STORAGE: { CERT_DIR: path.join(__dirname, '..', '..', config.get('storage.certs')), @@ -64,17 +66,19 @@ const CONSTRAINTS_FIELDS = { USERNAME: { min: 3, max: 20 }, // Length PASSWORD: { min: 6, max: 255 } // Length }, + VIDEO_ABUSES: { + REASON: { min: 2, max: 300 } // Length + }, VIDEOS: { NAME: { min: 3, max: 50 }, // Length DESCRIPTION: { min: 3, max: 250 }, // Length - MAGNET: { - INFO_HASH: { min: 10, max: 50 } // Length - }, + EXTNAME: [ '.mp4', '.ogv', '.webm' ], + INFO_HASH: { min: 40, max: 40 }, // Length, infohash is 20 bytes length but we represent it in hexa so 20 * 2 DURATION: { min: 1, max: 7200 }, // Number TAGS: { min: 1, max: 3 }, // Number of total tags TAG: { min: 2, max: 10 }, // Length THUMBNAIL: { min: 2, max: 30 }, - THUMBNAIL64: { min: 0, max: 20000 } // Bytes + THUMBNAIL_DATA: { min: 0, max: 20000 } // Bytes } } @@ -88,41 +92,7 @@ const FRIEND_SCORE = { // --------------------------------------------------------------------------- -const MONGO_MIGRATION_SCRIPTS = [ - { - script: '0005-create-application', - version: 5 - }, - { - script: '0010-users-password', - version: 10 - }, - { - script: '0015-admin-role', - version: 15 - }, - { - script: '0020-requests-endpoint', - version: 20 - }, - { - script: '0025-video-filenames', - version: 25 - }, - { - script: '0030-video-magnet', - version: 30 - }, - { - script: '0035-url-to-host', - version: 35 - }, - { - script: '0040-video-remote-id', - version: 40 - } -] -const LAST_MONGO_SCHEMA_VERSION = (maxBy(MONGO_MIGRATION_SCRIPTS, 'version'))['version'] +const LAST_MIGRATION_VERSION = 0 // --------------------------------------------------------------------------- @@ -138,8 +108,10 @@ let REQUESTS_INTERVAL = 600000 // Number of requests in parallel we can make const REQUESTS_IN_PARALLEL = 10 -// How many requests we put in request -const REQUESTS_LIMIT = 10 +// To how many pods we send requests +const REQUESTS_LIMIT_PODS = 10 +// How many requests we send to a pod per interval +const REQUESTS_LIMIT_PER_POD = 5 // Number of requests to retry for replay requests module const RETRY_REQUESTS = 5 @@ -148,16 +120,21 @@ const REQUEST_ENDPOINTS = { VIDEOS: 'videos' } -// --------------------------------------------------------------------------- - const REMOTE_SCHEME = { HTTP: 'https', WS: 'wss' } +// --------------------------------------------------------------------------- + +const SIGNATURE_ALGORITHM = 'RSA-SHA256' +const SIGNATURE_ENCODING = 'hex' + // Password encryption const BCRYPT_SALT_SIZE = 10 +// --------------------------------------------------------------------------- + // Express static paths (router) const STATIC_PATHS = { PREVIEWS: '/static/previews/', @@ -173,6 +150,8 @@ let STATIC_MAX_AGE = '30d' const THUMBNAILS_SIZE = '200x110' const PREVIEWS_SIZE = '640x480' +// --------------------------------------------------------------------------- + const USER_ROLES = { ADMIN: 'admin', USER: 'user' @@ -198,8 +177,7 @@ module.exports = { CONFIG, CONSTRAINTS_FIELDS, FRIEND_SCORE, - LAST_MONGO_SCHEMA_VERSION, - MONGO_MIGRATION_SCRIPTS, + LAST_MIGRATION_VERSION, OAUTH_LIFETIME, PAGINATION_COUNT_DEFAULT, PODS_SCORE, @@ -208,9 +186,12 @@ module.exports = { REQUEST_ENDPOINTS, REQUESTS_IN_PARALLEL, REQUESTS_INTERVAL, - REQUESTS_LIMIT, + REQUESTS_LIMIT_PODS, + REQUESTS_LIMIT_PER_POD, RETRY_REQUESTS, SEARCHABLE_COLUMNS, + SIGNATURE_ALGORITHM, + SIGNATURE_ENCODING, SORTABLE_COLUMNS, STATIC_MAX_AGE, STATIC_PATHS, diff --git a/server/initializers/database.js b/server/initializers/database.js index 0564e4e77..f8f68adeb 100644 --- a/server/initializers/database.js +++ b/server/initializers/database.js @@ -1,37 +1,77 @@ 'use strict' -const mongoose = require('mongoose') +const fs = require('fs') +const path = require('path') +const Sequelize = require('sequelize') const constants = require('../initializers/constants') const logger = require('../helpers/logger') +const utils = require('../helpers/utils') -// Bootstrap models -require('../models/application') -require('../models/oauth-token') -require('../models/user') -require('../models/oauth-client') -require('../models/video') -// Request model needs Video model -require('../models/pods') -// Request model needs Pod model -require('../models/request') +const database = {} -const database = { - connect: connect -} +const dbname = constants.CONFIG.DATABASE.DBNAME +const username = constants.CONFIG.DATABASE.USERNAME +const password = constants.CONFIG.DATABASE.PASSWORD -function connect () { - mongoose.Promise = global.Promise - mongoose.connect('mongodb://' + constants.CONFIG.DATABASE.HOSTNAME + ':' + constants.CONFIG.DATABASE.PORT + '/' + constants.CONFIG.DATABASE.DBNAME) - mongoose.connection.on('error', function () { - throw new Error('Mongodb connection error.') - }) +const sequelize = new Sequelize(dbname, username, password, { + dialect: 'postgres', + host: constants.CONFIG.DATABASE.HOSTNAME, + port: constants.CONFIG.DATABASE.PORT, + benchmark: utils.isTestInstance(), - mongoose.connection.on('open', function () { - logger.info('Connected to mongodb.') - }) -} + logging: function (message, benchmark) { + let newMessage = message + if (benchmark !== undefined) { + newMessage += ' | ' + benchmark + 'ms' + } + + logger.debug(newMessage) + } +}) + +database.sequelize = sequelize +database.Sequelize = Sequelize +database.init = init // --------------------------------------------------------------------------- module.exports = database + +// --------------------------------------------------------------------------- + +function init (silent, callback) { + if (!callback) { + callback = silent + silent = false + } + + if (!callback) callback = function () {} + + const modelDirectory = path.join(__dirname, '..', 'models') + fs.readdir(modelDirectory, function (err, files) { + if (err) throw err + + files.filter(function (file) { + // For all models but not utils.js + if (file === 'utils.js') return false + + return true + }) + .forEach(function (file) { + const model = sequelize.import(path.join(modelDirectory, file)) + + database[model.name] = model + }) + + Object.keys(database).forEach(function (modelName) { + if ('associate' in database[modelName]) { + database[modelName].associate(database) + } + }) + + if (!silent) logger.info('Database is ready.') + + return callback(null) + }) +} diff --git a/server/initializers/installer.js b/server/initializers/installer.js index 1df300ba8..fb63b81ac 100644 --- a/server/initializers/installer.js +++ b/server/initializers/installer.js @@ -3,26 +3,27 @@ const config = require('config') const each = require('async/each') const mkdirp = require('mkdirp') -const mongoose = require('mongoose') const passwordGenerator = require('password-generator') const path = require('path') const series = require('async/series') const checker = require('./checker') const constants = require('./constants') +const db = require('./database') const logger = require('../helpers/logger') const peertubeCrypto = require('../helpers/peertube-crypto') -const Application = mongoose.model('Application') -const Client = mongoose.model('OAuthClient') -const User = mongoose.model('User') - const installer = { installApplication } function installApplication (callback) { series([ + function createDatabase (callbackAsync) { + db.sequelize.sync().asCallback(callbackAsync) + // db.sequelize.sync({ force: true }).asCallback(callbackAsync) + }, + function createDirectories (callbackAsync) { createDirectoriesIfNotExist(callbackAsync) }, @@ -65,16 +66,18 @@ function createOAuthClientIfNotExist (callback) { logger.info('Creating a default OAuth Client.') - const secret = passwordGenerator(32, false) - const client = new Client({ + const id = passwordGenerator(32, false, /[a-z0-9]/) + const secret = passwordGenerator(32, false, /[a-zA-Z0-9]/) + const client = db.OAuthClient.build({ + clientId: id, clientSecret: secret, grants: [ 'password', 'refresh_token' ] }) - client.save(function (err, createdClient) { + client.save().asCallback(function (err, createdClient) { if (err) return callback(err) - logger.info('Client id: ' + createdClient._id) + logger.info('Client id: ' + createdClient.clientId) logger.info('Client secret: ' + createdClient.clientSecret) return callback(null) @@ -93,6 +96,7 @@ function createOAuthAdminIfNotExist (callback) { const username = 'root' const role = constants.USER_ROLES.ADMIN + const createOptions = {} let password = '' // Do not generate a random password for tests @@ -102,25 +106,27 @@ function createOAuthAdminIfNotExist (callback) { if (process.env.NODE_APP_INSTANCE) { password += process.env.NODE_APP_INSTANCE } + + // Our password is weak so do not validate it + createOptions.validate = false } else { password = passwordGenerator(8, true) } - const user = new User({ + const userData = { username, password, role - }) + } - user.save(function (err, createdUser) { + db.User.create(userData, createOptions).asCallback(function (err, createdUser) { if (err) return callback(err) logger.info('Username: ' + username) logger.info('User password: ' + password) - logger.info('Creating Application collection.') - const application = new Application({ mongoSchemaVersion: constants.LAST_MONGO_SCHEMA_VERSION }) - application.save(callback) + logger.info('Creating Application table.') + db.Application.create({ migrationVersion: constants.LAST_MIGRATION_VERSION }).asCallback(callback) }) }) } diff --git a/server/initializers/migrations/0005-create-application.js b/server/initializers/migrations/0005-create-application.js deleted file mode 100644 index e99dec019..000000000 --- a/server/initializers/migrations/0005-create-application.js +++ /dev/null @@ -1,17 +0,0 @@ -/* - Create the application collection in MongoDB. - Used to store the actual MongoDB scheme version. -*/ - -const mongoose = require('mongoose') - -const Application = mongoose.model('Application') - -exports.up = function (callback) { - const application = new Application() - application.save(callback) -} - -exports.down = function (callback) { - throw new Error('Not implemented.') -} diff --git a/server/initializers/migrations/0005-example.js b/server/initializers/migrations/0005-example.js new file mode 100644 index 000000000..cedc42919 --- /dev/null +++ b/server/initializers/migrations/0005-example.js @@ -0,0 +1,14 @@ +// /* +// This is just an example. +// */ + +// const db = require('../database') + +// // options contains the transaction +// exports.up = function (options, callback) { +// db.Application.create({ migrationVersion: 42 }, { transaction: options.transaction }).asCallback(callback) +// } + +// exports.down = function (options, callback) { +// throw new Error('Not implemented.') +// } diff --git a/server/initializers/migrations/0010-users-password.js b/server/initializers/migrations/0010-users-password.js deleted file mode 100644 index a0616a269..000000000 --- a/server/initializers/migrations/0010-users-password.js +++ /dev/null @@ -1,22 +0,0 @@ -/* - Convert plain user password to encrypted user password. -*/ - -const eachSeries = require('async/eachSeries') -const mongoose = require('mongoose') - -const User = mongoose.model('User') - -exports.up = function (callback) { - User.list(function (err, users) { - if (err) return callback(err) - - eachSeries(users, function (user, callbackEach) { - user.save(callbackEach) - }, callback) - }) -} - -exports.down = function (callback) { - throw new Error('Not implemented.') -} diff --git a/server/initializers/migrations/0015-admin-role.js b/server/initializers/migrations/0015-admin-role.js deleted file mode 100644 index af06dca9e..000000000 --- a/server/initializers/migrations/0015-admin-role.js +++ /dev/null @@ -1,16 +0,0 @@ -/* - Set the admin role to the root user. -*/ - -const constants = require('../constants') -const mongoose = require('mongoose') - -const User = mongoose.model('User') - -exports.up = function (callback) { - User.update({ username: 'root' }, { role: constants.USER_ROLES.ADMIN }, callback) -} - -exports.down = function (callback) { - throw new Error('Not implemented.') -} diff --git a/server/initializers/migrations/0020-requests-endpoint.js b/server/initializers/migrations/0020-requests-endpoint.js deleted file mode 100644 index 55feec571..000000000 --- a/server/initializers/migrations/0020-requests-endpoint.js +++ /dev/null @@ -1,15 +0,0 @@ -/* - Set the endpoint videos for requests. -*/ - -const mongoose = require('mongoose') - -const Request = mongoose.model('Request') - -exports.up = function (callback) { - Request.update({ }, { endpoint: 'videos' }, callback) -} - -exports.down = function (callback) { - throw new Error('Not implemented.') -} diff --git a/server/initializers/migrations/0025-video-filenames.js b/server/initializers/migrations/0025-video-filenames.js deleted file mode 100644 index df21494d7..000000000 --- a/server/initializers/migrations/0025-video-filenames.js +++ /dev/null @@ -1,57 +0,0 @@ -/* - Rename thumbnails and video filenames to _id.extension -*/ - -const each = require('async/each') -const fs = require('fs') -const path = require('path') -const mongoose = require('mongoose') - -const constants = require('../constants') -const logger = require('../../helpers/logger') - -const Video = mongoose.model('Video') - -exports.up = function (callback) { - // Use of lean because the new Video scheme does not have filename field - Video.find({ filename: { $ne: null } }).lean().exec(function (err, videos) { - if (err) throw err - - each(videos, function (video, callbackEach) { - const torrentName = video.filename + '.torrent' - const thumbnailName = video.thumbnail - const thumbnailExtension = path.extname(thumbnailName) - const videoName = video.filename - const videoExtension = path.extname(videoName) - - const newTorrentName = video._id + '.torrent' - const newThumbnailName = video._id + thumbnailExtension - const newVideoName = video._id + videoExtension - - const torrentsDir = constants.CONFIG.STORAGE.TORRENTS_DIR - const thumbnailsDir = constants.CONFIG.STORAGE.THUMBNAILS_DIR - const videosDir = constants.CONFIG.STORAGE.VIDEOS_DIR - - logger.info('Renaming %s to %s.', torrentsDir + torrentName, torrentsDir + newTorrentName) - fs.renameSync(torrentsDir + torrentName, torrentsDir + newTorrentName) - - logger.info('Renaming %s to %s.', thumbnailsDir + thumbnailName, thumbnailsDir + newThumbnailName) - fs.renameSync(thumbnailsDir + thumbnailName, thumbnailsDir + newThumbnailName) - - logger.info('Renaming %s to %s.', videosDir + videoName, videosDir + newVideoName) - fs.renameSync(videosDir + videoName, videosDir + newVideoName) - - Video.load(video._id, function (err, videoObj) { - if (err) return callbackEach(err) - - videoObj.extname = videoExtension - videoObj.remoteId = null - videoObj.save(callbackEach) - }) - }, callback) - }) -} - -exports.down = function (callback) { - throw new Error('Not implemented.') -} diff --git a/server/initializers/migrations/0030-video-magnet.js b/server/initializers/migrations/0030-video-magnet.js deleted file mode 100644 index b9119d61c..000000000 --- a/server/initializers/migrations/0030-video-magnet.js +++ /dev/null @@ -1,32 +0,0 @@ -/* - Change video magnet structures -*/ - -const each = require('async/each') -const magnet = require('magnet-uri') -const mongoose = require('mongoose') - -const Video = mongoose.model('Video') - -exports.up = function (callback) { - // Use of lean because the new Video scheme does not have magnetUri field - Video.find({ }).lean().exec(function (err, videos) { - if (err) throw err - - each(videos, function (video, callbackEach) { - const parsed = magnet.decode(video.magnetUri) - const infoHash = parsed.infoHash - - Video.load(video._id, function (err, videoObj) { - if (err) return callbackEach(err) - - videoObj.magnet.infoHash = infoHash - videoObj.save(callbackEach) - }) - }, callback) - }) -} - -exports.down = function (callback) { - throw new Error('Not implemented.') -} diff --git a/server/initializers/migrations/0035-url-to-host.js b/server/initializers/migrations/0035-url-to-host.js deleted file mode 100644 index 6243304d5..000000000 --- a/server/initializers/migrations/0035-url-to-host.js +++ /dev/null @@ -1,30 +0,0 @@ -/* - Change video magnet structures -*/ - -const each = require('async/each') -const mongoose = require('mongoose') - -const Video = mongoose.model('Video') - -exports.up = function (callback) { - // Use of lean because the new Video scheme does not have podUrl field - Video.find({ }).lean().exec(function (err, videos) { - if (err) throw err - - each(videos, function (video, callbackEach) { - Video.load(video._id, function (err, videoObj) { - if (err) return callbackEach(err) - - const host = video.podUrl.split('://')[1] - - videoObj.podHost = host - videoObj.save(callbackEach) - }) - }, callback) - }) -} - -exports.down = function (callback) { - throw new Error('Not implemented.') -} diff --git a/server/initializers/migrations/0040-video-remote-id.js b/server/initializers/migrations/0040-video-remote-id.js deleted file mode 100644 index 46a14a689..000000000 --- a/server/initializers/migrations/0040-video-remote-id.js +++ /dev/null @@ -1,59 +0,0 @@ -/* - Use remote id as identifier -*/ - -const map = require('lodash/map') -const mongoose = require('mongoose') -const readline = require('readline') - -const rl = readline.createInterface({ - input: process.stdin, - output: process.stdout -}) - -const logger = require('../../helpers/logger') -const friends = require('../../lib/friends') - -const Pod = mongoose.model('Pod') -const Video = mongoose.model('Video') - -exports.up = function (callback) { - Pod.find({}).lean().exec(function (err, pods) { - if (err) return callback(err) - - // We need to quit friends first - if (pods.length === 0) { - return setVideosRemoteId(callback) - } - - const timeout = setTimeout(function () { - throw new Error('You need to enter a value!') - }, 10000) - - rl.question('I am sorry but I need to quit friends for upgrading. Do you want to continue? (yes/*)', function (answer) { - if (answer !== 'yes') throw new Error('I cannot continue.') - - clearTimeout(timeout) - rl.close() - - const urls = map(pods, 'url') - logger.info('Saying goodbye to: ' + urls.join(', ')) - - setVideosRemoteId(function () { - friends.quitFriends(callback) - }) - }) - }) -} - -exports.down = function (callback) { - throw new Error('Not implemented.') -} - -function setVideosRemoteId (callback) { - Video.update({ filename: { $ne: null } }, { remoteId: null }, function (err) { - if (err) throw err - - Video.update({ filename: null }, { remoteId: mongoose.Types.ObjectId() }, callback) - }) -} diff --git a/server/initializers/migrator.js b/server/initializers/migrator.js index 6b31d994f..e5288b615 100644 --- a/server/initializers/migrator.js +++ b/server/initializers/migrator.js @@ -1,48 +1,36 @@ 'use strict' const eachSeries = require('async/eachSeries') -const mongoose = require('mongoose') +const fs = require('fs') const path = require('path') const constants = require('./constants') +const db = require('./database') const logger = require('../helpers/logger') -const Application = mongoose.model('Application') - const migrator = { migrate: migrate } function migrate (callback) { - Application.loadMongoSchemaVersion(function (err, actualVersion) { + db.Application.loadMigrationVersion(function (err, actualVersion) { if (err) return callback(err) - // If there are a new mongo schemas - if (!actualVersion || actualVersion < constants.LAST_MONGO_SCHEMA_VERSION) { + // If there are a new migration scripts + if (actualVersion < constants.LAST_MIGRATION_VERSION) { logger.info('Begin migrations.') - eachSeries(constants.MONGO_MIGRATION_SCRIPTS, function (entity, callbackEach) { - const versionScript = entity.version - - // Do not execute old migration scripts - if (versionScript <= actualVersion) return callbackEach(null) - - // Load the migration module and run it - const migrationScriptName = entity.script - logger.info('Executing %s migration script.', migrationScriptName) - - const migrationScript = require(path.join(__dirname, 'migrations', migrationScriptName)) - migrationScript.up(function (err) { - if (err) return callbackEach(err) - - // Update the new mongo version schema - Application.updateMongoSchemaVersion(versionScript, callbackEach) - }) - }, function (err) { + getMigrationScripts(function (err, migrationScripts) { if (err) return callback(err) - logger.info('Migrations finished. New mongo version schema: %s', constants.LAST_MONGO_SCHEMA_VERSION) - return callback(null) + eachSeries(migrationScripts, function (entity, callbackEach) { + executeMigration(actualVersion, entity, callbackEach) + }, function (err) { + if (err) return callback(err) + + logger.info('Migrations finished. New migration version schema: %s', constants.LAST_MIGRATION_VERSION) + return callback(null) + }) }) } else { return callback(null) @@ -54,3 +42,57 @@ function migrate (callback) { module.exports = migrator +// --------------------------------------------------------------------------- + +function getMigrationScripts (callback) { + fs.readdir(path.join(__dirname, 'migrations'), function (err, files) { + if (err) return callback(err) + + const filesToMigrate = [] + + files.forEach(function (file) { + // Filename is something like 'version-blabla.js' + const version = file.split('-')[0] + filesToMigrate.push({ + version, + script: file + }) + }) + + return callback(err, filesToMigrate) + }) +} + +function executeMigration (actualVersion, entity, callback) { + const versionScript = entity.version + + // Do not execute old migration scripts + if (versionScript <= actualVersion) return callback(null) + + // Load the migration module and run it + const migrationScriptName = entity.script + logger.info('Executing %s migration script.', migrationScriptName) + + const migrationScript = require(path.join(__dirname, 'migrations', migrationScriptName)) + + db.sequelize.transaction().asCallback(function (err, t) { + if (err) return callback(err) + + migrationScript.up({ transaction: t }, function (err) { + if (err) { + t.rollback() + return callback(err) + } + + // Update the new migration version + db.Application.updateMigrationVersion(versionScript, t, function (err) { + if (err) { + t.rollback() + return callback(err) + } + + t.commit().asCallback(callback) + }) + }) + }) +} diff --git a/server/lib/friends.js b/server/lib/friends.js index eaea040ca..f0575ff2f 100644 --- a/server/lib/friends.js +++ b/server/lib/friends.js @@ -4,20 +4,18 @@ const each = require('async/each') const eachLimit = require('async/eachLimit') const eachSeries = require('async/eachSeries') const fs = require('fs') -const mongoose = require('mongoose') const request = require('request') const waterfall = require('async/waterfall') const constants = require('../initializers/constants') +const db = require('../initializers/database') const logger = require('../helpers/logger') const requests = require('../helpers/requests') -const Pod = mongoose.model('Pod') -const Request = mongoose.model('Request') -const Video = mongoose.model('Video') - const friends = { addVideoToFriends, + updateVideoToFriends, + reportAbuseVideoToFriend, hasFriends, getMyCertificate, makeFriends, @@ -26,12 +24,47 @@ const friends = { sendOwnedVideosToPod } -function addVideoToFriends (video) { - createRequest('add', constants.REQUEST_ENDPOINTS.VIDEOS, video) +function addVideoToFriends (videoData, transaction, callback) { + const options = { + type: 'add', + endpoint: constants.REQUEST_ENDPOINTS.VIDEOS, + data: videoData, + transaction + } + createRequest(options, callback) +} + +function updateVideoToFriends (videoData, transaction, callback) { + const options = { + type: 'update', + endpoint: constants.REQUEST_ENDPOINTS.VIDEOS, + data: videoData, + transaction + } + createRequest(options, callback) +} + +function removeVideoToFriends (videoParams) { + const options = { + type: 'remove', + endpoint: constants.REQUEST_ENDPOINTS.VIDEOS, + data: videoParams + } + createRequest(options) +} + +function reportAbuseVideoToFriend (reportData, video) { + const options = { + type: 'report-abuse', + endpoint: constants.REQUEST_ENDPOINTS.VIDEOS, + data: reportData, + toIds: [ video.Author.podId ] + } + createRequest(options) } function hasFriends (callback) { - Pod.countAll(function (err, count) { + db.Pod.countAll(function (err, count) { if (err) return callback(err) const hasFriends = (count !== 0) @@ -69,13 +102,15 @@ function makeFriends (hosts, callback) { function quitFriends (callback) { // Stop pool requests - Request.deactivate() - // Flush pool requests - Request.flush() + db.Request.deactivate() waterfall([ + function flushRequests (callbackAsync) { + db.Request.flush(callbackAsync) + }, + function getPodsList (callbackAsync) { - return Pod.list(callbackAsync) + return db.Pod.list(callbackAsync) }, function announceIQuitMyFriends (pods, callbackAsync) { @@ -103,12 +138,12 @@ function quitFriends (callback) { function removePodsFromDB (pods, callbackAsync) { each(pods, function (pod, callbackEach) { - pod.remove(callbackEach) + pod.destroy().asCallback(callbackEach) }, callbackAsync) } ], function (err) { // Don't forget to re activate the scheduler, even if there was an error - Request.activate() + db.Request.activate() if (err) return callback(err) @@ -117,26 +152,28 @@ function quitFriends (callback) { }) } -function removeVideoToFriends (videoParams) { - createRequest('remove', constants.REQUEST_ENDPOINTS.VIDEOS, videoParams) -} - function sendOwnedVideosToPod (podId) { - Video.listOwned(function (err, videosList) { + db.Video.listOwnedAndPopulateAuthorAndTags(function (err, videosList) { if (err) { logger.error('Cannot get the list of videos we own.') return } videosList.forEach(function (video) { - video.toRemoteJSON(function (err, remoteVideo) { + video.toAddRemoteJSON(function (err, remoteVideo) { if (err) { logger.error('Cannot convert video to remote.', { error: err }) // Don't break the process return } - createRequest('add', constants.REQUEST_ENDPOINTS.VIDEOS, remoteVideo, [ podId ]) + const options = { + type: 'add', + endpoint: constants.REQUEST_ENDPOINTS.VIDEOS, + data: remoteVideo, + toIds: [ podId ] + } + createRequest(options) }) }) }) @@ -149,10 +186,10 @@ module.exports = friends // --------------------------------------------------------------------------- function computeForeignPodsList (host, podsScore, callback) { - getForeignPodsList(host, function (err, foreignPodsList) { + getForeignPodsList(host, function (err, res) { if (err) return callback(err) - if (!foreignPodsList) foreignPodsList = [] + const foreignPodsList = res.data // Let's give 1 point to the pod we ask the friends list foreignPodsList.push({ host }) @@ -200,9 +237,9 @@ function getForeignPodsList (host, callback) { function makeRequestsToWinningPods (cert, podsList, callback) { // Stop pool requests - Request.deactivate() + db.Request.deactivate() // Flush pool requests - Request.forceSend() + db.Request.forceSend() eachLimit(podsList, constants.REQUESTS_IN_PARALLEL, function (pod, callbackEach) { const params = { @@ -222,15 +259,15 @@ function makeRequestsToWinningPods (cert, podsList, callback) { } if (res.statusCode === 200) { - const podObj = new Pod({ host: pod.host, publicKey: body.cert }) - podObj.save(function (err, podCreated) { + const podObj = db.Pod.build({ host: pod.host, publicKey: body.cert }) + podObj.save().asCallback(function (err, podCreated) { if (err) { logger.error('Cannot add friend %s pod.', pod.host, { error: err }) return callbackEach() } // Add our videos to the request scheduler - sendOwnedVideosToPod(podCreated._id) + sendOwnedVideosToPod(podCreated.id) return callbackEach() }) @@ -242,28 +279,64 @@ function makeRequestsToWinningPods (cert, podsList, callback) { }, function endRequests () { // Final callback, we've ended all the requests // Now we made new friends, we can re activate the pool of requests - Request.activate() + db.Request.activate() logger.debug('makeRequestsToWinningPods finished.') return callback() }) } -function createRequest (type, endpoint, data, to) { - const req = new Request({ +// Wrapper that populate "toIds" argument with all our friends if it is not specified +// { type, endpoint, data, toIds, transaction } +function createRequest (options, callback) { + if (!callback) callback = function () {} + if (options.toIds) return _createRequest(options, callback) + + // If the "toIds" pods is not specified, we send the request to all our friends + db.Pod.listAllIds(options.transaction, function (err, podIds) { + if (err) { + logger.error('Cannot get pod ids', { error: err }) + return + } + + const newOptions = Object.assign(options, { toIds: podIds }) + return _createRequest(newOptions, callback) + }) +} + +// { type, endpoint, data, toIds, transaction } +function _createRequest (options, callback) { + const type = options.type + const endpoint = options.endpoint + const data = options.data + const toIds = options.toIds + const transaction = options.transaction + + const pods = [] + + // If there are no destination pods abort + if (toIds.length === 0) return callback(null) + + toIds.forEach(function (toPod) { + pods.push(db.Pod.build({ id: toPod })) + }) + + const createQuery = { endpoint, request: { type: type, data: data } - }) - - if (to) { - req.to = to } - req.save(function (err) { - if (err) logger.error('Cannot save the request.', { error: err }) + const dbRequestOptions = { + transaction + } + + return db.Request.create(createQuery, dbRequestOptions).asCallback(function (err, request) { + if (err) return callback(err) + + return request.setPods(pods, dbRequestOptions).asCallback(callback) }) } diff --git a/server/lib/oauth-model.js b/server/lib/oauth-model.js index d011c4b72..1c12f1b14 100644 --- a/server/lib/oauth-model.js +++ b/server/lib/oauth-model.js @@ -1,11 +1,6 @@ -const mongoose = require('mongoose') - +const db = require('../initializers/database') const logger = require('../helpers/logger') -const OAuthClient = mongoose.model('OAuthClient') -const OAuthToken = mongoose.model('OAuthToken') -const User = mongoose.model('User') - // See https://github.com/oauthjs/node-oauth2-server/wiki/Model-specification for the model specifications const OAuthModel = { getAccessToken, @@ -21,27 +16,25 @@ const OAuthModel = { function getAccessToken (bearerToken) { logger.debug('Getting access token (bearerToken: ' + bearerToken + ').') - return OAuthToken.getByTokenAndPopulateUser(bearerToken) + return db.OAuthToken.getByTokenAndPopulateUser(bearerToken) } function getClient (clientId, clientSecret) { logger.debug('Getting Client (clientId: ' + clientId + ', clientSecret: ' + clientSecret + ').') - // TODO req validator - const mongoId = new mongoose.mongo.ObjectID(clientId) - return OAuthClient.getByIdAndSecret(mongoId, clientSecret) + return db.OAuthClient.getByIdAndSecret(clientId, clientSecret) } function getRefreshToken (refreshToken) { logger.debug('Getting RefreshToken (refreshToken: ' + refreshToken + ').') - return OAuthToken.getByRefreshTokenAndPopulateClient(refreshToken) + return db.OAuthToken.getByRefreshTokenAndPopulateClient(refreshToken) } function getUser (username, password) { logger.debug('Getting User (username: ' + username + ', password: ' + password + ').') - return User.getByUsername(username).then(function (user) { + return db.User.getByUsername(username).then(function (user) { if (!user) return null // We need to return a promise @@ -60,8 +53,8 @@ function getUser (username, password) { } function revokeToken (token) { - return OAuthToken.getByRefreshTokenAndPopulateUser(token.refreshToken).then(function (tokenDB) { - if (tokenDB) tokenDB.remove() + return db.OAuthToken.getByRefreshTokenAndPopulateUser(token.refreshToken).then(function (tokenDB) { + if (tokenDB) tokenDB.destroy() /* * Thanks to https://github.com/manjeshpv/node-oauth2-server-implementation/blob/master/components/oauth/mongo-models.js @@ -80,18 +73,19 @@ function revokeToken (token) { function saveToken (token, client, user) { logger.debug('Saving token ' + token.accessToken + ' for client ' + client.id + ' and user ' + user.id + '.') - const tokenObj = new OAuthToken({ + const tokenToCreate = { accessToken: token.accessToken, accessTokenExpiresAt: token.accessTokenExpiresAt, - client: client.id, refreshToken: token.refreshToken, refreshTokenExpiresAt: token.refreshTokenExpiresAt, - user: user.id - }) + oAuthClientId: client.id, + userId: user.id + } - return tokenObj.save().then(function (tokenCreated) { + return db.OAuthToken.create(tokenToCreate).then(function (tokenCreated) { tokenCreated.client = client tokenCreated.user = user + return tokenCreated }).catch(function (err) { throw err diff --git a/server/middlewares/pods.js b/server/middlewares/pods.js index 487ea1259..e38fb341d 100644 --- a/server/middlewares/pods.js +++ b/server/middlewares/pods.js @@ -44,7 +44,6 @@ module.exports = podsMiddleware function getHostWithPort (host) { const splitted = host.split(':') - console.log(splitted) // The port was not specified if (splitted.length === 1) { if (constants.REMOTE_SCHEME.HTTP === 'https') return host + ':443' diff --git a/server/middlewares/secure.js b/server/middlewares/secure.js index ee836beed..b6e6d818b 100644 --- a/server/middlewares/secure.js +++ b/server/middlewares/secure.js @@ -1,18 +1,16 @@ 'use strict' +const db = require('../initializers/database') const logger = require('../helpers/logger') -const mongoose = require('mongoose') const peertubeCrypto = require('../helpers/peertube-crypto') -const Pod = mongoose.model('Pod') - const secureMiddleware = { checkSignature } function checkSignature (req, res, next) { const host = req.body.signature.host - Pod.loadByHost(host, function (err, pod) { + db.Pod.loadByHost(host, function (err, pod) { if (err) { logger.error('Cannot get signed host in body.', { error: err }) return res.sendStatus(500) @@ -25,9 +23,20 @@ function checkSignature (req, res, next) { logger.debug('Checking signature from %s.', host) - const signatureOk = peertubeCrypto.checkSignature(pod.publicKey, host, req.body.signature.signature) + let signatureShouldBe + if (req.body.data) { + signatureShouldBe = req.body.data + } else { + signatureShouldBe = host + } + + const signatureOk = peertubeCrypto.checkSignature(pod.publicKey, signatureShouldBe, req.body.signature.signature) if (signatureOk === true) { + res.locals.secure = { + pod + } + return next() } diff --git a/server/middlewares/sort.js b/server/middlewares/sort.js index f0b7274eb..39e167265 100644 --- a/server/middlewares/sort.js +++ b/server/middlewares/sort.js @@ -2,17 +2,24 @@ const sortMiddleware = { setUsersSort, + setVideoAbusesSort, setVideosSort } function setUsersSort (req, res, next) { - if (!req.query.sort) req.query.sort = '-createdDate' + if (!req.query.sort) req.query.sort = '-createdAt' + + return next() +} + +function setVideoAbusesSort (req, res, next) { + if (!req.query.sort) req.query.sort = '-createdAt' return next() } function setVideosSort (req, res, next) { - if (!req.query.sort) req.query.sort = '-createdDate' + if (!req.query.sort) req.query.sort = '-createdAt' return next() } diff --git a/server/middlewares/validators/remote.js b/server/middlewares/validators/remote.js deleted file mode 100644 index 858d193cc..000000000 --- a/server/middlewares/validators/remote.js +++ /dev/null @@ -1,30 +0,0 @@ -'use strict' - -const checkErrors = require('./utils').checkErrors -const logger = require('../../helpers/logger') - -const validatorsRemote = { - remoteVideos, - signature -} - -function remoteVideos (req, res, next) { - req.checkBody('data').isEachRemoteVideosValid() - - logger.debug('Checking remoteVideos parameters', { parameters: req.body }) - - checkErrors(req, res, next) -} - -function signature (req, res, next) { - req.checkBody('signature.host', 'Should have a signature host').isURL() - req.checkBody('signature.signature', 'Should have a signature').notEmpty() - - logger.debug('Checking signature parameters', { parameters: { signatureHost: req.body.signature.host } }) - - checkErrors(req, res, next) -} - -// --------------------------------------------------------------------------- - -module.exports = validatorsRemote diff --git a/server/middlewares/validators/remote/index.js b/server/middlewares/validators/remote/index.js new file mode 100644 index 000000000..022a2fe50 --- /dev/null +++ b/server/middlewares/validators/remote/index.js @@ -0,0 +1,13 @@ +'use strict' + +const remoteSignatureValidators = require('./signature') +const remoteVideosValidators = require('./videos') + +const validators = { + signature: remoteSignatureValidators, + videos: remoteVideosValidators +} + +// --------------------------------------------------------------------------- + +module.exports = validators diff --git a/server/middlewares/validators/remote/signature.js b/server/middlewares/validators/remote/signature.js new file mode 100644 index 000000000..002232c05 --- /dev/null +++ b/server/middlewares/validators/remote/signature.js @@ -0,0 +1,21 @@ +'use strict' + +const checkErrors = require('../utils').checkErrors +const logger = require('../../../helpers/logger') + +const validatorsRemoteSignature = { + signature +} + +function signature (req, res, next) { + req.checkBody('signature.host', 'Should have a signature host').isURL() + req.checkBody('signature.signature', 'Should have a signature').notEmpty() + + logger.debug('Checking signature parameters', { parameters: { signature: req.body.signature } }) + + checkErrors(req, res, next) +} + +// --------------------------------------------------------------------------- + +module.exports = validatorsRemoteSignature diff --git a/server/middlewares/validators/remote/videos.js b/server/middlewares/validators/remote/videos.js new file mode 100644 index 000000000..cf9925b6c --- /dev/null +++ b/server/middlewares/validators/remote/videos.js @@ -0,0 +1,20 @@ +'use strict' + +const checkErrors = require('../utils').checkErrors +const logger = require('../../../helpers/logger') + +const validatorsRemoteVideos = { + remoteVideos +} + +function remoteVideos (req, res, next) { + req.checkBody('data').isEachRemoteRequestVideosValid() + + logger.debug('Checking remoteVideos parameters', { parameters: req.body }) + + checkErrors(req, res, next) +} + +// --------------------------------------------------------------------------- + +module.exports = validatorsRemoteVideos diff --git a/server/middlewares/validators/sort.js b/server/middlewares/validators/sort.js index 431d3fffd..b7eec0316 100644 --- a/server/middlewares/validators/sort.js +++ b/server/middlewares/validators/sort.js @@ -6,29 +6,38 @@ const logger = require('../../helpers/logger') const validatorsSort = { usersSort, + videoAbusesSort, videosSort } function usersSort (req, res, next) { const sortableColumns = constants.SORTABLE_COLUMNS.USERS - req.checkQuery('sort', 'Should have correct sortable column').optional().isIn(sortableColumns) + checkSort(req, res, next, sortableColumns) +} - logger.debug('Checking sort parameters', { parameters: req.query }) +function videoAbusesSort (req, res, next) { + const sortableColumns = constants.SORTABLE_COLUMNS.VIDEO_ABUSES - checkErrors(req, res, next) + checkSort(req, res, next, sortableColumns) } function videosSort (req, res, next) { const sortableColumns = constants.SORTABLE_COLUMNS.VIDEOS + checkSort(req, res, next, sortableColumns) +} + +// --------------------------------------------------------------------------- + +module.exports = validatorsSort + +// --------------------------------------------------------------------------- + +function checkSort (req, res, next, sortableColumns) { req.checkQuery('sort', 'Should have correct sortable column').optional().isIn(sortableColumns) logger.debug('Checking sort parameters', { parameters: req.query }) checkErrors(req, res, next) } - -// --------------------------------------------------------------------------- - -module.exports = validatorsSort diff --git a/server/middlewares/validators/users.js b/server/middlewares/validators/users.js index 02e4f34cb..0629550bc 100644 --- a/server/middlewares/validators/users.js +++ b/server/middlewares/validators/users.js @@ -1,12 +1,9 @@ 'use strict' -const mongoose = require('mongoose') - const checkErrors = require('./utils').checkErrors +const db = require('../../initializers/database') const logger = require('../../helpers/logger') -const User = mongoose.model('User') - const validatorsUsers = { usersAdd, usersRemove, @@ -20,7 +17,7 @@ function usersAdd (req, res, next) { logger.debug('Checking usersAdd parameters', { parameters: req.body }) checkErrors(req, res, function () { - User.loadByUsername(req.body.username, function (err, user) { + db.User.loadByUsername(req.body.username, function (err, user) { if (err) { logger.error('Error in usersAdd request validator.', { error: err }) return res.sendStatus(500) @@ -34,12 +31,12 @@ function usersAdd (req, res, next) { } function usersRemove (req, res, next) { - req.checkParams('id', 'Should have a valid id').notEmpty().isMongoId() + req.checkParams('id', 'Should have a valid id').notEmpty().isInt() logger.debug('Checking usersRemove parameters', { parameters: req.params }) checkErrors(req, res, function () { - User.loadById(req.params.id, function (err, user) { + db.User.loadById(req.params.id, function (err, user) { if (err) { logger.error('Error in usersRemove request validator.', { error: err }) return res.sendStatus(500) @@ -55,7 +52,7 @@ function usersRemove (req, res, next) { } function usersUpdate (req, res, next) { - req.checkParams('id', 'Should have a valid id').notEmpty().isMongoId() + req.checkParams('id', 'Should have a valid id').notEmpty().isInt() // Add old password verification req.checkBody('password', 'Should have a valid password').isUserPasswordValid() diff --git a/server/middlewares/validators/videos.js b/server/middlewares/validators/videos.js index 76e943e77..4fe6dcd8b 100644 --- a/server/middlewares/validators/videos.js +++ b/server/middlewares/validators/videos.js @@ -1,19 +1,19 @@ 'use strict' -const mongoose = require('mongoose') - const checkErrors = require('./utils').checkErrors const constants = require('../../initializers/constants') const customVideosValidators = require('../../helpers/custom-validators').videos +const db = require('../../initializers/database') const logger = require('../../helpers/logger') -const Video = mongoose.model('Video') - const validatorsVideos = { videosAdd, + videosUpdate, videosGet, videosRemove, - videosSearch + videosSearch, + + videoAbuseReport } function videosAdd (req, res, next) { @@ -29,7 +29,7 @@ function videosAdd (req, res, next) { checkErrors(req, res, function () { const videoFile = req.files.videofile[0] - Video.getDurationFromFile(videoFile.path, function (err, duration) { + db.Video.getDurationFromFile(videoFile.path, function (err, duration) { if (err) { return res.status(400).send('Cannot retrieve metadata of the file.') } @@ -44,40 +44,56 @@ function videosAdd (req, res, next) { }) } -function videosGet (req, res, next) { - req.checkParams('id', 'Should have a valid id').notEmpty().isMongoId() +function videosUpdate (req, res, next) { + req.checkParams('id', 'Should have a valid id').notEmpty().isUUID(4) + req.checkBody('name', 'Should have a valid name').optional().isVideoNameValid() + req.checkBody('description', 'Should have a valid description').optional().isVideoDescriptionValid() + req.checkBody('tags', 'Should have correct tags').optional().isVideoTagsValid() - logger.debug('Checking videosGet parameters', { parameters: req.params }) + logger.debug('Checking videosUpdate parameters', { parameters: req.body }) checkErrors(req, res, function () { - Video.load(req.params.id, function (err, video) { - if (err) { - logger.error('Error in videosGet request validator.', { error: err }) - return res.sendStatus(500) + checkVideoExists(req.params.id, res, function () { + // We need to make additional checks + if (res.locals.video.isOwned() === false) { + return res.status(403).send('Cannot update video of another pod') } - if (!video) return res.status(404).send('Video not found') + if (res.locals.video.Author.userId !== res.locals.oauth.token.User.id) { + return res.status(403).send('Cannot update video of another user') + } next() }) }) } +function videosGet (req, res, next) { + req.checkParams('id', 'Should have a valid id').notEmpty().isUUID(4) + + logger.debug('Checking videosGet parameters', { parameters: req.params }) + + checkErrors(req, res, function () { + checkVideoExists(req.params.id, res, next) + }) +} + function videosRemove (req, res, next) { - req.checkParams('id', 'Should have a valid id').notEmpty().isMongoId() + req.checkParams('id', 'Should have a valid id').notEmpty().isUUID(4) logger.debug('Checking videosRemove parameters', { parameters: req.params }) checkErrors(req, res, function () { - Video.load(req.params.id, function (err, video) { - if (err) { - logger.error('Error in videosRemove request validator.', { error: err }) - return res.sendStatus(500) + checkVideoExists(req.params.id, res, function () { + // We need to make additional checks + + if (res.locals.video.isOwned() === false) { + return res.status(403).send('Cannot remove video of another pod') } - if (!video) return res.status(404).send('Video not found') - else if (video.isOwned() === false) return res.status(403).send('Cannot remove video of another pod') - else if (video.author !== res.locals.oauth.token.user.username) return res.status(403).send('Cannot remove video of another user') + if (res.locals.video.Author.userId !== res.locals.oauth.token.User.id) { + return res.status(403).send('Cannot remove video of another user') + } next() }) @@ -94,6 +110,33 @@ function videosSearch (req, res, next) { checkErrors(req, res, next) } +function videoAbuseReport (req, res, next) { + req.checkParams('id', 'Should have a valid id').notEmpty().isUUID(4) + req.checkBody('reason', 'Should have a valid reason').isVideoAbuseReasonValid() + + logger.debug('Checking videoAbuseReport parameters', { parameters: req.body }) + + checkErrors(req, res, function () { + checkVideoExists(req.params.id, res, next) + }) +} + // --------------------------------------------------------------------------- module.exports = validatorsVideos + +// --------------------------------------------------------------------------- + +function checkVideoExists (id, res, callback) { + db.Video.loadAndPopulateAuthorAndPodAndTags(id, function (err, video) { + if (err) { + logger.error('Error in video request validator.', { error: err }) + return res.sendStatus(500) + } + + if (!video) return res.status(404).send('Video not found') + + res.locals.video = video + callback() + }) +} diff --git a/server/models/application.js b/server/models/application.js index 452ac4283..46dcfde33 100644 --- a/server/models/application.js +++ b/server/models/application.js @@ -1,31 +1,52 @@ -const mongoose = require('mongoose') +'use strict' -// --------------------------------------------------------------------------- +module.exports = function (sequelize, DataTypes) { + const Application = sequelize.define('Application', + { + migrationVersion: { + type: DataTypes.INTEGER, + defaultValue: 0, + allowNull: false, + validate: { + isInt: true + } + } + }, + { + classMethods: { + loadMigrationVersion, + updateMigrationVersion + } + } + ) -const ApplicationSchema = mongoose.Schema({ - mongoSchemaVersion: { - type: Number, - default: 0 - } -}) - -ApplicationSchema.statics = { - loadMongoSchemaVersion, - updateMongoSchemaVersion + return Application } -mongoose.model('Application', ApplicationSchema) - // --------------------------------------------------------------------------- -function loadMongoSchemaVersion (callback) { - return this.findOne({}, { mongoSchemaVersion: 1 }, function (err, data) { - const version = data ? data.mongoSchemaVersion : 0 +function loadMigrationVersion (callback) { + const query = { + attributes: [ 'migrationVersion' ] + } + + return this.findOne(query).asCallback(function (err, data) { + const version = data ? data.migrationVersion : 0 return callback(err, version) }) } -function updateMongoSchemaVersion (newVersion, callback) { - return this.update({}, { mongoSchemaVersion: newVersion }, callback) +function updateMigrationVersion (newVersion, transaction, callback) { + const options = { + where: {} + } + + if (!callback) { + transaction = callback + } else { + options.transaction = transaction + } + + return this.update({ migrationVersion: newVersion }, options).asCallback(callback) } diff --git a/server/models/author.js b/server/models/author.js new file mode 100644 index 000000000..7d15fb6ec --- /dev/null +++ b/server/models/author.js @@ -0,0 +1,85 @@ +'use strict' + +const customUsersValidators = require('../helpers/custom-validators').users + +module.exports = function (sequelize, DataTypes) { + const Author = sequelize.define('Author', + { + name: { + type: DataTypes.STRING, + allowNull: false, + validate: { + usernameValid: function (value) { + const res = customUsersValidators.isUserUsernameValid(value) + if (res === false) throw new Error('Username is not valid.') + } + } + } + }, + { + indexes: [ + { + fields: [ 'name' ] + }, + { + fields: [ 'podId' ] + }, + { + fields: [ 'userId' ] + } + ], + classMethods: { + associate, + + findOrCreateAuthor + } + } + ) + + return Author +} + +// --------------------------------------------------------------------------- + +function associate (models) { + this.belongsTo(models.Pod, { + foreignKey: { + name: 'podId', + allowNull: true + }, + onDelete: 'cascade' + }) + + this.belongsTo(models.User, { + foreignKey: { + name: 'userId', + allowNull: true + }, + onDelete: 'cascade' + }) +} + +function findOrCreateAuthor (name, podId, userId, transaction, callback) { + if (!callback) { + callback = transaction + transaction = null + } + + const author = { + name, + podId, + userId + } + + const query = { + where: author, + defaults: author + } + + if (transaction) query.transaction = transaction + + this.findOrCreate(query).asCallback(function (err, result) { + // [ instance, wasCreated ] + return callback(err, result[0]) + }) +} diff --git a/server/models/oauth-client.js b/server/models/oauth-client.js index a1aefa985..021a34007 100644 --- a/server/models/oauth-client.js +++ b/server/models/oauth-client.js @@ -1,33 +1,62 @@ -const mongoose = require('mongoose') +'use strict' -// --------------------------------------------------------------------------- +module.exports = function (sequelize, DataTypes) { + const OAuthClient = sequelize.define('OAuthClient', + { + clientId: { + type: DataTypes.STRING, + allowNull: false + }, + clientSecret: { + type: DataTypes.STRING, + allowNull: false + }, + grants: { + type: DataTypes.ARRAY(DataTypes.STRING) + }, + redirectUris: { + type: DataTypes.ARRAY(DataTypes.STRING) + } + }, + { + indexes: [ + { + fields: [ 'clientId' ], + unique: true + }, + { + fields: [ 'clientId', 'clientSecret' ], + unique: true + } + ], + classMethods: { + countTotal, + getByIdAndSecret, + loadFirstClient + } + } + ) -const OAuthClientSchema = mongoose.Schema({ - clientSecret: String, - grants: Array, - redirectUris: Array -}) - -OAuthClientSchema.path('clientSecret').required(true) - -OAuthClientSchema.statics = { - getByIdAndSecret, - list, - loadFirstClient + return OAuthClient } -mongoose.model('OAuthClient', OAuthClientSchema) - // --------------------------------------------------------------------------- -function list (callback) { - return this.find(callback) +function countTotal (callback) { + return this.count().asCallback(callback) } function loadFirstClient (callback) { - return this.findOne({}, callback) + return this.findOne().asCallback(callback) } -function getByIdAndSecret (id, clientSecret) { - return this.findOne({ _id: id, clientSecret: clientSecret }).exec() +function getByIdAndSecret (clientId, clientSecret) { + const query = { + where: { + clientId: clientId, + clientSecret: clientSecret + } + } + + return this.findOne(query) } diff --git a/server/models/oauth-token.js b/server/models/oauth-token.js index aff73bfb1..68e7c9ff7 100644 --- a/server/models/oauth-token.js +++ b/server/models/oauth-token.js @@ -1,42 +1,96 @@ -const mongoose = require('mongoose') +'use strict' const logger = require('../helpers/logger') // --------------------------------------------------------------------------- -const OAuthTokenSchema = mongoose.Schema({ - accessToken: String, - accessTokenExpiresAt: Date, - client: { type: mongoose.Schema.Types.ObjectId, ref: 'OAuthClient' }, - refreshToken: String, - refreshTokenExpiresAt: Date, - user: { type: mongoose.Schema.Types.ObjectId, ref: 'User' } -}) +module.exports = function (sequelize, DataTypes) { + const OAuthToken = sequelize.define('OAuthToken', + { + accessToken: { + type: DataTypes.STRING, + allowNull: false + }, + accessTokenExpiresAt: { + type: DataTypes.DATE, + allowNull: false + }, + refreshToken: { + type: DataTypes.STRING, + allowNull: false + }, + refreshTokenExpiresAt: { + type: DataTypes.DATE, + allowNull: false + } + }, + { + indexes: [ + { + fields: [ 'refreshToken' ], + unique: true + }, + { + fields: [ 'accessToken' ], + unique: true + }, + { + fields: [ 'userId' ] + }, + { + fields: [ 'oAuthClientId' ] + } + ], + classMethods: { + associate, -OAuthTokenSchema.path('accessToken').required(true) -OAuthTokenSchema.path('client').required(true) -OAuthTokenSchema.path('user').required(true) + getByRefreshTokenAndPopulateClient, + getByTokenAndPopulateUser, + getByRefreshTokenAndPopulateUser, + removeByUserId + } + } + ) -OAuthTokenSchema.statics = { - getByRefreshTokenAndPopulateClient, - getByTokenAndPopulateUser, - getByRefreshTokenAndPopulateUser, - removeByUserId + return OAuthToken } -mongoose.model('OAuthToken', OAuthTokenSchema) - // --------------------------------------------------------------------------- +function associate (models) { + this.belongsTo(models.User, { + foreignKey: { + name: 'userId', + allowNull: false + }, + onDelete: 'cascade' + }) + + this.belongsTo(models.OAuthClient, { + foreignKey: { + name: 'oAuthClientId', + allowNull: false + }, + onDelete: 'cascade' + }) +} + function getByRefreshTokenAndPopulateClient (refreshToken) { - return this.findOne({ refreshToken: refreshToken }).populate('client').exec().then(function (token) { + const query = { + where: { + refreshToken: refreshToken + }, + include: [ this.associations.OAuthClient ] + } + + return this.findOne(query).then(function (token) { if (!token) return token const tokenInfos = { refreshToken: token.refreshToken, refreshTokenExpiresAt: token.refreshTokenExpiresAt, client: { - id: token.client._id.toString() + id: token.client.id }, user: { id: token.user @@ -50,13 +104,41 @@ function getByRefreshTokenAndPopulateClient (refreshToken) { } function getByTokenAndPopulateUser (bearerToken) { - return this.findOne({ accessToken: bearerToken }).populate('user').exec() + const query = { + where: { + accessToken: bearerToken + }, + include: [ this.sequelize.models.User ] + } + + return this.findOne(query).then(function (token) { + if (token) token.user = token.User + + return token + }) } function getByRefreshTokenAndPopulateUser (refreshToken) { - return this.findOne({ refreshToken: refreshToken }).populate('user').exec() + const query = { + where: { + refreshToken: refreshToken + }, + include: [ this.sequelize.models.User ] + } + + return this.findOne(query).then(function (token) { + token.user = token.User + + return token + }) } function removeByUserId (userId, callback) { - return this.remove({ user: userId }, callback) + const query = { + where: { + userId: userId + } + } + + return this.destroy(query).asCallback(callback) } diff --git a/server/models/pod.js b/server/models/pod.js new file mode 100644 index 000000000..b3c6db8e8 --- /dev/null +++ b/server/models/pod.js @@ -0,0 +1,200 @@ +'use strict' + +const map = require('lodash/map') + +const constants = require('../initializers/constants') +const customPodsValidators = require('../helpers/custom-validators').pods + +// --------------------------------------------------------------------------- + +module.exports = function (sequelize, DataTypes) { + const Pod = sequelize.define('Pod', + { + host: { + type: DataTypes.STRING, + allowNull: false, + validate: { + isHost: function (value) { + const res = customPodsValidators.isHostValid(value) + if (res === false) throw new Error('Host not valid.') + } + } + }, + publicKey: { + type: DataTypes.STRING(5000), + allowNull: false + }, + score: { + type: DataTypes.INTEGER, + defaultValue: constants.FRIEND_SCORE.BASE, + allowNull: false, + validate: { + isInt: true, + max: constants.FRIEND_SCORE.MAX + } + } + }, + { + indexes: [ + { + fields: [ 'host' ] + }, + { + fields: [ 'score' ] + } + ], + classMethods: { + associate, + + countAll, + incrementScores, + list, + listAllIds, + listRandomPodIdsWithRequest, + listBadPods, + load, + loadByHost, + removeAll + }, + instanceMethods: { + toFormatedJSON + } + } + ) + + return Pod +} + +// ------------------------------ METHODS ------------------------------ + +function toFormatedJSON () { + const json = { + id: this.id, + host: this.host, + score: this.score, + createdAt: this.createdAt + } + + return json +} + +// ------------------------------ Statics ------------------------------ + +function associate (models) { + this.belongsToMany(models.Request, { + foreignKey: 'podId', + through: models.RequestToPod, + onDelete: 'cascade' + }) +} + +function countAll (callback) { + return this.count().asCallback(callback) +} + +function incrementScores (ids, value, callback) { + if (!callback) callback = function () {} + + const update = { + score: this.sequelize.literal('score +' + value) + } + + const options = { + where: { + id: { + $in: ids + } + }, + // In this case score is a literal and not an integer so we do not validate it + validate: false + } + + return this.update(update, options).asCallback(callback) +} + +function list (callback) { + return this.findAll().asCallback(callback) +} + +function listAllIds (transaction, callback) { + if (!callback) { + callback = transaction + transaction = null + } + + const query = { + attributes: [ 'id' ] + } + + if (transaction) query.transaction = transaction + + return this.findAll(query).asCallback(function (err, pods) { + if (err) return callback(err) + + return callback(null, map(pods, 'id')) + }) +} + +function listRandomPodIdsWithRequest (limit, callback) { + const self = this + + self.count().asCallback(function (err, count) { + if (err) return callback(err) + + // Optimization... + if (count === 0) return callback(null, []) + + let start = Math.floor(Math.random() * count) - limit + if (start < 0) start = 0 + + const query = { + attributes: [ 'id' ], + order: [ + [ 'id', 'ASC' ] + ], + offset: start, + limit: limit, + where: { + id: { + $in: [ + this.sequelize.literal('SELECT "podId" FROM "RequestToPods"') + ] + } + } + } + + return this.findAll(query).asCallback(function (err, pods) { + if (err) return callback(err) + + return callback(null, map(pods, 'id')) + }) + }) +} + +function listBadPods (callback) { + const query = { + where: { + score: { $lte: 0 } + } + } + + return this.findAll(query).asCallback(callback) +} + +function load (id, callback) { + return this.findById(id).asCallback(callback) +} + +function loadByHost (host, callback) { + const query = { + where: { + host: host + } + } + + return this.findOne(query).asCallback(callback) +} + +function removeAll (callback) { + return this.destroy().asCallback(callback) +} diff --git a/server/models/pods.js b/server/models/pods.js deleted file mode 100644 index 49c73472a..000000000 --- a/server/models/pods.js +++ /dev/null @@ -1,119 +0,0 @@ -'use strict' - -const each = require('async/each') -const mongoose = require('mongoose') -const map = require('lodash/map') -const validator = require('express-validator').validator - -const constants = require('../initializers/constants') - -const Video = mongoose.model('Video') - -// --------------------------------------------------------------------------- - -const PodSchema = mongoose.Schema({ - host: String, - publicKey: String, - score: { type: Number, max: constants.FRIEND_SCORE.MAX }, - createdDate: { - type: Date, - default: Date.now - } -}) - -PodSchema.path('host').validate(validator.isURL) -PodSchema.path('publicKey').required(true) -PodSchema.path('score').validate(function (value) { return !isNaN(value) }) - -PodSchema.methods = { - toFormatedJSON -} - -PodSchema.statics = { - countAll, - incrementScores, - list, - listAllIds, - listBadPods, - load, - loadByHost, - removeAll -} - -PodSchema.pre('save', function (next) { - const self = this - - Pod.loadByHost(this.host, function (err, pod) { - if (err) return next(err) - - if (pod) return next(new Error('Pod already exists.')) - - self.score = constants.FRIEND_SCORE.BASE - return next() - }) -}) - -PodSchema.pre('remove', function (next) { - // Remove the videos owned by this pod too - Video.listByHost(this.host, function (err, videos) { - if (err) return next(err) - - each(videos, function (video, callbackEach) { - video.remove(callbackEach) - }, next) - }) -}) - -const Pod = mongoose.model('Pod', PodSchema) - -// ------------------------------ METHODS ------------------------------ - -function toFormatedJSON () { - const json = { - id: this._id, - host: this.host, - score: this.score, - createdDate: this.createdDate - } - - return json -} - -// ------------------------------ Statics ------------------------------ - -function countAll (callback) { - return this.count(callback) -} - -function incrementScores (ids, value, callback) { - if (!callback) callback = function () {} - return this.update({ _id: { $in: ids } }, { $inc: { score: value } }, { multi: true }, callback) -} - -function list (callback) { - return this.find(callback) -} - -function listAllIds (callback) { - return this.find({}, { _id: 1 }, function (err, pods) { - if (err) return callback(err) - - return callback(null, map(pods, '_id')) - }) -} - -function listBadPods (callback) { - return this.find({ score: 0 }, callback) -} - -function load (id, callback) { - return this.findById(id, callback) -} - -function loadByHost (host, callback) { - return this.findOne({ host }, callback) -} - -function removeAll (callback) { - return this.remove({}, callback) -} diff --git a/server/models/request-to-pod.js b/server/models/request-to-pod.js new file mode 100644 index 000000000..f42a53458 --- /dev/null +++ b/server/models/request-to-pod.js @@ -0,0 +1,42 @@ +'use strict' + +// --------------------------------------------------------------------------- + +module.exports = function (sequelize, DataTypes) { + const RequestToPod = sequelize.define('RequestToPod', {}, { + indexes: [ + { + fields: [ 'requestId' ] + }, + { + fields: [ 'podId' ] + }, + { + fields: [ 'requestId', 'podId' ], + unique: true + } + ], + classMethods: { + removePodOf + } + }) + + return RequestToPod +} + +// --------------------------------------------------------------------------- + +function removePodOf (requestsIds, podId, callback) { + if (!callback) callback = function () {} + + const query = { + where: { + requestId: { + $in: requestsIds + }, + podId: podId + } + } + + this.destroy(query).asCallback(callback) +} diff --git a/server/models/request.js b/server/models/request.js index c2cfe83ce..cd52ea767 100644 --- a/server/models/request.js +++ b/server/models/request.js @@ -2,66 +2,60 @@ const each = require('async/each') const eachLimit = require('async/eachLimit') -const values = require('lodash/values') -const mongoose = require('mongoose') const waterfall = require('async/waterfall') +const values = require('lodash/values') const constants = require('../initializers/constants') const logger = require('../helpers/logger') const requests = require('../helpers/requests') -const Pod = mongoose.model('Pod') - let timer = null let lastRequestTimestamp = 0 // --------------------------------------------------------------------------- -const RequestSchema = mongoose.Schema({ - request: mongoose.Schema.Types.Mixed, - endpoint: { - type: String, - enum: [ values(constants.REQUEST_ENDPOINTS) ] - }, - to: [ +module.exports = function (sequelize, DataTypes) { + const Request = sequelize.define('Request', { - type: mongoose.Schema.Types.ObjectId, - ref: 'Pod' - } - ] -}) + request: { + type: DataTypes.JSON, + allowNull: false + }, + endpoint: { + type: DataTypes.ENUM(values(constants.REQUEST_ENDPOINTS)), + allowNull: false + } + }, + { + classMethods: { + associate, -RequestSchema.statics = { - activate, - deactivate, - flush, - forceSend, - list, - remainingMilliSeconds + activate, + countTotalRequests, + deactivate, + flush, + forceSend, + remainingMilliSeconds + } + } + ) + + return Request } -RequestSchema.pre('save', function (next) { - const self = this - - if (self.to.length === 0) { - Pod.listAllIds(function (err, podIds) { - if (err) return next(err) - - // No friends - if (podIds.length === 0) return - - self.to = podIds - return next() - }) - } else { - return next() - } -}) - -mongoose.model('Request', RequestSchema) - // ------------------------------ STATICS ------------------------------ +function associate (models) { + this.belongsToMany(models.Pod, { + foreignKey: { + name: 'requestId', + allowNull: false + }, + through: models.RequestToPod, + onDelete: 'CASCADE' + }) +} + function activate () { logger.info('Requests scheduler activated.') lastRequestTimestamp = Date.now() @@ -73,15 +67,25 @@ function activate () { }, constants.REQUESTS_INTERVAL) } +function countTotalRequests (callback) { + const query = { + include: [ this.sequelize.models.Pod ] + } + + return this.count(query).asCallback(callback) +} + function deactivate () { logger.info('Requests scheduler deactivated.') clearInterval(timer) timer = null } -function flush () { +function flush (callback) { removeAll.call(this, function (err) { if (err) logger.error('Cannot flush the requests.', { error: err }) + + return callback(err) }) } @@ -90,10 +94,6 @@ function forceSend () { makeRequests.call(this) } -function list (callback) { - this.find({ }, callback) -} - function remainingMilliSeconds () { if (timer === null) return -1 @@ -122,7 +122,7 @@ function makeRequest (toPod, requestEndpoint, requestsToMake, callback) { 'Error sending secure request to %s pod.', toPod.host, { - error: err || new Error('Status code not 20x : ' + res.statusCode) + error: err ? err.message : 'Status code not 20x : ' + res.statusCode } ) @@ -136,10 +136,11 @@ function makeRequest (toPod, requestEndpoint, requestsToMake, callback) { // Make all the requests of the scheduler function makeRequests () { const self = this + const RequestToPod = this.sequelize.models.RequestToPod - // We limit the size of the requests (REQUESTS_LIMIT) + // We limit the size of the requests // We don't want to stuck with the same failing requests so we get a random list - listWithLimitAndRandom.call(self, constants.REQUESTS_LIMIT, function (err, requests) { + listWithLimitAndRandom.call(self, constants.REQUESTS_LIMIT_PODS, constants.REQUESTS_LIMIT_PER_POD, function (err, requests) { if (err) { logger.error('Cannot get the list of requests.', { err: err }) return // Abort @@ -151,78 +152,77 @@ function makeRequests () { return } - logger.info('Making requests to friends.') - // We want to group requests by destinations pod and endpoint const requestsToMakeGrouped = {} + Object.keys(requests).forEach(function (toPodId) { + requests[toPodId].forEach(function (data) { + const request = data.request + const pod = data.pod + const hashKey = toPodId + request.endpoint - requests.forEach(function (poolRequest) { - poolRequest.to.forEach(function (toPodId) { - const hashKey = toPodId + poolRequest.endpoint if (!requestsToMakeGrouped[hashKey]) { requestsToMakeGrouped[hashKey] = { - toPodId, - endpoint: poolRequest.endpoint, - ids: [], // pool request ids, to delete them from the DB in the future + toPod: pod, + endpoint: request.endpoint, + ids: [], // request ids, to delete them from the DB in the future datas: [] // requests data, } } - requestsToMakeGrouped[hashKey].ids.push(poolRequest._id) - requestsToMakeGrouped[hashKey].datas.push(poolRequest.request) + requestsToMakeGrouped[hashKey].ids.push(request.id) + requestsToMakeGrouped[hashKey].datas.push(request.request) }) }) + logger.info('Making requests to friends.') + const goodPods = [] const badPods = [] eachLimit(Object.keys(requestsToMakeGrouped), constants.REQUESTS_IN_PARALLEL, function (hashKey, callbackEach) { const requestToMake = requestsToMakeGrouped[hashKey] + const toPod = requestToMake.toPod - // FIXME: mongodb request inside a loop :/ - Pod.load(requestToMake.toPodId, function (err, toPod) { - if (err) { - logger.error('Error finding pod by id.', { err: err }) - return callbackEach() + // Maybe the pod is not our friend anymore so simply remove it + if (!toPod) { + const requestIdsToDelete = requestToMake.ids + + logger.info('Removing %d requests of unexisting pod %s.', requestIdsToDelete.length, requestToMake.toPod.id) + RequestToPod.removePodOf.call(self, requestIdsToDelete, requestToMake.toPod.id) + return callbackEach() + } + + makeRequest(toPod, requestToMake.endpoint, requestToMake.datas, function (success) { + if (success === true) { + logger.debug('Removing requests for pod %s.', requestToMake.toPod.id, { requestsIds: requestToMake.ids }) + + goodPods.push(requestToMake.toPod.id) + + // Remove the pod id of these request ids + RequestToPod.removePodOf(requestToMake.ids, requestToMake.toPod.id, callbackEach) + } else { + badPods.push(requestToMake.toPod.id) + callbackEach() } - - // Maybe the pod is not our friend anymore so simply remove it - if (!toPod) { - const requestIdsToDelete = requestToMake.ids - - logger.info('Removing %d requests of unexisting pod %s.', requestIdsToDelete.length, requestToMake.toPodId) - removePodOf.call(self, requestIdsToDelete, requestToMake.toPodId) - return callbackEach() - } - - makeRequest(toPod, requestToMake.endpoint, requestToMake.datas, function (success) { - if (success === true) { - logger.debug('Removing requests for %s pod.', requestToMake.toPodId, { requestsIds: requestToMake.ids }) - - goodPods.push(requestToMake.toPodId) - - // Remove the pod id of these request ids - removePodOf.call(self, requestToMake.ids, requestToMake.toPodId, callbackEach) - } else { - badPods.push(requestToMake.toPodId) - callbackEach() - } - }) }) }, function () { // All the requests were made, we update the pods score - updatePodsScore(goodPods, badPods) + updatePodsScore.call(self, goodPods, badPods) // Flush requests with no pod - removeWithEmptyTo.call(self) + removeWithEmptyTo.call(self, function (err) { + if (err) logger.error('Error when removing requests with no pods.', { error: err }) + }) }) }) } // Remove pods with a score of 0 (too many requests where they were unreachable) function removeBadPods () { + const self = this + waterfall([ function findBadPods (callback) { - Pod.listBadPods(function (err, pods) { + self.sequelize.models.Pod.listBadPods(function (err, pods) { if (err) { logger.error('Cannot find bad pods.', { error: err }) return callback(err) @@ -233,10 +233,8 @@ function removeBadPods () { }, function removeTheseBadPods (pods, callback) { - if (pods.length === 0) return callback(null, 0) - each(pods, function (pod, callbackEach) { - pod.remove(callbackEach) + pod.destroy().asCallback(callbackEach) }, function (err) { return callback(err, pods.length) }) @@ -253,43 +251,98 @@ function removeBadPods () { } function updatePodsScore (goodPods, badPods) { + const self = this + const Pod = this.sequelize.models.Pod + logger.info('Updating %d good pods and %d bad pods scores.', goodPods.length, badPods.length) - Pod.incrementScores(goodPods, constants.PODS_SCORE.BONUS, function (err) { - if (err) logger.error('Cannot increment scores of good pods.') - }) + if (goodPods.length !== 0) { + Pod.incrementScores(goodPods, constants.PODS_SCORE.BONUS, function (err) { + if (err) logger.error('Cannot increment scores of good pods.', { error: err }) + }) + } - Pod.incrementScores(badPods, constants.PODS_SCORE.MALUS, function (err) { - if (err) logger.error('Cannot decrement scores of bad pods.') - removeBadPods() + if (badPods.length !== 0) { + Pod.incrementScores(badPods, constants.PODS_SCORE.MALUS, function (err) { + if (err) logger.error('Cannot decrement scores of bad pods.', { error: err }) + removeBadPods.call(self) + }) + } +} + +function listWithLimitAndRandom (limitPods, limitRequestsPerPod, callback) { + const self = this + const Pod = this.sequelize.models.Pod + + Pod.listRandomPodIdsWithRequest(limitPods, function (err, podIds) { + if (err) return callback(err) + + // We don't have friends that have requests + if (podIds.length === 0) return callback(null, []) + + // The the first x requests of these pods + // It is very important to sort by id ASC to keep the requests order! + const query = { + order: [ + [ 'id', 'ASC' ] + ], + include: [ + { + model: self.sequelize.models.Pod, + where: { + id: { + $in: podIds + } + } + } + ] + } + + self.findAll(query).asCallback(function (err, requests) { + if (err) return callback(err) + + const requestsGrouped = groupAndTruncateRequests(requests, limitRequestsPerPod) + return callback(err, requestsGrouped) + }) }) } -function listWithLimitAndRandom (limit, callback) { - const self = this +function groupAndTruncateRequests (requests, limitRequestsPerPod) { + const requestsGrouped = {} - self.count(function (err, count) { - if (err) return callback(err) + requests.forEach(function (request) { + request.Pods.forEach(function (pod) { + if (!requestsGrouped[pod.id]) requestsGrouped[pod.id] = [] - let start = Math.floor(Math.random() * count) - limit - if (start < 0) start = 0 - - self.find().sort({ _id: 1 }).skip(start).limit(limit).exec(callback) + if (requestsGrouped[pod.id].length < limitRequestsPerPod) { + requestsGrouped[pod.id].push({ + request, + pod + }) + } + }) }) + + return requestsGrouped } function removeAll (callback) { - this.remove({ }, callback) -} - -function removePodOf (requestsIds, podId, callback) { - if (!callback) callback = function () {} - - this.update({ _id: { $in: requestsIds } }, { $pull: { to: podId } }, { multi: true }, callback) + // Delete all requests + this.truncate({ cascade: true }).asCallback(callback) } function removeWithEmptyTo (callback) { if (!callback) callback = function () {} - this.remove({ to: { $size: 0 } }, callback) + const query = { + where: { + id: { + $notIn: [ + this.sequelize.literal('SELECT "requestId" FROM "RequestToPods"') + ] + } + } + } + + this.destroy(query).asCallback(callback) } diff --git a/server/models/tag.js b/server/models/tag.js new file mode 100644 index 000000000..145e090c1 --- /dev/null +++ b/server/models/tag.js @@ -0,0 +1,76 @@ +'use strict' + +const each = require('async/each') + +// --------------------------------------------------------------------------- + +module.exports = function (sequelize, DataTypes) { + const Tag = sequelize.define('Tag', + { + name: { + type: DataTypes.STRING, + allowNull: false + } + }, + { + timestamps: false, + indexes: [ + { + fields: [ 'name' ], + unique: true + } + ], + classMethods: { + associate, + + findOrCreateTags + } + } + ) + + return Tag +} + +// --------------------------------------------------------------------------- + +function associate (models) { + this.belongsToMany(models.Video, { + foreignKey: 'tagId', + through: models.VideoTag, + onDelete: 'cascade' + }) +} + +function findOrCreateTags (tags, transaction, callback) { + if (!callback) { + callback = transaction + transaction = null + } + + const self = this + const tagInstances = [] + + each(tags, function (tag, callbackEach) { + const query = { + where: { + name: tag + }, + defaults: { + name: tag + } + } + + if (transaction) query.transaction = transaction + + self.findOrCreate(query).asCallback(function (err, res) { + if (err) return callbackEach(err) + + // res = [ tag, isCreated ] + const tag = res[0] + tagInstances.push(tag) + return callbackEach() + }) + }, function (err) { + return callback(err, tagInstances) + }) +} diff --git a/server/models/user.js b/server/models/user.js index a19de7072..36ed723cc 100644 --- a/server/models/user.js +++ b/server/models/user.js @@ -1,60 +1,81 @@ -const mongoose = require('mongoose') +'use strict' + +const values = require('lodash/values') -const customUsersValidators = require('../helpers/custom-validators').users const modelUtils = require('./utils') +const constants = require('../initializers/constants') const peertubeCrypto = require('../helpers/peertube-crypto') - -const OAuthToken = mongoose.model('OAuthToken') +const customUsersValidators = require('../helpers/custom-validators').users // --------------------------------------------------------------------------- -const UserSchema = mongoose.Schema({ - createdDate: { - type: Date, - default: Date.now - }, - password: String, - username: String, - role: String -}) +module.exports = function (sequelize, DataTypes) { + const User = sequelize.define('User', + { + password: { + type: DataTypes.STRING, + allowNull: false, + validate: { + passwordValid: function (value) { + const res = customUsersValidators.isUserPasswordValid(value) + if (res === false) throw new Error('Password not valid.') + } + } + }, + username: { + type: DataTypes.STRING, + allowNull: false, + validate: { + usernameValid: function (value) { + const res = customUsersValidators.isUserUsernameValid(value) + if (res === false) throw new Error('Username not valid.') + } + } + }, + role: { + type: DataTypes.ENUM(values(constants.USER_ROLES)), + allowNull: false + } + }, + { + indexes: [ + { + fields: [ 'username' ] + } + ], + classMethods: { + associate, -UserSchema.path('password').required(customUsersValidators.isUserPasswordValid) -UserSchema.path('username').required(customUsersValidators.isUserUsernameValid) -UserSchema.path('role').validate(customUsersValidators.isUserRoleValid) + countTotal, + getByUsername, + list, + listForApi, + loadById, + loadByUsername + }, + instanceMethods: { + isPasswordMatch, + toFormatedJSON + }, + hooks: { + beforeCreate: beforeCreateOrUpdate, + beforeUpdate: beforeCreateOrUpdate + } + } + ) -UserSchema.methods = { - isPasswordMatch, - toFormatedJSON + return User } -UserSchema.statics = { - countTotal, - getByUsername, - list, - listForApi, - loadById, - loadByUsername -} - -UserSchema.pre('save', function (next) { - const user = this - - peertubeCrypto.cryptPassword(this.password, function (err, hash) { +function beforeCreateOrUpdate (user, options, next) { + peertubeCrypto.cryptPassword(user.password, function (err, hash) { if (err) return next(err) user.password = hash return next() }) -}) - -UserSchema.pre('remove', function (next) { - const user = this - - OAuthToken.removeByUserId(user._id, next) -}) - -mongoose.model('User', UserSchema) +} // ------------------------------ METHODS ------------------------------ @@ -64,35 +85,68 @@ function isPasswordMatch (password, callback) { function toFormatedJSON () { return { - id: this._id, + id: this.id, username: this.username, role: this.role, - createdDate: this.createdDate + createdAt: this.createdAt } } // ------------------------------ STATICS ------------------------------ +function associate (models) { + this.hasOne(models.Author, { + foreignKey: 'userId', + onDelete: 'cascade' + }) + + this.hasMany(models.OAuthToken, { + foreignKey: 'userId', + onDelete: 'cascade' + }) +} + function countTotal (callback) { - return this.count(callback) + return this.count().asCallback(callback) } function getByUsername (username) { - return this.findOne({ username: username }) + const query = { + where: { + username: username + } + } + + return this.findOne(query) } function list (callback) { - return this.find(callback) + return this.find().asCallback(callback) } function listForApi (start, count, sort, callback) { - const query = {} - return modelUtils.listForApiWithCount.call(this, query, start, count, sort, callback) + const query = { + offset: start, + limit: count, + order: [ modelUtils.getSort(sort) ] + } + + return this.findAndCountAll(query).asCallback(function (err, result) { + if (err) return callback(err) + + return callback(null, result.rows, result.count) + }) } function loadById (id, callback) { - return this.findById(id, callback) + return this.findById(id).asCallback(callback) } function loadByUsername (username, callback) { - return this.findOne({ username: username }, callback) + const query = { + where: { + username: username + } + } + + return this.findOne(query).asCallback(callback) } diff --git a/server/models/utils.js b/server/models/utils.js index e798aabe6..49636b3d8 100644 --- a/server/models/utils.js +++ b/server/models/utils.js @@ -1,28 +1,23 @@ 'use strict' -const parallel = require('async/parallel') - const utils = { - listForApiWithCount + getSort } -function listForApiWithCount (query, start, count, sort, callback) { - const self = this +// Translate for example "-name" to [ 'name', 'DESC' ] +function getSort (value) { + let field + let direction - parallel([ - function (asyncCallback) { - self.find(query).skip(start).limit(count).sort(sort).exec(asyncCallback) - }, - function (asyncCallback) { - self.count(query, asyncCallback) - } - ], function (err, results) { - if (err) return callback(err) + if (value.substring(0, 1) === '-') { + direction = 'DESC' + field = value.substring(1) + } else { + direction = 'ASC' + field = value + } - const data = results[0] - const total = results[1] - return callback(null, data, total) - }) + return [ field, direction ] } // --------------------------------------------------------------------------- diff --git a/server/models/video-abuse.js b/server/models/video-abuse.js new file mode 100644 index 000000000..766a7568d --- /dev/null +++ b/server/models/video-abuse.js @@ -0,0 +1,113 @@ +'use strict' + +const constants = require('../initializers/constants') +const modelUtils = require('./utils') +const customVideosValidators = require('../helpers/custom-validators').videos + +module.exports = function (sequelize, DataTypes) { + const VideoAbuse = sequelize.define('VideoAbuse', + { + reporterUsername: { + type: DataTypes.STRING, + allowNull: false, + validate: { + reporterUsernameValid: function (value) { + const res = customVideosValidators.isVideoAbuseReporterUsernameValid(value) + if (res === false) throw new Error('Video abuse reporter username is not valid.') + } + } + }, + reason: { + type: DataTypes.STRING, + allowNull: false, + validate: { + reasonValid: function (value) { + const res = customVideosValidators.isVideoAbuseReasonValid(value) + if (res === false) throw new Error('Video abuse reason is not valid.') + } + } + } + }, + { + indexes: [ + { + fields: [ 'videoId' ] + }, + { + fields: [ 'reporterPodId' ] + } + ], + classMethods: { + associate, + + listForApi + }, + instanceMethods: { + toFormatedJSON + } + } + ) + + return VideoAbuse +} + +// --------------------------------------------------------------------------- + +function associate (models) { + this.belongsTo(models.Pod, { + foreignKey: { + name: 'reporterPodId', + allowNull: true + }, + onDelete: 'cascade' + }) + + this.belongsTo(models.Video, { + foreignKey: { + name: 'videoId', + allowNull: false + }, + onDelete: 'cascade' + }) +} + +function listForApi (start, count, sort, callback) { + const query = { + offset: start, + limit: count, + order: [ modelUtils.getSort(sort) ], + include: [ + { + model: this.sequelize.models.Pod, + required: false + } + ] + } + + return this.findAndCountAll(query).asCallback(function (err, result) { + if (err) return callback(err) + + return callback(null, result.rows, result.count) + }) +} + +function toFormatedJSON () { + let reporterPodHost + + if (this.Pod) { + reporterPodHost = this.Pod.host + } else { + // It means it's our video + reporterPodHost = constants.CONFIG.WEBSERVER.HOST + } + + const json = { + id: this.id, + reporterPodHost, + reason: this.reason, + reporterUsername: this.reporterUsername, + videoId: this.videoId + } + + return json +} diff --git a/server/models/video-tag.js b/server/models/video-tag.js new file mode 100644 index 000000000..cd9277a6e --- /dev/null +++ b/server/models/video-tag.js @@ -0,0 +1,18 @@ +'use strict' + +// --------------------------------------------------------------------------- + +module.exports = function (sequelize, DataTypes) { + const VideoTag = sequelize.define('VideoTag', {}, { + indexes: [ + { + fields: [ 'videoId' ] + }, + { + fields: [ 'tagId' ] + } + ] + }) + + return VideoTag +} diff --git a/server/models/video.js b/server/models/video.js index 330067cdf..17eff6428 100644 --- a/server/models/video.js +++ b/server/models/video.js @@ -1,108 +1,160 @@ 'use strict' +const Buffer = require('safe-buffer').Buffer const createTorrent = require('create-torrent') const ffmpeg = require('fluent-ffmpeg') const fs = require('fs') const magnetUtil = require('magnet-uri') +const map = require('lodash/map') const parallel = require('async/parallel') const parseTorrent = require('parse-torrent') const pathUtils = require('path') -const mongoose = require('mongoose') +const values = require('lodash/values') const constants = require('../initializers/constants') -const customVideosValidators = require('../helpers/custom-validators').videos const logger = require('../helpers/logger') +const friends = require('../lib/friends') const modelUtils = require('./utils') +const customVideosValidators = require('../helpers/custom-validators').videos // --------------------------------------------------------------------------- -// TODO: add indexes on searchable columns -const VideoSchema = mongoose.Schema({ - name: String, - extname: { - type: String, - enum: [ '.mp4', '.webm', '.ogv' ] - }, - remoteId: mongoose.Schema.Types.ObjectId, - description: String, - magnet: { - infoHash: String - }, - podHost: String, - author: String, - duration: Number, - tags: [ String ], - createdDate: { - type: Date, - default: Date.now - } -}) +module.exports = function (sequelize, DataTypes) { + const Video = sequelize.define('Video', + { + id: { + type: DataTypes.UUID, + defaultValue: DataTypes.UUIDV4, + primaryKey: true, + validate: { + isUUID: 4 + } + }, + name: { + type: DataTypes.STRING, + allowNull: false, + validate: { + nameValid: function (value) { + const res = customVideosValidators.isVideoNameValid(value) + if (res === false) throw new Error('Video name is not valid.') + } + } + }, + extname: { + type: DataTypes.ENUM(values(constants.CONSTRAINTS_FIELDS.VIDEOS.EXTNAME)), + allowNull: false + }, + remoteId: { + type: DataTypes.UUID, + allowNull: true, + validate: { + isUUID: 4 + } + }, + description: { + type: DataTypes.STRING, + allowNull: false, + validate: { + descriptionValid: function (value) { + const res = customVideosValidators.isVideoDescriptionValid(value) + if (res === false) throw new Error('Video description is not valid.') + } + } + }, + infoHash: { + type: DataTypes.STRING, + allowNull: false, + validate: { + infoHashValid: function (value) { + const res = customVideosValidators.isVideoInfoHashValid(value) + if (res === false) throw new Error('Video info hash is not valid.') + } + } + }, + duration: { + type: DataTypes.INTEGER, + allowNull: false, + validate: { + durationValid: function (value) { + const res = customVideosValidators.isVideoDurationValid(value) + if (res === false) throw new Error('Video duration is not valid.') + } + } + } + }, + { + indexes: [ + { + fields: [ 'authorId' ] + }, + { + fields: [ 'remoteId' ] + }, + { + fields: [ 'name' ] + }, + { + fields: [ 'createdAt' ] + }, + { + fields: [ 'duration' ] + }, + { + fields: [ 'infoHash' ] + } + ], + classMethods: { + associate, -VideoSchema.path('name').validate(customVideosValidators.isVideoNameValid) -VideoSchema.path('description').validate(customVideosValidators.isVideoDescriptionValid) -VideoSchema.path('podHost').validate(customVideosValidators.isVideoPodHostValid) -VideoSchema.path('author').validate(customVideosValidators.isVideoAuthorValid) -VideoSchema.path('duration').validate(customVideosValidators.isVideoDurationValid) -VideoSchema.path('tags').validate(customVideosValidators.isVideoTagsValid) - -VideoSchema.methods = { - generateMagnetUri, - getVideoFilename, - getThumbnailName, - getPreviewName, - getTorrentName, - isOwned, - toFormatedJSON, - toRemoteJSON -} - -VideoSchema.statics = { - generateThumbnailFromBase64, - getDurationFromFile, - listForApi, - listByHostAndRemoteId, - listByHost, - listOwned, - listOwnedByAuthor, - listRemotes, - load, - search -} - -VideoSchema.pre('remove', function (next) { - const video = this - const tasks = [] - - tasks.push( - function (callback) { - removeThumbnail(video, callback) + generateThumbnailFromData, + getDurationFromFile, + list, + listForApi, + listOwnedAndPopulateAuthorAndTags, + listOwnedByAuthor, + load, + loadByHostAndRemoteId, + loadAndPopulateAuthor, + loadAndPopulateAuthorAndPodAndTags, + searchAndPopulateAuthorAndPodAndTags + }, + instanceMethods: { + generateMagnetUri, + getVideoFilename, + getThumbnailName, + getPreviewName, + getTorrentName, + isOwned, + toFormatedJSON, + toAddRemoteJSON, + toUpdateRemoteJSON + }, + hooks: { + beforeValidate, + beforeCreate, + afterDestroy + } } ) - if (video.isOwned()) { - tasks.push( - function (callback) { - removeFile(video, callback) - }, - function (callback) { - removeTorrent(video, callback) - }, - function (callback) { - removePreview(video, callback) - } - ) + return Video +} + +function beforeValidate (video, options, next) { + // Put a fake infoHash if it does not exists yet + if (video.isOwned() && !video.infoHash) { + // 40 hexa length + video.infoHash = '0123456789abcdef0123456789abcdef01234567' } - parallel(tasks, next) -}) + return next(null) +} -VideoSchema.pre('save', function (next) { - const video = this +function beforeCreate (video, options, next) { const tasks = [] if (video.isOwned()) { const videoPath = pathUtils.join(constants.CONFIG.STORAGE.VIDEOS_DIR, video.getVideoFilename()) - this.podHost = constants.CONFIG.WEBSERVER.HOST tasks.push( // TODO: refractoring @@ -123,9 +175,8 @@ VideoSchema.pre('save', function (next) { if (err) return callback(err) const parsedTorrent = parseTorrent(torrent) - video.magnet.infoHash = parsedTorrent.infoHash - - callback(null) + video.set('infoHash', parsedTorrent.infoHash) + video.validate().asCallback(callback) }) }) }, @@ -141,12 +192,72 @@ VideoSchema.pre('save', function (next) { } return next() -}) +} -mongoose.model('Video', VideoSchema) +function afterDestroy (video, options, next) { + const tasks = [] + + tasks.push( + function (callback) { + removeThumbnail(video, callback) + } + ) + + if (video.isOwned()) { + tasks.push( + function (callback) { + removeFile(video, callback) + }, + + function (callback) { + removeTorrent(video, callback) + }, + + function (callback) { + removePreview(video, callback) + }, + + function (callback) { + const params = { + remoteId: video.id + } + + friends.removeVideoToFriends(params) + + return callback() + } + ) + } + + parallel(tasks, next) +} // ------------------------------ METHODS ------------------------------ +function associate (models) { + this.belongsTo(models.Author, { + foreignKey: { + name: 'authorId', + allowNull: false + }, + onDelete: 'cascade' + }) + + this.belongsToMany(models.Tag, { + foreignKey: 'videoId', + through: models.VideoTag, + onDelete: 'cascade' + }) + + this.hasMany(models.VideoAbuse, { + foreignKey: { + name: 'videoId', + allowNull: false + }, + onDelete: 'cascade' + }) +} + function generateMagnetUri () { let baseUrlHttp, baseUrlWs @@ -154,8 +265,8 @@ function generateMagnetUri () { baseUrlHttp = constants.CONFIG.WEBSERVER.URL baseUrlWs = constants.CONFIG.WEBSERVER.WS + '://' + constants.CONFIG.WEBSERVER.HOSTNAME + ':' + constants.CONFIG.WEBSERVER.PORT } else { - baseUrlHttp = constants.REMOTE_SCHEME.HTTP + '://' + this.podHost - baseUrlWs = constants.REMOTE_SCHEME.WS + '://' + this.podHost + baseUrlHttp = constants.REMOTE_SCHEME.HTTP + '://' + this.Author.Pod.host + baseUrlWs = constants.REMOTE_SCHEME.WS + '://' + this.Author.Pod.host } const xs = baseUrlHttp + constants.STATIC_PATHS.TORRENTS + this.getTorrentName() @@ -166,7 +277,7 @@ function generateMagnetUri () { xs, announce, urlList, - infoHash: this.magnet.infoHash, + infoHash: this.infoHash, name: this.name } @@ -174,20 +285,20 @@ function generateMagnetUri () { } function getVideoFilename () { - if (this.isOwned()) return this._id + this.extname + if (this.isOwned()) return this.id + this.extname return this.remoteId + this.extname } function getThumbnailName () { // We always have a copy of the thumbnail - return this._id + '.jpg' + return this.id + '.jpg' } function getPreviewName () { const extension = '.jpg' - if (this.isOwned()) return this._id + extension + if (this.isOwned()) return this.id + extension return this.remoteId + extension } @@ -195,7 +306,7 @@ function getPreviewName () { function getTorrentName () { const extension = '.torrent' - if (this.isOwned()) return this._id + extension + if (this.isOwned()) return this.id + extension return this.remoteId + extension } @@ -205,27 +316,37 @@ function isOwned () { } function toFormatedJSON () { + let podHost + + if (this.Author.Pod) { + podHost = this.Author.Pod.host + } else { + // It means it's our video + podHost = constants.CONFIG.WEBSERVER.HOST + } + const json = { - id: this._id, + id: this.id, name: this.name, description: this.description, - podHost: this.podHost, + podHost, isLocal: this.isOwned(), magnetUri: this.generateMagnetUri(), - author: this.author, + author: this.Author.name, duration: this.duration, - tags: this.tags, - thumbnailPath: constants.STATIC_PATHS.THUMBNAILS + '/' + this.getThumbnailName(), - createdDate: this.createdDate + tags: map(this.Tags, 'name'), + thumbnailPath: pathUtils.join(constants.STATIC_PATHS.THUMBNAILS, this.getThumbnailName()), + createdAt: this.createdAt, + updatedAt: this.updatedAt } return json } -function toRemoteJSON (callback) { +function toAddRemoteJSON (callback) { const self = this - // Convert thumbnail to base64 + // Get thumbnail data to send to the other pod const thumbnailPath = pathUtils.join(constants.CONFIG.STORAGE.THUMBNAILS_DIR, this.getThumbnailName()) fs.readFile(thumbnailPath, function (err, thumbnailData) { if (err) { @@ -236,13 +357,14 @@ function toRemoteJSON (callback) { const remoteVideo = { name: self.name, description: self.description, - magnet: self.magnet, - remoteId: self._id, - author: self.author, + infoHash: self.infoHash, + remoteId: self.id, + author: self.Author.name, duration: self.duration, - thumbnailBase64: new Buffer(thumbnailData).toString('base64'), - tags: self.tags, - createdDate: self.createdDate, + thumbnailData: thumbnailData.toString('binary'), + tags: map(self.Tags, 'name'), + createdAt: self.createdAt, + updatedAt: self.updatedAt, extname: self.extname } @@ -250,14 +372,31 @@ function toRemoteJSON (callback) { }) } +function toUpdateRemoteJSON (callback) { + const json = { + name: this.name, + description: this.description, + infoHash: this.infoHash, + remoteId: this.id, + author: this.Author.name, + duration: this.duration, + tags: map(this.Tags, 'name'), + createdAt: this.createdAt, + updatedAt: this.updatedAt, + extname: this.extname + } + + return json +} + // ------------------------------ STATICS ------------------------------ -function generateThumbnailFromBase64 (video, thumbnailData, callback) { +function generateThumbnailFromData (video, thumbnailData, callback) { // Creating the thumbnail for a remote video const thumbnailName = video.getThumbnailName() const thumbnailPath = constants.CONFIG.STORAGE.THUMBNAILS_DIR + thumbnailName - fs.writeFile(thumbnailPath, thumbnailData, { encoding: 'base64' }, function (err) { + fs.writeFile(thumbnailPath, Buffer.from(thumbnailData, 'binary'), function (err) { if (err) return callback(err) return callback(null, thumbnailName) @@ -272,51 +411,186 @@ function getDurationFromFile (videoPath, callback) { }) } +function list (callback) { + return this.find().asCallback() +} + function listForApi (start, count, sort, callback) { - const query = {} - return modelUtils.listForApiWithCount.call(this, query, start, count, sort, callback) + const query = { + offset: start, + limit: count, + distinct: true, // For the count, a video can have many tags + order: [ modelUtils.getSort(sort), [ this.sequelize.models.Tag, 'name', 'ASC' ] ], + include: [ + { + model: this.sequelize.models.Author, + include: [ { model: this.sequelize.models.Pod, required: false } ] + }, + + this.sequelize.models.Tag + ] + } + + return this.findAndCountAll(query).asCallback(function (err, result) { + if (err) return callback(err) + + return callback(null, result.rows, result.count) + }) } -function listByHostAndRemoteId (fromHost, remoteId, callback) { - this.find({ podHost: fromHost, remoteId: remoteId }, callback) +function loadByHostAndRemoteId (fromHost, remoteId, callback) { + const query = { + where: { + remoteId: remoteId + }, + include: [ + { + model: this.sequelize.models.Author, + include: [ + { + model: this.sequelize.models.Pod, + required: true, + where: { + host: fromHost + } + } + ] + } + ] + } + + return this.findOne(query).asCallback(callback) } -function listByHost (fromHost, callback) { - this.find({ podHost: fromHost }, callback) -} - -function listOwned (callback) { +function listOwnedAndPopulateAuthorAndTags (callback) { // If remoteId is null this is *our* video - this.find({ remoteId: null }, callback) + const query = { + where: { + remoteId: null + }, + include: [ this.sequelize.models.Author, this.sequelize.models.Tag ] + } + + return this.findAll(query).asCallback(callback) } function listOwnedByAuthor (author, callback) { - this.find({ remoteId: null, author: author }, callback) -} + const query = { + where: { + remoteId: null + }, + include: [ + { + model: this.sequelize.models.Author, + where: { + name: author + } + } + ] + } -function listRemotes (callback) { - this.find({ remoteId: { $ne: null } }, callback) + return this.findAll(query).asCallback(callback) } function load (id, callback) { - this.findById(id, callback) + return this.findById(id).asCallback(callback) } -function search (value, field, start, count, sort, callback) { - const query = {} +function loadAndPopulateAuthor (id, callback) { + const options = { + include: [ this.sequelize.models.Author ] + } + + return this.findById(id, options).asCallback(callback) +} + +function loadAndPopulateAuthorAndPodAndTags (id, callback) { + const options = { + include: [ + { + model: this.sequelize.models.Author, + include: [ { model: this.sequelize.models.Pod, required: false } ] + }, + this.sequelize.models.Tag + ] + } + + return this.findById(id, options).asCallback(callback) +} + +function searchAndPopulateAuthorAndPodAndTags (value, field, start, count, sort, callback) { + const podInclude = { + model: this.sequelize.models.Pod, + required: false + } + + const authorInclude = { + model: this.sequelize.models.Author, + include: [ + podInclude + ] + } + + const tagInclude = { + model: this.sequelize.models.Tag + } + + const query = { + where: {}, + offset: start, + limit: count, + distinct: true, // For the count, a video can have many tags + order: [ modelUtils.getSort(sort), [ this.sequelize.models.Tag, 'name', 'ASC' ] ] + } + // Make an exact search with the magnet if (field === 'magnetUri') { const infoHash = magnetUtil.decode(value).infoHash - query.magnet = { - infoHash - } + query.where.infoHash = infoHash } else if (field === 'tags') { - query[field] = value + const escapedValue = this.sequelize.escape('%' + value + '%') + query.where = { + id: { + $in: this.sequelize.literal( + '(SELECT "VideoTags"."videoId" FROM "Tags" INNER JOIN "VideoTags" ON "Tags"."id" = "VideoTags"."tagId" WHERE name LIKE ' + escapedValue + ')' + ) + } + } + } else if (field === 'host') { + // FIXME: Include our pod? (not stored in the database) + podInclude.where = { + host: { + $like: '%' + value + '%' + } + } + podInclude.required = true + } else if (field === 'author') { + authorInclude.where = { + name: { + $like: '%' + value + '%' + } + } + + // authorInclude.or = true } else { - query[field] = new RegExp(value, 'i') + query.where[field] = { + $like: '%' + value + '%' + } } - modelUtils.listForApiWithCount.call(this, query, start, count, sort, callback) + query.include = [ + authorInclude, tagInclude + ] + + if (tagInclude.where) { + // query.include.push([ this.sequelize.models.Tag ]) + } + + return this.findAndCountAll(query).asCallback(function (err, result) { + if (err) return callback(err) + + return callback(null, result.rows, result.count) + }) } // --------------------------------------------------------------------------- diff --git a/server/tests/api/check-params.js b/server/tests/api/check-params.js deleted file mode 100644 index 444c2fc55..000000000 --- a/server/tests/api/check-params.js +++ /dev/null @@ -1,764 +0,0 @@ -'use strict' - -const chai = require('chai') -const expect = chai.expect -const pathUtils = require('path') -const request = require('supertest') -const series = require('async/series') - -const loginUtils = require('../utils/login') -const requestsUtils = require('../utils/requests') -const serversUtils = require('../utils/servers') -const usersUtils = require('../utils/users') - -describe('Test parameters validator', function () { - let server = null - let userAccessToken = null - - // --------------------------------------------------------------- - - before(function (done) { - this.timeout(20000) - - series([ - function (next) { - serversUtils.flushTests(next) - }, - function (next) { - serversUtils.runServer(1, function (server1) { - server = server1 - - next() - }) - }, - function (next) { - loginUtils.loginAndGetAccessToken(server, function (err, token) { - if (err) throw err - server.accessToken = token - - next() - }) - } - ], done) - }) - - describe('Of the pods API', function () { - const path = '/api/v1/pods/' - - describe('When making friends', function () { - let userAccessToken = null - - before(function (done) { - usersUtils.createUser(server.url, server.accessToken, 'user1', 'password', function () { - server.user = { - username: 'user1', - password: 'password' - } - - loginUtils.loginAndGetAccessToken(server, function (err, accessToken) { - if (err) throw err - - userAccessToken = accessToken - - done() - }) - }) - }) - - describe('When making friends', function () { - const body = { - hosts: [ 'localhost:9002' ] - } - - it('Should fail without hosts', function (done) { - request(server.url) - .post(path + '/makefriends') - .set('Authorization', 'Bearer ' + server.accessToken) - .set('Accept', 'application/json') - .expect(400, done) - }) - - it('Should fail if hosts is not an array', function (done) { - request(server.url) - .post(path + '/makefriends') - .send({ hosts: 'localhost:9002' }) - .set('Authorization', 'Bearer ' + server.accessToken) - .set('Accept', 'application/json') - .expect(400, done) - }) - - it('Should fail if the array is not composed by hosts', function (done) { - request(server.url) - .post(path + '/makefriends') - .send({ hosts: [ 'localhost:9002', 'localhost:coucou' ] }) - .set('Authorization', 'Bearer ' + server.accessToken) - .set('Accept', 'application/json') - .expect(400, done) - }) - - it('Should fail if the array is composed with http schemes', function (done) { - request(server.url) - .post(path + '/makefriends') - .send({ hosts: [ 'localhost:9002', 'http://localhost:9003' ] }) - .set('Authorization', 'Bearer ' + server.accessToken) - .set('Accept', 'application/json') - .expect(400, done) - }) - - it('Should fail if hosts are not unique', function (done) { - request(server.url) - .post(path + '/makefriends') - .send({ urls: [ 'localhost:9002', 'localhost:9002' ] }) - .set('Authorization', 'Bearer ' + server.accessToken) - .set('Accept', 'application/json') - .expect(400, done) - }) - - it('Should fail with a invalid token', function (done) { - request(server.url) - .post(path + '/makefriends') - .send(body) - .set('Authorization', 'Bearer faketoken') - .set('Accept', 'application/json') - .expect(401, done) - }) - - it('Should fail if the user is not an administrator', function (done) { - request(server.url) - .post(path + '/makefriends') - .send(body) - .set('Authorization', 'Bearer ' + userAccessToken) - .set('Accept', 'application/json') - .expect(403, done) - }) - }) - - describe('When quitting friends', function () { - it('Should fail with a invalid token', function (done) { - request(server.url) - .get(path + '/quitfriends') - .query({ start: 'hello' }) - .set('Authorization', 'Bearer faketoken') - .set('Accept', 'application/json') - .expect(401, done) - }) - - it('Should fail if the user is not an administrator', function (done) { - request(server.url) - .get(path + '/quitfriends') - .query({ start: 'hello' }) - .set('Authorization', 'Bearer ' + userAccessToken) - .set('Accept', 'application/json') - .expect(403, done) - }) - }) - }) - - describe('When adding a pod', function () { - it('Should fail with nothing', function (done) { - const data = {} - requestsUtils.makePostBodyRequest(server.url, path, null, data, done) - }) - - it('Should fail without public key', function (done) { - const data = { - host: 'coucou.com' - } - requestsUtils.makePostBodyRequest(server.url, path, null, data, done) - }) - - it('Should fail without an host', function (done) { - const data = { - publicKey: 'mysuperpublickey' - } - requestsUtils.makePostBodyRequest(server.url, path, null, data, done) - }) - - it('Should fail with an incorrect host', function (done) { - const data = { - host: 'http://coucou.com', - publicKey: 'mysuperpublickey' - } - requestsUtils.makePostBodyRequest(server.url, path, null, data, function () { - data.host = 'http://coucou' - requestsUtils.makePostBodyRequest(server.url, path, null, data, function () { - data.host = 'coucou' - requestsUtils.makePostBodyRequest(server.url, path, null, data, done) - }) - }) - }) - - it('Should succeed with the correct parameters', function (done) { - const data = { - host: 'coucou.com', - publicKey: 'mysuperpublickey' - } - requestsUtils.makePostBodyRequest(server.url, path, null, data, done, 200) - }) - }) - }) - - describe('Of the videos API', function () { - const path = '/api/v1/videos/' - - describe('When listing a video', function () { - it('Should fail with a bad start pagination', function (done) { - request(server.url) - .get(path) - .query({ start: 'hello' }) - .set('Accept', 'application/json') - .expect(400, done) - }) - - it('Should fail with a bad count pagination', function (done) { - request(server.url) - .get(path) - .query({ count: 'hello' }) - .set('Accept', 'application/json') - .expect(400, done) - }) - - it('Should fail with an incorrect sort', function (done) { - request(server.url) - .get(path) - .query({ sort: 'hello' }) - .set('Accept', 'application/json') - .expect(400, done) - }) - }) - - describe('When searching a video', function () { - it('Should fail with nothing', function (done) { - request(server.url) - .get(pathUtils.join(path, 'search')) - .set('Accept', 'application/json') - .expect(400, done) - }) - - it('Should fail with a bad start pagination', function (done) { - request(server.url) - .get(pathUtils.join(path, 'search', 'test')) - .query({ start: 'hello' }) - .set('Accept', 'application/json') - .expect(400, done) - }) - - it('Should fail with a bad count pagination', function (done) { - request(server.url) - .get(pathUtils.join(path, 'search', 'test')) - .query({ count: 'hello' }) - .set('Accept', 'application/json') - .expect(400, done) - }) - - it('Should fail with an incorrect sort', function (done) { - request(server.url) - .get(pathUtils.join(path, 'search', 'test')) - .query({ sort: 'hello' }) - .set('Accept', 'application/json') - .expect(400, done) - }) - }) - - describe('When adding a video', function () { - it('Should fail with nothing', function (done) { - const data = {} - const attach = {} - requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, done) - }) - - it('Should fail without name', function (done) { - const data = { - description: 'my super description', - tags: [ 'tag1', 'tag2' ] - } - const attach = { - 'videofile': pathUtils.join(__dirname, 'fixtures', 'video_short.webm') - } - requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, done) - }) - - it('Should fail with a long name', function (done) { - const data = { - name: 'My very very very very very very very very very very very very very very very very long name', - description: 'my super description', - tags: [ 'tag1', 'tag2' ] - } - const attach = { - 'videofile': pathUtils.join(__dirname, 'fixtures', 'video_short.webm') - } - requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, done) - }) - - it('Should fail without description', function (done) { - const data = { - name: 'my super name', - tags: [ 'tag1', 'tag2' ] - } - const attach = { - 'videofile': pathUtils.join(__dirname, 'fixtures', 'video_short.webm') - } - requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, done) - }) - - it('Should fail with a long description', function (done) { - const data = { - name: 'my super name', - description: 'my super description which is very very very very very very very very very very very very very very' + - 'very very very very very very very very very very very very very very very very very very very very very' + - 'very very very very very very very very very very very very very very very long', - tags: [ 'tag1', 'tag2' ] - } - const attach = { - 'videofile': pathUtils.join(__dirname, 'fixtures', 'video_short.webm') - } - requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, done) - }) - - it('Should fail without tags', function (done) { - const data = { - name: 'my super name', - description: 'my super description' - } - const attach = { - 'videofile': pathUtils.join(__dirname, 'fixtures', 'video_short.webm') - } - requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, done) - }) - - it('Should fail with too many tags', function (done) { - const data = { - name: 'my super name', - description: 'my super description', - tags: [ 'tag1', 'tag2', 'tag3', 'tag4' ] - } - const attach = { - 'videofile': pathUtils.join(__dirname, 'fixtures', 'video_short.webm') - } - requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, done) - }) - - it('Should fail with not enough tags', function (done) { - const data = { - name: 'my super name', - description: 'my super description', - tags: [ ] - } - const attach = { - 'videofile': pathUtils.join(__dirname, 'fixtures', 'video_short.webm') - } - requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, done) - }) - - it('Should fail with a tag length too low', function (done) { - const data = { - name: 'my super name', - description: 'my super description', - tags: [ 'tag1', 't' ] - } - const attach = { - 'videofile': pathUtils.join(__dirname, 'fixtures', 'video_short.webm') - } - requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, done) - }) - - it('Should fail with a tag length too big', function (done) { - const data = { - name: 'my super name', - description: 'my super description', - tags: [ 'mysupertagtoolong', 'tag1' ] - } - const attach = { - 'videofile': pathUtils.join(__dirname, 'fixtures', 'video_short.webm') - } - requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, done) - }) - - it('Should fail with malformed tags', function (done) { - const data = { - name: 'my super name', - description: 'my super description', - tags: [ 'my tag' ] - } - const attach = { - 'videofile': pathUtils.join(__dirname, 'fixtures', 'video_short.webm') - } - requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, done) - }) - - it('Should fail without an input file', function (done) { - const data = { - name: 'my super name', - description: 'my super description', - tags: [ 'tag1', 'tag2' ] - } - const attach = {} - requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, done) - }) - - it('Should fail without an incorrect input file', function (done) { - const data = { - name: 'my super name', - description: 'my super description', - tags: [ 'tag1', 'tag2' ] - } - const attach = { - 'videofile': pathUtils.join(__dirname, 'fixtures', 'video_short_fake.webm') - } - requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, done) - }) - - it('Should fail with a too big duration', function (done) { - const data = { - name: 'my super name', - description: 'my super description', - tags: [ 'tag1', 'tag2' ] - } - const attach = { - 'videofile': pathUtils.join(__dirname, 'fixtures', 'video_too_long.webm') - } - requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, done) - }) - - it('Should succeed with the correct parameters', function (done) { - const data = { - name: 'my super name', - description: 'my super description', - tags: [ 'tag1', 'tag2' ] - } - const attach = { - 'videofile': pathUtils.join(__dirname, 'fixtures', 'video_short.webm') - } - requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, function () { - attach.videofile = pathUtils.join(__dirname, 'fixtures', 'video_short.mp4') - requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, function () { - attach.videofile = pathUtils.join(__dirname, 'fixtures', 'video_short.ogv') - requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, done, 204) - }, false) - }, false) - }) - }) - - describe('When getting a video', function () { - it('Should return the list of the videos with nothing', function (done) { - request(server.url) - .get(path) - .set('Accept', 'application/json') - .expect(200) - .expect('Content-Type', /json/) - .end(function (err, res) { - if (err) throw err - - expect(res.body.data).to.be.an('array') - expect(res.body.data.length).to.equal(3) - - done() - }) - }) - - it('Should fail without a mongodb id', function (done) { - request(server.url) - .get(path + 'coucou') - .set('Accept', 'application/json') - .expect(400, done) - }) - - it('Should return 404 with an incorrect video', function (done) { - request(server.url) - .get(path + '123456789012345678901234') - .set('Accept', 'application/json') - .expect(404, done) - }) - - it('Should succeed with the correct parameters') - }) - - describe('When removing a video', function () { - it('Should have 404 with nothing', function (done) { - request(server.url) - .delete(path) - .set('Authorization', 'Bearer ' + server.accessToken) - .expect(400, done) - }) - - it('Should fail without a mongodb id', function (done) { - request(server.url) - .delete(path + 'hello') - .set('Authorization', 'Bearer ' + server.accessToken) - .expect(400, done) - }) - - it('Should fail with a video which does not exist', function (done) { - request(server.url) - .delete(path + '123456789012345678901234') - .set('Authorization', 'Bearer ' + server.accessToken) - .expect(404, done) - }) - - it('Should fail with a video of another user') - - it('Should fail with a video of another pod') - - it('Should succeed with the correct parameters') - }) - }) - - describe('Of the users API', function () { - const path = '/api/v1/users/' - let userId = null - let rootId = null - - describe('When listing users', function () { - it('Should fail with a bad start pagination', function (done) { - request(server.url) - .get(path) - .query({ start: 'hello' }) - .set('Accept', 'application/json') - .expect(400, done) - }) - - it('Should fail with a bad count pagination', function (done) { - request(server.url) - .get(path) - .query({ count: 'hello' }) - .set('Accept', 'application/json') - .expect(400, done) - }) - - it('Should fail with an incorrect sort', function (done) { - request(server.url) - .get(path) - .query({ sort: 'hello' }) - .set('Accept', 'application/json') - .expect(400, done) - }) - }) - - describe('When adding a new user', function () { - it('Should fail with a too small username', function (done) { - const data = { - username: 'ji', - password: 'mysuperpassword' - } - - requestsUtils.makePostBodyRequest(server.url, path, server.accessToken, data, done) - }) - - it('Should fail with a too long username', function (done) { - const data = { - username: 'mysuperusernamewhichisverylong', - password: 'mysuperpassword' - } - - requestsUtils.makePostBodyRequest(server.url, path, server.accessToken, data, done) - }) - - it('Should fail with an incorrect username', function (done) { - const data = { - username: 'my username', - password: 'mysuperpassword' - } - - requestsUtils.makePostBodyRequest(server.url, path, server.accessToken, data, done) - }) - - it('Should fail with a too small password', function (done) { - const data = { - username: 'myusername', - password: 'bla' - } - - requestsUtils.makePostBodyRequest(server.url, path, server.accessToken, data, done) - }) - - it('Should fail with a too long password', function (done) { - const data = { - username: 'myusername', - password: 'my super long password which is very very very very very very very very very very very very very very' + - 'very very very very very very very very very very very very very very very veryv very very very very' + - 'very very very very very very very very very very very very very very very very very very very very long' - } - - requestsUtils.makePostBodyRequest(server.url, path, server.accessToken, data, done) - }) - - it('Should fail with an non authenticated user', function (done) { - const data = { - username: 'myusername', - password: 'my super password' - } - - requestsUtils.makePostBodyRequest(server.url, path, 'super token', data, done, 401) - }) - - it('Should fail if we add a user with the same username', function (done) { - const data = { - username: 'user1', - password: 'my super password' - } - - requestsUtils.makePostBodyRequest(server.url, path, server.accessToken, data, done, 409) - }) - - it('Should succeed with the correct params', function (done) { - const data = { - username: 'user2', - password: 'my super password' - } - - requestsUtils.makePostBodyRequest(server.url, path, server.accessToken, data, done, 204) - }) - - it('Should fail with a non admin user', function (done) { - server.user = { - username: 'user1', - password: 'password' - } - - loginUtils.loginAndGetAccessToken(server, function (err, accessToken) { - if (err) throw err - - userAccessToken = accessToken - - const data = { - username: 'user3', - password: 'my super password' - } - - requestsUtils.makePostBodyRequest(server.url, path, userAccessToken, data, done, 403) - }) - }) - }) - - describe('When updating a user', function () { - before(function (done) { - usersUtils.getUsersList(server.url, function (err, res) { - if (err) throw err - - userId = res.body.data[1].id - rootId = res.body.data[2].id - done() - }) - }) - - it('Should fail with a too small password', function (done) { - const data = { - password: 'bla' - } - - requestsUtils.makePutBodyRequest(server.url, path + userId, userAccessToken, data, done) - }) - - it('Should fail with a too long password', function (done) { - const data = { - password: 'my super long password which is very very very very very very very very very very very very very very' + - 'very very very very very very very very very very very very very very very veryv very very very very' + - 'very very very very very very very very very very very very very very very very very very very very long' - } - - requestsUtils.makePutBodyRequest(server.url, path + userId, userAccessToken, data, done) - }) - - it('Should fail with an non authenticated user', function (done) { - const data = { - password: 'my super password' - } - - requestsUtils.makePutBodyRequest(server.url, path + userId, 'super token', data, done, 401) - }) - - it('Should succeed with the correct params', function (done) { - const data = { - password: 'my super password' - } - - requestsUtils.makePutBodyRequest(server.url, path + userId, userAccessToken, data, done, 204) - }) - }) - - describe('When getting my information', function () { - it('Should fail with a non authenticated user', function (done) { - request(server.url) - .get(path + 'me') - .set('Authorization', 'Bearer faketoken') - .set('Accept', 'application/json') - .expect(401, done) - }) - - it('Should success with the correct parameters', function (done) { - request(server.url) - .get(path + 'me') - .set('Authorization', 'Bearer ' + userAccessToken) - .set('Accept', 'application/json') - .expect(200, done) - }) - }) - - describe('When removing an user', function () { - it('Should fail with an incorrect id', function (done) { - request(server.url) - .delete(path + 'bla-bla') - .set('Authorization', 'Bearer ' + server.accessToken) - .expect(400, done) - }) - - it('Should fail with the root user', function (done) { - request(server.url) - .delete(path + rootId) - .set('Authorization', 'Bearer ' + server.accessToken) - .expect(400, done) - }) - - it('Should return 404 with a non existing id', function (done) { - request(server.url) - .delete(path + '579f982228c99c221d8092b8') - .set('Authorization', 'Bearer ' + server.accessToken) - .expect(404, done) - }) - }) - }) - - describe('Of the remote videos API', function () { - describe('When making a secure request', function () { - it('Should check a secure request') - }) - - describe('When adding a video', function () { - it('Should check when adding a video') - }) - - describe('When removing a video', function () { - it('Should check when removing a video') - }) - }) - - describe('Of the requests API', function () { - const path = '/api/v1/requests/stats' - - it('Should fail with an non authenticated user', function (done) { - request(server.url) - .get(path) - .set('Accept', 'application/json') - .expect(401, done) - }) - - it('Should fail with a non admin user', function (done) { - request(server.url) - .get(path) - .set('Authorization', 'Bearer ' + userAccessToken) - .set('Accept', 'application/json') - .expect(403, done) - }) - }) - - after(function (done) { - process.kill(-server.app.pid) - - // Keep the logs if the test failed - if (this.ok) { - serversUtils.flushTests(done) - } else { - done() - } - }) -}) diff --git a/server/tests/api/check-params/index.js b/server/tests/api/check-params/index.js new file mode 100644 index 000000000..d0824f08a --- /dev/null +++ b/server/tests/api/check-params/index.js @@ -0,0 +1,9 @@ +'use strict' + +// Order of the tests we want to execute +require('./pods') +require('./remotes') +require('./users') +require('./requests') +require('./videos') +require('./video-abuses') diff --git a/server/tests/api/check-params/pods.js b/server/tests/api/check-params/pods.js new file mode 100644 index 000000000..2f85af644 --- /dev/null +++ b/server/tests/api/check-params/pods.js @@ -0,0 +1,204 @@ +'use strict' + +const request = require('supertest') +const series = require('async/series') + +const loginUtils = require('../../utils/login') +const requestsUtils = require('../../utils/requests') +const serversUtils = require('../../utils/servers') +const usersUtils = require('../../utils/users') + +describe('Test pods API validators', function () { + const path = '/api/v1/pods/' + let server = null + + // --------------------------------------------------------------- + + before(function (done) { + this.timeout(20000) + + series([ + function (next) { + serversUtils.flushTests(next) + }, + function (next) { + serversUtils.runServer(1, function (server1) { + server = server1 + + next() + }) + }, + function (next) { + loginUtils.loginAndGetAccessToken(server, function (err, token) { + if (err) throw err + server.accessToken = token + + next() + }) + } + ], done) + }) + + describe('When making friends', function () { + let userAccessToken = null + + before(function (done) { + usersUtils.createUser(server.url, server.accessToken, 'user1', 'password', function () { + server.user = { + username: 'user1', + password: 'password' + } + + loginUtils.loginAndGetAccessToken(server, function (err, accessToken) { + if (err) throw err + + userAccessToken = accessToken + + done() + }) + }) + }) + + describe('When making friends', function () { + const body = { + hosts: [ 'localhost:9002' ] + } + + it('Should fail without hosts', function (done) { + request(server.url) + .post(path + '/makefriends') + .set('Authorization', 'Bearer ' + server.accessToken) + .set('Accept', 'application/json') + .expect(400, done) + }) + + it('Should fail if hosts is not an array', function (done) { + request(server.url) + .post(path + '/makefriends') + .send({ hosts: 'localhost:9002' }) + .set('Authorization', 'Bearer ' + server.accessToken) + .set('Accept', 'application/json') + .expect(400, done) + }) + + it('Should fail if the array is not composed by hosts', function (done) { + request(server.url) + .post(path + '/makefriends') + .send({ hosts: [ 'localhost:9002', 'localhost:coucou' ] }) + .set('Authorization', 'Bearer ' + server.accessToken) + .set('Accept', 'application/json') + .expect(400, done) + }) + + it('Should fail if the array is composed with http schemes', function (done) { + request(server.url) + .post(path + '/makefriends') + .send({ hosts: [ 'localhost:9002', 'http://localhost:9003' ] }) + .set('Authorization', 'Bearer ' + server.accessToken) + .set('Accept', 'application/json') + .expect(400, done) + }) + + it('Should fail if hosts are not unique', function (done) { + request(server.url) + .post(path + '/makefriends') + .send({ urls: [ 'localhost:9002', 'localhost:9002' ] }) + .set('Authorization', 'Bearer ' + server.accessToken) + .set('Accept', 'application/json') + .expect(400, done) + }) + + it('Should fail with a invalid token', function (done) { + request(server.url) + .post(path + '/makefriends') + .send(body) + .set('Authorization', 'Bearer faketoken') + .set('Accept', 'application/json') + .expect(401, done) + }) + + it('Should fail if the user is not an administrator', function (done) { + request(server.url) + .post(path + '/makefriends') + .send(body) + .set('Authorization', 'Bearer ' + userAccessToken) + .set('Accept', 'application/json') + .expect(403, done) + }) + }) + + describe('When quitting friends', function () { + it('Should fail with a invalid token', function (done) { + request(server.url) + .get(path + '/quitfriends') + .query({ start: 'hello' }) + .set('Authorization', 'Bearer faketoken') + .set('Accept', 'application/json') + .expect(401, done) + }) + + it('Should fail if the user is not an administrator', function (done) { + request(server.url) + .get(path + '/quitfriends') + .query({ start: 'hello' }) + .set('Authorization', 'Bearer ' + userAccessToken) + .set('Accept', 'application/json') + .expect(403, done) + }) + }) + }) + + describe('When adding a pod', function () { + it('Should fail with nothing', function (done) { + const data = {} + requestsUtils.makePostBodyRequest(server.url, path, null, data, done) + }) + + it('Should fail without public key', function (done) { + const data = { + host: 'coucou.com' + } + requestsUtils.makePostBodyRequest(server.url, path, null, data, done) + }) + + it('Should fail without an host', function (done) { + const data = { + publicKey: 'mysuperpublickey' + } + requestsUtils.makePostBodyRequest(server.url, path, null, data, done) + }) + + it('Should fail with an incorrect host', function (done) { + const data = { + host: 'http://coucou.com', + publicKey: 'mysuperpublickey' + } + requestsUtils.makePostBodyRequest(server.url, path, null, data, function () { + data.host = 'http://coucou' + requestsUtils.makePostBodyRequest(server.url, path, null, data, function () { + data.host = 'coucou' + requestsUtils.makePostBodyRequest(server.url, path, null, data, done) + }) + }) + }) + + it('Should succeed with the correct parameters', function (done) { + const data = { + host: 'coucou.com', + publicKey: 'mysuperpublickey' + } + requestsUtils.makePostBodyRequest(server.url, path, null, data, done, 200) + }) + }) + + after(function (done) { + process.kill(-server.app.pid) + + // Keep the logs if the test failed + if (this.ok) { + serversUtils.flushTests(done) + } else { + done() + } + }) +}) diff --git a/server/tests/api/check-params/remotes.js b/server/tests/api/check-params/remotes.js new file mode 100644 index 000000000..c1ab9fb2b --- /dev/null +++ b/server/tests/api/check-params/remotes.js @@ -0,0 +1,64 @@ +'use strict' + +const series = require('async/series') + +const loginUtils = require('../../utils/login') +const serversUtils = require('../../utils/servers') + +describe('Test remote videos API validators', function () { + let server = null + + // --------------------------------------------------------------- + + before(function (done) { + this.timeout(20000) + + series([ + function (next) { + serversUtils.flushTests(next) + }, + function (next) { + serversUtils.runServer(1, function (server1) { + server = server1 + + next() + }) + }, + function (next) { + loginUtils.loginAndGetAccessToken(server, function (err, token) { + if (err) throw err + server.accessToken = token + + next() + }) + } + ], done) + }) + + describe('When making a secure request', function () { + it('Should check a secure request') + }) + + describe('When adding a video', function () { + it('Should check when adding a video') + }) + + describe('When removing a video', function () { + it('Should check when removing a video') + }) + + describe('When reporting abuse on a video', function () { + it('Should check when reporting a video abuse') + }) + + after(function (done) { + process.kill(-server.app.pid) + + // Keep the logs if the test failed + if (this.ok) { + serversUtils.flushTests(done) + } else { + done() + } + }) +}) diff --git a/server/tests/api/check-params/requests.js b/server/tests/api/check-params/requests.js new file mode 100644 index 000000000..08f58db43 --- /dev/null +++ b/server/tests/api/check-params/requests.js @@ -0,0 +1,87 @@ +'use strict' + +const request = require('supertest') +const series = require('async/series') + +const loginUtils = require('../../utils/login') +const usersUtils = require('../../utils/users') +const serversUtils = require('../../utils/servers') + +describe('Test requests API validators', function () { + const path = '/api/v1/requests/stats' + let server = null + let userAccessToken = null + + // --------------------------------------------------------------- + + before(function (done) { + this.timeout(20000) + + series([ + function (next) { + serversUtils.flushTests(next) + }, + function (next) { + serversUtils.runServer(1, function (server1) { + server = server1 + + next() + }) + }, + function (next) { + loginUtils.loginAndGetAccessToken(server, function (err, token) { + if (err) throw err + server.accessToken = token + + next() + }) + }, + function (next) { + const username = 'user' + const password = 'my super password' + + usersUtils.createUser(server.url, server.accessToken, username, password, next) + }, + function (next) { + const user = { + username: 'user', + password: 'my super password' + } + + loginUtils.getUserAccessToken(server, user, function (err, accessToken) { + if (err) throw err + + userAccessToken = accessToken + + next() + }) + } + ], done) + }) + + it('Should fail with an non authenticated user', function (done) { + request(server.url) + .get(path) + .set('Accept', 'application/json') + .expect(401, done) + }) + + it('Should fail with a non admin user', function (done) { + request(server.url) + .get(path) + .set('Authorization', 'Bearer ' + userAccessToken) + .set('Accept', 'application/json') + .expect(403, done) + }) + + after(function (done) { + process.kill(-server.app.pid) + + // Keep the logs if the test failed + if (this.ok) { + serversUtils.flushTests(done) + } else { + done() + } + }) +}) diff --git a/server/tests/api/check-params/users.js b/server/tests/api/check-params/users.js new file mode 100644 index 000000000..c1fcf34a4 --- /dev/null +++ b/server/tests/api/check-params/users.js @@ -0,0 +1,284 @@ +'use strict' + +const request = require('supertest') +const series = require('async/series') + +const loginUtils = require('../../utils/login') +const requestsUtils = require('../../utils/requests') +const serversUtils = require('../../utils/servers') +const usersUtils = require('../../utils/users') + +describe('Test users API validators', function () { + const path = '/api/v1/users/' + let userId = null + let rootId = null + let server = null + let userAccessToken = null + + // --------------------------------------------------------------- + + before(function (done) { + this.timeout(20000) + + series([ + function (next) { + serversUtils.flushTests(next) + }, + function (next) { + serversUtils.runServer(1, function (server1) { + server = server1 + + next() + }) + }, + function (next) { + loginUtils.loginAndGetAccessToken(server, function (err, token) { + if (err) throw err + server.accessToken = token + + next() + }) + }, + function (next) { + const username = 'user1' + const password = 'my super password' + + usersUtils.createUser(server.url, server.accessToken, username, password, next) + }, + function (next) { + const user = { + username: 'user1', + password: 'my super password' + } + + loginUtils.getUserAccessToken(server, user, function (err, accessToken) { + if (err) throw err + + userAccessToken = accessToken + + next() + }) + } + ], done) + }) + + describe('When listing users', function () { + it('Should fail with a bad start pagination', function (done) { + request(server.url) + .get(path) + .query({ start: 'hello' }) + .set('Accept', 'application/json') + .expect(400, done) + }) + + it('Should fail with a bad count pagination', function (done) { + request(server.url) + .get(path) + .query({ count: 'hello' }) + .set('Accept', 'application/json') + .expect(400, done) + }) + + it('Should fail with an incorrect sort', function (done) { + request(server.url) + .get(path) + .query({ sort: 'hello' }) + .set('Accept', 'application/json') + .expect(400, done) + }) + }) + + describe('When adding a new user', function () { + it('Should fail with a too small username', function (done) { + const data = { + username: 'ji', + password: 'mysuperpassword' + } + + requestsUtils.makePostBodyRequest(server.url, path, server.accessToken, data, done) + }) + + it('Should fail with a too long username', function (done) { + const data = { + username: 'mysuperusernamewhichisverylong', + password: 'mysuperpassword' + } + + requestsUtils.makePostBodyRequest(server.url, path, server.accessToken, data, done) + }) + + it('Should fail with an incorrect username', function (done) { + const data = { + username: 'my username', + password: 'mysuperpassword' + } + + requestsUtils.makePostBodyRequest(server.url, path, server.accessToken, data, done) + }) + + it('Should fail with a too small password', function (done) { + const data = { + username: 'myusername', + password: 'bla' + } + + requestsUtils.makePostBodyRequest(server.url, path, server.accessToken, data, done) + }) + + it('Should fail with a too long password', function (done) { + const data = { + username: 'myusername', + password: 'my super long password which is very very very very very very very very very very very very very very' + + 'very very very very very very very very very very very very very very very veryv very very very very' + + 'very very very very very very very very very very very very very very very very very very very very long' + } + + requestsUtils.makePostBodyRequest(server.url, path, server.accessToken, data, done) + }) + + it('Should fail with an non authenticated user', function (done) { + const data = { + username: 'myusername', + password: 'my super password' + } + + requestsUtils.makePostBodyRequest(server.url, path, 'super token', data, done, 401) + }) + + it('Should fail if we add a user with the same username', function (done) { + const data = { + username: 'user1', + password: 'my super password' + } + + requestsUtils.makePostBodyRequest(server.url, path, server.accessToken, data, done, 409) + }) + + it('Should succeed with the correct params', function (done) { + const data = { + username: 'user2', + password: 'my super password' + } + + requestsUtils.makePostBodyRequest(server.url, path, server.accessToken, data, done, 204) + }) + + it('Should fail with a non admin user', function (done) { + server.user = { + username: 'user1', + password: 'my super password' + } + + loginUtils.loginAndGetAccessToken(server, function (err, accessToken) { + if (err) throw err + + userAccessToken = accessToken + + const data = { + username: 'user3', + password: 'my super password' + } + + requestsUtils.makePostBodyRequest(server.url, path, userAccessToken, data, done, 403) + }) + }) + }) + + describe('When updating a user', function () { + before(function (done) { + usersUtils.getUsersList(server.url, function (err, res) { + if (err) throw err + + userId = res.body.data[1].id + rootId = res.body.data[2].id + done() + }) + }) + + it('Should fail with a too small password', function (done) { + const data = { + password: 'bla' + } + + requestsUtils.makePutBodyRequest(server.url, path + userId, userAccessToken, data, done) + }) + + it('Should fail with a too long password', function (done) { + const data = { + password: 'my super long password which is very very very very very very very very very very very very very very' + + 'very very very very very very very very very very very very very very very veryv very very very very' + + 'very very very very very very very very very very very very very very very very very very very very long' + } + + requestsUtils.makePutBodyRequest(server.url, path + userId, userAccessToken, data, done) + }) + + it('Should fail with an non authenticated user', function (done) { + const data = { + password: 'my super password' + } + + requestsUtils.makePutBodyRequest(server.url, path + userId, 'super token', data, done, 401) + }) + + it('Should succeed with the correct params', function (done) { + const data = { + password: 'my super password' + } + + requestsUtils.makePutBodyRequest(server.url, path + userId, userAccessToken, data, done, 204) + }) + }) + + describe('When getting my information', function () { + it('Should fail with a non authenticated user', function (done) { + request(server.url) + .get(path + 'me') + .set('Authorization', 'Bearer faketoken') + .set('Accept', 'application/json') + .expect(401, done) + }) + + it('Should success with the correct parameters', function (done) { + request(server.url) + .get(path + 'me') + .set('Authorization', 'Bearer ' + userAccessToken) + .set('Accept', 'application/json') + .expect(200, done) + }) + }) + + describe('When removing an user', function () { + it('Should fail with an incorrect id', function (done) { + request(server.url) + .delete(path + 'bla-bla') + .set('Authorization', 'Bearer ' + server.accessToken) + .expect(400, done) + }) + + it('Should fail with the root user', function (done) { + request(server.url) + .delete(path + rootId) + .set('Authorization', 'Bearer ' + server.accessToken) + .expect(400, done) + }) + + it('Should return 404 with a non existing id', function (done) { + request(server.url) + .delete(path + '45') + .set('Authorization', 'Bearer ' + server.accessToken) + .expect(404, done) + }) + }) + + after(function (done) { + process.kill(-server.app.pid) + + // Keep the logs if the test failed + if (this.ok) { + serversUtils.flushTests(done) + } else { + done() + } + }) +}) diff --git a/server/tests/api/check-params/video-abuses.js b/server/tests/api/check-params/video-abuses.js new file mode 100644 index 000000000..8cb4ccdc1 --- /dev/null +++ b/server/tests/api/check-params/video-abuses.js @@ -0,0 +1,180 @@ +'use strict' + +const request = require('supertest') +const series = require('async/series') + +const loginUtils = require('../../utils/login') +const requestsUtils = require('../../utils/requests') +const serversUtils = require('../../utils/servers') +const usersUtils = require('../../utils/users') +const videosUtils = require('../../utils/videos') + +describe('Test video abuses API validators', function () { + let server = null + let userAccessToken = null + + // --------------------------------------------------------------- + + before(function (done) { + this.timeout(20000) + + series([ + function (next) { + serversUtils.flushTests(next) + }, + function (next) { + serversUtils.runServer(1, function (server1) { + server = server1 + + next() + }) + }, + function (next) { + loginUtils.loginAndGetAccessToken(server, function (err, token) { + if (err) throw err + server.accessToken = token + + next() + }) + }, + function (next) { + const username = 'user1' + const password = 'my super password' + + usersUtils.createUser(server.url, server.accessToken, username, password, next) + }, + function (next) { + const user = { + username: 'user1', + password: 'my super password' + } + + loginUtils.getUserAccessToken(server, user, function (err, accessToken) { + if (err) throw err + + userAccessToken = accessToken + + next() + }) + }, + // Upload some videos on each pods + function (next) { + const name = 'my super name for pod' + const description = 'my super description for pod' + const tags = [ 'tag' ] + const file = 'video_short2.webm' + videosUtils.uploadVideo(server.url, server.accessToken, name, description, tags, file, next) + }, + function (next) { + videosUtils.getVideosList(server.url, function (err, res) { + if (err) throw err + + const videos = res.body.data + server.video = videos[0] + + next() + }) + } + ], done) + }) + + describe('When listing video abuses', function () { + const path = '/api/v1/videos/abuse' + + it('Should fail with a bad start pagination', function (done) { + request(server.url) + .get(path) + .query({ start: 'hello' }) + .set('Authorization', 'Bearer ' + server.accessToken) + .set('Accept', 'application/json') + .expect(400, done) + }) + + it('Should fail with a bad count pagination', function (done) { + request(server.url) + .get(path) + .query({ count: 'hello' }) + .set('Accept', 'application/json') + .set('Authorization', 'Bearer ' + server.accessToken) + .expect(400, done) + }) + + it('Should fail with an incorrect sort', function (done) { + request(server.url) + .get(path) + .query({ sort: 'hello' }) + .set('Accept', 'application/json') + .set('Authorization', 'Bearer ' + server.accessToken) + .expect(400, done) + }) + + it('Should fail with a non authenticated user', function (done) { + request(server.url) + .get(path) + .query({ sort: 'hello' }) + .set('Accept', 'application/json') + .expect(401, done) + }) + + it('Should fail with a non admin user', function (done) { + request(server.url) + .get(path) + .query({ sort: 'hello' }) + .set('Accept', 'application/json') + .set('Authorization', 'Bearer ' + userAccessToken) + .expect(403, done) + }) + }) + + describe('When reporting a video abuse', function () { + const basePath = '/api/v1/videos/' + + it('Should fail with nothing', function (done) { + const path = basePath + server.video + '/abuse' + const data = {} + requestsUtils.makePostBodyRequest(server.url, path, server.accessToken, data, done) + }) + + it('Should fail with a wrong video', function (done) { + const wrongPath = '/api/v1/videos/blabla/abuse' + const data = {} + requestsUtils.makePostBodyRequest(server.url, wrongPath, server.accessToken, data, done) + }) + + it('Should fail with a non authenticated user', function (done) { + const data = {} + const path = basePath + server.video + '/abuse' + requestsUtils.makePostBodyRequest(server.url, path, 'hello', data, done, 401) + }) + + it('Should fail with a reason too short', function (done) { + const data = { + reason: 'h' + } + const path = basePath + server.video + '/abuse' + requestsUtils.makePostBodyRequest(server.url, path, server.accessToken, data, done) + }) + + it('Should fail with a reason too big', function (done) { + const data = { + reason: '0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef' + + '0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef' + + '0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef' + + '0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef' + } + const path = basePath + server.video + '/abuse' + requestsUtils.makePostBodyRequest(server.url, path, server.accessToken, data, done) + }) + }) + + after(function (done) { + process.kill(-server.app.pid) + + // Keep the logs if the test failed + if (this.ok) { + serversUtils.flushTests(done) + } else { + done() + } + }) +}) diff --git a/server/tests/api/check-params/videos.js b/server/tests/api/check-params/videos.js new file mode 100644 index 000000000..fac903715 --- /dev/null +++ b/server/tests/api/check-params/videos.js @@ -0,0 +1,460 @@ +'use strict' + +const chai = require('chai') +const expect = chai.expect +const pathUtils = require('path') +const request = require('supertest') +const series = require('async/series') + +const loginUtils = require('../../utils/login') +const requestsUtils = require('../../utils/requests') +const serversUtils = require('../../utils/servers') +const videosUtils = require('../../utils/videos') + +describe('Test videos API validator', function () { + const path = '/api/v1/videos/' + let server = null + + // --------------------------------------------------------------- + + before(function (done) { + this.timeout(20000) + + series([ + function (next) { + serversUtils.flushTests(next) + }, + function (next) { + serversUtils.runServer(1, function (server1) { + server = server1 + + next() + }) + }, + function (next) { + loginUtils.loginAndGetAccessToken(server, function (err, token) { + if (err) throw err + server.accessToken = token + + next() + }) + } + ], done) + }) + + describe('When listing a video', function () { + it('Should fail with a bad start pagination', function (done) { + request(server.url) + .get(path) + .query({ start: 'hello' }) + .set('Accept', 'application/json') + .expect(400, done) + }) + + it('Should fail with a bad count pagination', function (done) { + request(server.url) + .get(path) + .query({ count: 'hello' }) + .set('Accept', 'application/json') + .expect(400, done) + }) + + it('Should fail with an incorrect sort', function (done) { + request(server.url) + .get(path) + .query({ sort: 'hello' }) + .set('Accept', 'application/json') + .expect(400, done) + }) + }) + + describe('When searching a video', function () { + it('Should fail with nothing', function (done) { + request(server.url) + .get(pathUtils.join(path, 'search')) + .set('Accept', 'application/json') + .expect(400, done) + }) + + it('Should fail with a bad start pagination', function (done) { + request(server.url) + .get(pathUtils.join(path, 'search', 'test')) + .query({ start: 'hello' }) + .set('Accept', 'application/json') + .expect(400, done) + }) + + it('Should fail with a bad count pagination', function (done) { + request(server.url) + .get(pathUtils.join(path, 'search', 'test')) + .query({ count: 'hello' }) + .set('Accept', 'application/json') + .expect(400, done) + }) + + it('Should fail with an incorrect sort', function (done) { + request(server.url) + .get(pathUtils.join(path, 'search', 'test')) + .query({ sort: 'hello' }) + .set('Accept', 'application/json') + .expect(400, done) + }) + }) + + describe('When adding a video', function () { + it('Should fail with nothing', function (done) { + const data = {} + const attach = {} + requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, done) + }) + + it('Should fail without name', function (done) { + const data = { + description: 'my super description', + tags: [ 'tag1', 'tag2' ] + } + const attach = { + 'videofile': pathUtils.join(__dirname, '..', 'fixtures', 'video_short.webm') + } + requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, done) + }) + + it('Should fail with a long name', function (done) { + const data = { + name: 'My very very very very very very very very very very very very very very very very long name', + description: 'my super description', + tags: [ 'tag1', 'tag2' ] + } + const attach = { + 'videofile': pathUtils.join(__dirname, '..', 'fixtures', 'video_short.webm') + } + requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, done) + }) + + it('Should fail without description', function (done) { + const data = { + name: 'my super name', + tags: [ 'tag1', 'tag2' ] + } + const attach = { + 'videofile': pathUtils.join(__dirname, '..', 'fixtures', 'video_short.webm') + } + requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, done) + }) + + it('Should fail with a long description', function (done) { + const data = { + name: 'my super name', + description: 'my super description which is very very very very very very very very very very very very very very' + + 'very very very very very very very very very very very very very very very very very very very very very' + + 'very very very very very very very very very very very very very very very long', + tags: [ 'tag1', 'tag2' ] + } + const attach = { + 'videofile': pathUtils.join(__dirname, '..', 'fixtures', 'video_short.webm') + } + requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, done) + }) + + it('Should fail without tags', function (done) { + const data = { + name: 'my super name', + description: 'my super description' + } + const attach = { + 'videofile': pathUtils.join(__dirname, '..', 'fixtures', 'video_short.webm') + } + requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, done) + }) + + it('Should fail with too many tags', function (done) { + const data = { + name: 'my super name', + description: 'my super description', + tags: [ 'tag1', 'tag2', 'tag3', 'tag4' ] + } + const attach = { + 'videofile': pathUtils.join(__dirname, '..', 'fixtures', 'video_short.webm') + } + requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, done) + }) + + it('Should fail with not enough tags', function (done) { + const data = { + name: 'my super name', + description: 'my super description', + tags: [ ] + } + const attach = { + 'videofile': pathUtils.join(__dirname, '..', 'fixtures', 'video_short.webm') + } + requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, done) + }) + + it('Should fail with a tag length too low', function (done) { + const data = { + name: 'my super name', + description: 'my super description', + tags: [ 'tag1', 't' ] + } + const attach = { + 'videofile': pathUtils.join(__dirname, '..', 'fixtures', 'video_short.webm') + } + requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, done) + }) + + it('Should fail with a tag length too big', function (done) { + const data = { + name: 'my super name', + description: 'my super description', + tags: [ 'mysupertagtoolong', 'tag1' ] + } + const attach = { + 'videofile': pathUtils.join(__dirname, '..', 'fixtures', 'video_short.webm') + } + requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, done) + }) + + it('Should fail with malformed tags', function (done) { + const data = { + name: 'my super name', + description: 'my super description', + tags: [ 'my tag' ] + } + const attach = { + 'videofile': pathUtils.join(__dirname, '..', 'fixtures', 'video_short.webm') + } + requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, done) + }) + + it('Should fail without an input file', function (done) { + const data = { + name: 'my super name', + description: 'my super description', + tags: [ 'tag1', 'tag2' ] + } + const attach = {} + requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, done) + }) + + it('Should fail without an incorrect input file', function (done) { + const data = { + name: 'my super name', + description: 'my super description', + tags: [ 'tag1', 'tag2' ] + } + const attach = { + 'videofile': pathUtils.join(__dirname, '..', 'fixtures', 'video_short_fake.webm') + } + requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, done) + }) + + it('Should fail with a too big duration', function (done) { + const data = { + name: 'my super name', + description: 'my super description', + tags: [ 'tag1', 'tag2' ] + } + const attach = { + 'videofile': pathUtils.join(__dirname, '..', 'fixtures', 'video_too_long.webm') + } + requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, done) + }) + + it('Should succeed with the correct parameters', function (done) { + const data = { + name: 'my super name', + description: 'my super description', + tags: [ 'tag1', 'tag2' ] + } + const attach = { + 'videofile': pathUtils.join(__dirname, '..', 'fixtures', 'video_short.webm') + } + requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, function () { + attach.videofile = pathUtils.join(__dirname, '..', 'fixtures', 'video_short.mp4') + requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, function () { + attach.videofile = pathUtils.join(__dirname, '..', 'fixtures', 'video_short.ogv') + requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, done, 204) + }, false) + }, false) + }) + }) + + describe('When updating a video', function () { + let videoId + + before(function (done) { + videosUtils.getVideosList(server.url, function (err, res) { + if (err) throw err + + videoId = res.body.data[0].id + + return done() + }) + }) + + it('Should fail with nothing', function (done) { + const data = {} + requestsUtils.makePutBodyRequest(server.url, path, server.accessToken, data, done) + }) + + it('Should fail without a valid uuid', function (done) { + const data = { + description: 'my super description', + tags: [ 'tag1', 'tag2' ] + } + requestsUtils.makePutBodyRequest(server.url, path + 'blabla', server.accessToken, data, done) + }) + + it('Should fail with an unknown id', function (done) { + const data = { + description: 'my super description', + tags: [ 'tag1', 'tag2' ] + } + requestsUtils.makePutBodyRequest(server.url, path + '4da6fde3-88f7-4d16-b119-108df5630b06', server.accessToken, data, done, 404) + }) + + it('Should fail with a long name', function (done) { + const data = { + name: 'My very very very very very very very very very very very very very very very very long name', + description: 'my super description', + tags: [ 'tag1', 'tag2' ] + } + requestsUtils.makePutBodyRequest(server.url, path + videoId, server.accessToken, data, done) + }) + + it('Should fail with a long description', function (done) { + const data = { + name: 'my super name', + description: 'my super description which is very very very very very very very very very very very very very very' + + 'very very very very very very very very very very very very very very very very very very very very very' + + 'very very very very very very very very very very very very very very very long', + tags: [ 'tag1', 'tag2' ] + } + requestsUtils.makePutBodyRequest(server.url, path + videoId, server.accessToken, data, done) + }) + + it('Should fail with too many tags', function (done) { + const data = { + name: 'my super name', + description: 'my super description', + tags: [ 'tag1', 'tag2', 'tag3', 'tag4' ] + } + requestsUtils.makePutBodyRequest(server.url, path + videoId, server.accessToken, data, done) + }) + + it('Should fail with not enough tags', function (done) { + const data = { + name: 'my super name', + description: 'my super description', + tags: [ ] + } + requestsUtils.makePutBodyRequest(server.url, path + videoId, server.accessToken, data, done) + }) + + it('Should fail with a tag length too low', function (done) { + const data = { + name: 'my super name', + description: 'my super description', + tags: [ 'tag1', 't' ] + } + requestsUtils.makePutBodyRequest(server.url, path + videoId, server.accessToken, data, done) + }) + + it('Should fail with a tag length too big', function (done) { + const data = { + name: 'my super name', + description: 'my super description', + tags: [ 'mysupertagtoolong', 'tag1' ] + } + requestsUtils.makePutBodyRequest(server.url, path + videoId, server.accessToken, data, done) + }) + + it('Should fail with malformed tags', function (done) { + const data = { + name: 'my super name', + description: 'my super description', + tags: [ 'my tag' ] + } + requestsUtils.makePutBodyRequest(server.url, path + videoId, server.accessToken, data, done) + }) + + it('Should fail with a video of another user') + + it('Should fail with a video of another pod') + }) + + describe('When getting a video', function () { + it('Should return the list of the videos with nothing', function (done) { + request(server.url) + .get(path) + .set('Accept', 'application/json') + .expect(200) + .expect('Content-Type', /json/) + .end(function (err, res) { + if (err) throw err + + expect(res.body.data).to.be.an('array') + expect(res.body.data.length).to.equal(3) + + done() + }) + }) + + it('Should fail without a correct uuid', function (done) { + request(server.url) + .get(path + 'coucou') + .set('Accept', 'application/json') + .expect(400, done) + }) + + it('Should return 404 with an incorrect video', function (done) { + request(server.url) + .get(path + '4da6fde3-88f7-4d16-b119-108df5630b06') + .set('Accept', 'application/json') + .expect(404, done) + }) + + it('Should succeed with the correct parameters') + }) + + describe('When removing a video', function () { + it('Should have 404 with nothing', function (done) { + request(server.url) + .delete(path) + .set('Authorization', 'Bearer ' + server.accessToken) + .expect(400, done) + }) + + it('Should fail without a correct uuid', function (done) { + request(server.url) + .delete(path + 'hello') + .set('Authorization', 'Bearer ' + server.accessToken) + .expect(400, done) + }) + + it('Should fail with a video which does not exist', function (done) { + request(server.url) + .delete(path + '4da6fde3-88f7-4d16-b119-108df5630b06') + .set('Authorization', 'Bearer ' + server.accessToken) + .expect(404, done) + }) + + it('Should fail with a video of another user') + + it('Should fail with a video of another pod') + + it('Should succeed with the correct parameters') + }) + + after(function (done) { + process.kill(-server.app.pid) + + // Keep the logs if the test failed + if (this.ok) { + serversUtils.flushTests(done) + } else { + done() + } + }) +}) diff --git a/server/tests/api/friends-advanced.js b/server/tests/api/friends-advanced.js index 0a2d58d82..708138bc9 100644 --- a/server/tests/api/friends-advanced.js +++ b/server/tests/api/friends-advanced.js @@ -86,7 +86,7 @@ describe('Test advanced friends', function () { getFriendsList(5, function (err, res) { if (err) throw err - expect(res.body.length).to.equal(0) + expect(res.body.data.length).to.equal(0) done() }) @@ -111,7 +111,7 @@ describe('Test advanced friends', function () { getFriendsList(i, function (err, res) { if (err) throw err - expect(res.body.length).to.equal(0) + expect(res.body.data.length).to.equal(0) callback() }) @@ -140,7 +140,7 @@ describe('Test advanced friends', function () { getFriendsList(i, function (err, res) { if (err) throw err - expect(res.body.length).to.equal(3) + expect(res.body.data.length).to.equal(3) callback() }) @@ -182,7 +182,7 @@ describe('Test advanced friends', function () { if (err) throw err // Pod 4 didn't know pod 1 and 2 removed it - expect(res.body.length).to.equal(3) + expect(res.body.data.length).to.equal(3) next() }) }, @@ -200,7 +200,7 @@ describe('Test advanced friends', function () { if (err) throw err // Pod 4 should not be our friend - const result = res.body + const result = res.body.data expect(result.length).to.equal(3) for (const pod of result) { expect(pod.host).not.equal(servers[3].host) diff --git a/server/tests/api/friends-basic.js b/server/tests/api/friends-basic.js index a871f9838..6f37ff291 100644 --- a/server/tests/api/friends-basic.js +++ b/server/tests/api/friends-basic.js @@ -28,7 +28,7 @@ describe('Test basic friends', function () { podsUtils.getFriendsList(serverToTest.url, function (err, res) { if (err) throw err - const result = res.body + const result = res.body.data expect(result).to.be.an('array') expect(result.length).to.equal(2) @@ -65,7 +65,7 @@ describe('Test basic friends', function () { podsUtils.getFriendsList(server.url, function (err, res) { if (err) throw err - const result = res.body + const result = res.body.data expect(result).to.be.an('array') expect(result.length).to.equal(0) callback() @@ -90,14 +90,14 @@ describe('Test basic friends', function () { podsUtils.getFriendsList(servers[1].url, function (err, res) { if (err) throw err - const result = res.body + const result = res.body.data expect(result).to.be.an('array') expect(result.length).to.equal(1) const pod = result[0] expect(pod.host).to.equal(servers[2].host) expect(pod.score).to.equal(20) - expect(miscsUtils.dateIsValid(pod.createdDate)).to.be.true + expect(miscsUtils.dateIsValid(pod.createdAt)).to.be.true next() }) @@ -107,14 +107,14 @@ describe('Test basic friends', function () { podsUtils.getFriendsList(servers[2].url, function (err, res) { if (err) throw err - const result = res.body + const result = res.body.data expect(result).to.be.an('array') expect(result.length).to.equal(1) const pod = result[0] expect(pod.host).to.equal(servers[1].host) expect(pod.score).to.equal(20) - expect(miscsUtils.dateIsValid(pod.createdDate)).to.be.true + expect(miscsUtils.dateIsValid(pod.createdAt)).to.be.true next() }) @@ -154,7 +154,7 @@ describe('Test basic friends', function () { podsUtils.getFriendsList(servers[1].url, function (err, res) { if (err) throw err - const result = res.body + const result = res.body.data expect(result).to.be.an('array') expect(result.length).to.equal(0) @@ -167,7 +167,7 @@ describe('Test basic friends', function () { podsUtils.getFriendsList(url, function (err, res) { if (err) throw err - const result = res.body + const result = res.body.data expect(result).to.be.an('array') expect(result.length).to.equal(1) expect(result[0].host).not.to.be.equal(servers[1].host) diff --git a/server/tests/api/multiple-pods.js b/server/tests/api/multiple-pods.js index be278d7c5..df12ba0e9 100644 --- a/server/tests/api/multiple-pods.js +++ b/server/tests/api/multiple-pods.js @@ -4,7 +4,8 @@ const chai = require('chai') const each = require('async/each') const expect = chai.expect const series = require('async/series') -const webtorrent = new (require('webtorrent'))() +const WebTorrent = require('webtorrent') +const webtorrent = new WebTorrent() const loginUtils = require('../utils/login') const miscsUtils = require('../utils/miscs') @@ -104,7 +105,8 @@ describe('Test multiple pods', function () { expect(video.magnetUri).to.exist expect(video.duration).to.equal(10) expect(video.tags).to.deep.equal([ 'tag1p1', 'tag2p1' ]) - expect(miscsUtils.dateIsValid(video.createdDate)).to.be.true + expect(miscsUtils.dateIsValid(video.createdAt)).to.be.true + expect(miscsUtils.dateIsValid(video.updatedAt)).to.be.true expect(video.author).to.equal('root') if (server.url !== 'http://localhost:9001') { @@ -166,7 +168,8 @@ describe('Test multiple pods', function () { expect(video.magnetUri).to.exist expect(video.duration).to.equal(5) expect(video.tags).to.deep.equal([ 'tag1p2', 'tag2p2', 'tag3p2' ]) - expect(miscsUtils.dateIsValid(video.createdDate)).to.be.true + expect(miscsUtils.dateIsValid(video.createdAt)).to.be.true + expect(miscsUtils.dateIsValid(video.updatedAt)).to.be.true expect(video.author).to.equal('root') if (server.url !== 'http://localhost:9002') { @@ -246,7 +249,8 @@ describe('Test multiple pods', function () { expect(video1.duration).to.equal(5) expect(video1.tags).to.deep.equal([ 'tag1p3' ]) expect(video1.author).to.equal('root') - expect(miscsUtils.dateIsValid(video1.createdDate)).to.be.true + expect(miscsUtils.dateIsValid(video1.createdAt)).to.be.true + expect(miscsUtils.dateIsValid(video1.updatedAt)).to.be.true expect(video2.name).to.equal('my super name for pod 3-2') expect(video2.description).to.equal('my super description for pod 3-2') @@ -255,7 +259,8 @@ describe('Test multiple pods', function () { expect(video2.duration).to.equal(5) expect(video2.tags).to.deep.equal([ 'tag2p3', 'tag3p3', 'tag4p3' ]) expect(video2.author).to.equal('root') - expect(miscsUtils.dateIsValid(video2.createdDate)).to.be.true + expect(miscsUtils.dateIsValid(video2.createdAt)).to.be.true + expect(miscsUtils.dateIsValid(video2.updatedAt)).to.be.true if (server.url !== 'http://localhost:9003') { expect(video1.isLocal).to.be.false @@ -299,8 +304,8 @@ describe('Test multiple pods', function () { if (err) throw err const video = res.body.data[0] - toRemove.push(res.body.data[2].id) - toRemove.push(res.body.data[3].id) + toRemove.push(res.body.data[2]) + toRemove.push(res.body.data[3]) webtorrent.add(video.magnetUri, function (torrent) { expect(torrent.files).to.exist @@ -368,16 +373,68 @@ describe('Test multiple pods', function () { }) }) }) + }) - it('Should remove the file 3 and 3-2 by asking pod 3', function (done) { + describe('Should manipulate these videos', function () { + it('Should update the video 3 by asking pod 3', function (done) { + this.timeout(15000) + + const name = 'my super video updated' + const description = 'my super description updated' + const tags = [ 'tagup1', 'tagup2' ] + + videosUtils.updateVideo(servers[2].url, servers[2].accessToken, toRemove[0].id, name, description, tags, function (err) { + if (err) throw err + + setTimeout(done, 11000) + }) + }) + + it('Should have the video 3 updated on each pod', function (done) { + this.timeout(200000) + + each(servers, function (server, callback) { + // Avoid "duplicate torrent" errors + const webtorrent = new WebTorrent() + + videosUtils.getVideosList(server.url, function (err, res) { + if (err) throw err + + const videos = res.body.data + const videoUpdated = videos.find(function (video) { + return video.name === 'my super video updated' + }) + + expect(!!videoUpdated).to.be.true + expect(videoUpdated.description).to.equal('my super description updated') + expect(videoUpdated.tags).to.deep.equal([ 'tagup1', 'tagup2' ]) + expect(miscsUtils.dateIsValid(videoUpdated.updatedAt, 20000)).to.be.true + + videosUtils.testVideoImage(server.url, 'video_short3.webm', videoUpdated.thumbnailPath, function (err, test) { + if (err) throw err + expect(test).to.equal(true) + + webtorrent.add(videoUpdated.magnetUri, function (torrent) { + expect(torrent.files).to.exist + expect(torrent.files.length).to.equal(1) + expect(torrent.files[0].path).to.exist.and.to.not.equal('') + + callback() + }) + }) + }) + }, done) + }) + + it('Should remove the videos 3 and 3-2 by asking pod 3', function (done) { this.timeout(15000) series([ function (next) { - videosUtils.removeVideo(servers[2].url, servers[2].accessToken, toRemove[0], next) + videosUtils.removeVideo(servers[2].url, servers[2].accessToken, toRemove[0].id, next) }, function (next) { - videosUtils.removeVideo(servers[2].url, servers[2].accessToken, toRemove[1], next) + videosUtils.removeVideo(servers[2].url, servers[2].accessToken, toRemove[1].id, next) }], function (err) { if (err) throw err @@ -394,11 +451,11 @@ describe('Test multiple pods', function () { const videos = res.body.data expect(videos).to.be.an('array') expect(videos.length).to.equal(2) - expect(videos[0].id).not.to.equal(videos[1].id) - expect(videos[0].id).not.to.equal(toRemove[0]) - expect(videos[1].id).not.to.equal(toRemove[0]) - expect(videos[0].id).not.to.equal(toRemove[1]) - expect(videos[1].id).not.to.equal(toRemove[1]) + expect(videos[0].name).not.to.equal(videos[1].name) + expect(videos[0].name).not.to.equal(toRemove[0].name) + expect(videos[1].name).not.to.equal(toRemove[0].name) + expect(videos[0].name).not.to.equal(toRemove[1].name) + expect(videos[1].name).not.to.equal(toRemove[1].name) callback() }) diff --git a/server/tests/api/requests.js b/server/tests/api/requests.js index af36f6e34..933ed29b4 100644 --- a/server/tests/api/requests.js +++ b/server/tests/api/requests.js @@ -69,7 +69,7 @@ describe('Test requests stats', function () { }) }) - it('Should have the correct request', function (done) { + it('Should have the correct total request', function (done) { this.timeout(15000) const server = servers[0] @@ -79,40 +79,16 @@ describe('Test requests stats', function () { uploadVideo(server, function (err) { if (err) throw err - getRequestsStats(server, function (err, res) { - if (err) throw err + setTimeout(function () { + getRequestsStats(server, function (err, res) { + if (err) throw err - const body = res.body - expect(body.requests).to.have.lengthOf(1) + const body = res.body + expect(body.totalRequests).to.equal(1) - const request = body.requests[0] - expect(request.to).to.have.lengthOf(1) - expect(request.request.type).to.equal('add') - - // Wait one cycle - setTimeout(done, 10000) - }) - }) - }) - - it('Should have the correct requests', function (done) { - const server = servers[0] - - uploadVideo(server, function (err) { - if (err) throw err - - getRequestsStats(server, function (err, res) { - if (err) throw err - - const body = res.body - expect(body.requests).to.have.lengthOf(2) - - const request = body.requests[1] - expect(request.to).to.have.lengthOf(1) - expect(request.request.type).to.equal('add') - - done() - }) + done() + }) + }, 1000) }) }) diff --git a/server/tests/api/single-pod.js b/server/tests/api/single-pod.js index 65d1a7a65..2db60448f 100644 --- a/server/tests/api/single-pod.js +++ b/server/tests/api/single-pod.js @@ -82,7 +82,8 @@ describe('Test a single pod', function () { expect(video.author).to.equal('root') expect(video.isLocal).to.be.true expect(video.tags).to.deep.equal([ 'tag1', 'tag2', 'tag3' ]) - expect(miscsUtils.dateIsValid(video.createdDate)).to.be.true + expect(miscsUtils.dateIsValid(video.createdAt)).to.be.true + expect(miscsUtils.dateIsValid(video.updatedAt)).to.be.true videosUtils.testVideoImage(server.url, 'video_short.webm', video.thumbnailPath, function (err, test) { if (err) throw err @@ -116,7 +117,8 @@ describe('Test a single pod', function () { expect(video.author).to.equal('root') expect(video.isLocal).to.be.true expect(video.tags).to.deep.equal([ 'tag1', 'tag2', 'tag3' ]) - expect(miscsUtils.dateIsValid(video.createdDate)).to.be.true + expect(miscsUtils.dateIsValid(video.createdAt)).to.be.true + expect(miscsUtils.dateIsValid(video.updatedAt)).to.be.true videosUtils.testVideoImage(server.url, 'video_short.webm', video.thumbnailPath, function (err, test) { if (err) throw err @@ -142,7 +144,8 @@ describe('Test a single pod', function () { expect(video.author).to.equal('root') expect(video.isLocal).to.be.true expect(video.tags).to.deep.equal([ 'tag1', 'tag2', 'tag3' ]) - expect(miscsUtils.dateIsValid(video.createdDate)).to.be.true + expect(miscsUtils.dateIsValid(video.createdAt)).to.be.true + expect(miscsUtils.dateIsValid(video.updatedAt)).to.be.true videosUtils.testVideoImage(server.url, 'video_short.webm', video.thumbnailPath, function (err, test) { if (err) throw err @@ -153,31 +156,33 @@ describe('Test a single pod', function () { }) }) - it('Should search the video by podHost', function (done) { - videosUtils.searchVideo(server.url, '9001', 'podHost', function (err, res) { - if (err) throw err + // Not implemented yet + // it('Should search the video by podHost', function (done) { + // videosUtils.searchVideo(server.url, '9001', 'host', function (err, res) { + // if (err) throw err - expect(res.body.total).to.equal(1) - expect(res.body.data).to.be.an('array') - expect(res.body.data.length).to.equal(1) + // expect(res.body.total).to.equal(1) + // expect(res.body.data).to.be.an('array') + // expect(res.body.data.length).to.equal(1) - const video = res.body.data[0] - expect(video.name).to.equal('my super name') - expect(video.description).to.equal('my super description') - expect(video.podHost).to.equal('localhost:9001') - expect(video.author).to.equal('root') - expect(video.isLocal).to.be.true - expect(video.tags).to.deep.equal([ 'tag1', 'tag2', 'tag3' ]) - expect(miscsUtils.dateIsValid(video.createdDate)).to.be.true + // const video = res.body.data[0] + // expect(video.name).to.equal('my super name') + // expect(video.description).to.equal('my super description') + // expect(video.podHost).to.equal('localhost:9001') + // expect(video.author).to.equal('root') + // expect(video.isLocal).to.be.true + // expect(video.tags).to.deep.equal([ 'tag1', 'tag2', 'tag3' ]) + // expect(miscsUtils.dateIsValid(video.createdAt)).to.be.true + // expect(miscsUtils.dateIsValid(video.updatedAt)).to.be.true - videosUtils.testVideoImage(server.url, 'video_short.webm', video.thumbnailPath, function (err, test) { - if (err) throw err - expect(test).to.equal(true) + // videosUtils.testVideoImage(server.url, 'video_short.webm', video.thumbnailPath, function (err, test) { + // if (err) throw err + // expect(test).to.equal(true) - done() - }) - }) - }) + // done() + // }) + // }) + // }) it('Should search the video by tag', function (done) { videosUtils.searchVideo(server.url, 'tag1', 'tags', function (err, res) { @@ -194,7 +199,8 @@ describe('Test a single pod', function () { expect(video.author).to.equal('root') expect(video.isLocal).to.be.true expect(video.tags).to.deep.equal([ 'tag1', 'tag2', 'tag3' ]) - expect(miscsUtils.dateIsValid(video.createdDate)).to.be.true + expect(miscsUtils.dateIsValid(video.createdAt)).to.be.true + expect(miscsUtils.dateIsValid(video.updatedAt)).to.be.true videosUtils.testVideoImage(server.url, 'video_short.webm', video.thumbnailPath, function (err, test) { if (err) throw err @@ -230,7 +236,7 @@ describe('Test a single pod', function () { }) it('Should not find a search by tag', function (done) { - videosUtils.searchVideo(server.url, 'tag', 'tags', function (err, res) { + videosUtils.searchVideo(server.url, 'hello', 'tags', function (err, res) { if (err) throw err expect(res.body.total).to.equal(0) @@ -332,69 +338,69 @@ describe('Test a single pod', function () { }) it('Should list only the two first videos', function (done) { - videosUtils.getVideosListPagination(server.url, 0, 2, function (err, res) { + videosUtils.getVideosListPagination(server.url, 0, 2, 'name', function (err, res) { if (err) throw err const videos = res.body.data expect(res.body.total).to.equal(6) expect(videos.length).to.equal(2) - expect(videos[0].name === videosListBase[0].name) - expect(videos[1].name === videosListBase[1].name) + expect(videos[0].name).to.equal(videosListBase[0].name) + expect(videos[1].name).to.equal(videosListBase[1].name) done() }) }) it('Should list only the next three videos', function (done) { - videosUtils.getVideosListPagination(server.url, 2, 3, function (err, res) { + videosUtils.getVideosListPagination(server.url, 2, 3, 'name', function (err, res) { if (err) throw err const videos = res.body.data expect(res.body.total).to.equal(6) expect(videos.length).to.equal(3) - expect(videos[0].name === videosListBase[2].name) - expect(videos[1].name === videosListBase[3].name) - expect(videos[2].name === videosListBase[4].name) + expect(videos[0].name).to.equal(videosListBase[2].name) + expect(videos[1].name).to.equal(videosListBase[3].name) + expect(videos[2].name).to.equal(videosListBase[4].name) done() }) }) it('Should list the last video', function (done) { - videosUtils.getVideosListPagination(server.url, 5, 6, function (err, res) { + videosUtils.getVideosListPagination(server.url, 5, 6, 'name', function (err, res) { if (err) throw err const videos = res.body.data expect(res.body.total).to.equal(6) expect(videos.length).to.equal(1) - expect(videos[0].name === videosListBase[5].name) + expect(videos[0].name).to.equal(videosListBase[5].name) done() }) }) it('Should search the first video', function (done) { - videosUtils.searchVideoWithPagination(server.url, 'webm', 'name', 0, 1, function (err, res) { + videosUtils.searchVideoWithPagination(server.url, 'webm', 'name', 0, 1, 'name', function (err, res) { if (err) throw err const videos = res.body.data expect(res.body.total).to.equal(4) expect(videos.length).to.equal(1) - expect(videos[0].name === 'video_short.webm name') + expect(videos[0].name).to.equal('video_short1.webm name') done() }) }) it('Should search the last two videos', function (done) { - videosUtils.searchVideoWithPagination(server.url, 'webm', 'name', 2, 2, function (err, res) { + videosUtils.searchVideoWithPagination(server.url, 'webm', 'name', 2, 2, 'name', function (err, res) { if (err) throw err const videos = res.body.data expect(res.body.total).to.equal(4) expect(videos.length).to.equal(2) - expect(videos[0].name === 'video_short2.webm name') - expect(videos[1].name === 'video_short3.webm name') + expect(videos[0].name).to.equal('video_short3.webm name') + expect(videos[1].name).to.equal('video_short.webm name') done() }) @@ -424,29 +430,30 @@ describe('Test a single pod', function () { }) }) - it('Should search all the 9001 port videos', function (done) { - videosUtils.searchVideoWithPagination(server.url, '9001', 'podHost', 0, 15, function (err, res) { - if (err) throw err + // Not implemented yet + // it('Should search all the 9001 port videos', function (done) { + // videosUtils.searchVideoWithPagination(server.url, '9001', 'host', 0, 15, function (err, res) { + // if (err) throw err - const videos = res.body.data - expect(res.body.total).to.equal(6) - expect(videos.length).to.equal(6) + // const videos = res.body.data + // expect(res.body.total).to.equal(6) + // expect(videos.length).to.equal(6) - done() - }) - }) + // done() + // }) + // }) - it('Should search all the localhost videos', function (done) { - videosUtils.searchVideoWithPagination(server.url, 'localhost', 'podHost', 0, 15, function (err, res) { - if (err) throw err + // it('Should search all the localhost videos', function (done) { + // videosUtils.searchVideoWithPagination(server.url, 'localhost', 'host', 0, 15, function (err, res) { + // if (err) throw err - const videos = res.body.data - expect(res.body.total).to.equal(6) - expect(videos.length).to.equal(6) + // const videos = res.body.data + // expect(res.body.total).to.equal(6) + // expect(videos.length).to.equal(6) - done() - }) - }) + // done() + // }) + // }) it('Should search the good magnetUri video', function (done) { const video = videosListBase[0] @@ -469,12 +476,12 @@ describe('Test a single pod', function () { const videos = res.body.data expect(res.body.total).to.equal(6) expect(videos.length).to.equal(6) - expect(videos[5].name === 'video_short.mp4 name') - expect(videos[4].name === 'video_short.ogv name') - expect(videos[3].name === 'video_short.webm name') - expect(videos[2].name === 'video_short1.webm name') - expect(videos[1].name === 'video_short2.webm name') - expect(videos[0].name === 'video_short3.webm name') + expect(videos[0].name).to.equal('video_short.webm name') + expect(videos[1].name).to.equal('video_short.ogv name') + expect(videos[2].name).to.equal('video_short.mp4 name') + expect(videos[3].name).to.equal('video_short3.webm name') + expect(videos[4].name).to.equal('video_short2.webm name') + expect(videos[5].name).to.equal('video_short1.webm name') done() }) @@ -488,15 +495,107 @@ describe('Test a single pod', function () { expect(res.body.total).to.equal(4) expect(videos.length).to.equal(4) - expect(videos[0].name === 'video_short.webm name') - expect(videos[1].name === 'video_short1.webm name') - expect(videos[2].name === 'video_short2.webm name') - expect(videos[3].name === 'video_short3.webm name') + expect(videos[0].name).to.equal('video_short1.webm name') + expect(videos[1].name).to.equal('video_short2.webm name') + expect(videos[2].name).to.equal('video_short3.webm name') + expect(videos[3].name).to.equal('video_short.webm name') + + videoId = videos[2].id done() }) }) + it('Should update a video', function (done) { + const name = 'my super video updated' + const description = 'my super description updated' + const tags = [ 'tagup1', 'tagup2' ] + + videosUtils.updateVideo(server.url, server.accessToken, videoId, name, description, tags, done) + }) + + it('Should have the video updated', function (done) { + this.timeout(60000) + + videosUtils.getVideo(server.url, videoId, function (err, res) { + if (err) throw err + + const video = res.body + + expect(video.name).to.equal('my super video updated') + expect(video.description).to.equal('my super description updated') + expect(video.podHost).to.equal('localhost:9001') + expect(video.author).to.equal('root') + expect(video.isLocal).to.be.true + expect(video.tags).to.deep.equal([ 'tagup1', 'tagup2' ]) + expect(miscsUtils.dateIsValid(video.createdAt)).to.be.true + expect(miscsUtils.dateIsValid(video.updatedAt)).to.be.true + + videosUtils.testVideoImage(server.url, 'video_short3.webm', video.thumbnailPath, function (err, test) { + if (err) throw err + expect(test).to.equal(true) + + webtorrent.add(video.magnetUri, function (torrent) { + expect(torrent.files).to.exist + expect(torrent.files.length).to.equal(1) + expect(torrent.files[0].path).to.exist.and.to.not.equal('') + + done() + }) + }) + }) + }) + + it('Should update only the tags of a video', function (done) { + const tags = [ 'tag1', 'tag2', 'supertag' ] + + videosUtils.updateVideo(server.url, server.accessToken, videoId, null, null, tags, function (err) { + if (err) throw err + + videosUtils.getVideo(server.url, videoId, function (err, res) { + if (err) throw err + + const video = res.body + + expect(video.name).to.equal('my super video updated') + expect(video.description).to.equal('my super description updated') + expect(video.podHost).to.equal('localhost:9001') + expect(video.author).to.equal('root') + expect(video.isLocal).to.be.true + expect(video.tags).to.deep.equal([ 'tag1', 'tag2', 'supertag' ]) + expect(miscsUtils.dateIsValid(video.createdAt)).to.be.true + expect(miscsUtils.dateIsValid(video.updatedAt)).to.be.true + + done() + }) + }) + }) + + it('Should update only the description of a video', function (done) { + const description = 'hello everybody' + + videosUtils.updateVideo(server.url, server.accessToken, videoId, null, description, null, function (err) { + if (err) throw err + + videosUtils.getVideo(server.url, videoId, function (err, res) { + if (err) throw err + + const video = res.body + + expect(video.name).to.equal('my super video updated') + expect(video.description).to.equal('hello everybody') + expect(video.podHost).to.equal('localhost:9001') + expect(video.author).to.equal('root') + expect(video.isLocal).to.be.true + expect(video.tags).to.deep.equal([ 'tag1', 'tag2', 'supertag' ]) + expect(miscsUtils.dateIsValid(video.createdAt)).to.be.true + expect(miscsUtils.dateIsValid(video.updatedAt)).to.be.true + + done() + }) + }) + }) + after(function (done) { process.kill(-server.app.pid) diff --git a/server/tests/api/users.js b/server/tests/api/users.js index 94267f104..e6d937eb0 100644 --- a/server/tests/api/users.js +++ b/server/tests/api/users.js @@ -261,8 +261,8 @@ describe('Test users', function () { }) }) - it('Should list only the second user by createdDate desc', function (done) { - usersUtils.getUsersListPaginationAndSort(server.url, 0, 1, '-createdDate', function (err, res) { + it('Should list only the second user by createdAt desc', function (done) { + usersUtils.getUsersListPaginationAndSort(server.url, 0, 1, '-createdAt', function (err, res) { if (err) throw err const result = res.body @@ -279,8 +279,8 @@ describe('Test users', function () { }) }) - it('Should list all the users by createdDate asc', function (done) { - usersUtils.getUsersListPaginationAndSort(server.url, 0, 2, 'createdDate', function (err, res) { + it('Should list all the users by createdAt asc', function (done) { + usersUtils.getUsersListPaginationAndSort(server.url, 0, 2, 'createdAt', function (err, res) { if (err) throw err const result = res.body diff --git a/server/tests/api/video-abuse.js b/server/tests/api/video-abuse.js new file mode 100644 index 000000000..58db17c42 --- /dev/null +++ b/server/tests/api/video-abuse.js @@ -0,0 +1,191 @@ +'use strict' + +const chai = require('chai') +const each = require('async/each') +const expect = chai.expect +const series = require('async/series') + +const loginUtils = require('../utils/login') +const podsUtils = require('../utils/pods') +const serversUtils = require('../utils/servers') +const videosUtils = require('../utils/videos') +const videoAbusesUtils = require('../utils/video-abuses') + +describe('Test video abuses', function () { + let servers = [] + + before(function (done) { + this.timeout(30000) + + series([ + // Run servers + function (next) { + serversUtils.flushAndRunMultipleServers(2, function (serversRun) { + servers = serversRun + next() + }) + }, + // Get the access tokens + function (next) { + each(servers, function (server, callbackEach) { + loginUtils.loginAndGetAccessToken(server, function (err, accessToken) { + if (err) return callbackEach(err) + + server.accessToken = accessToken + callbackEach() + }) + }, next) + }, + // Pod 1 make friends too + function (next) { + const server = servers[0] + podsUtils.makeFriends(server.url, server.accessToken, next) + }, + // Upload some videos on each pods + function (next) { + const name = 'my super name for pod 1' + const description = 'my super description for pod 1' + const tags = [ 'tag' ] + const file = 'video_short2.webm' + videosUtils.uploadVideo(servers[0].url, servers[0].accessToken, name, description, tags, file, next) + }, + function (next) { + const name = 'my super name for pod 2' + const description = 'my super description for pod 2' + const tags = [ 'tag' ] + const file = 'video_short2.webm' + videosUtils.uploadVideo(servers[1].url, servers[1].accessToken, name, description, tags, file, next) + }, + // Wait videos propagation + function (next) { + setTimeout(next, 11000) + }, + function (next) { + videosUtils.getVideosList(servers[0].url, function (err, res) { + if (err) throw err + + const videos = res.body.data + + expect(videos.length).to.equal(2) + + servers[0].video = videos.find(function (video) { return video.name === 'my super name for pod 1' }) + servers[1].video = videos.find(function (video) { return video.name === 'my super name for pod 2' }) + + next() + }) + } + ], done) + }) + + it('Should not have video abuses', function (done) { + videoAbusesUtils.getVideoAbusesList(servers[0].url, servers[0].accessToken, function (err, res) { + if (err) throw err + + expect(res.body.total).to.equal(0) + expect(res.body.data).to.be.an('array') + expect(res.body.data.length).to.equal(0) + + done() + }) + }) + + it('Should report abuse on a local video', function (done) { + this.timeout(15000) + + const reason = 'my super bad reason' + videoAbusesUtils.reportVideoAbuse(servers[0].url, servers[0].accessToken, servers[0].video.id, reason, function (err) { + if (err) throw err + + // We wait requests propagation, even if the pod 1 is not supposed to make a request to pod 2 + setTimeout(done, 11000) + }) + }) + + it('Should have 1 video abuses on pod 1 and 0 on pod 2', function (done) { + videoAbusesUtils.getVideoAbusesList(servers[0].url, servers[0].accessToken, function (err, res) { + if (err) throw err + + expect(res.body.total).to.equal(1) + expect(res.body.data).to.be.an('array') + expect(res.body.data.length).to.equal(1) + + const abuse = res.body.data[0] + expect(abuse.reason).to.equal('my super bad reason') + expect(abuse.reporterUsername).to.equal('root') + expect(abuse.reporterPodHost).to.equal('localhost:9001') + expect(abuse.videoId).to.equal(servers[0].video.id) + + videoAbusesUtils.getVideoAbusesList(servers[1].url, servers[1].accessToken, function (err, res) { + if (err) throw err + + expect(res.body.total).to.equal(0) + expect(res.body.data).to.be.an('array') + expect(res.body.data.length).to.equal(0) + + done() + }) + }) + }) + + it('Should report abuse on a remote video', function (done) { + this.timeout(15000) + + const reason = 'my super bad reason 2' + videoAbusesUtils.reportVideoAbuse(servers[0].url, servers[0].accessToken, servers[1].video.id, reason, function (err) { + if (err) throw err + + // We wait requests propagation + setTimeout(done, 11000) + }) + }) + + it('Should have 2 video abuse on pod 1 and 1 on pod 2', function (done) { + videoAbusesUtils.getVideoAbusesList(servers[0].url, servers[0].accessToken, function (err, res) { + if (err) throw err + + expect(res.body.total).to.equal(2) + expect(res.body.data).to.be.an('array') + expect(res.body.data.length).to.equal(2) + + let abuse = res.body.data[0] + expect(abuse.reason).to.equal('my super bad reason') + expect(abuse.reporterUsername).to.equal('root') + expect(abuse.reporterPodHost).to.equal('localhost:9001') + expect(abuse.videoId).to.equal(servers[0].video.id) + + abuse = res.body.data[1] + expect(abuse.reason).to.equal('my super bad reason 2') + expect(abuse.reporterUsername).to.equal('root') + expect(abuse.reporterPodHost).to.equal('localhost:9001') + expect(abuse.videoId).to.equal(servers[1].video.id) + + videoAbusesUtils.getVideoAbusesList(servers[1].url, servers[1].accessToken, function (err, res) { + if (err) throw err + + expect(res.body.total).to.equal(1) + expect(res.body.data).to.be.an('array') + expect(res.body.data.length).to.equal(1) + + let abuse = res.body.data[0] + expect(abuse.reason).to.equal('my super bad reason 2') + expect(abuse.reporterUsername).to.equal('root') + expect(abuse.reporterPodHost).to.equal('localhost:9001') + + done() + }) + }) + }) + + after(function (done) { + servers.forEach(function (server) { + process.kill(-server.app.pid) + }) + + // Keep the logs if the test failed + if (this.ok) { + serversUtils.flushTests(done) + } else { + done() + } + }) +}) diff --git a/server/tests/real-world/real-world.js b/server/tests/real-world/real-world.js index dba1970c5..12ab06d6d 100644 --- a/server/tests/real-world/real-world.js +++ b/server/tests/real-world/real-world.js @@ -2,6 +2,7 @@ const each = require('async/each') const isEqual = require('lodash/isEqual') +const differenceWith = require('lodash/differenceWith') const program = require('commander') const series = require('async/series') @@ -16,32 +17,41 @@ const videosUtils = require('../utils/videos') program .option('-c, --create [weight]', 'Weight for creating videos') .option('-r, --remove [weight]', 'Weight for removing videos') + .option('-u, --update [weight]', 'Weight for updating videos') .option('-p, --pods [n]', 'Number of pods to run (3 or 6)', /^3|6$/, 3) .option('-a, --action [interval]', 'Interval in ms for an action') .option('-i, --integrity [interval]', 'Interval in ms for an integrity check') .option('-f, --flush', 'Flush datas on exit') + .option('-d, --difference', 'Display difference if integrity is not okay') .parse(process.argv) -const createWeight = parseInt(program.create) || 5 -const removeWeight = parseInt(program.remove) || 4 +const createWeight = program.create !== undefined ? parseInt(program.create) : 5 +const removeWeight = program.remove !== undefined ? parseInt(program.remove) : 4 +const updateWeight = program.update !== undefined ? parseInt(program.update) : 4 const flushAtExit = program.flush || false -const actionInterval = parseInt(program.action) || 500 -let integrityInterval = parseInt(program.integrity) || 60000 +const actionInterval = program.action !== undefined ? parseInt(program.action) : 500 +const integrityInterval = program.integrity !== undefined ? parseInt(program.integrity) : 60000 +const displayDiffOnFail = program.integrity || false const numberOfPods = 6 + // Wait requests between pods -const requestsMaxPerInterval = constants.INTERVAL / actionInterval -const intervalsToMakeAllRequests = Math.ceil(requestsMaxPerInterval / constants.REQUESTS_LIMIT) -const waitForBeforeIntegrityCheck = (intervalsToMakeAllRequests * constants.INTERVAL) + 1000 +const baseRequestInterval = integrityInterval < constants.REQUESTS_INTERVAL ? constants.REQUESTS_INTERVAL : integrityInterval +const requestsMaxPerInterval = baseRequestInterval / actionInterval +const intervalsToMakeAllRequests = Math.ceil(requestsMaxPerInterval / constants.REQUESTS_LIMIT_PER_POD) +const waitForBeforeIntegrityCheck = (intervalsToMakeAllRequests * constants.REQUESTS_INTERVAL) - integrityInterval + 1000 -integrityInterval += waitForBeforeIntegrityCheck - -console.log('Create weight: %d, remove weight: %d.', createWeight, removeWeight) +console.log('Create weight: %d, update weight: %d, remove weight: %d.', createWeight, updateWeight, removeWeight) if (flushAtExit) { console.log('Program will flush data on exit.') } else { console.log('Program will not flush data on exit.') } +if (displayDiffOnFail) { + console.log('Program will display diff on failure.') +} else { + console.log('Program will not display diff on failure') +} console.log('Interval in ms for each action: %d.', actionInterval) console.log('Interval in ms for each integrity check: %d.', integrityInterval) console.log('Will wait %d ms before an integrity check.', waitForBeforeIntegrityCheck) @@ -63,16 +73,20 @@ runServers(numberOfPods, function (err, servers) { setInterval(function () { if (checking === true) return - const rand = getRandomInt(0, createWeight + removeWeight) + const rand = getRandomInt(0, createWeight + updateWeight + removeWeight) if (rand < createWeight) { upload(servers, getRandomNumServer(servers)) + } else if (rand < createWeight + updateWeight) { + update(servers, getRandomNumServer(servers)) } else { remove(servers, getRandomNumServer(servers)) } }, actionInterval) setInterval(function () { + if (checking === true) return + console.log('Checking integrity...') checking = true @@ -160,9 +174,9 @@ function exitServers (servers, callback) { function upload (servers, numServer, callback) { if (!callback) callback = function () {} - const name = 'my super name for pod 1' - const description = 'my super description for pod 1' - const tags = [ 'tag1p1', 'tag2p1' ] + const name = Date.now() + ' name' + const description = Date.now() + ' description' + const tags = [ Date.now().toString().substring(0, 5) + 't1', Date.now().toString().substring(0, 5) + 't2' ] const file = 'video_short1.webm' console.log('Upload video to server ' + numServer) @@ -170,6 +184,26 @@ function upload (servers, numServer, callback) { videosUtils.uploadVideo(servers[numServer].url, servers[numServer].accessToken, name, description, tags, file, callback) } +function update (servers, numServer, callback) { + if (!callback) callback = function () {} + + videosUtils.getVideosList(servers[numServer].url, function (err, res) { + if (err) throw err + + const videos = res.body.data.filter(function (video) { return video.isLocal }) + if (videos.length === 0) return callback() + + const toUpdate = videos[getRandomInt(0, videos.length)].id + const name = Date.now() + ' name' + const description = Date.now() + ' description' + const tags = [ Date.now().toString().substring(0, 5) + 't1', Date.now().toString().substring(0, 5) + 't2' ] + + console.log('Updating video of server ' + numServer) + + videosUtils.updateVideo(servers[numServer].url, servers[numServer].accessToken, toUpdate, name, description, tags, callback) + }) +} + function remove (servers, numServer, callback) { if (!callback) callback = function () {} @@ -196,6 +230,7 @@ function checkIntegrity (servers, callback) { delete serverVideo.id delete serverVideo.isLocal delete serverVideo.thumbnailPath + delete serverVideo.updatedAt } videos.push(serverVideos) @@ -205,6 +240,11 @@ function checkIntegrity (servers, callback) { for (const video of videos) { if (!isEqual(video, videos[0])) { console.error('Integrity not ok!') + + if (displayDiffOnFail) { + console.log(differenceWith(videos[0], video, isEqual)) + } + process.exit(-1) } } diff --git a/server/tests/utils/login.js b/server/tests/utils/login.js index 465564e14..c984c0baf 100644 --- a/server/tests/utils/login.js +++ b/server/tests/utils/login.js @@ -4,7 +4,8 @@ const request = require('supertest') const loginUtils = { login, - loginAndGetAccessToken + loginAndGetAccessToken, + getUserAccessToken } // ---------------------- Export functions -------------------- @@ -43,6 +44,14 @@ function loginAndGetAccessToken (server, callback) { }) } +function getUserAccessToken (server, user, callback) { + login(server.url, server.client, user, 200, function (err, res) { + if (err) return callback(err) + + return callback(null, res.body.access_token) + }) +} + // --------------------------------------------------------------------------- module.exports = loginUtils diff --git a/server/tests/utils/miscs.js b/server/tests/utils/miscs.js index 4ceff65df..c4b661496 100644 --- a/server/tests/utils/miscs.js +++ b/server/tests/utils/miscs.js @@ -6,12 +6,14 @@ const miscsUtils = { // ---------------------- Export functions -------------------- -function dateIsValid (dateString) { +function dateIsValid (dateString, interval) { const dateToCheck = new Date(dateString) const now = new Date() // Check if the interval is more than 2 minutes - if (now - dateToCheck > 120000) return false + if (!interval) interval = 120000 + + if (now - dateToCheck > interval) return false return true } diff --git a/server/tests/utils/servers.js b/server/tests/utils/servers.js index 01c9a2f39..1946ef49a 100644 --- a/server/tests/utils/servers.js +++ b/server/tests/utils/servers.js @@ -34,7 +34,7 @@ function flushAndRunMultipleServers (totalServers, serversRun) { runServer(j, function (app, url) { anotherServerDone(j, app, url) }) - }, 1000 * j) + }, 1000 * (j - 1)) } }) } @@ -60,12 +60,12 @@ function runServer (number, callback) { // These actions are async so we need to be sure that they have both been done const serverRunString = { - 'Connected to mongodb': false, + 'Database is ready': false, 'Server listening on port': false } const regexps = { - client_id: 'Client id: ([a-f0-9]+)', + client_id: 'Client id: (.+)', client_secret: 'Client secret: (.+)', user_username: 'Username: (.+)', user_password: 'User password: (.+)' @@ -103,7 +103,7 @@ function runServer (number, callback) { if (serverRunString[key] === false) dontContinue = true } - // If no, there is maybe one thing not already initialized (mongodb...) + // If no, there is maybe one thing not already initialized (client/user credentials generation...) if (dontContinue === true) return server.app.stdout.removeListener('data', onStdout) diff --git a/server/tests/utils/video-abuses.js b/server/tests/utils/video-abuses.js new file mode 100644 index 000000000..596c824b3 --- /dev/null +++ b/server/tests/utils/video-abuses.js @@ -0,0 +1,73 @@ +'use strict' + +const request = require('supertest') + +const videosUtils = { + getVideoAbusesList, + getVideoAbusesListPagination, + getVideoAbusesListSort, + reportVideoAbuse +} + +// ---------------------- Export functions -------------------- + +function reportVideoAbuse (url, token, videoId, reason, specialStatus, end) { + if (!end) { + end = specialStatus + specialStatus = 204 + } + + const path = '/api/v1/videos/' + videoId + '/abuse' + + request(url) + .post(path) + .set('Accept', 'application/json') + .set('Authorization', 'Bearer ' + token) + .send({ reason }) + .expect(specialStatus) + .end(end) +} + +function getVideoAbusesList (url, token, end) { + const path = '/api/v1/videos/abuse' + + request(url) + .get(path) + .query({ sort: 'createdAt' }) + .set('Accept', 'application/json') + .set('Authorization', 'Bearer ' + token) + .expect(200) + .expect('Content-Type', /json/) + .end(end) +} + +function getVideoAbusesListPagination (url, token, start, count, end) { + const path = '/api/v1/videos/abuse' + + request(url) + .get(path) + .query({ start: start }) + .query({ count: count }) + .set('Accept', 'application/json') + .set('Authorization', 'Bearer ' + token) + .expect(200) + .expect('Content-Type', /json/) + .end(end) +} + +function getVideoAbusesListSort (url, token, sort, end) { + const path = '/api/v1/videos/abuse' + + request(url) + .get(path) + .query({ sort: sort }) + .set('Accept', 'application/json') + .set('Authorization', 'Bearer ' + token) + .expect(200) + .expect('Content-Type', /json/) + .end(end) +} + +// --------------------------------------------------------------------------- + +module.exports = videosUtils diff --git a/server/tests/utils/videos.js b/server/tests/utils/videos.js index 536093db1..f94368437 100644 --- a/server/tests/utils/videos.js +++ b/server/tests/utils/videos.js @@ -15,7 +15,8 @@ const videosUtils = { searchVideoWithPagination, searchVideoWithSort, testVideoImage, - uploadVideo + uploadVideo, + updateVideo } // ---------------------- Export functions -------------------- @@ -25,7 +26,7 @@ function getAllVideosListBy (url, end) { request(url) .get(path) - .query({ sort: 'createdDate' }) + .query({ sort: 'createdAt' }) .query({ start: 0 }) .query({ count: 10000 }) .set('Accept', 'application/json') @@ -57,17 +58,25 @@ function getVideosList (url, end) { .end(end) } -function getVideosListPagination (url, start, count, end) { +function getVideosListPagination (url, start, count, sort, end) { + if (!end) { + end = sort + sort = null + } + const path = '/api/v1/videos' - request(url) - .get(path) - .query({ start: start }) - .query({ count: count }) - .set('Accept', 'application/json') - .expect(200) - .expect('Content-Type', /json/) - .end(end) + const req = request(url) + .get(path) + .query({ start: start }) + .query({ count: count }) + + if (sort) req.query({ sort }) + + req.set('Accept', 'application/json') + .expect(200) + .expect('Content-Type', /json/) + .end(end) } function getVideosListSort (url, sort, end) { @@ -115,18 +124,26 @@ function searchVideo (url, search, field, end) { .end(end) } -function searchVideoWithPagination (url, search, field, start, count, end) { +function searchVideoWithPagination (url, search, field, start, count, sort, end) { + if (!end) { + end = sort + sort = null + } + const path = '/api/v1/videos' - request(url) - .get(path + '/search/' + search) - .query({ start: start }) - .query({ count: count }) - .query({ field: field }) - .set('Accept', 'application/json') - .expect(200) - .expect('Content-Type', /json/) - .end(end) + const req = request(url) + .get(path + '/search/' + search) + .query({ start: start }) + .query({ count: count }) + .query({ field: field }) + + if (sort) req.query({ sort }) + + req.set('Accept', 'application/json') + .expect(200) + .expect('Content-Type', /json/) + .end(end) } function searchVideoWithSort (url, search, sort, end) { @@ -194,6 +211,31 @@ function uploadVideo (url, accessToken, name, description, tags, fixture, specia .end(end) } +function updateVideo (url, accessToken, id, name, description, tags, specialStatus, end) { + if (!end) { + end = specialStatus + specialStatus = 204 + } + + const path = '/api/v1/videos/' + id + + const req = request(url) + .put(path) + .set('Accept', 'application/json') + .set('Authorization', 'Bearer ' + accessToken) + + if (name) req.field('name', name) + if (description) req.field('description', description) + + if (tags) { + for (let i = 0; i < tags.length; i++) { + req.field('tags[' + i + ']', tags[i]) + } + } + + req.expect(specialStatus).end(end) +} + // --------------------------------------------------------------------------- module.exports = videosUtils diff --git a/support/doc/client/code.md b/support/doc/client/code.md index 3c2029847..f629af32f 100644 --- a/support/doc/client/code.md +++ b/support/doc/client/code.md @@ -54,7 +54,7 @@ Uses [TSLint](https://palantir.github.io/tslint/) for TypeScript linting and [An * Install [the dependencies](https://github.com/Chocobozzz/PeerTube#dependencies) * Run `npm install` at the root directory to install all the dependencies - * Run MongoDB + * Run PostgreSQL and create the database `peertube_dev`. * Run `npm run dev` to compile the client and automatically run the server. Then the server will watch and compile the client files automatically. You just need to refresh the browser to see your modifications. In a Angular 2 application, we create components that we put together. Each component is defined by an HTML structure, a TypeScript file and optionnaly a SASS file. diff --git a/support/doc/server/code.md b/support/doc/server/code.md index 652bd53df..c15885c8c 100644 --- a/support/doc/server/code.md +++ b/support/doc/server/code.md @@ -6,9 +6,9 @@ The server is a web server developed with [NodeJS](https://nodejs.org)/[Express] ## Technologies * [NodeJS](https://nodejs.org) -> Language - * [MongoDB](https://www.mongodb.com/) -> Database + * [PostgreSQL](https://www.postgresql.org/) -> Database * [Express](http://expressjs.com) -> Web server framework - * [Mongoose](http://mongoosejs.com/) -> MongoDB object modeling + * [Sequelize](http://docs.sequelizejs.com/en/v3/) -> SQL ORM * [WebTorrent](https://webtorrent.io/) -> BitTorrent tracker and torrent creation * [Mocha](https://mochajs.org/) -> Test framework @@ -28,7 +28,7 @@ All other server files are in the [server](https://github.com/Chocobozzz/PeerTub |__ initializers -> functions used at the server startup (installer, database, constants...) |__ lib -> library function (WebTorrent, OAuth2, friends logic...) |__ middlewares -> middlewares for controllers (requests validators, requests pagination...) - |__ models -> Mongoose models for each MongoDB collection (videos, users, pods...) + |__ models -> Sequelize models for each SQL tables (videos, users, pods...) |__ tests -> API tests and real world simulations (to test the decentralized feature...) @@ -41,7 +41,7 @@ Uses [JavaScript Standard Style](http://standardjs.com/). * Install [the dependencies](https://github.com/Chocobozzz/PeerTube#dependencies) * Run `npm install` at the root directory to install all the dependencies - * Run MongoDB + * Run PostgreSQL and create the database `peertube_dev`. * Run `npm run dev` to compile the client and automatically run the server. If the client files are already compiled you can simply run `NODE_ENV=test node server` The `NODE_ENV=test` is set to speed up communications between pods (see [constants.js](https://github.com/Chocobozzz/PeerTube/blob/master/server/initializers/constants.js)). @@ -62,7 +62,7 @@ A video is seeded by the server throught the [WebSeed](http://www.bittorrent.org ![Architecture scheme](https://github.com/Chocobozzz/PeerTube/blob/master/support/doc/server/upload-video.png) -When a user uploads a video, the rest API create the torrent file and then adds it to its Mongo database. +When a user uploads a video, the rest API create the torrent file and then adds it to its database. If a user wants to watch the video, the tracker will indicate all other users that are watching the video + the HTTP url for the WebSeed.