Merge pull request #859 from syuilo/improve/serializers

improve performance & use GridFS for files

package.json

@@ -18,7 +18,7 @@
     "clean": "gulp clean",
     "cleanall": "gulp cleanall",
     "lint": "gulp lint",
-    "test": "gulp test"
+		"test": "gulp test"
   },
   "devDependencies": {
     "@types/bcryptjs": "2.4.0",
@@ -95,6 +95,7 @@
     "webpack": "3.8.1"
   },
   "dependencies": {
+    "@prezzemolo/rap": "0.1.2",
     "accesses": "2.5.0",
     "animejs": "2.2.0",
     "autwh": "0.0.1",

src/api/common/add-file-to-drive.ts

@@ -4,14 +4,27 @@ import * as gm from 'gm';
 import * as debug from 'debug';
 import fileType = require('file-type');
 import prominence = require('prominence');
-import DriveFile from '../models/drive-file';
+import DriveFile, { getGridFSBucket } from '../models/drive-file';
 import DriveFolder from '../models/drive-folder';
 import serialize from '../serializers/drive-file';
 import event from '../event';
 import config from '../../conf';
+import { Duplex } from 'stream';
 
 const log = debug('misskey:register-drive-file');
 
+const addToGridFS = (name, binary, metadata): Promise<any> => new Promise(async (resolve, reject) => {
+	const dataStream = new Duplex();
+	dataStream.push(binary);
+	dataStream.push(null);
+
+	const bucket = await getGridFSBucket();
+	const writeStream = bucket.openUploadStream(name, { metadata });
+	writeStream.once('finish', (doc) => { resolve(doc); });
+	writeStream.on('error', reject);
+	dataStream.pipe(writeStream);
+});
+
 /**
  * Add file to drive
  *
@@ -58,7 +71,7 @@ export default (
 
 	// Generate hash
 	const hash = crypto
-		.createHash('sha256')
+		.createHash('md5')
 		.update(data)
 		.digest('hex') as string;
 
@@ -67,8 +80,8 @@ export default (
 	if (!force) {
 		// Check if there is a file with the same hash
 		const much = await DriveFile.findOne({
-			user_id: user._id,
-			hash: hash
+			md5: hash,
+			'metadata.user_id': user._id
 		});
 
 		if (much !== null) {
@@ -82,13 +95,13 @@ export default (
 	// Calculate drive usage
 	const usage = ((await DriveFile
 		.aggregate([
-			{ $match: { user_id: user._id } },
+			{ $match: { 'metadata.user_id': user._id } },
 			{ $project: {
-				datasize: true
+				length: true
 			}},
 			{ $group: {
 				_id: null,
-				usage: { $sum: '$datasize' }
+				usage: { $sum: '$length' }
 			}}
 		]))[0] || {
 			usage: 0
@@ -131,21 +144,15 @@ export default (
 	}
 
 	// Create DriveFile document
-	const file = await DriveFile.insert({
-		created_at: new Date(),
+	const file = await addToGridFS(`${user._id}/${name}`, data, {
 		user_id: user._id,
 		folder_id: folder !== null ? folder._id : null,
-		data: data,
-		datasize: size,
 		type: mime,
 		name: name,
 		comment: comment,
-		hash: hash,
 		properties: properties
 	});
 
-	delete file.data;
-
 	log(`drive file has been created ${file._id}`);
 
 	resolve(file);

src/api/endpoints/drive.ts

@@ -14,16 +14,16 @@ module.exports = (params, user) => new Promise(async (res, rej) => {
 	// Calculate drive usage
 	const usage = ((await DriveFile
 		.aggregate([
-			{ $match: { user_id: user._id } },
+			{ $match: { 'metadata.user_id': user._id } },
 			{
 				$project: {
-					datasize: true
+					length: true
 				}
 			},
 			{
 				$group: {
 					_id: null,
-					usage: { $sum: '$datasize' }
+					usage: { $sum: '$length' }
 				}
 			}
 		]))[0] || {

src/api/endpoints/drive/files.ts

@@ -13,35 +13,35 @@ import serialize from '../../serializers/drive-file';
  * @param {any} app
  * @return {Promise<any>}
  */
-module.exports = (params, user, app) => new Promise(async (res, rej) => {
+module.exports = async (params, user, app) => {
 	// Get 'limit' parameter
 	const [limit = 10, limitErr] = $(params.limit).optional.number().range(1, 100).$;
-	if (limitErr) return rej('invalid limit param');
+	if (limitErr) throw 'invalid limit param';
 
 	// Get 'since_id' parameter
 	const [sinceId, sinceIdErr] = $(params.since_id).optional.id().$;
-	if (sinceIdErr) return rej('invalid since_id param');
+	if (sinceIdErr) throw 'invalid since_id param';
 
 	// Get 'max_id' parameter
 	const [maxId, maxIdErr] = $(params.max_id).optional.id().$;
-	if (maxIdErr) return rej('invalid max_id param');
+	if (maxIdErr) throw 'invalid max_id param';
 
 	// Check if both of since_id and max_id is specified
 	if (sinceId && maxId) {
-		return rej('cannot set since_id and max_id');
+		throw 'cannot set since_id and max_id';
 	}
 
 	// Get 'folder_id' parameter
 	const [folderId = null, folderIdErr] = $(params.folder_id).optional.nullable.id().$;
-	if (folderIdErr) return rej('invalid folder_id param');
+	if (folderIdErr) throw 'invalid folder_id param';
 
 	// Construct query
 	const sort = {
 		_id: -1
 	};
 	const query = {
-		user_id: user._id,
-		folder_id: folderId
+		'metadata.user_id': user._id,
+		'metadata.folder_id': folderId
 	} as any;
 	if (sinceId) {
 		sort._id = 1;
@@ -57,14 +57,11 @@ module.exports = (params, user, app) => new Promise(async (res, rej) => {
 	// Issue query
 	const files = await DriveFile
 		.find(query, {
-			fields: {
-				data: false
-			},
 			limit: limit,
 			sort: sort
 		});
 
 	// Serialize
-	res(await Promise.all(files.map(async file =>
-		await serialize(file))));
-});
+	const _files = await Promise.all(files.map(file => serialize(file)));
+	return _files;
+};

src/api/endpoints/drive/files/find.ts

@@ -24,13 +24,9 @@ module.exports = (params, user) => new Promise(async (res, rej) => {
 	// Issue query
 	const files = await DriveFile
 		.find({
-			name: name,
-			user_id: user._id,
-			folder_id: folderId
-		}, {
-			fields: {
-				data: false
-			}
+			'metadata.name': name,
+			'metadata.user_id': user._id,
+			'metadata.folder_id': folderId
 		});
 
 	// Serialize

src/api/endpoints/drive/files/show.ts

@@ -12,28 +12,26 @@ import serialize from '../../../serializers/drive-file';
  * @param {any} user
  * @return {Promise<any>}
  */
-module.exports = (params, user) => new Promise(async (res, rej) => {
+module.exports = async (params, user) => {
 	// Get 'file_id' parameter
 	const [fileId, fileIdErr] = $(params.file_id).id().$;
-	if (fileIdErr) return rej('invalid file_id param');
+	if (fileIdErr) throw 'invalid file_id param';
 
 	// Fetch file
 	const file = await DriveFile
 		.findOne({
 			_id: fileId,
-			user_id: user._id
-		}, {
-			fields: {
-				data: false
-			}
+			'metadata.user_id': user._id
 		});
 
 	if (file === null) {
-		return rej('file-not-found');
+		throw 'file-not-found';
 	}
 
 	// Serialize
-	res(await serialize(file, {
+	const _file = await serialize(file, {
 		detail: true
-	}));
-});
+	});
+
+	return _file;
+};

src/api/endpoints/drive/files/update.ts

@@ -24,11 +24,7 @@ module.exports = (params, user) => new Promise(async (res, rej) => {
 	const file = await DriveFile
 		.findOne({
 			_id: fileId,
-			user_id: user._id
-		}, {
-			fields: {
-				data: false
-			}
+			'metadata.user_id': user._id
 		});
 
 	if (file === null) {
@@ -38,7 +34,7 @@ module.exports = (params, user) => new Promise(async (res, rej) => {
 	// Get 'name' parameter
 	const [name, nameErr] = $(params.name).optional.string().pipe(validateFileName).$;
 	if (nameErr) return rej('invalid name param');
-	if (name) file.name = name;
+	if (name) file.metadata.name = name;
 
 	// Get 'folder_id' parameter
 	const [folderId, folderIdErr] = $(params.folder_id).optional.nullable.id().$;
@@ -46,7 +42,7 @@ module.exports = (params, user) => new Promise(async (res, rej) => {
 
 	if (folderId !== undefined) {
 		if (folderId === null) {
-			file.folder_id = null;
+			file.metadata.folder_id = null;
 		} else {
 			// Fetch folder
 			const folder = await DriveFolder
@@ -59,14 +55,14 @@ module.exports = (params, user) => new Promise(async (res, rej) => {
 				return rej('folder-not-found');
 			}
 
-			file.folder_id = folder._id;
+			file.metadata.folder_id = folder._id;
 		}
 	}
 
-	DriveFile.update(file._id, {
+	await DriveFile.update(file._id, {
 		$set: {
-			name: file.name,
-			folder_id: file.folder_id
+			'metadata.name': file.metadata.name,
+			'metadata.folder_id': file.metadata.folder_id
 		}
 	});
 

src/api/endpoints/drive/folders/find.ts

@@ -30,6 +30,5 @@ module.exports = (params, user) => new Promise(async (res, rej) => {
 		});
 
 	// Serialize
-	res(await Promise.all(folders.map(async folder =>
-		await serialize(folder))));
+	res(await Promise.all(folders.map(folder => serialize(folder))));
 });

src/api/endpoints/drive/folders/update.ts

@@ -4,7 +4,7 @@
 import $ from 'cafy';
 import DriveFolder from '../../../models/drive-folder';
 import { isValidFolderName } from '../../../models/drive-folder';
-import serialize from '../../../serializers/drive-file';
+import serialize from '../../../serializers/drive-folder';
 import event from '../../../event';
 
 /**

src/api/endpoints/messaging/messages/create.ts

@@ -54,9 +54,7 @@ module.exports = (params, user) => new Promise(async (res, rej) => {
 	if (fileId !== undefined) {
 		file = await DriveFile.findOne({
 			_id: fileId,
-			user_id: user._id
-		}, {
-			data: false
+			'metadata.user_id': user._id
 		});
 
 		if (file === null) {

src/api/endpoints/posts/create.ts

@@ -44,9 +44,7 @@ module.exports = (params, user: IUser, app) => new Promise(async (res, rej) => {
 			// SELECT _id
 			const entity = await DriveFile.findOne({
 				_id: mediaId,
-				user_id: user._id
-			}, {
-				_id: true
+				'metadata.user_id': user._id
 			});
 
 			if (entity === null) {

src/api/endpoints/posts/timeline.ts

@@ -2,6 +2,7 @@
  * Module dependencies
  */
 import $ from 'cafy';
+import rap from '@prezzemolo/rap';
 import Post from '../../models/post';
 import ChannelWatching from '../../models/channel-watching';
 import getFriends from '../../common/get-friends';
@@ -15,32 +16,33 @@ import serialize from '../../serializers/post';
  * @param {any} app
  * @return {Promise<any>}
  */
-module.exports = (params, user, app) => new Promise(async (res, rej) => {
+module.exports = async (params, user, app) => {
 	// Get 'limit' parameter
 	const [limit = 10, limitErr] = $(params.limit).optional.number().range(1, 100).$;
-	if (limitErr) return rej('invalid limit param');
+	if (limitErr) throw 'invalid limit param';
 
 	// Get 'since_id' parameter
 	const [sinceId, sinceIdErr] = $(params.since_id).optional.id().$;
-	if (sinceIdErr) return rej('invalid since_id param');
+	if (sinceIdErr) throw 'invalid since_id param';
 
 	// Get 'max_id' parameter
 	const [maxId, maxIdErr] = $(params.max_id).optional.id().$;
-	if (maxIdErr) return rej('invalid max_id param');
+	if (maxIdErr) throw 'invalid max_id param';
 
 	// Check if both of since_id and max_id is specified
 	if (sinceId && maxId) {
-		return rej('cannot set since_id and max_id');
+		throw 'cannot set since_id and max_id';
 	}
 
-	// ID list of the user itself and other users who the user follows
-	const followingIds = await getFriends(user._id);
-
-	// Watchしているチャンネルを取得
-	const watches = await ChannelWatching.find({
-		user_id: user._id,
-		// 削除されたドキュメントは除く
-		deleted_at: { $exists: false }
+	const { followingIds, watchChannelIds } = await rap({
+		// ID list of the user itself and other users who the user follows
+		followingIds: getFriends(user._id),
+		// Watchしているチャンネルを取得
+		watchChannelIds: ChannelWatching.find({
+			user_id: user._id,
+			// 削除されたドキュメントは除く
+			deleted_at: { $exists: false }
+		}).then(watches => watches.map(w => w.channel_id))
 	});
 
 	//#region Construct query
@@ -65,7 +67,7 @@ module.exports = (params, user, app) => new Promise(async (res, rej) => {
 		}, {
 			// Watchしているチャンネルへの投稿
 			channel_id: {
-				$in: watches.map(w => w.channel_id)
+				$in: watchChannelIds
 			}
 		}]
 	} as any;
@@ -90,7 +92,6 @@ module.exports = (params, user, app) => new Promise(async (res, rej) => {
 		});
 
 	// Serialize
-	res(await Promise.all(timeline.map(async post =>
-		await serialize(post, user)
-	)));
-});
+	const _timeline = await Promise.all(timeline.map(post => serialize(post, user)));
+	return _timeline;
+};

src/api/models/drive-file.ts

@@ -1,11 +1,22 @@
-import db from '../../db/mongodb';
+import * as mongodb from 'mongodb';
+import monkDb, { nativeDbConn } from '../../db/mongodb';
 
-const collection = db.get('drive_files');
+const collection = monkDb.get('drive_files.files');
 
 (collection as any).createIndex('hash'); // fuck type definition
 
 export default collection as any; // fuck type definition
 
+const getGridFSBucket = async (): Promise<mongodb.GridFSBucket> => {
+	const db = await nativeDbConn();
+	const bucket = new mongodb.GridFSBucket(db, {
+		bucketName: 'drive_files'
+	});
+	return bucket;
+};
+
+export { getGridFSBucket };
+
 export function validateFileName(name: string): boolean {
 	return (
 		(name.trim().length > 0) &&

src/api/serializers/drive-file.ts

@@ -31,44 +31,40 @@ export default (
 	if (mongo.ObjectID.prototype.isPrototypeOf(file)) {
 		_file = await DriveFile.findOne({
 			_id: file
-		}, {
-				fields: {
-					data: false
-				}
-			});
+		});
 	} else if (typeof file === 'string') {
 		_file = await DriveFile.findOne({
 			_id: new mongo.ObjectID(file)
-		}, {
-				fields: {
-					data: false
-				}
-			});
+		});
 	} else {
 		_file = deepcopy(file);
 	}
 
-	// Rename _id to id
-	_file.id = _file._id;
-	delete _file._id;
+	if (!_file) return reject('invalid file arg.');
 
-	delete _file.data;
+	// rendered target
+	let _target: any = {};
 
-	_file.url = `${config.drive_url}/${_file.id}/${encodeURIComponent(_file.name)}`;
+	_target.id = _file._id;
+	_target.created_at = _file.uploadDate;
 
-	if (opts.detail && _file.folder_id) {
+	_target = Object.assign(_target, _file.metadata);
+
+	_target.url = `${config.drive_url}/${_target.id}/${encodeURIComponent(_target.name)}`;
+
+	if (opts.detail && _target.folder_id) {
 		// Populate folder
-		_file.folder = await serializeDriveFolder(_file.folder_id, {
+		_target.folder = await serializeDriveFolder(_target.folder_id, {
 			detail: true
 		});
 	}
 
-	if (opts.detail && _file.tags) {
+	if (opts.detail && _target.tags) {
 		// Populate tags
-		_file.tags = await _file.tags.map(async (tag: any) =>
+		_target.tags = await _target.tags.map(async (tag: any) =>
 			await serializeDriveTag(tag)
 		);
 	}
 
-	resolve(_file);
+	resolve(_target);
 });

src/api/serializers/drive-folder.ts

@@ -44,7 +44,7 @@ const self = (
 		});
 
 		const childFilesCount = await DriveFile.count({
-			folder_id: _folder.id
+			'metadata.folder_id': _folder.id
 		});
 
 		_folder.folders_count = childFoldersCount;

src/api/serializers/post.ts

@@ -12,6 +12,7 @@ import serializeChannel from './channel';
 import serializeUser from './user';
 import serializeDriveFile from './drive-file';
 import parse from '../common/text';
+import rap from '@prezzemolo/rap';
 
 /**
  * Serialize a post
@@ -21,13 +22,13 @@ import parse from '../common/text';
  * @param options? serialize options
  * @return response
  */
-const self = (
+const self = async (
 	post: string | mongo.ObjectID | IPost,
 	me?: string | mongo.ObjectID | IUser,
 	options?: {
 		detail: boolean
 	}
-) => new Promise<any>(async (resolve, reject) => {
+) => {
 	const opts = options || {
 		detail: true,
 	};
@@ -56,6 +57,8 @@ const self = (
 		_post = deepcopy(post);
 	}
 
+	if (!_post) throw 'invalid post arg.';
+
 	const id = _post._id;
 
 	// Rename _id to id
@@ -70,105 +73,120 @@ const self = (
 	}
 
 	// Populate user
-	_post.user = await serializeUser(_post.user_id, meId);
+	_post.user = serializeUser(_post.user_id, meId);
 
 	// Populate app
 	if (_post.app_id) {
-		_post.app = await serializeApp(_post.app_id);
+		_post.app = serializeApp(_post.app_id);
 	}
 
 	// Populate channel
 	if (_post.channel_id) {
-		_post.channel = await serializeChannel(_post.channel_id);
+		_post.channel = serializeChannel(_post.channel_id);
 	}
 
 	// Populate media
 	if (_post.media_ids) {
-		_post.media = await Promise.all(_post.media_ids.map(async fileId =>
-			await serializeDriveFile(fileId)
+		_post.media = Promise.all(_post.media_ids.map(fileId =>
+			serializeDriveFile(fileId)
 		));
 	}
 
 	// When requested a detailed post data
 	if (opts.detail) {
 		// Get previous post info
-		const prev = await Post.findOne({
-			user_id: _post.user_id,
-			_id: {
-				$lt: id
-			}
-		}, {
-			fields: {
-				_id: true
-			},
-			sort: {
-				_id: -1
-			}
-		});
-		_post.prev = prev ? prev._id : null;
+		_post.prev = (async () => {
+			const prev = await Post.findOne({
+				user_id: _post.user_id,
+				_id: {
+					$lt: id
+				}
+			}, {
+				fields: {
+					_id: true
+				},
+				sort: {
+					_id: -1
+				}
+			});
+			return prev ? prev._id : null;
+		})();
 
 		// Get next post info
-		const next = await Post.findOne({
-			user_id: _post.user_id,
-			_id: {
-				$gt: id
-			}
-		}, {
-			fields: {
-				_id: true
-			},
-			sort: {
-				_id: 1
-			}
-		});
-		_post.next = next ? next._id : null;
+		_post.next = (async () => {
+			const next = await Post.findOne({
+				user_id: _post.user_id,
+				_id: {
+					$gt: id
+				}
+			}, {
+				fields: {
+					_id: true
+				},
+				sort: {
+					_id: 1
+				}
+			});
+			return next ? next._id : null;
+		})();
 
 		if (_post.reply_id) {
 			// Populate reply to post
-			_post.reply = await self(_post.reply_id, meId, {
+			_post.reply = self(_post.reply_id, meId, {
 				detail: false
 			});
 		}
 
 		if (_post.repost_id) {
 			// Populate repost
-			_post.repost = await self(_post.repost_id, meId, {
+			_post.repost = self(_post.repost_id, meId, {
 				detail: _post.text == null
 			});
 		}
 
 		// Poll
 		if (meId && _post.poll) {
-			const vote = await Vote
-				.findOne({
-					user_id: meId,
-					post_id: id
-				});
+			_post.poll = (async (poll) => {
+				const vote = await Vote
+					.findOne({
+						user_id: meId,
+						post_id: id
+					});
 
-			if (vote != null) {
-				const myChoice = _post.poll.choices
-					.filter(c => c.id == vote.choice)[0];
+				if (vote != null) {
+					const myChoice = poll.choices
+						.filter(c => c.id == vote.choice)[0];
 
-				myChoice.is_voted = true;
-			}
+					myChoice.is_voted = true;
+				}
+
+				return poll;
+			})(_post.poll);
 		}
 
 		// Fetch my reaction
 		if (meId) {
-			const reaction = await Reaction
-				.findOne({
-					user_id: meId,
-					post_id: id,
-					deleted_at: { $exists: false }
-				});
+			_post.my_reaction = (async () => {
+				const reaction = await Reaction
+					.findOne({
+						user_id: meId,
+						post_id: id,
+						deleted_at: { $exists: false }
+					});
 
-			if (reaction) {
-				_post.my_reaction = reaction.reaction;
-			}
+				if (reaction) {
+					return reaction.reaction;
+				}
+
+				return null;
+			})();
 		}
 	}
 
-	resolve(_post);
-});
+	// resolve promises in _post object
+	_post = await rap(_post);
+
+	return _post;
+};
 
 export default self;

src/api/serializers/user.ts

@@ -8,6 +8,7 @@ import serializePost from './post';
 import Following from '../models/following';
 import getFriends from '../common/get-friends';
 import config from '../../conf';
+import rap from '@prezzemolo/rap';
 
 /**
  * Serialize a user
@@ -55,6 +56,8 @@ export default (
 		_user = deepcopy(user);
 	}
 
+	if (!_user) return reject('invalid user arg.');
+
 	// Me
 	const meId: mongo.ObjectID = me
 		? mongo.ObjectID.prototype.isPrototypeOf(me)
@@ -104,26 +107,30 @@ export default (
 
 	if (meId && !meId.equals(_user.id)) {
 		// If the user is following
-		const follow = await Following.findOne({
-			follower_id: meId,
-			followee_id: _user.id,
-			deleted_at: { $exists: false }
-		});
-		_user.is_following = follow !== null;
+		_user.is_following = (async () => {
+			const follow = await Following.findOne({
+				follower_id: meId,
+				followee_id: _user.id,
+				deleted_at: { $exists: false }
+			});
+			return follow !== null;
+		})();
 
 		// If the user is followed
-		const follow2 = await Following.findOne({
-			follower_id: _user.id,
-			followee_id: meId,
-			deleted_at: { $exists: false }
-		});
-		_user.is_followed = follow2 !== null;
+		_user.is_followed = (async () => {
+			const follow2 = await Following.findOne({
+				follower_id: _user.id,
+				followee_id: meId,
+				deleted_at: { $exists: false }
+			});
+			return follow2 !== null;
+		})();
 	}
 
 	if (opts.detail) {
 		if (_user.pinned_post_id) {
 			// Populate pinned post
-			_user.pinned_post = await serializePost(_user.pinned_post_id, meId, {
+			_user.pinned_post = serializePost(_user.pinned_post_id, meId, {
 				detail: true
 			});
 		}
@@ -132,23 +139,24 @@ export default (
 			const myFollowingIds = await getFriends(meId);
 
 			// Get following you know count
-			const followingYouKnowCount = await Following.count({
+			_user.following_you_know_count = Following.count({
 				followee_id: { $in: myFollowingIds },
 				follower_id: _user.id,
 				deleted_at: { $exists: false }
 			});
-			_user.following_you_know_count = followingYouKnowCount;
 
 			// Get followers you know count
-			const followersYouKnowCount = await Following.count({
+			_user.followers_you_know_count = Following.count({
 				followee_id: _user.id,
 				follower_id: { $in: myFollowingIds },
 				deleted_at: { $exists: false }
 			});
-			_user.followers_you_know_count = followersYouKnowCount;
 		}
 	}
 
+	// resolve promises in _user object
+	_user = await rap(_user);
+
 	resolve(_user);
 });
 /*

src/db/mongodb.ts

@@ -1,11 +1,38 @@
-import * as mongo from 'monk';
-
 import config from '../conf';
 
 const uri = config.mongodb.user && config.mongodb.pass
-	? `mongodb://${config.mongodb.user}:${config.mongodb.pass}@${config.mongodb.host}:${config.mongodb.port}/${config.mongodb.db}`
-	: `mongodb://${config.mongodb.host}:${config.mongodb.port}/${config.mongodb.db}`;
+? `mongodb://${config.mongodb.user}:${config.mongodb.pass}@${config.mongodb.host}:${config.mongodb.port}/${config.mongodb.db}`
+: `mongodb://${config.mongodb.host}:${config.mongodb.port}/${config.mongodb.db}`;
+
+/**
+ * monk
+ */
+import * as mongo from 'monk';
 
 const db = mongo(uri);
 
 export default db;
+
+/**
+ * MongoDB native module (officialy)
+ */
+import * as mongodb from 'mongodb';
+
+let mdb: mongodb.Db;
+
+const nativeDbConn = async (): Promise<mongodb.Db> => {
+	if (mdb) return mdb;
+
+	const db = await ((): Promise<mongodb.Db> => new Promise((resolve, reject) => {
+		mongodb.MongoClient.connect(uri, (e, db) => {
+			if (e) return reject(e);
+			resolve(db);
+		});
+	}))();
+
+	mdb = db;
+
+	return db;
+};
+
+export { nativeDbConn };

src/file/server.ts

@@ -9,7 +9,7 @@ import * as cors from 'cors';
 import * as mongodb from 'mongodb';
 import * as gm from 'gm';
 
-import File from '../api/models/drive-file';
+import DriveFile, { getGridFSBucket } from '../api/models/drive-file';
 
 /**
  * Init app
@@ -97,17 +97,28 @@ app.get('/:id', async (req, res) => {
 		return;
 	}
 
-	const file = await File.findOne({ _id: new mongodb.ObjectID(req.params.id) });
+	const fileId = new mongodb.ObjectID(req.params.id);
+	const file = await DriveFile.findOne({ _id: fileId });
 
 	if (file == null) {
-		res.status(404).sendFile(`${__dirname} / assets / dummy.png`);
-		return;
-	} else if (file.data == null) {
-		res.sendStatus(400);
+		res.status(404).sendFile(`${__dirname}/assets/dummy.png`);
 		return;
 	}
 
-	send(file.data.buffer, file.type, req, res);
+	const bucket = await getGridFSBucket();
+
+	const buffer = await ((id): Promise<Buffer> => new Promise((resolve, reject) => {
+		const chunks = [];
+		const readableStream = bucket.openDownloadStream(id);
+	 readableStream.on('data', chunk => {
+			chunks.push(chunk);
+		});
+		readableStream.on('end', () => {
+			resolve(Buffer.concat(chunks));
+		});
+	}))(fileId);
+
+	send(buffer, file.metadata.type, req, res);
 });
 
 app.get('/:id/:name', async (req, res) => {
@@ -117,17 +128,28 @@ app.get('/:id/:name', async (req, res) => {
 		return;
 	}
 
-	const file = await File.findOne({ _id: new mongodb.ObjectID(req.params.id) });
+	const fileId = new mongodb.ObjectID(req.params.id);
+	const file = await DriveFile.findOne({ _id: fileId });
 
 	if (file == null) {
 		res.status(404).sendFile(`${__dirname}/assets/dummy.png`);
 		return;
-	} else if (file.data == null) {
-		res.sendStatus(400);
-		return;
 	}
 
-	send(file.data.buffer, file.type, req, res);
+	const bucket = await getGridFSBucket();
+
+	const buffer = await ((id): Promise<Buffer> => new Promise((resolve, reject) => {
+		const chunks = [];
+		const readableStream = bucket.openDownloadStream(id);
+	 readableStream.on('data', chunk => {
+			chunks.push(chunk);
+		});
+		readableStream.on('end', () => {
+			resolve(Buffer.concat(chunks));
+		});
+	}))(fileId);
+
+	send(buffer, file.metadata.type, req, res);
 });
 
 module.exports = app;

test/api.js

@@ -1152,9 +1152,12 @@ async function insertHimawari(opts) {
 }
 
 async function insertDriveFile(opts) {
-	return await db.get('drive_files').insert(Object.assign({
-		name: 'strawberry-pasta.png'
-	}, opts));
+	return await db.get('drive_files.files').insert({
+		length: opts.datasize,
+		metadata: Object.assign({
+			name: 'strawberry-pasta.png'
+		}, opts)
+	});
 }
 
 async function insertDriveFolder(opts) {

tools/migration/use-gridfs.js

@@ -0,0 +1,49 @@
+// for Node.js interpret
+
+const { default: db } = require('../../built/db/mongodb')
+const { default: DriveFile, getGridFSBucket } = require('../../built/api/models/drive-file')
+const { Duplex } = require('stream')
+
+const writeToGridFS = (bucket, buffer, ...rest) => new Promise((resolve, reject) => {
+	const writeStream = bucket.openUploadStreamWithId(...rest)
+	
+	const dataStream = new Duplex()
+	dataStream.push(buffer)
+	dataStream.push(null)
+
+	writeStream.once('finish', resolve)
+	writeStream.on('error', reject)
+
+	dataStream.pipe(writeStream)
+})
+
+const migrateToGridFS = async (doc) => {
+	const id = doc._id
+	const buffer = doc.data.buffer
+	const created_at = doc.created_at
+
+	delete doc._id
+	delete doc.created_at
+	delete doc.datasize
+	delete doc.hash
+	delete doc.data
+
+	const bucket = await getGridFSBucket()
+	const added = await writeToGridFS(bucket, buffer, id, `${id}/${doc.name}`, { metadata: doc })
+
+	const result = await DriveFile.update(id, {
+		$set: {
+			uploadDate: created_at
+		}
+	})
+
+	return added && result.ok === 1
+}
+
+const main = async () => {
+	const docs = await db.get('drive_files').find()
+	const all = await Promise.all(docs.map(migrateToGridFS))
+	return all
+}
+
+main().then(console.dir).catch(console.error)