2017-11-16 22:49:58 +09:00
|
|
|
import { Buffer } from 'buffer';
|
|
|
|
import * as fs from 'fs';
|
|
|
|
import * as stream from 'stream';
|
|
|
|
|
2016-12-29 07:49:51 +09:00
|
|
|
import * as mongodb from 'mongodb';
|
|
|
|
import * as crypto from 'crypto';
|
2017-02-06 21:11:09 +09:00
|
|
|
import * as debug from 'debug';
|
2017-01-03 06:03:19 +09:00
|
|
|
import fileType = require('file-type');
|
2018-07-24 01:58:11 +09:00
|
|
|
import * as Minio from 'minio';
|
|
|
|
import * as uuid from 'uuid';
|
2018-07-25 00:29:18 +09:00
|
|
|
import * as sharp from 'sharp';
|
2017-11-16 22:49:58 +09:00
|
|
|
|
2018-06-15 09:53:30 +09:00
|
|
|
import DriveFile, { IMetadata, getDriveFileBucket, IDriveFile } from '../../models/drive-file';
|
2018-04-05 03:21:11 +09:00
|
|
|
import DriveFolder from '../../models/drive-folder';
|
|
|
|
import { pack } from '../../models/drive-file';
|
2018-07-07 19:19:00 +09:00
|
|
|
import event, { publishDriveStream } from '../../stream';
|
2018-05-26 09:57:31 +09:00
|
|
|
import { isLocalUser, IUser, IRemoteUser } from '../../models/user';
|
2018-06-15 09:53:30 +09:00
|
|
|
import delFile from './delete-file';
|
2018-07-21 19:17:15 +09:00
|
|
|
import config from '../../config';
|
2016-12-29 07:49:51 +09:00
|
|
|
|
2018-03-27 16:51:12 +09:00
|
|
|
const log = debug('misskey:drive:add-file');
|
2017-02-06 21:11:09 +09:00
|
|
|
|
2018-07-24 01:58:11 +09:00
|
|
|
async function save(readable: stream.Readable, name: string, type: string, hash: string, size: number, metadata: any): Promise<IDriveFile> {
|
2018-07-25 08:01:12 +09:00
|
|
|
if (config.drive && config.drive.storage == 'minio') {
|
|
|
|
const minio = new Minio.Client(config.drive.config);
|
|
|
|
const id = uuid.v4();
|
|
|
|
const obj = `${config.drive.prefix}/${id}`;
|
2018-07-26 17:29:05 +09:00
|
|
|
const baseUrl = config.drive.baseUrl
|
|
|
|
|| `${ config.drive.config.secure ? 'https' : 'http' }://${ config.drive.config.endPoint }${ config.drive.config.port ? ':' + config.drive.config.port : '' }/${ config.drive.bucket }`;
|
2018-07-26 16:06:43 +09:00
|
|
|
await minio.putObject(config.drive.bucket, obj, readable, size, { 'Content-Type': type, 'Cache-Control': 'max-age=31536000, immutable' });
|
2018-07-24 01:58:11 +09:00
|
|
|
|
2018-07-25 08:01:12 +09:00
|
|
|
Object.assign(metadata, {
|
|
|
|
withoutChunks: true,
|
|
|
|
storage: 'minio',
|
|
|
|
storageProps: {
|
|
|
|
id: id
|
|
|
|
},
|
2018-07-26 17:29:05 +09:00
|
|
|
url: `${ baseUrl }/${ obj }`
|
2018-07-25 08:01:12 +09:00
|
|
|
});
|
2018-07-24 01:58:11 +09:00
|
|
|
|
2018-07-25 08:01:12 +09:00
|
|
|
const file = await DriveFile.insert({
|
|
|
|
length: size,
|
|
|
|
uploadDate: new Date(),
|
|
|
|
md5: hash,
|
|
|
|
filename: name,
|
|
|
|
metadata: metadata,
|
|
|
|
contentType: type
|
|
|
|
});
|
2018-07-24 01:58:11 +09:00
|
|
|
|
2018-07-25 08:01:12 +09:00
|
|
|
return file;
|
2018-07-24 01:58:11 +09:00
|
|
|
} else {
|
|
|
|
// Get MongoDB GridFS bucket
|
|
|
|
const bucket = await getDriveFileBucket();
|
|
|
|
|
|
|
|
return new Promise<IDriveFile>((resolve, reject) => {
|
|
|
|
const writeStream = bucket.openUploadStream(name, { contentType: type, metadata });
|
2018-05-03 20:03:14 +09:00
|
|
|
writeStream.once('finish', resolve);
|
|
|
|
writeStream.on('error', reject);
|
|
|
|
readable.pipe(writeStream);
|
2018-07-24 01:58:11 +09:00
|
|
|
});
|
|
|
|
}
|
|
|
|
}
|
2018-05-03 20:03:14 +09:00
|
|
|
|
2018-05-25 18:41:49 +09:00
|
|
|
async function deleteOldFile(user: IRemoteUser) {
|
|
|
|
const oldFile = await DriveFile.findOne({
|
|
|
|
_id: {
|
|
|
|
$nin: [user.avatarId, user.bannerId]
|
|
|
|
}
|
|
|
|
}, {
|
|
|
|
sort: {
|
|
|
|
_id: 1
|
|
|
|
}
|
|
|
|
});
|
|
|
|
|
|
|
|
if (oldFile) {
|
2018-06-15 09:53:30 +09:00
|
|
|
delFile(oldFile, true);
|
2018-05-25 18:41:49 +09:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Add file to drive
|
|
|
|
*
|
|
|
|
* @param user User who wish to add file
|
|
|
|
* @param path File path
|
|
|
|
* @param name Name
|
|
|
|
* @param comment Comment
|
|
|
|
* @param folderId Folder ID
|
|
|
|
* @param force If set to true, forcibly upload the file even if there is a file with the same hash.
|
|
|
|
* @return Created drive file
|
|
|
|
*/
|
|
|
|
export default async function(
|
2018-05-26 09:57:31 +09:00
|
|
|
user: IUser,
|
2017-11-14 05:10:28 +09:00
|
|
|
path: string,
|
2016-12-29 07:49:51 +09:00
|
|
|
name: string = null,
|
|
|
|
comment: string = null,
|
|
|
|
folderId: mongodb.ObjectID = null,
|
2018-04-03 23:45:13 +09:00
|
|
|
force: boolean = false,
|
2018-07-24 05:04:43 +09:00
|
|
|
isLink: boolean = false,
|
2018-05-04 18:32:03 +09:00
|
|
|
url: string = null,
|
2018-07-20 02:40:37 +09:00
|
|
|
uri: string = null,
|
|
|
|
sensitive = false
|
2018-05-25 18:41:49 +09:00
|
|
|
): Promise<IDriveFile> {
|
|
|
|
// Calc md5 hash
|
|
|
|
const calcHash = new Promise<string>((res, rej) => {
|
|
|
|
const readable = fs.createReadStream(path);
|
|
|
|
const hash = crypto.createHash('md5');
|
2018-06-17 20:04:19 +09:00
|
|
|
const chunks: Buffer[] = [];
|
2018-05-25 18:41:49 +09:00
|
|
|
readable
|
|
|
|
.on('error', rej)
|
|
|
|
.pipe(hash)
|
|
|
|
.on('error', rej)
|
|
|
|
.on('data', chunk => chunks.push(chunk))
|
|
|
|
.on('end', () => {
|
|
|
|
const buffer = Buffer.concat(chunks);
|
|
|
|
res(buffer.toString('hex'));
|
2017-11-14 04:54:47 +09:00
|
|
|
});
|
2018-05-25 18:41:49 +09:00
|
|
|
});
|
|
|
|
|
|
|
|
// Detect content type
|
|
|
|
const detectMime = new Promise<[string, string]>((res, rej) => {
|
|
|
|
const readable = fs.createReadStream(path);
|
|
|
|
readable
|
|
|
|
.on('error', rej)
|
|
|
|
.once('data', (buffer: Buffer) => {
|
|
|
|
readable.destroy();
|
|
|
|
const type = fileType(buffer);
|
|
|
|
if (type) {
|
|
|
|
res([type.mime, type.ext]);
|
|
|
|
} else {
|
|
|
|
// 種類が同定できなかったら application/octet-stream にする
|
|
|
|
res(['application/octet-stream', null]);
|
|
|
|
}
|
|
|
|
});
|
|
|
|
});
|
|
|
|
|
|
|
|
// Get file size
|
|
|
|
const getFileSize = new Promise<number>((res, rej) => {
|
|
|
|
fs.stat(path, (err, stats) => {
|
|
|
|
if (err) return rej(err);
|
|
|
|
res(stats.size);
|
|
|
|
});
|
|
|
|
});
|
|
|
|
|
|
|
|
const [hash, [mime, ext], size] = await Promise.all([calcHash, detectMime, getFileSize]);
|
2017-11-14 03:46:30 +09:00
|
|
|
|
2017-11-14 04:54:47 +09:00
|
|
|
log(`hash: ${hash}, mime: ${mime}, ext: ${ext}, size: ${size}`);
|
2016-12-29 07:49:51 +09:00
|
|
|
|
2017-11-14 04:54:47 +09:00
|
|
|
// detect name
|
2018-05-25 18:41:49 +09:00
|
|
|
const detectedName = name || (ext ? `untitled.${ext}` : 'untitled');
|
2017-11-14 04:54:47 +09:00
|
|
|
|
|
|
|
if (!force) {
|
|
|
|
// Check if there is a file with the same hash
|
|
|
|
const much = await DriveFile.findOne({
|
|
|
|
md5: hash,
|
2018-05-13 16:52:47 +09:00
|
|
|
'metadata.userId': user._id,
|
2018-05-14 12:47:33 +09:00
|
|
|
'metadata.deletedAt': { $exists: false }
|
2017-11-14 04:54:47 +09:00
|
|
|
});
|
|
|
|
|
2018-06-08 22:03:14 +09:00
|
|
|
if (much) {
|
|
|
|
log(`file with same hash is found: ${much._id}`);
|
2017-11-14 04:54:47 +09:00
|
|
|
return much;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-05-25 18:41:49 +09:00
|
|
|
//#region Check drive usage
|
2018-07-24 05:04:43 +09:00
|
|
|
if (!isLink) {
|
2018-05-25 20:19:14 +09:00
|
|
|
const usage = await DriveFile
|
|
|
|
.aggregate([{
|
|
|
|
$match: {
|
|
|
|
'metadata.userId': user._id,
|
|
|
|
'metadata.deletedAt': { $exists: false }
|
|
|
|
}
|
|
|
|
}, {
|
|
|
|
$project: {
|
|
|
|
length: true
|
|
|
|
}
|
|
|
|
}, {
|
|
|
|
$group: {
|
|
|
|
_id: null,
|
|
|
|
usage: { $sum: '$length' }
|
|
|
|
}
|
|
|
|
}])
|
|
|
|
.then((aggregates: any[]) => {
|
|
|
|
if (aggregates.length > 0) {
|
|
|
|
return aggregates[0].usage;
|
|
|
|
}
|
|
|
|
return 0;
|
|
|
|
});
|
2017-11-16 22:49:58 +09:00
|
|
|
|
2018-05-25 20:19:14 +09:00
|
|
|
log(`drive usage is ${usage}`);
|
2017-11-16 22:49:58 +09:00
|
|
|
|
2018-07-21 19:17:15 +09:00
|
|
|
const driveCapacity = 1024 * 1024 * (isLocalUser(user) ? config.localDriveCapacityMb : config.remoteDriveCapacityMb);
|
|
|
|
|
2018-05-25 20:19:14 +09:00
|
|
|
// If usage limit exceeded
|
2018-07-21 19:17:15 +09:00
|
|
|
if (usage + size > driveCapacity) {
|
2018-05-25 20:19:14 +09:00
|
|
|
if (isLocalUser(user)) {
|
|
|
|
throw 'no-free-space';
|
|
|
|
} else {
|
|
|
|
// (アバターまたはバナーを含まず)最も古いファイルを削除する
|
|
|
|
deleteOldFile(user);
|
|
|
|
}
|
2018-05-25 18:41:49 +09:00
|
|
|
}
|
|
|
|
}
|
|
|
|
//#endregion
|
|
|
|
|
|
|
|
const fetchFolder = async () => {
|
|
|
|
if (!folderId) {
|
|
|
|
return null;
|
|
|
|
}
|
|
|
|
|
|
|
|
const driveFolder = await DriveFolder.findOne({
|
|
|
|
_id: folderId,
|
|
|
|
userId: user._id
|
|
|
|
});
|
|
|
|
|
|
|
|
if (driveFolder == null) throw 'folder-not-found';
|
|
|
|
|
|
|
|
return driveFolder;
|
|
|
|
};
|
|
|
|
|
2018-06-17 20:04:19 +09:00
|
|
|
const properties: {[key: string]: any} = {};
|
2018-05-25 18:41:49 +09:00
|
|
|
|
2018-05-27 23:19:57 +09:00
|
|
|
let propPromises: Array<Promise<void>> = [];
|
2017-11-16 22:49:58 +09:00
|
|
|
|
2018-07-25 00:29:18 +09:00
|
|
|
const isImage = ['image/jpeg', 'image/gif', 'image/png', 'image/webp'].includes(mime);
|
2018-05-25 18:41:49 +09:00
|
|
|
|
|
|
|
if (isImage) {
|
2018-07-25 00:29:18 +09:00
|
|
|
const img = sharp(path);
|
|
|
|
|
2018-05-25 18:41:49 +09:00
|
|
|
// Calc width and height
|
|
|
|
const calcWh = async () => {
|
2017-12-11 13:33:33 +09:00
|
|
|
log('calculate image width and height...');
|
|
|
|
|
2017-12-08 19:42:02 +09:00
|
|
|
// Calculate width and height
|
2018-07-25 00:29:18 +09:00
|
|
|
const meta = await img.metadata();
|
2017-11-16 22:49:58 +09:00
|
|
|
|
2018-07-25 00:29:18 +09:00
|
|
|
log(`image width and height is calculated: ${meta.width}, ${meta.height}`);
|
2017-11-16 22:49:58 +09:00
|
|
|
|
2018-07-25 00:29:18 +09:00
|
|
|
properties['width'] = meta.width;
|
|
|
|
properties['height'] = meta.height;
|
2018-05-25 18:41:49 +09:00
|
|
|
};
|
2017-12-11 13:33:33 +09:00
|
|
|
|
2018-05-25 18:41:49 +09:00
|
|
|
// Calc average color
|
|
|
|
const calcAvg = async () => {
|
2017-12-11 13:33:33 +09:00
|
|
|
log('calculate average color...');
|
|
|
|
|
2018-07-26 17:10:43 +09:00
|
|
|
try {
|
|
|
|
const info = await (img as any).stats();
|
2017-12-11 13:33:33 +09:00
|
|
|
|
2018-07-26 17:10:43 +09:00
|
|
|
const r = Math.round(info.channels[0].mean);
|
|
|
|
const g = Math.round(info.channels[1].mean);
|
|
|
|
const b = Math.round(info.channels[2].mean);
|
2017-12-11 13:33:33 +09:00
|
|
|
|
2018-07-26 17:10:43 +09:00
|
|
|
log(`average color is calculated: ${r}, ${g}, ${b}`);
|
2017-12-11 13:33:33 +09:00
|
|
|
|
2018-07-26 17:10:43 +09:00
|
|
|
const value = info.isOpaque ? [r, g, b] : [r, g, b, 255];
|
2017-11-14 03:46:30 +09:00
|
|
|
|
2018-07-26 17:10:43 +09:00
|
|
|
properties['avgColor'] = value;
|
|
|
|
} catch (e) { }
|
2018-05-25 18:41:49 +09:00
|
|
|
};
|
2017-12-08 19:42:02 +09:00
|
|
|
|
2018-05-25 18:41:49 +09:00
|
|
|
propPromises = [calcWh(), calcAvg()];
|
2017-12-08 19:42:02 +09:00
|
|
|
}
|
|
|
|
|
2018-05-27 23:19:57 +09:00
|
|
|
const [folder] = await Promise.all([fetchFolder(), Promise.all(propPromises)]);
|
2018-05-25 18:41:49 +09:00
|
|
|
|
2018-04-03 23:45:13 +09:00
|
|
|
const metadata = {
|
2018-03-29 14:48:47 +09:00
|
|
|
userId: user._id,
|
2018-05-04 15:30:56 +09:00
|
|
|
_user: {
|
|
|
|
host: user.host
|
|
|
|
},
|
2018-03-29 14:48:47 +09:00
|
|
|
folderId: folder !== null ? folder._id : null,
|
2017-11-14 04:54:47 +09:00
|
|
|
comment: comment,
|
2018-05-25 20:19:14 +09:00
|
|
|
properties: properties,
|
2018-07-24 05:04:43 +09:00
|
|
|
withoutChunks: isLink,
|
2018-07-25 00:29:18 +09:00
|
|
|
isRemote: isLink,
|
2018-07-20 02:40:37 +09:00
|
|
|
isSensitive: sensitive
|
2018-04-03 23:45:13 +09:00
|
|
|
} as IMetadata;
|
|
|
|
|
2018-05-04 18:32:03 +09:00
|
|
|
if (url !== null) {
|
2018-07-24 05:04:43 +09:00
|
|
|
metadata.src = url;
|
|
|
|
|
|
|
|
if (isLink) {
|
|
|
|
metadata.url = url;
|
|
|
|
}
|
2018-05-04 18:32:03 +09:00
|
|
|
}
|
|
|
|
|
2018-04-03 23:45:13 +09:00
|
|
|
if (uri !== null) {
|
|
|
|
metadata.uri = uri;
|
|
|
|
}
|
|
|
|
|
2018-07-24 05:04:43 +09:00
|
|
|
const driveFile = isLink
|
2018-05-25 20:19:14 +09:00
|
|
|
? await DriveFile.insert({
|
|
|
|
length: 0,
|
|
|
|
uploadDate: new Date(),
|
|
|
|
md5: hash,
|
|
|
|
filename: detectedName,
|
|
|
|
metadata: metadata,
|
|
|
|
contentType: mime
|
|
|
|
})
|
2018-07-24 01:58:11 +09:00
|
|
|
: await (save(fs.createReadStream(path), detectedName, mime, hash, size, metadata));
|
2018-05-25 18:41:49 +09:00
|
|
|
|
|
|
|
log(`drive file has been created ${driveFile._id}`);
|
|
|
|
|
|
|
|
pack(driveFile).then(packedFile => {
|
|
|
|
// Publish drive_file_created event
|
|
|
|
event(user._id, 'drive_file_created', packedFile);
|
|
|
|
publishDriveStream(user._id, 'file_created', packedFile);
|
|
|
|
});
|
2018-05-03 20:03:14 +09:00
|
|
|
|
2018-07-24 01:58:11 +09:00
|
|
|
// TODO: サムネイル生成
|
2018-05-03 20:03:14 +09:00
|
|
|
|
2018-05-25 18:41:49 +09:00
|
|
|
return driveFile;
|
|
|
|
}
|