misskey/src/services/drive/add-file.ts

309 lines
7.7 KiB
TypeScript
Raw Normal View History

import { Buffer } from 'buffer';
import * as fs from 'fs';
import * as stream from 'stream';
2016-12-29 07:49:51 +09:00
import * as mongodb from 'mongodb';
import * as crypto from 'crypto';
import * as _gm from 'gm';
2017-02-06 21:11:09 +09:00
import * as debug from 'debug';
2017-01-03 06:03:19 +09:00
import fileType = require('file-type');
const prominence = require('prominence');
import DriveFile, { IMetadata, getDriveFileBucket, IDriveFile } from '../../models/drive-file';
2018-04-05 03:21:11 +09:00
import DriveFolder from '../../models/drive-folder';
import { pack } from '../../models/drive-file';
2018-07-07 19:19:00 +09:00
import event, { publishDriveStream } from '../../stream';
2018-05-26 09:57:31 +09:00
import { isLocalUser, IUser, IRemoteUser } from '../../models/user';
import { getDriveFileThumbnailBucket } from '../../models/drive-file-thumbnail';
import genThumbnail from '../../drive/gen-thumbnail';
import delFile from './delete-file';
2018-07-21 19:17:15 +09:00
import config from '../../config';
2016-12-29 07:49:51 +09:00
const gm = _gm.subClass({
imageMagick: true
});
2018-03-27 16:51:12 +09:00
const log = debug('misskey:drive:add-file');
2017-02-06 21:11:09 +09:00
const writeChunks = (name: string, readable: stream.Readable, type: string, metadata: any) =>
getDriveFileBucket()
.then(bucket => new Promise((resolve, reject) => {
const writeStream = bucket.openUploadStream(name, { contentType: type, metadata });
2018-04-10 02:34:47 +09:00
writeStream.once('finish', resolve);
writeStream.on('error', reject);
readable.pipe(writeStream);
2017-11-14 03:47:42 +09:00
}));
2017-11-06 14:37:00 +09:00
const writeThumbnailChunks = (name: string, readable: stream.Readable, originalId: mongodb.ObjectID) =>
getDriveFileThumbnailBucket()
.then(bucket => new Promise((resolve, reject) => {
const writeStream = bucket.openUploadStream(name, {
contentType: 'image/jpeg',
metadata: {
originalId
}
});
writeStream.once('finish', resolve);
writeStream.on('error', reject);
readable.pipe(writeStream);
}));
2018-05-25 18:41:49 +09:00
async function deleteOldFile(user: IRemoteUser) {
const oldFile = await DriveFile.findOne({
_id: {
$nin: [user.avatarId, user.bannerId]
}
}, {
sort: {
_id: 1
}
});
if (oldFile) {
delFile(oldFile, true);
2018-05-25 18:41:49 +09:00
}
}
/**
* Add file to drive
*
* @param user User who wish to add file
* @param path File path
* @param name Name
* @param comment Comment
* @param folderId Folder ID
* @param force If set to true, forcibly upload the file even if there is a file with the same hash.
* @return Created drive file
*/
export default async function(
2018-05-26 09:57:31 +09:00
user: IUser,
path: string,
2016-12-29 07:49:51 +09:00
name: string = null,
comment: string = null,
folderId: mongodb.ObjectID = null,
2018-04-03 23:45:13 +09:00
force: boolean = false,
metaOnly: boolean = false,
url: string = null,
2018-07-20 02:40:37 +09:00
uri: string = null,
sensitive = false
2018-05-25 18:41:49 +09:00
): Promise<IDriveFile> {
// Calc md5 hash
const calcHash = new Promise<string>((res, rej) => {
const readable = fs.createReadStream(path);
const hash = crypto.createHash('md5');
const chunks: Buffer[] = [];
2018-05-25 18:41:49 +09:00
readable
.on('error', rej)
.pipe(hash)
.on('error', rej)
.on('data', chunk => chunks.push(chunk))
.on('end', () => {
const buffer = Buffer.concat(chunks);
res(buffer.toString('hex'));
});
2018-05-25 18:41:49 +09:00
});
// Detect content type
const detectMime = new Promise<[string, string]>((res, rej) => {
const readable = fs.createReadStream(path);
readable
.on('error', rej)
.once('data', (buffer: Buffer) => {
readable.destroy();
const type = fileType(buffer);
if (type) {
res([type.mime, type.ext]);
} else {
// 種類が同定できなかったら application/octet-stream にする
res(['application/octet-stream', null]);
}
});
});
// Get file size
const getFileSize = new Promise<number>((res, rej) => {
fs.stat(path, (err, stats) => {
if (err) return rej(err);
res(stats.size);
});
});
const [hash, [mime, ext], size] = await Promise.all([calcHash, detectMime, getFileSize]);
log(`hash: ${hash}, mime: ${mime}, ext: ${ext}, size: ${size}`);
2016-12-29 07:49:51 +09:00
// detect name
2018-05-25 18:41:49 +09:00
const detectedName = name || (ext ? `untitled.${ext}` : 'untitled');
if (!force) {
// Check if there is a file with the same hash
const much = await DriveFile.findOne({
md5: hash,
2018-05-13 16:52:47 +09:00
'metadata.userId': user._id,
2018-05-14 12:47:33 +09:00
'metadata.deletedAt': { $exists: false }
});
2018-06-08 22:03:14 +09:00
if (much) {
log(`file with same hash is found: ${much._id}`);
return much;
}
}
2018-05-25 18:41:49 +09:00
//#region Check drive usage
if (!metaOnly) {
const usage = await DriveFile
.aggregate([{
$match: {
'metadata.userId': user._id,
'metadata.deletedAt': { $exists: false }
}
}, {
$project: {
length: true
}
}, {
$group: {
_id: null,
usage: { $sum: '$length' }
}
}])
.then((aggregates: any[]) => {
if (aggregates.length > 0) {
return aggregates[0].usage;
}
return 0;
});
log(`drive usage is ${usage}`);
2018-07-21 19:17:15 +09:00
const driveCapacity = 1024 * 1024 * (isLocalUser(user) ? config.localDriveCapacityMb : config.remoteDriveCapacityMb);
// If usage limit exceeded
2018-07-21 19:17:15 +09:00
if (usage + size > driveCapacity) {
if (isLocalUser(user)) {
throw 'no-free-space';
} else {
// (アバターまたはバナーを含まず)最も古いファイルを削除する
deleteOldFile(user);
}
2018-05-25 18:41:49 +09:00
}
}
//#endregion
const fetchFolder = async () => {
if (!folderId) {
return null;
}
const driveFolder = await DriveFolder.findOne({
_id: folderId,
userId: user._id
});
if (driveFolder == null) throw 'folder-not-found';
return driveFolder;
};
const properties: {[key: string]: any} = {};
2018-05-25 18:41:49 +09:00
2018-05-27 23:19:57 +09:00
let propPromises: Array<Promise<void>> = [];
2018-05-25 18:41:49 +09:00
const isImage = ['image/jpeg', 'image/gif', 'image/png'].includes(mime);
if (isImage) {
// Calc width and height
const calcWh = async () => {
2017-12-11 13:33:33 +09:00
log('calculate image width and height...');
2017-12-08 19:42:02 +09:00
// Calculate width and height
const g = gm(fs.createReadStream(path), name);
const size = await prominence(g).size();
2017-12-11 13:33:33 +09:00
log(`image width and height is calculated: ${size.width}, ${size.height}`);
2018-05-25 18:41:49 +09:00
properties['width'] = size.width;
properties['height'] = size.height;
};
2017-12-11 13:33:33 +09:00
2018-05-25 18:41:49 +09:00
// Calc average color
const calcAvg = async () => {
2017-12-11 13:33:33 +09:00
log('calculate average color...');
const info = await prominence(gm(fs.createReadStream(path), name)).identify();
const isTransparent = info ? info['Channel depth'].Alpha != null : false;
2017-12-11 13:33:33 +09:00
const buffer = await prominence(gm(fs.createReadStream(path), name)
.setFormat('ppm')
.resize(1, 1)) // 1pxのサイズに縮小して平均色を取得するというハック
.toBuffer();
const r = buffer.readUInt8(buffer.length - 3);
const g = buffer.readUInt8(buffer.length - 2);
const b = buffer.readUInt8(buffer.length - 1);
log(`average color is calculated: ${r}, ${g}, ${b}`);
2018-05-25 18:41:49 +09:00
const value = isTransparent ? [r, g, b, 255] : [r, g, b];
2018-05-25 18:41:49 +09:00
properties['avgColor'] = value;
};
2017-12-08 19:42:02 +09:00
2018-05-25 18:41:49 +09:00
propPromises = [calcWh(), calcAvg()];
2017-12-08 19:42:02 +09:00
}
2018-05-27 23:19:57 +09:00
const [folder] = await Promise.all([fetchFolder(), Promise.all(propPromises)]);
2018-05-25 18:41:49 +09:00
2018-04-03 23:45:13 +09:00
const metadata = {
2018-03-29 14:48:47 +09:00
userId: user._id,
_user: {
host: user.host
},
2018-03-29 14:48:47 +09:00
folderId: folder !== null ? folder._id : null,
comment: comment,
properties: properties,
2018-07-20 02:40:37 +09:00
isMetaOnly: metaOnly,
isSensitive: sensitive
2018-04-03 23:45:13 +09:00
} as IMetadata;
if (url !== null) {
metadata.url = url;
}
2018-04-03 23:45:13 +09:00
if (uri !== null) {
metadata.uri = uri;
}
const driveFile = metaOnly
? await DriveFile.insert({
length: 0,
uploadDate: new Date(),
md5: hash,
filename: detectedName,
metadata: metadata,
contentType: mime
})
: await (writeChunks(detectedName, fs.createReadStream(path), mime, metadata) as Promise<IDriveFile>);
2018-05-25 18:41:49 +09:00
log(`drive file has been created ${driveFile._id}`);
pack(driveFile).then(packedFile => {
// Publish drive_file_created event
event(user._id, 'drive_file_created', packedFile);
publishDriveStream(user._id, 'file_created', packedFile);
});
if (!metaOnly) {
try {
const thumb = await genThumbnail(driveFile);
if (thumb) {
await writeThumbnailChunks(detectedName, thumb, driveFile._id);
}
} catch (e) {
// noop
}
}
2018-05-25 18:41:49 +09:00
return driveFile;
}