Add emoji from Twemoji 15.0 to the emoji picker/completion (#33395)

app/javascript/mastodon/features/emoji/emoji_compressed.js

@@ -9,18 +9,91 @@
 
 // This version comment should be bumped each time the emoji data is changed
 // to ensure that the prevaled file is regenerated by Babel
-// version: 2
+// version: 3
 
-const { emojiIndex } = require('emoji-mart');
-let data = require('emoji-mart/data/all.json');
+// This json file contains the names of the categories.
+const emojiMart5LocalesData = require('@emoji-mart/data/i18n/en.json');
+const emojiMart5Data = require('@emoji-mart/data/sets/15/all.json');
 const { uncompress: emojiMartUncompress } = require('emoji-mart/dist/utils/data');
+const _ = require('lodash');
+
 
 const emojiMap = require('./emoji_map.json');
+// This json file is downloaded from https://github.com/iamcal/emoji-data/
+// and is used to correct the sheet coordinates since we're using that repo's sheet
+const emojiSheetData = require('./emoji_sheet.json');
 const { unicodeToFilename } = require('./unicode_to_filename');
 const { unicodeToUnifiedName } = require('./unicode_to_unified_name');
 
-if(data.compressed) {
-  data = emojiMartUncompress(data);
+// Grabbed from `emoji_utils` to avoid circular dependency
+function unifiedToNative(unified) {
+  let unicodes = unified.split('-'),
+    codePoints = unicodes.map((u) => `0x${u}`);
+
+  return String.fromCodePoint(...codePoints);
+}
+
+let data = {
+  compressed: true,
+  categories: emojiMart5Data.categories.map(cat => {
+    return {
+      ...cat,
+      name: emojiMart5LocalesData.categories[cat.id]
+    };
+  }),
+  aliases: emojiMart5Data.aliases,
+  emojis: _(emojiMart5Data.emojis).values().map(emoji => {
+    let skin_variations = {};
+    const unified = emoji.skins[0].unified.toUpperCase();
+    const emojiFromRawData = emojiSheetData.find(e => e.unified === unified);
+
+    if (!emojiFromRawData) {
+      return undefined;
+    }
+
+    if (emoji.skins.length > 1) {
+      const [, ...nonDefaultSkins] = emoji.skins;
+      nonDefaultSkins.forEach(skin => {
+        const [matchingRawCodePoints,matchingRawEmoji] = Object.entries(emojiFromRawData.skin_variations).find((pair) => {
+          const [, value] = pair;
+          return value.unified.toLowerCase() === skin.unified;
+        });
+
+        if (matchingRawEmoji && matchingRawCodePoints) {
+          // At the time of writing, the json from `@emoji-mart/data` doesn't have data
+          // for emoji like `woman-heart-woman` with two different skin tones.
+          const skinToneCode = matchingRawCodePoints.split('-')[0];
+          skin_variations[skinToneCode] = {
+            unified: matchingRawEmoji.unified.toUpperCase(),
+            non_qualified: null,
+            sheet_x: matchingRawEmoji.sheet_x,
+            sheet_y: matchingRawEmoji.sheet_y,
+            has_img_twitter: true,
+            native: unifiedToNative(matchingRawEmoji.unified.toUpperCase())
+          };
+        }
+      });
+    }
+
+    return {
+      a: emoji.name,
+      b: unified,
+      c: undefined,
+      f: true,
+      j: [emoji.id, ...emoji.keywords],
+      k: [emojiFromRawData.sheet_x, emojiFromRawData.sheet_y],
+      m: emoji.emoticons?.[0],
+      l: emoji.emoticons,
+      o: emoji.version,
+      id: emoji.id,
+      skin_variations,
+      native: unifiedToNative(unified.toUpperCase())
+    };
+  }).compact().keyBy(e => e.id).mapValues(e => _.omit(e, 'id')).value()
+};
+
+if (data.compressed) {
+  emojiMartUncompress(data);
 }
 
 const emojiMartData = data;
@@ -32,15 +105,10 @@ const shortcodeMap   = {};
 const shortCodesToEmojiData = {};
 const emojisWithoutShortCodes = [];
 
-Object.keys(emojiIndex.emojis).forEach(key => {
-  let emoji = emojiIndex.emojis[key];
+Object.keys(emojiMart5Data.emojis).forEach(key => {
+  let emoji = emojiMart5Data.emojis[key];
 
-  // Emojis with skin tone modifiers are stored like this
-  if (Object.hasOwn(emoji, '1')) {
-    emoji = emoji['1'];
-  }
-
-  shortcodeMap[emoji.native] = emoji.id;
+  shortcodeMap[emoji.skins[0].native] = emoji.id;
 });
 
 const stripModifiers = unicode => {
@@ -84,13 +152,9 @@ Object.keys(emojiMap).forEach(key => {
   }
 });
 
-Object.keys(emojiIndex.emojis).forEach(key => {
-  let emoji = emojiIndex.emojis[key];
+Object.keys(emojiMartData.emojis).forEach(key => {
+  let emoji = emojiMartData.emojis[key];
 
-  // Emojis with skin tone modifiers are stored like this
-  if (Object.hasOwn(emoji, '1')) {
-    emoji = emoji['1'];
-  }
 
   const { native } = emoji;
   let { short_names, search, unified } = emojiMartData.emojis[key];
@@ -135,4 +199,5 @@ module.exports = JSON.parse(JSON.stringify([
   emojiMartData.categories,
   emojiMartData.aliases,
   emojisWithoutShortCodes,
+  emojiMartData
 ]));