Merge branch 'update-emoji-digests-with-latest-from-gemojione' into 'master'

Update Emoji digests and assets with latest from Gemojione v3.3.0

Closes #32634

See merge request gitlab-org/gitlab-ce!15390

app/assets/javascripts/emoji/support/is_emoji_unicode_supported.js

@@ -7,6 +7,17 @@ function isFlagEmoji(emojiUnicode) {
   return emojiUnicode.length === 4 && cp >= flagACodePoint && cp <= flagZCodePoint;
 }
 
+// Tested on mac OS 10.12.6 and Windows 10 FCU, it renders as two separate characters
+const baseFlagCodePoint = 127987; // parseInt('1F3F3', 16)
+const rainbowCodePoint = 127752; // parseInt('1F308', 16)
+function isRainbowFlagEmoji(emojiUnicode) {
+  const characters = Array.from(emojiUnicode);
+  // Length 4 because flags are made of 2 characters which are surrogate pairs
+  return emojiUnicode.length === 4 &&
+    characters[0].codePointAt(0) === baseFlagCodePoint &&
+    characters[1].codePointAt(0) === rainbowCodePoint;
+}
+
 // Chrome <57 renders keycaps oddly
 // See https://bugs.chromium.org/p/chromium/issues/detail?id=632294
 // Same issue on Windows also fixed in Chrome 57, http://i.imgur.com/rQF7woO.png
@@ -57,9 +68,11 @@ function isPersonZwjEmoji(emojiUnicode) {
 // in `isEmojiUnicodeSupported` logic
 function checkFlagEmojiSupport(unicodeSupportMap, emojiUnicode) {
   const isFlagResult = isFlagEmoji(emojiUnicode);
+  const isRainbowFlagResult = isRainbowFlagEmoji(emojiUnicode);
   return (
     (unicodeSupportMap.flag && isFlagResult) ||
-    !isFlagResult
+    (unicodeSupportMap.rainbowFlag && isRainbowFlagResult) ||
+    (!isFlagResult && !isRainbowFlagResult)
   );
 }
 
@@ -113,6 +126,7 @@ function isEmojiUnicodeSupported(unicodeSupportMap = {}, emojiUnicode, unicodeVe
 export {
   isEmojiUnicodeSupported as default,
   isFlagEmoji,
+  isRainbowFlagEmoji,
   isKeycapEmoji,
   isSkinToneComboEmoji,
   isHorceRacingSkinToneComboEmoji,

app/assets/javascripts/emoji/support/unicode_support_map.js

@@ -1,5 +1,7 @@
 import AccessorUtilities from '../../lib/utils/accessor';
 
+const GL_EMOJI_VERSION = '0.2.0';
+
 const unicodeSupportTestMap = {
   // man, student (emojione does not have any of these yet), http://emojipedia.org/emoji-zwj-sequences/
   // occupationZwj: '\u{1F468}\u{200D}\u{1F393}',
@@ -13,6 +15,7 @@ const unicodeSupportTestMap = {
   horseRacing: '\u{1F3C7}\u{1F3FF}',
   // US flag, http://emojipedia.org/flags/
   flag: '\u{1F1FA}\u{1F1F8}',
+  rainbowFlag: '\u{1F3F3}\u{1F308}',
   // http://emojipedia.org/modifiers/
   skinToneModifier: [
     // spy_tone5
@@ -141,23 +144,31 @@ function generateUnicodeSupportMap(testMap) {
 }
 
 export default function getUnicodeSupportMap() {
-  let unicodeSupportMap;
-  let userAgentFromCache;
-
   const isLocalStorageAvailable = AccessorUtilities.isLocalStorageAccessSafe();
 
-  if (isLocalStorageAvailable) userAgentFromCache = window.localStorage.getItem('gl-emoji-user-agent');
+  let glEmojiVersionFromCache;
+  let userAgentFromCache;
+  if (isLocalStorageAvailable) {
+    glEmojiVersionFromCache = window.localStorage.getItem('gl-emoji-version');
+    userAgentFromCache = window.localStorage.getItem('gl-emoji-user-agent');
+  }
 
+  let unicodeSupportMap;
   try {
     unicodeSupportMap = JSON.parse(window.localStorage.getItem('gl-emoji-unicode-support-map'));
   } catch (err) {
     // swallow
   }
 
-  if (!unicodeSupportMap || userAgentFromCache !== navigator.userAgent) {
+  if (
+    !unicodeSupportMap ||
+    glEmojiVersionFromCache !== GL_EMOJI_VERSION ||
+    userAgentFromCache !== navigator.userAgent
+  ) {
     unicodeSupportMap = generateUnicodeSupportMap(unicodeSupportTestMap);
 
     if (isLocalStorageAvailable) {
+      window.localStorage.setItem('gl-emoji-version', GL_EMOJI_VERSION);
       window.localStorage.setItem('gl-emoji-user-agent', navigator.userAgent);
       window.localStorage.setItem('gl-emoji-unicode-support-map', JSON.stringify(unicodeSupportMap));
     }

changelogs/unreleased/update-emoji-digests-with-latest-from-gemojione.yml

@@ -0,0 +1,6 @@
+---
+title: 'Update emojis. Add :gay_pride_flag: and :speech_left:. Remove extraneous comma
+  in :cartwheel_tone4:'
+merge_request:
+author:
+type: changed

doc/development/README.md

@@ -22,6 +22,7 @@ comments: false
 
 - [UX guide](ux_guide/index.md) for building GitLab with existing CSS styles and elements
 - [Frontend guidelines](fe_guide/index.md)
+- [Emoji guide](fe_guide/emojis.md)
 
 ## Backend guides
 

doc/development/fe_guide/emojis.md

@@ -0,0 +1,27 @@
+# Emojis
+
+GitLab supports native unicode emojis and fallsback to image-based emojis selectively
+when your platform does not support it.
+
+# How to update Emojis
+
+ 1. Update the `gemojione` gem
+ 1. Update `fixtures/emojis/index.json` from [Gemojione](https://github.com/jonathanwiesel/gemojione/blob/master/config/index.json).
+    In the future, we could grab the file directly from the gem.
+    We should probably make a PR on the Gemojione project to get access to
+    all emojis after being parsed or just a raw path to the `json` file itself.
+ 1. Ensure [`emoji-unicode-version`](https://www.npmjs.com/package/emoji-unicode-version)
+    is up to date with the latest version.
+ 1. Run `bundle exec rake gemojione:aliases`
+ 1. Run `bundle exec rake gemojione:digests`
+ 1. Run `bundle exec rake gemojione:sprite`
+ 1. Ensure new sprite sheets generated for 1x and 2x
+    - `app/assets/images/emoji.png`
+    - `app/assets/images/emoji@2x.png`
+ 1. Ensure you see new individual images copied into `app/assets/images/emoji/`
+ 1. Ensure you can see the new emojis and their aliases in the GFM Autocomplete
+ 1. Ensure you can see the new emojis and their aliases in the award emoji menu
+ 1. You might need to add new emoji unicode support checks and rules for platforms
+    that do not support a certain emoji and we need to fallback to an image.
+    See `app/assets/javascripts/emoji/support/is_emoji_unicode_supported.js`
+    and `app/assets/javascripts/emoji/support/unicode_support_map.js`

doc/development/rake_tasks.md

@@ -122,6 +122,15 @@ they can be easily inspected.
 bundle exec rake services:doc
 ```
 
+## Updating Emoji Aliases
+
+To update the Emoji aliases file (used for Emoji autocomplete) you must run the
+following:
+
+```
+bundle exec rake gemojione:aliases
+```
+
 ## Updating Emoji Digests
 
 To update the Emoji digests file (used for Emoji autocomplete) you must run the
@@ -131,6 +140,7 @@ following:
 bundle exec rake gemojione:digests
 ```
 
+
 This will update the file `fixtures/emojis/digests.json` based on the currently
 available Emoji.
 

fixtures/emojis/aliases.json

@@ -339,6 +339,7 @@
    "baguette_bread":"french_bread",
    "anguished":"frowning",
    "white_frowning_face":"frowning2",
+   "rainbow_flag":"gay_pride_flag",
    "goal_net":"goal",
    "hammer_and_pick":"hammer_pick",
    "raised_hand_with_fingers_splayed":"hand_splayed",
@@ -488,6 +489,7 @@
    "slightly_smiling_face":"slight_smile",
    "sneeze":"sneezing_face",
    "speaking_head_in_silhouette":"speaking_head",
+   "left_speech_bubble":"speech_left",
    "sleuth_or_spy":"spy",
    "sleuth_or_spy_tone1":"spy_tone1",
    "sleuth_or_spy_tone2":"spy_tone2",
@@ -537,4 +539,4 @@
    "wrestling_tone4":"wrestlers_tone4",
    "wrestling_tone5":"wrestlers_tone5",
    "zipper_mouth_face":"zipper_mouth"
-}
+}

fixtures/emojis/digests.json

@@ -1478,7 +1478,7 @@
   },
   "cartwheel_tone4": {
     "category": "activity",
-    "moji": "🤸🏾,",
+    "moji": "🤸🏾",
     "description": "person doing cartwheel tone 4",
     "unicodeVersion": "9.0",
     "digest": "8253afb672431c84e498014c30babb00b9284bec773009e79f7f06aa7108643e"
@@ -5375,6 +5375,13 @@
     "unicodeVersion": "6.0",
     "digest": "180e66f19d9285e02d0a5e859722c608206826e80323942b9938fc49d44973b1"
   },
+  "gay_pride_flag": {
+    "category": "flags",
+    "moji": "🏳🌈",
+    "description": "gay_pride_flag",
+    "unicodeVersion": "6.0",
+    "digest": "924e668c559db61b7f4724a661223081c2fc60d55169f3fe1ad6156934d1d37f"
+  },
   "gemini": {
     "category": "symbols",
     "moji": "♊",
@@ -7578,7 +7585,7 @@
     "moji": "🤶",
     "description": "mother christmas",
     "unicodeVersion": "9.0",
-    "digest": "1f72f586ca75bd7ebb4150cdcc8199a930c32fa4b81510cb8d200f1b3ddd4076"
+    "digest": "357d769371305a8584f46d6087a962d647b6af22fab363a44702f38ab7814091"
   },
   "mrs_claus_tone1": {
     "category": "people",
@@ -10709,6 +10716,13 @@
     "unicodeVersion": "6.0",
     "digest": "817100d9979456e7d2f253ac22e13b7a2302dc1590566214915b003e403c53ca"
   },
+  "speech_left": {
+    "category": "symbols",
+    "moji": "🗨",
+    "description": "left speech bubble",
+    "unicodeVersion": "7.0",
+    "digest": "912797107d574f5665411498b6e349dbdec69846f085b6dc356548c4155e90b0"
+  },
   "speedboat": {
     "category": "travel",
     "moji": "🚤",

fixtures/emojis/generate_aliases.rb

@@ -1,18 +0,0 @@
-#!/usr/bin/env ruby
-
-require 'json'
-
-aliases = {}
-
-index_file = File.expand_path("./index.json")
-index = JSON.parse(File.read(index_file))
-
-index.each_pair do |key, data|
-  data['aliases'].each do |a|
-    a.tr!(':', '')
-
-    aliases[a] = key
-  end
-end
-
-puts JSON.pretty_generate(aliases, indent: '   ', space: '', space_before: '')

fixtures/emojis/index.json

@@ -4023,7 +4023,7 @@
     ],
     "aliases_ascii": [],
     "keywords": [],
-    "moji": "🤸🏾,"
+    "moji": "🤸🏾"
   },
   "cartwheel_tone5": {
     "unicode": "1F938-1F3FF",
@@ -14475,6 +14475,19 @@
     ],
     "moji": "💎"
   },
+  "gay_pride_flag": {
+    "unicode": "1F3F3-1F308",
+    "unicode_alternates": [],
+    "name": "gay_pride_flag",
+    "shortname": ":gay_pride_flag:",
+    "category": "extras",
+    "aliases": [
+      ":rainbow_flag:"
+    ],
+    "aliases_ascii": [],
+    "keywords": [],
+    "moji": "🏳🌈"
+  },
   "gemini": {
     "unicode": "264A",
     "unicode_alternates": [
@@ -16830,7 +16843,6 @@
       "0:-)",
       "0:)",
       "0;^)",
-      "O:-)",
       "O:)",
       "O;-)",
       "O=)",
@@ -28506,6 +28518,21 @@
     ],
     "moji": "💬"
   },
+  "speech_left": {
+    "unicode": "1F5E8",
+    "unicode_alternates": [
+      "1F5E8-FE0F"
+    ],
+    "name": "left speech bubble",
+    "shortname": ":speech_left:",
+    "category": "symbols",
+    "aliases": [
+      ":left_speech_bubble:"
+    ],
+    "aliases_ascii": [],
+    "keywords": [],
+    "moji": "🗨"
+  },
   "speedboat": {
     "unicode": "1F6A4",
     "unicode_alternates": [],
@@ -33477,4 +33504,4 @@
     ],
     "moji": "💤"
   }
-}
+}

lib/tasks/gemojione.rake

@@ -1,5 +1,28 @@
 namespace :gemojione do
   desc 'Generates Emoji SHA256 digests'
+
+  task aliases: ['yarn:check', 'environment'] do
+    require 'json'
+
+    aliases = {}
+
+    index_file = File.join(Rails.root, 'fixtures', 'emojis', 'index.json')
+    index = JSON.parse(File.read(index_file))
+
+    index.each_pair do |key, data|
+      data['aliases'].each do |a|
+        a.tr!(':', '')
+
+        aliases[a] = key
+      end
+    end
+
+    out = File.join(Rails.root, 'fixtures', 'emojis', 'aliases.json')
+    File.open(out, 'w') do |handle|
+      handle.write(JSON.pretty_generate(aliases, indent: '   ', space: '', space_before: ''))
+    end
+  end
+
   task digests: ['yarn:check', 'environment'] do
     require 'digest/sha2'
     require 'json'
@@ -16,8 +39,13 @@ namespace :gemojione do
         fpath = File.join(dir, "#{emoji_hash['unicode']}.png")
         hash_digest = Digest::SHA256.file(fpath).hexdigest
 
+        category = emoji_hash['category']
+        if name == 'gay_pride_flag'
+          category = 'flags'
+        end
+
         entry = {
-          category: emoji_hash['category'],
+          category: category,
           moji: emoji_hash['moji'],
           description: emoji_hash['description'],
           unicodeVersion: Gitlab::Emoji.emoji_unicode_version(name),
@@ -29,7 +57,6 @@ namespace :gemojione do
     end
 
     out = File.join(Rails.root, 'fixtures', 'emojis', 'digests.json')
-
     File.open(out, 'w') do |handle|
       handle.write(JSON.pretty_generate(resultant_emoji_map))
     end

spec/javascripts/behaviors/gl_emoji/unicode_support_map_spec.js

@@ -21,13 +21,18 @@ describe('Unicode Support Map', () => {
       });
 
       it('should call .getItem and .setItem', () => {
-        const allArgs = window.localStorage.setItem.calls.allArgs();
+        const getArgs = window.localStorage.getItem.calls.allArgs();
+        const setArgs = window.localStorage.setItem.calls.allArgs();
 
-        expect(window.localStorage.getItem).toHaveBeenCalledWith('gl-emoji-user-agent');
-        expect(allArgs[0][0]).toBe('gl-emoji-user-agent');
-        expect(allArgs[0][1]).toBe(navigator.userAgent);
-        expect(allArgs[1][0]).toBe('gl-emoji-unicode-support-map');
-        expect(allArgs[1][1]).toBe(stringSupportMap);
+        expect(getArgs[0][0]).toBe('gl-emoji-version');
+        expect(getArgs[1][0]).toBe('gl-emoji-user-agent');
+
+        expect(setArgs[0][0]).toBe('gl-emoji-version');
+        expect(setArgs[0][1]).toBe('0.2.0');
+        expect(setArgs[1][0]).toBe('gl-emoji-user-agent');
+        expect(setArgs[1][1]).toBe(navigator.userAgent);
+        expect(setArgs[2][0]).toBe('gl-emoji-unicode-support-map');
+        expect(setArgs[2][1]).toBe(stringSupportMap);
       });
     });
 

spec/javascripts/emoji_spec.js

@@ -1,6 +1,7 @@
 import { glEmojiTag } from '~/emoji';
 import isEmojiUnicodeSupported, {
   isFlagEmoji,
+  isRainbowFlagEmoji,
   isKeycapEmoji,
   isSkinToneComboEmoji,
   isHorceRacingSkinToneComboEmoji,
@@ -217,6 +218,24 @@ describe('gl_emoji', () => {
     });
   });
 
+  describe('isRainbowFlagEmoji', () => {
+    it('should gracefully handle empty string', () => {
+      expect(isRainbowFlagEmoji('')).toBeFalsy();
+    });
+    it('should detect rainbow_flag', () => {
+      expect(isRainbowFlagEmoji('🏳🌈')).toBeTruthy();
+    });
+    it('should not detect flag_white on its\' own', () => {
+      expect(isRainbowFlagEmoji('🏳')).toBeFalsy();
+    });
+    it('should not detect rainbow on its\' own', () => {
+      expect(isRainbowFlagEmoji('🌈')).toBeFalsy();
+    });
+    it('should not detect flag_white with something else', () => {
+      expect(isRainbowFlagEmoji('🏳🔵')).toBeFalsy();
+    });
+  });
+
   describe('isKeycapEmoji', () => {
     it('should gracefully handle empty string', () => {
       expect(isKeycapEmoji('')).toBeFalsy();