java - 用高/低代理计算java中的单词和字符？

Question

我知道有一些关于这个主题的SO，但是所提供的所有解决方案似乎都采用了与我在 javascript 中看到的示例不同的方法。

下面是计算文本字符串中输入的段落、句子、单词和字符的 javascript 示例，其中包括检查高/低代理项以专门计算字符:

javascript版本

count(text);

function count(original) {
    var trimmed = original.replace(/[\u200B]+/, '').trim();
    return {
        paragraphs: trimmed ? (trimmed.match(/\n+/g) || []).length + 1 : 0,
        sentences: trimmed ? (trimmed.match(/[.?!…\n]+./g) || []).length + 1 : 0,
        words: trimmed ? (trimmed.replace(/['";:,.?¿\-!¡]+/g, '').match(/\S+/g) || []).length : 0,
        characters: trimmed ? _decode(trimmed.replace(/\s/g, '')).length : 0,
        all: _decode(original).length
    };
};

function _decode(string) {
    var output = [],
        counter = 0,
        length = string.length,
        value, extra;
    while (counter < length) {
        value = string.charCodeAt(counter++);
        if (value >= 0xD800 && value <= 0xDBFF && counter < length) {
            // High surrogate, and there is a next character.
            extra = string.charCodeAt(counter++);
            if ((extra & 0xFC00) === 0xDC00) {
                // Low surrogate.
                output.push(((value & 0x3FF) << 10) + (extra & 0x3FF) + 0x10000);
            } else {
                // unmatched surrogate; only append this code unit, in case the next
                // code unit is the high surrogate of a surrogate pair
                output.push(value, extra);
                counter--;
            }
        } else {
            output.push(value);
        }
    }
    return output;
}

下面和 jsfiddle 中的演示

var text = 'This is a paragraph. This is the 2nd sentence in the 1st paragraph.\nThis is another paragraph.';
var count = doCount(text);

document.body.innerHTML = '<pre>' + text + '</pre><hr>';
for (i in count) {
	document.body.innerHTML += '<p>'+ i +': ' + count[i] + '</p>';
}

/* COUNTING LIBRARY */

/**
 * Extracted from https://github.com/RadLikeWhoa/Countable/, which in 
 * turn uses `ucs2decode` function from the punycode.js library.
 */
function doCount(original) {
    var trimmed = original.replace(/[\u200B]+/, '').trim();

    return {
        paragraphs: trimmed ? (trimmed.match(/\n+/g) || []).length + 1 : 0,
        sentences: trimmed ? (trimmed.match(/[.?!…\n]+./g) || []).length + 1 : 0,
        words: trimmed ? (trimmed.replace(/['";:,.?¿\-!¡]+/g, '').match(/\S+/g) || []).length : 0,
        characters: trimmed ? _decode(trimmed.replace(/\s/g, '')).length : 0,
        all: _decode(original).length
    };
};

/**
 * `ucs2decode` function from the punycode.js library.
 *
 * Creates an array containing the decimal code points of each Unicode
 * character in the string. While JavaScript uses UCS-2 internally, this
 * function will convert a pair of surrogate halves (each of which UCS-2
 * exposes as separate characters) into a single code point, matching
 * UTF-16.
 *
 * @see     <http://goo.gl/8M09r>
 * @see     <http://goo.gl/u4UUC>
 *
 * @param   {String}  string   The Unicode input string (UCS-2).
 *
 * @return  {Array}   The new array of code points.
 */
function _decode(string) {
    var output = [],
        counter = 0,
        length = string.length,
        value, extra;

    while (counter < length) {
        value = string.charCodeAt(counter++);

        if (value >= 0xD800 && value <= 0xDBFF && counter < length) {
            // High surrogate, and there is a next character.
            extra = string.charCodeAt(counter++);

            if ((extra & 0xFC00) === 0xDC00) {
                // Low surrogate.
                output.push(((value & 0x3FF) << 10) + (extra & 0x3FF) + 0x10000);
            } else {
                // unmatched surrogate; only append this code unit, in case the next
                // code unit is the high surrogate of a surrogate pair
                output.push(value, extra);
                counter--;
            }
        } else {
            output.push(value);
        }
    }

    return output;
}

我不太熟悉字符编码方案和高/低代理项之类的东西，但是使用 java 计数时不需要这样做吗？

我对 javascript 实现的结果很满意，我想在我的 java 后端上进行计数，但我不确定是否需要相同的方法或应该如何完成。

Answer 1

因此，javascript 版本的作用是，如果代理项对出现在正在解码的文本中，则将它们读取为一个字符。这在 Javascript 中是可能的，因为取决于 Javascript 引擎 both UCS-2 and UTF-16是允许的，并且 UTF-16 支持高代理项，这意味着单个可见字符使用代码点进行编码。为了正确计算长度，库会考虑额外的代码点，以便将它们计为 1。

在 Java 中你也有类似的问题，只不过在 Java 中你可以有更多的编码方案。幸运的是，Java 已经为包含高代理项的字符串返回正确的长度。不过，如果您想分离组合的代码点甚至删除它们，Java 提供了 Normalizer ( example of removing diacritics 来自文本)。

string = Normalizer.normalize(string, Normalizer.Form.NFD);