keep combining marks with their letter when tokenizing, fixes #1099

This commit is contained in:
dualfroz committed 2026-10-14 19:39:16 +02:00
1 parent 77f2d74e3d
commit 236b97ae91
2 files changed
+26 -1

No files matched your search

@@ -2,6 +2,9 @@
//we have slightly different rules for start/end - like #hashtags.
const isLetter = /\p{Letter}/u
const hasMark = /\p{Mark}/u
// a letter with its combining marks, or any one char
const splitChars = /\p{Letter}\p{Mark}*|./gsu
const isNumber = /[\p{Number}\p{Currency_Symbol}]/u
const hasAcronym = /^[a-z]\.([a-z]\.)+/i
const chillin = /[sn]['’]$/
@@ -28,7 +31,8 @@ const normalizePunctuation = function (str, model) {
const original = str
let pre = ''
let post = ''
const chars = Array.from(str)
// keep combining marks on their letter - 'cafe\u0301'
const chars = hasMark.test(str) ? str.match(splitChars) : Array.from(str)
let start = 0
let end = chars.length