@@ -562,6 +562,28 @@ private extension String {
562562 }
563563}
564564
565+ /// The CJK punctuation primitives, internal because they are the single source of truth shared by
566+ /// this file's acceptance policy and `SentenceBoundaryClassifier`'s sentence-end detection. Adding a
567+ /// codepoint here updates phrase boundaries, chunk binding, and the generation stop in one edit.
568+ extension Character {
569+ /// The CJK sentence terminators: ideographic full stop `。`, fullwidth `!` `?`, and the halfwidth
570+ /// ideographic stop `。`. Unlike the ASCII period these are unambiguous (they never mark decimals,
571+ /// list numbers, or abbreviations), so every consumer treats them as terminal without classifier
572+ /// disambiguation.
573+ var isCJKSentenceTerminator : Bool {
574+ self == " \u{3002} " || self == " \u{FF01} " || self == " \u{FF1F} " || self == " \u{FF61} "
575+ }
576+
577+ /// The CJK closing punctuation: corner brackets `」` `』` (and the halfwidth corner `」`),
578+ /// fullwidth parenthesis `)`, lenticular bracket `】`, and angle brackets `〉` `》`. Walk-backs
579+ /// skip a run of these to find the real terminator underneath, and chunk binding attaches them to
580+ /// the word they close.
581+ var isCJKClosingPunctuation : Bool {
582+ self == " \u{300D} " || self == " \u{300F} " || self == " \u{FF09} "
583+ || self == " \u{3011} " || self == " \u{3009} " || self == " \u{300B} " || self == " \u{FF63} "
584+ }
585+ }
586+
565587private extension Character {
566588 /// True when the character begins a word of a space-less script (Han, Hiragana, Katakana, Hangul,
567589 /// Thai, Lao, Khmer, Myanmar, ...). These scripts write words without separating spaces, so the
@@ -597,24 +619,6 @@ private extension Character {
597619 isLetter || isNumber
598620 }
599621
600- /// The CJK sentence terminators: ideographic full stop `。`, fullwidth `!` `?`, and the halfwidth
601- /// ideographic stop `。`. Declared once so `isPhraseSentenceTerminator` (phrase ends) and
602- /// `bindsToPrecedingSpacelessWord` (chunk binding) share one list instead of each restating the
603- /// four codepoints and silently drifting when one is updated.
604- var isCJKSentenceTerminator : Bool {
605- self == " \u{3002} " || self == " \u{FF01} " || self == " \u{FF1F} " || self == " \u{FF61} "
606- }
607-
608- /// The CJK closing punctuation: corner brackets `」` `』` (and the halfwidth corner `」`),
609- /// fullwidth parenthesis `)`, lenticular bracket `】`, and angle brackets `〉` `》`. Declared
610- /// once so `isPhraseClosingPunctuation` (the closer walk-back) and
611- /// `bindsToPrecedingSpacelessWord` (chunk binding) share one list instead of each restating the
612- /// codepoints.
613- var isCJKClosingPunctuation : Bool {
614- self == " \u{300D} " || self == " \u{300F} " || self == " \u{FF09} "
615- || self == " \u{3011} " || self == " \u{3009} " || self == " \u{300B} " || self == " \u{FF63} "
616- }
617-
618622 /// The CJK opening brackets: corner brackets `「` `『` (and the halfwidth corner `「`), fullwidth
619623 /// parenthesis `(`, lenticular bracket `【`, and angle brackets `〈` `《`. These lead the word
620624 /// they quote, so the trailing-binding rule stops before them while the punctuation-led peel
0 commit comments