Skip to content

Commit d74f63e

Browse files
authored
Merge pull request #115 from mattn/hegel-fixes
Fix width mismatches found by the Hegel property tests
2 parents 791e45e + ea10d6a commit d74f63e

2 files changed

Lines changed: 170 additions & 35 deletions

File tree

‎runewidth.go‎

Lines changed: 89 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -514,19 +514,77 @@ func isJoiner(r rune) bool {
514514
return inTable(r, joiner)
515515
}
516516

517+
// clusterText returns s to segment in place of s. When s ends in a lead
518+
// byte short of the bytes it promises, the segmenter gives up there and
519+
// glues everything from it onto the cluster before, where the two-cell cap
520+
// hides it, although the same bytes followed by anything else form
521+
// clusters of their own. Such a string gets NULs after it, enough for any
522+
// lead byte to be decoded; as controls they are clusters of their own, and
523+
// the loops over the clusters stop at len(s) before reaching them.
524+
func clusterText(s string) string {
525+
if endsShort(s) {
526+
return s + "\x00\x00\x00"
527+
}
528+
return s
529+
}
530+
531+
// endsShort reports whether one of the last three bytes of s is a lead
532+
// byte that needs more bytes than are left, which is where the segmenter
533+
// stops decoding. It only looks at the length the lead byte announces, not
534+
// at whether the bytes after it could continue it, as the segmenter does.
535+
func endsShort(s string) bool {
536+
for i := max(len(s)-3, 0); i < len(s); i++ {
537+
b, left := s[i], len(s)-i
538+
if b >= 0xC2 && b < 0xE0 && left < 2 ||
539+
b >= 0xE0 && b < 0xF0 && left < 3 ||
540+
b >= 0xF0 && b < 0xF8 && left < 4 {
541+
return true
542+
}
543+
}
544+
return false
545+
}
546+
517547
// graphemeWidth returns the width of a single grapheme cluster: the sum of
518-
// the widths of its runes, capped at 2 cells. The cap keeps multi-rune
519-
// sequences that render as a single glyph (ZWJ emoji, flags, Hangul jamo)
520-
// from being counted wider than the two cells terminals give them.
548+
// the widths of its runes, with the part from the first rune that starts a
549+
// single glyph (an emoji, a regional indicator, a Hangul jamo or syllable)
550+
// capped at 2 cells. The cap keeps multi-rune sequences that render as one
551+
// glyph (ZWJ emoji, skin tones, flags, Hangul jamo) from being counted
552+
// wider than the two cells terminals give them. What comes before it, such
553+
// as a Prepend rune, and a cluster with no such rune, like a letter with a
554+
// skin tone modifier after it, are shown rune by rune and summed.
521555
func (c *Condition) graphemeWidth(cluster string) int {
522-
width := 0
556+
if r, size := utf8.DecodeRuneInString(cluster); size == len(cluster) {
557+
return c.RuneWidth(r) // one rune, never over the cap
558+
}
559+
width, glyph := 0, -1
523560
for _, r := range cluster {
524-
width += c.RuneWidth(r)
561+
w := c.RuneWidth(r)
562+
if glyph < 0 && r >= 0x1100 && startsGlyph(r) {
563+
glyph = 0
564+
}
565+
if glyph >= 0 {
566+
glyph += w
567+
} else {
568+
width += w
569+
}
525570
}
526-
if width > 2 {
527-
width = 2
571+
return width + min(max(glyph, 0), 2)
572+
}
573+
574+
// startsGlyph reports whether r begins a sequence that a terminal draws as
575+
// a single glyph however many runes it has.
576+
func startsGlyph(r rune) bool {
577+
switch {
578+
case 0x1100 <= r && r <= 0x11FF, // Hangul jamo
579+
0xA960 <= r && r <= 0xA97F, // Hangul jamo extended-A
580+
0xAC00 <= r && r <= 0xD7FF, // Hangul syllables, jamo extended-B
581+
0x1F1E6 <= r && r <= 0x1F1FF: // regional indicators
582+
return true
583+
case r < 0x203C, 0x3299 < r && r < 0x1F004:
584+
// Outside the emoji table: spares CJK text the search.
585+
return false
528586
}
529-
return width
587+
return inTable(r, emoji)
530588
}
531589

532590
// StringWidth return width as you can see
@@ -536,6 +594,11 @@ func (c *Condition) StringWidth(s string) (width int) {
536594
if b < 0x20 || b == 0x7F {
537595
return 0
538596
}
597+
if b >= 0x80 {
598+
// Not UTF-8 on its own: measure it as the U+FFFD every other
599+
// path decodes it to, which is ambiguous width.
600+
return c.RuneWidth(utf8.RuneError)
601+
}
539602
return 1
540603
}
541604
if len(s) > 0 && len(s) <= utf8.UTFMax {
@@ -565,8 +628,8 @@ graphemes:
565628
return
566629
}
567630
width = 0
568-
g := graphemes.FromString(s)
569-
for g.Next() {
631+
g := graphemes.FromString(clusterText(s))
632+
for g.Next() && g.Start() < len(s) {
570633
width += c.graphemeWidth(g.Value())
571634
}
572635
return
@@ -597,8 +660,8 @@ func (c *Condition) Truncate(s string, w int, tail string) string {
597660
}
598661
var width int
599662
pos := len(s)
600-
g := graphemes.FromString(s)
601-
for g.Next() {
663+
g := graphemes.FromString(clusterText(s))
664+
for g.Next() && g.Start() < len(s) {
602665
chWidth := c.graphemeWidth(g.Value())
603666
if width+chWidth > w {
604667
pos = g.Start()
@@ -651,8 +714,8 @@ func (c *Condition) TruncateLeft(s string, w int, prefix string) string {
651714
var width int
652715
pos := len(s)
653716

654-
g := graphemes.FromString(s)
655-
for g.Next() {
717+
g := graphemes.FromString(clusterText(s))
718+
for g.Next() && g.Start() < len(s) {
656719
chWidth := c.graphemeWidth(g.Value())
657720

658721
if width+chWidth > w {
@@ -714,8 +777,8 @@ func (c *Condition) TruncatePrefix(s string, w int, prefix string) string {
714777
}
715778
var width int
716779
var pos int
717-
g := graphemes.FromString(s)
718-
for g.Next() {
780+
g := graphemes.FromString(clusterText(s))
781+
for g.Next() && g.Start() < len(s) {
719782
chWidth := c.graphemeWidth(g.Value())
720783
if sw-(width+chWidth) <= w {
721784
pos = g.End()
@@ -791,6 +854,14 @@ func (c *Condition) Wrap(s string, w int) string {
791854
width = 0
792855
continue
793856
}
857+
// CRLF is one cluster that ends the line, as on the
858+
// segmenter, so it is never broken before.
859+
if b == '\r' && i+1 < len(s) && s[i+1] == '\n' {
860+
out.WriteString("\r\n")
861+
width = 0
862+
i++
863+
continue
864+
}
794865
// Same rule as the StringWidth fast path: no ASCII byte is
795866
// wide or ambiguous, so the flags in c cannot change this.
796867
cw := 0
@@ -815,8 +886,8 @@ func (c *Condition) Wrap(s string, w int) string {
815886
width := 0
816887
var out strings.Builder
817888
out.Grow(len(s) + len(s)/max(w, 1) + 1)
818-
g := graphemes.FromString(s)
819-
for g.Next() {
889+
g := graphemes.FromString(clusterText(s))
890+
for g.Next() && g.Start() < len(s) {
820891
cluster := g.Value()
821892
// LF and CRLF are each a single cluster
822893
if strings.HasSuffix(cluster, "\n") {

‎runewidth_test.go‎

Lines changed: 81 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -632,6 +632,19 @@ func TestWrapNonPositiveWidth(t *testing.T) {
632632
}
633633
}
634634

635+
func TestWrapCRLFNonPositiveWidth(t *testing.T) {
636+
// The ASCII fast path has to keep CRLF together as the segmenter
637+
// does, even at a width that breaks before every other cluster.
638+
c := &Condition{}
639+
for _, s := range []string{"a\r\nb", "\r\n", "ab\r\n\r\ncd", "a\rb", "a\r"} {
640+
for _, w := range []int{0, -1, 1, 2} {
641+
if got, want := c.Wrap(s, w), clusterWrap(c, s, w); got != want {
642+
t.Errorf("Wrap(%q, %d) = %q, cluster loop = %q", s, w, got, want)
643+
}
644+
}
645+
}
646+
}
647+
635648
func TestCreateLUTRebuildsAfterFlagChange(t *testing.T) {
636649
savedEA, savedLut := DefaultCondition.EastAsianWidth, DefaultCondition.combinedLut
637650
defer func() {
@@ -699,8 +712,8 @@ func TestWrapGraphemeCluster(t *testing.T) {
699712
// taken out, the behaviour the fast path has to reproduce exactly.
700713
func clusterWidth(c *Condition, s string) int {
701714
width := 0
702-
g := graphemes.FromString(s)
703-
for g.Next() {
715+
g := graphemes.FromString(clusterText(s))
716+
for g.Next() && g.Start() < len(s) {
704717
width += c.graphemeWidth(g.Value())
705718
}
706719
return width
@@ -709,8 +722,8 @@ func clusterWidth(c *Condition, s string) int {
709722
func clusterWrap(c *Condition, s string, w int) string {
710723
width := 0
711724
var out strings.Builder
712-
g := graphemes.FromString(s)
713-
for g.Next() {
725+
g := graphemes.FromString(clusterText(s))
726+
for g.Next() && g.Start() < len(s) {
714727
cluster := g.Value()
715728
if strings.HasSuffix(cluster, "\n") {
716729
out.WriteString(cluster)
@@ -831,13 +844,8 @@ func TestInvalidUTF8MatchesClusterLoop(t *testing.T) {
831844
b.WriteString(invalidPieces[r.Intn(len(invalidPieces))])
832845
}
833846
s := b.String()
834-
// StringWidth answers a single byte from a shortcut of its
835-
// own, which has never agreed with the segmenter on a byte
836-
// that is not valid UTF-8.
837-
if len(s) > 1 {
838-
if got, want := c.StringWidth(s), clusterWidth(c, s); got != want {
839-
t.Fatalf("StringWidth(%q) = %d, cluster loop = %d", s, got, want)
840-
}
847+
if got, want := c.StringWidth(s), clusterWidth(c, s); got != want {
848+
t.Fatalf("StringWidth(%q) = %d, cluster loop = %d", s, got, want)
841849
}
842850
w := 1 + r.Intn(6)
843851
if got, want := c.Wrap(s, w), clusterWrap(c, s, w); got != want {
@@ -858,6 +866,62 @@ func TestInvalidUTF8MatchesClusterLoop(t *testing.T) {
858866
}
859867
}
860868

869+
func TestIncompleteSequenceAtTheEnd(t *testing.T) {
870+
// The bytes of a sequence cut short at the end are measured as they
871+
// are when anything follows them, rather than vanishing into the
872+
// cluster before them.
873+
r := rand.New(rand.NewSource(10))
874+
for _, c := range []*Condition{{}, {EastAsianWidth: true, StrictEmojiNeutral: true}} {
875+
for i := 0; i < 20000; i++ {
876+
var b strings.Builder
877+
for j := 1 + r.Intn(6); j > 0; j-- {
878+
b.WriteString(invalidPieces[r.Intn(len(invalidPieces))])
879+
}
880+
s := b.String()
881+
if got, want := c.StringWidth(s), c.StringWidth(s+" ")-1; got != want {
882+
t.Fatalf("StringWidth(%q) = %d, want %d as with a space after it", s, got, want)
883+
}
884+
}
885+
}
886+
if got := Truncate("あ\xe3\x81", 2, ""); got != "あ" {
887+
t.Errorf("Truncate(%q, 2, \"\") = %q, want %q", "あ\xe3\x81", got, "あ")
888+
}
889+
}
890+
891+
func TestClusterCapCoversOnlyTheGlyph(t *testing.T) {
892+
// The two-cell cap is for sequences drawn as one glyph. A letter in
893+
// front of a skin tone modifier, or a Prepend rune in front of the
894+
// rest, is drawn on its own and keeps its cell.
895+
c := &Condition{}
896+
for _, tt := range []struct {
897+
s string
898+
want int
899+
}{
900+
{"👍🏼", 2},
901+
{"🇯🇵", 2},
902+
{"👨\u200d👩\u200d👧\u200d👦", 2},
903+
{"🏳\ufe0f\u200d🌈", 2},
904+
{"\u1100\u1161\u11a8", 2},
905+
{"🏼", 2},
906+
{" 🏼", 3},
907+
{"a🏼", 3},
908+
{"\u0890\u0601 ", 3},
909+
{"\u0601👍", 3},
910+
} {
911+
if got := c.StringWidth(tt.s); got != tt.want {
912+
t.Errorf("StringWidth(%+q) = %d, want %d", tt.s, got, tt.want)
913+
}
914+
}
915+
for _, s := range []string{"🏼", "\u0890\u0601"} {
916+
if got := c.StringWidth(c.FillLeft(s, 4)); got != 4 {
917+
t.Errorf("StringWidth(FillLeft(%+q, 4)) = %d, want 4", s, got)
918+
}
919+
if got := c.StringWidth(c.FillRight(s, 4)); got != 4 {
920+
t.Errorf("StringWidth(FillRight(%+q, 4)) = %d, want 4", s, got)
921+
}
922+
}
923+
}
924+
861925
// The three Truncate functions with the fast path taken out, the behaviour
862926
// it has to reproduce exactly.
863927
func clusterTruncate(c *Condition, s string, w int, tail string) string {
@@ -867,8 +931,8 @@ func clusterTruncate(c *Condition, s string, w int, tail string) string {
867931
w -= c.StringWidth(tail)
868932
var width int
869933
pos := len(s)
870-
g := graphemes.FromString(s)
871-
for g.Next() {
934+
g := graphemes.FromString(clusterText(s))
935+
for g.Next() && g.Start() < len(s) {
872936
chWidth := c.graphemeWidth(g.Value())
873937
if width+chWidth > w {
874938
pos = g.Start()
@@ -885,8 +949,8 @@ func clusterTruncateLeft(c *Condition, s string, w int, prefix string) string {
885949
}
886950
var width int
887951
pos := len(s)
888-
g := graphemes.FromString(s)
889-
for g.Next() {
952+
g := graphemes.FromString(clusterText(s))
953+
for g.Next() && g.Start() < len(s) {
890954
chWidth := c.graphemeWidth(g.Value())
891955
if width+chWidth > w {
892956
if width < w {
@@ -913,8 +977,8 @@ func clusterTruncatePrefix(c *Condition, s string, w int, prefix string) string
913977
w -= c.StringWidth(prefix)
914978
var width int
915979
var pos int
916-
g := graphemes.FromString(s)
917-
for g.Next() {
980+
g := graphemes.FromString(clusterText(s))
981+
for g.Next() && g.Start() < len(s) {
918982
chWidth := c.graphemeWidth(g.Value())
919983
if sw-(width+chWidth) <= w {
920984
pos = g.End()

0 commit comments

Comments
 (0)
⌂ 换网址