From c6af0c6e806d6292826d18a8029da19b2d31b818 Mon Sep 17 00:00:00 2001 From: lionel-rowe Date: Fri, 12 Jul 2024 14:23:13 +0800 Subject: [PATCH] refactor(csv): rename `graphemeLength` to `codePointLength` (#5421) --- csv/_io.ts | 10 +++++----- csv/_shared.ts | 14 ++++++-------- csv/parse.ts | 10 +++++----- csv/parse_test.ts | 2 +- 4 files changed, 17 insertions(+), 19 deletions(-) diff --git a/csv/_io.ts b/csv/_io.ts index 15491a1ca..862042a9c 100644 --- a/csv/_io.ts +++ b/csv/_io.ts @@ -4,7 +4,7 @@ // https://github.com/golang/go/blob/master/LICENSE // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. -import { graphemeLength } from "./_shared.ts"; +import { codePointLength } from "./_shared.ts"; /** Options for {@linkcode parseRecord}. */ export interface ReadOptions { @@ -98,7 +98,7 @@ export async function parseRecord( if (!options.lazyQuotes) { const j = field.indexOf(quote); if (j >= 0) { - const col = graphemeLength( + const col = codePointLength( fullLine.slice(0, fullLine.length - line.slice(j).length), ); throw new ParseError(startLine + 1, lineIndex, col, ERR_BARE_QUOTE); @@ -138,7 +138,7 @@ export async function parseRecord( recordBuffer += quote; } else { // `"*` sequence (invalid non-escaped quote). - const col = graphemeLength( + const col = codePointLength( fullLine.slice(0, fullLine.length - line.length - quoteLen), ); throw new ParseError(startLine + 1, lineIndex, col, ERR_QUOTE); @@ -153,7 +153,7 @@ export async function parseRecord( if (r === null) { // Abrupt end of file (EOF or error). if (!options.lazyQuotes) { - const col = graphemeLength(fullLine); + const col = codePointLength(fullLine); throw new ParseError(startLine + 1, lineIndex, col, ERR_QUOTE); } fieldIndexes.push(recordBuffer.length); @@ -163,7 +163,7 @@ export async function parseRecord( } else { // Abrupt end of file (EOF on error). if (!options.lazyQuotes) { - const col = graphemeLength(fullLine); + const col = codePointLength(fullLine); throw new ParseError(startLine + 1, lineIndex, col, ERR_QUOTE); } fieldIndexes.push(recordBuffer.length); diff --git a/csv/_shared.ts b/csv/_shared.ts index f2b34dc37..9d3d876f7 100644 --- a/csv/_shared.ts +++ b/csv/_shared.ts @@ -1,15 +1,13 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. /** - * returns length of a string considering surrogate pairs - * ```ts - * function graphemeLength(s: string) { - * return Array.from(s).length; - * } - * graphemeLength("🐱") // 1 - * "🐱".length // 2 + * Gives the length of a string in Unicode code points + * + * ``` + * codePointLength("🐱"); // 1 + * "🐱".length; // 2 * ``` */ -export function graphemeLength(s: string) { +export function codePointLength(s: string) { return Array.from(s).length; } diff --git a/csv/parse.ts b/csv/parse.ts index d610b6a44..27812daff 100644 --- a/csv/parse.ts +++ b/csv/parse.ts @@ -12,7 +12,7 @@ import { type ReadOptions, type RecordWithColumn, } from "./_io.ts"; -import { graphemeLength } from "./_shared.ts"; +import { codePointLength } from "./_shared.ts"; export { ParseError, type ParseResult, type RecordWithColumn }; @@ -109,7 +109,7 @@ class Parser { if (!this.#options.lazyQuotes) { const j = field.indexOf(quote); if (j >= 0) { - const col = graphemeLength( + const col = codePointLength( fullLine.slice(0, fullLine.length - line.slice(j).length), ); throw new ParseError(startLine + 1, lineIndex, col, ERR_BARE_QUOTE); @@ -149,7 +149,7 @@ class Parser { recordBuffer += quote; } else { // `"*` sequence (invalid non-escaped quote). - const col = graphemeLength( + const col = codePointLength( fullLine.slice(0, fullLine.length - line.length - quoteLen), ); throw new ParseError(startLine + 1, lineIndex, col, ERR_QUOTE); @@ -164,7 +164,7 @@ class Parser { if (r === null) { // Abrupt end of file (EOF or error). if (!this.#options.lazyQuotes) { - const col = graphemeLength(fullLine); + const col = codePointLength(fullLine); throw new ParseError(startLine + 1, lineIndex, col, ERR_QUOTE); } fieldIndexes.push(recordBuffer.length); @@ -174,7 +174,7 @@ class Parser { } else { // Abrupt end of file (EOF on error). if (!this.#options.lazyQuotes) { - const col = graphemeLength(fullLine); + const col = codePointLength(fullLine); throw new ParseError(startLine + 1, lineIndex, col, ERR_QUOTE); } fieldIndexes.push(recordBuffer.length); diff --git a/csv/parse_test.ts b/csv/parse_test.ts index 18b390609..092b675da 100644 --- a/csv/parse_test.ts +++ b/csv/parse_test.ts @@ -199,7 +199,7 @@ Deno.test({ }, }); await t.step({ - name: "error column grapheme number", + name: "error column Unicode code point number", fn() { const input = `a,b,🐱"`; assertThrows(