'\" '\" Copyright (c) 1997 Sun Microsystems, Inc. '\" '\" See the file "license.terms" for information on usage and redistribution '\" of this file, and for a DISCLAIMER OF ALL WARRANTIES. '\" .TH Utf 3 "8.1" Tcl "Tcl Library Procedures" .\" The -*- nroff -*- definitions below are for supplemental macros used .\" in Tcl/Tk manual entries. .\" .\" .AP type name in/out ?indent? .\" Start paragraph describing an argument to a library procedure. .\" type is type of argument (int, etc.), in/out is either "in", "out", .\" or "in/out" to describe whether procedure reads or modifies arg, .\" and indent is equivalent to second arg of .IP (shouldn't ever be .\" needed; use .AS below instead) .\" .\" .AS ?type? ?name? .\" Give maximum sizes of arguments for setting tab stops. Type and .\" name are examples of largest possible arguments that will be passed .\" to .AP later. If args are omitted, default tab stops are used. .\" .\" .BS .\" Start box enclosure. From here until next .BE, everything will be .\" enclosed in one large box. .\" .\" .BE .\" End of box enclosure. .\" .\" .CS .\" Begin code excerpt. .\" .\" .CE .\" End code excerpt. .\" .\" .VS ?version? ?br? .\" Begin vertical sidebar, for use in marking newly-changed parts .\" of man pages. The first argument is ignored and used for recording .\" the version when the .VS was added, so that the sidebars can be .\" found and removed when they reach a certain age. If another argument .\" is present, then a line break is forced before starting the sidebar. .\" .\" .VE .\" End of vertical sidebar. .\" .\" .DS .\" Begin an indented unfilled display. .\" .\" .DE .\" End of indented unfilled display. .\" .\" .SO ?manpage? .\" Start of list of standard options for a Tk widget. The manpage .\" argument defines where to look up the standard options; if .\" omitted, defaults to "options". The options follow on successive .\" lines, in three columns separated by tabs. .\" .\" .SE .\" End of list of standard options for a Tk widget. .\" .\" .OP cmdName dbName dbClass .\" Start of description of a specific option. cmdName gives the .\" option's name as specified in the class command, dbName gives .\" the option's name in the option database, and dbClass gives .\" the option's class in the option database. .\" .\" .UL arg1 arg2 .\" Print arg1 underlined, then print arg2 normally. .\" .\" .QW arg1 ?arg2? .\" Print arg1 in quotes, then arg2 normally (for trailing punctuation). .\" .\" .PQ arg1 ?arg2? .\" Print an open parenthesis, arg1 in quotes, then arg2 normally .\" (for trailing punctuation) and then a closing parenthesis. .\" .\" # Set up traps and other miscellaneous stuff for Tcl/Tk man pages. .if t .wh -1.3i ^B .nr ^l \n(.l .ad b .\" # Start an argument description .de AP .ie !"\\$4"" .TP \\$4 .el \{\ . ie !"\\$2"" .TP \\n()Cu . el .TP 15 .\} .ta \\n()Au \\n()Bu .ie !"\\$3"" \{\ \&\\$1 \\fI\\$2\\fP (\\$3) .\".b .\} .el \{\ .br .ie !"\\$2"" \{\ \&\\$1 \\fI\\$2\\fP .\} .el \{\ \&\\fI\\$1\\fP .\} .\} .. .\" # define tabbing values for .AP .de AS .nr )A 10n .if !"\\$1"" .nr )A \\w'\\$1'u+3n .nr )B \\n()Au+15n .\" .if !"\\$2"" .nr )B \\w'\\$2'u+\\n()Au+3n .nr )C \\n()Bu+\\w'(in/out)'u+2n .. .AS Tcl_Interp Tcl_CreateInterp in/out .\" # BS - start boxed text .\" # ^y = starting y location .\" # ^b = 1 .de BS .br .mk ^y .nr ^b 1u .if n .nf .if n .ti 0 .if n \l'\\n(.lu\(ul' .if n .fi .. .\" # BE - end boxed text (draw box now) .de BE .nf .ti 0 .mk ^t .ie n \l'\\n(^lu\(ul' .el \{\ .\" Draw four-sided box normally, but don't draw top of .\" box if the box started on an earlier page. .ie !\\n(^b-1 \{\ \h'-1.5n'\L'|\\n(^yu-1v'\l'\\n(^lu+3n\(ul'\L'\\n(^tu+1v-\\n(^yu'\l'|0u-1.5n\(ul' .\} .el \}\ \h'-1.5n'\L'|\\n(^yu-1v'\h'\\n(^lu+3n'\L'\\n(^tu+1v-\\n(^yu'\l'|0u-1.5n\(ul' .\} .\} .fi .br .nr ^b 0 .. .\" # VS - start vertical sidebar .\" # ^Y = starting y location .\" # ^v = 1 (for troff; for nroff this doesn't matter) .de VS .if !"\\$2"" .br .mk ^Y .ie n 'mc \s12\(br\s0 .el .nr ^v 1u .. .\" # VE - end of vertical sidebar .de VE .ie n 'mc .el \{\ .ev 2 .nf .ti 0 .mk ^t \h'|\\n(^lu+3n'\L'|\\n(^Yu-1v\(bv'\v'\\n(^tu+1v-\\n(^Yu'\h'-|\\n(^lu+3n' .sp -1 .fi .ev .\} .nr ^v 0 .. .\" # Special macro to handle page bottom: finish off current .\" # box/sidebar if in box/sidebar mode, then invoked standard .\" # page bottom macro. .de ^B .ev 2 'ti 0 'nf .mk ^t .if \\n(^b \{\ .\" Draw three-sided box if this is the box's first page, .\" draw two sides but no top otherwise. .ie !\\n(^b-1 \h'-1.5n'\L'|\\n(^yu-1v'\l'\\n(^lu+3n\(ul'\L'\\n(^tu+1v-\\n(^yu'\h'|0u'\c .el \h'-1.5n'\L'|\\n(^yu-1v'\h'\\n(^lu+3n'\L'\\n(^tu+1v-\\n(^yu'\h'|0u'\c .\} .if \\n(^v \{\ .nr ^x \\n(^tu+1v-\\n(^Yu \kx\h'-\\nxu'\h'|\\n(^lu+3n'\ky\L'-\\n(^xu'\v'\\n(^xu'\h'|0u'\c .\} .bp 'fi .ev .if \\n(^b \{\ .mk ^y .nr ^b 2 .\} .if \\n(^v \{\ .mk ^Y .\} .. .\" # DS - begin display .de DS .RS .nf .sp .. .\" # DE - end display .de DE .fi .RE .sp .. .\" # SO - start of list of standard options .de SO 'ie '\\$1'' .ds So \\fBoptions\\fR 'el .ds So \\fB\\$1\\fR .SH "STANDARD OPTIONS" .LP .nf .ta 5.5c 11c .ft B .. .\" # SE - end of list of standard options .de SE .fi .ft R .LP See the \\*(So manual entry for details on the standard options. .. .\" # OP - start of full description for a single option .de OP .LP .nf .ta 4c Command-Line Name: \\fB\\$1\\fR Database Name: \\fB\\$2\\fR Database Class: \\fB\\$3\\fR .fi .IP .. .\" # CS - begin code excerpt .de CS .RS .nf .ta .25i .5i .75i 1i .. .\" # CE - end code excerpt .de CE .fi .RE .. .\" # UL - underline word .de UL \\$1\l'|0\(ul'\\$2 .. .\" # QW - apply quotation marks to word .de QW .ie '\\*(lq'"' ``\\$1''\\$2 .\"" fix emacs highlighting .el \\*(lq\\$1\\*(rq\\$2 .. .\" # PQ - apply parens and quotation marks to word .de PQ .ie '\\*(lq'"' (``\\$1''\\$2)\\$3 .\"" fix emacs highlighting .el (\\*(lq\\$1\\*(rq\\$2)\\$3 .. .\" # QR - quoted range .de QR .ie '\\*(lq'"' ``\\$1''\\-``\\$2''\\$3 .\"" fix emacs highlighting .el \\*(lq\\$1\\*(rq\\-\\*(lq\\$2\\*(rq\\$3 .. .\" # MT - "empty" string .de MT .QW "" .. .BS .SH NAME Tcl_UniChar, Tcl_UniCharToUtf, Tcl_UtfToUniChar, Tcl_UtfToChar16, Tcl_UtfToWChar, Tcl_UniCharToUtfDString, Tcl_UtfToUniCharDString, Tcl_Char16ToUtfDString, Tcl_UtfToWCharDString, Tcl_UtfToChar16DString, Tcl_WCharToUtfDString, Tcl_WCharLen, Tcl_Char16Len, Tcl_UniCharLen, Tcl_UtfNcmp, Tcl_UtfNcasecmp, Tcl_UtfCharComplete, Tcl_NumUtfChars, Tcl_UtfFindFirst, Tcl_UtfFindLast, Tcl_UtfNext, Tcl_UtfPrev, Tcl_UniCharAtIndex, Tcl_UtfAtIndex, Tcl_UtfBackslash \- routines for manipulating TUTF-8 encoded byte sequences .SH SYNOPSIS .nf \fB#include \fR .sp typedef ... \fBTcl_UniChar\fR; .sp Tcl_Size \fBTcl_UniCharToUtf\fR(\fIch, buf\fR) .sp Tcl_Size \fBTcl_UtfToUniChar\fR(\fIsrc, chPtr\fR) .sp Tcl_Size \fBTcl_UtfToChar16\fR(\fIsrc, uPtr\fR) .sp Tcl_Size \fBTcl_UtfToWChar\fR(\fIsrc, wPtr\fR) .sp char * \fBTcl_UniCharToUtfDString\fR(\fIuniStr, numUniChars, dsPtr\fR) .sp char * \fBTcl_Char16ToUtfDString\fR(\fIutf16, numUtf16, dsPtr\fR) .sp char * \fBTcl_WCharToUtfDString\fR(\fIwcharStr, numWChars, dsPtr\fR) .sp Tcl_UniChar * \fBTcl_UtfToUniCharDString\fR(\fIsrc, numBytes, dsPtr\fR) .sp unsigned short * \fBTcl_UtfToChar16DString\fR(\fIsrc, numBytes, dsPtr\fR) .sp wchar_t * \fBTcl_UtfToWCharDString\fR(\fIsrc, numBytes, dsPtr\fR) .sp Tcl_Size \fBTcl_Char16Len\fR(\fIutf16\fR) .sp Tcl_Size \fBTcl_WCharLen\fR(\fIwcharStr\fR) .sp Tcl_Size \fBTcl_UniCharLen\fR(\fIuniStr\fR) .sp int \fBTcl_UtfNcmp\fR(\fIcs, ct, length\fR) .sp int \fBTcl_UtfNcasecmp\fR(\fIcs, ct, length\fR) .sp int \fBTcl_UtfCharComplete\fR(\fIsrc, numBytes\fR) .sp Tcl_Size \fBTcl_NumUtfChars\fR(\fIsrc, numBytes\fR) .sp const char * \fBTcl_UtfFindFirst\fR(\fIsrc, ch\fR) .sp const char * \fBTcl_UtfFindLast\fR(\fIsrc, ch\fR) .sp const char * \fBTcl_UtfNext\fR(\fIsrc\fR) .sp const char * \fBTcl_UtfPrev\fR(\fIsrc, start\fR) .sp int \fBTcl_UniCharAtIndex\fR(\fIsrc, index\fR) .sp const char * \fBTcl_UtfAtIndex\fR(\fIsrc, index\fR) .sp Tcl_Size \fBTcl_UtfBackslash\fR(\fIsrc, readPtr, dst\fR) .fi .SH ARGUMENTS .AS "const Tcl_UniChar" *uniPattern in/out .AP char *buf out Buffer in which the TUTF-8 representation of the Tcl_UniChar is stored. At most \fBTCL_UTF_MAX\fR bytes are stored in the buffer. .AP int ch in The Unicode character to be converted or examined. .AP Tcl_UniChar *chPtr out Filled with the Tcl_UniChar represented by the head of the TUTF-8 byte sequence. .AP unsigned short *uPtr out Filled with the utf-16 represented by the head of the TUTF-8 byte sequence. .AP wchar_t *wPtr out Filled with the wchar_t represented by the head of the TUTF-8 byte sequence. .AP "const char" *src in Pointer to a TUTF-8 byte sequence. .AP "const char" *cs in Pointer to a TUTF-8 byte sequence. .AP "const char" *ct in Pointer to a TUTF-8 byte sequence. .AP "const Tcl_UniChar" *uniStr in A sequence of \fBTcl_UniChar\fR units with null-termination optional depending on function. .AP "const Tcl_UniChar" *ucs in A null-terminated sequence of \fBTcl_UniChar\fR. .AP "const Tcl_UniChar" *uct in A null-terminated sequence of \fBTcl_UniChar\fR. .AP "const Tcl_UniChar" *uniPattern in A null-terminated sequence of \fBTcl_UniChar\fR. .AP "const unsigned short" *utf16 in A sequence of UTF-16 units with null-termination optional depending on function. .AP "const wchar_t" *wcharStr in A sequence of \fBwchar_t\fR units with null-termination optional depending on function. .AP Tcl_Size numBytes in The length of the TUTF-8 input in bytes. If negative, the length includes all bytes until the first null byte. .AP Tcl_Size numUtf16 in The length of the input in UTF-16 units. If negative, the length includes all bytes until the first null. .AP Tcl_Size numUniChars in The length of the input in Tcl_UniChar units. If negative, the length includes all bytes until the first null. .AP Tcl_Size numWChars in The length of the input in wchar_t units. If negative, the length includes all bytes until the first null. .AP "Tcl_DString" *dsPtr in/out A pointer to a previously initialized \fBTcl_DString\fR. .AP "const char" *start in Pointer to the beginning of a TUTF-8 byte sequence. .AP Tcl_Size index in The index of a character (not byte) in the TUTF-8 byte sequence. .AP int *readPtr out If non-NULL, filled with the number of bytes in the backslash sequence, including the backslash character. .AP char *dst out Buffer in which the bytes represented by the backslash sequence are stored. At most \fBTCL_UTF_MAX\fR bytes are stored in the buffer. .AP int nocase in Specifies whether the match should be done case-sensitive (0) or case-insensitive (1). .BE .SH DESCRIPTION .PP N.B. The use of the term \fITUTF-8\fR in this documentation refers to the modified UTF-8 encoding used internally by Tcl. This differs from the UTF-8 encoding defined in the Unicode standard with respect to the encoding of the NUL character U+0000 which Tcl encodes internally as the byte sequence 0xC0 0x80 and not a single 0x00 byte as per the standard. The term \fITUTF-8 byte sequence\fR refers to a byte sequence representing one or more Unicode code points encoded in TUTF-8. The term \fIcharacter\fR refers to a Unicode code point and is used interchangeably with it. .PP The routines described here convert between TUTF-8 encoded byte sequences and other representation forms. .PP The \fBTcl_UniChar\fR type is an C integer type wide enough to hold a single Unicode code point value. A TUTF-8 byte sequence encoding a single code point may have a maximum length of 4 bytes, defined as the C preprocessor symbol \fBTCL_UTF_MAX\fR. This is also the maximum number of bytes that \fBTcl_UtfToUniChar\fR can consume in a single call. .PP \fBTcl_UniCharToUtf\fR encodes the character \fIch\fR as a TUTF-8 byte sequence, storing it starting at \fIbuf\fR. The return value is the number of bytes stored in \fIbuf\fR. The character \fIch\fR can be or'ed with the value TCL_COMBINE to enable special behavior, compatible with Tcl 8.x. Then, if ch is a high surrogate (range U+D800 - U+DBFF), the return value will be 1 and a single byte in the range 0xF0 - 0xF4 will be stored. If \fIch\fR is a low surrogate (range U+DC00 - U+DFFF), an attempt is made to combine the result with the earlier produced bytes, resulting in a 4-byte TUTF-8 byte sequence. .PP \fBTcl_UtfToUniChar\fR reads a TUTF-8 byte sequence starting at \fIsrc\fR and encoding a single code point, and stores it as a Tcl_UniChar in \fI*chPtr\fR. The return value is the number of bytes read from \fIsrc\fR. The caller must ensure that the source buffer is long enough such that this routine does not run off the end and dereference non-existent or random memory; if the source buffer is known to be null-terminated, this will not happen. If the input starts with a byte in the range 0x80 - 0x9F, \fBTcl_UtfToUniChar\fR assumes the cp1252 encoding, stores the corresponding Tcl_UniChar in \fI*chPtr\fR and returns 1. If the input is otherwise not in proper TUTF-8 format, \fBTcl_UtfToUniChar\fR will store the first byte of \fIsrc\fR in \fI*chPtr\fR as a Tcl_UniChar between 0x00A0 and 0x00FF and return 1. .PP \fBTcl_UniCharToUtfDString\fR converts the input in the form of a sequence of \fBTcl_UniChar\fR code points to TUTF-8, appending the result to the previously initialized output \fBTcl_DString\fR. The return value is a pointer to the TUTF-8 encoded representation of the \fBappended\fR string. .PP \fBTcl_UtfToUniCharDString\fR converts the input in the form of a TUTF-8 byte sequence to a \fBTcl_UniChar\fR sequence appending the result in the previously initialized \fBTcl_DString\fR. The return value is a pointer to the appended result which is also terminated with a \fBTcl_UniChar\fR NUL character. .PP \fBTcl_WCharToUtfDString\fR and \fBTcl_UtfToWCharDString\fR are similar to \fBTcl_UniCharToUtfDString\fR and \fBTcl_UtfToUniCharDString\fR except they operate on sequences of \fBwchar_t\fR instead of \fBTcl_UniChar\fR. .PP \fBTcl_Char16ToUtfDString\fR and \fBTcl_UtfToChar16DString\fR are similar to \fBTcl_UniCharToUtfDString\fR and \fBTcl_UtfToUniCharDString\fR except they operate on sequences of \fBUTF-16\fR units instead of \fBTcl_UniChar\fR. .PP \fBTcl_Char16Len\fR corresponds to \fBstrlen\fR for UTF-16 characters. It accepts a null-terminated UTF-16 sequence and returns the number of UTF-16 units until the null. .PP \fBTcl_WCharLen\fR corresponds to \fBstrlen\fR for \fBwchar_t\fR characters. It accepts a null-terminated \fBwchar_t\fR sequence and returns the number of \fBwchar_t\fR units until the null. .PP \fBTcl_UniCharLen\fR corresponds to \fBstrlen\fR for Tcl_UniChar characters. It accepts a null-terminated Tcl_UniChar string and returns the number of Tcl_UniChar's (not bytes) in that string. .PP \fBTcl_UtfNcmp\fR corresponds to \fBstrncmp\fR and accepts two null-terminated TUTF-8 encoded strings each of which should represent a sequence of at least \fIlength\fR code points. \fBTcl_UtfNcmp\fR compares the code points represented by each of the encoded strings in order. It returns an integer greater than, equal to, or less than 0 if the first string is greater than, equal to, or less than the second string respectively. .PP \fBTcl_UtfNcasecmp\fR corresponds to \fBstrncasecmp\fR for TUTF-8 encoded strings. It is similar to \fBTcl_UtfNcmp\fR except comparisons ignore differences in case when comparing upper, lower or title case characters. .PP \fBTcl_UtfCharComplete\fR returns 1 if the source TUTF-8 byte sequence \fIsrc\fR of \fInumBytes\fR bytes is long enough to be decoded by \fBTcl_UtfToUniChar\fR/\fBTcl_UtfNext\fR, or 0 otherwise. This function does not guarantee that the TUTF-8 byte sequence is properly formed. This routine is used by procedures that are operating on a byte at a time and need to know if a full Unicode character has been seen. .PP \fBTcl_NumUtfChars\fR corresponds to \fBstrlen\fR for TUTF-8 byte sequences. It returns the number of Tcl_UniChars that are represented by the TUTF-8 byte sequence \fIsrc\fR. The length of the source string is \fIlength\fR bytes. If the length is negative, all bytes up to the first null byte are used. .PP \fBTcl_UtfFindFirst\fR corresponds to \fBstrchr\fR for TUTF-8 byte sequences. It returns a pointer to the first occurrence of the Unicode character \fIch\fR in the null-terminated TUTF-8 byte sequence \fIsrc\fR. The null terminator is considered part of the byte sequence. .PP \fBTcl_UtfFindLast\fR corresponds to \fBstrrchr\fR for TUTF-8 byte sequences. It returns a pointer to the last occurrence of the Unicode character \fIch\fR in the null-terminated TUTF-8 byte sequence \fIsrc\fR. The null terminator is considered part of the byte sequence. .PP Given \fIsrc\fR, a pointer to some location in a TUTF-8 byte sequence, \fBTcl_UtfNext\fR returns a pointer to the start of the TUTF-8 byte sequence corresponding to the next character The caller must not ask for the next character after the last character in the string if the string is not terminated by a null character. \fBTcl_UtfCharComplete\fR can be used in that case to make sure enough bytes are available before calling \fBTcl_UtfNext\fR. .PP \fBTcl_UtfPrev\fR is used to step backward through but not beyond the TUTF-8 byte sequence that begins at \fIstart\fR. If the byte sequence is made up entirely of complete and well-formed characters, and \fIsrc\fR points to the lead byte of one of those characters (or to the location one byte past the end of the string), then repeated calls of \fBTcl_UtfPrev\fR will return pointers to the lead bytes of each character in the string, one character at a time, terminating when it returns \fIstart\fR. .PP When the conditions of completeness and well-formedness may not be satisfied, a more precise description of the function of \fBTcl_UtfPrev\fR is necessary. It always returns a pointer greater than or equal to \fIstart\fR; that is, always a pointer to a location in the string. It always returns a pointer to a byte that begins a character when scanning for characters beginning from \fIstart\fR. When \fIsrc\fR is greater than \fIstart\fR, it always returns a pointer less than \fIsrc\fR and greater than or equal to (\fIsrc\fR - 4). The character that begins at the returned pointer is the first one that either includes the byte \fIsrc[-1]\fR, or might include it if the right trail bytes are present at \fIsrc\fR and greater. \fBTcl_UtfPrev\fR never reads the byte \fIsrc[0]\fR nor the byte \fIstart[-1]\fR nor the byte \fIsrc[-5]\fR. .PP \fBTcl_UniCharAtIndex\fR corresponds to a C string array dereference or the Pascal Ord() function. It returns the Unicode code point represented at the specified character (not byte) \fIindex\fR in the TUTF-8 byte sequence \fIsrc\fR. The source string must contain at least \fIindex\fR characters. If \fIindex\fR is negative it returns -1. .PP \fBTcl_UtfAtIndex\fR returns a pointer to the specified character (not byte) \fIindex\fR in the TUTF-8 byte sequence \fIsrc\fR. The source must contain at least \fIindex\fR characters. This is equivalent to calling \fBTcl_UtfToUniChar\fR \fIindex\fR times. If \fIindex\fR is negative, the return pointer points to the first character in the source. .PP \fBTcl_UtfBackslash\fR is a utility procedure used by several of the Tcl commands. It parses a backslash sequence and stores the properly formed TUTF-8 encoding of the character represented by the backslash sequence in the output buffer \fIdst\fR. At most \fBTCL_UTF_MAX\fR bytes are stored in the buffer. \fBTcl_UtfBackslash\fR modifies \fI*readPtr\fR to contain the number of bytes in the backslash sequence, including the backslash character. The return value is the number of bytes stored in the output buffer. See the \fBTcl\fR manual entry for information on the valid backslash sequences. All of the sequences described in the Tcl manual entry are supported by \fBTcl_UtfBackslash\fR. .SH KEYWORDS utf, unicode, backslash