1	/* CPP Library - charsets
2	Copyright (C) 1998-2023 Free Software Foundation, Inc.
3
4	Broken out of c-lex.cc Apr 2003, adding valid C99 UCN ranges.
5
6	This program is free software; you can redistribute it and/or modify it
7	under the terms of the GNU General Public License as published by the
8	Free Software Foundation; either version 3, or (at your option) any
9	later version.
10
11	This program is distributed in the hope that it will be useful,
12	but WITHOUT ANY WARRANTY; without even the implied warranty of
13	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14	GNU General Public License for more details.
15
16	You should have received a copy of the GNU General Public License
17	along with this program; see the file COPYING3. If not see
18	<http://www.gnu.org/licenses/>. */
19
20	#include "config.h"
21	#include "system.h"
22	#include "cpplib.h"
23	#include "internal.h"
24
25	/* Character set handling for C-family languages.
26
27	Terminological note: In what follows, "charset" or "character set"
28	will be taken to mean both an abstract set of characters and an
29	encoding for that set.
30
31	The C99 standard discusses two character sets: source and execution.
32	The source character set is used for internal processing in translation
33	phases 1 through 4; the execution character set is used thereafter.
34	Both are required by 5.2.1.2p1 to be multibyte encodings, not wide
35	character encodings (see 3.7.2, 3.7.3 for the standardese meanings
36	of these terms). Furthermore, the "basic character set" (listed in
37	5.2.1p3) is to be encoded in each with values one byte wide, and is
38	to appear in the initial shift state.
39
40	It is not explicitly mentioned, but there is also a "wide execution
41	character set" used to encode wide character constants and wide
42	string literals; this is supposed to be the result of applying the
43	standard library function mbstowcs() to an equivalent narrow string
44	(6.4.5p5). However, the behavior of hexadecimal and octal
45	\-escapes is at odds with this; they are supposed to be translated
46	directly to wchar_t values (6.4.4.4p5,6).
47
48	The source character set is not necessarily the character set used
49	to encode physical source files on disk; translation phase 1 converts
50	from whatever that encoding is to the source character set.
51
52	The presence of universal character names in C99 (6.4.3 et seq.)
53	forces the source character set to be isomorphic to ISO 10646,
54	that is, Unicode. There is no such constraint on the execution
55	character set; note also that the conversion from source to
56	execution character set does not occur for identifiers (5.1.1.2p1#5).
57
58	For convenience of implementation, the source character set's
59	encoding of the basic character set should be identical to the
60	execution character set OF THE HOST SYSTEM's encoding of the basic
61	character set, and it should not be a state-dependent encoding.
62
63	cpplib uses UTF-8 or UTF-EBCDIC for the source character set,
64	depending on whether the host is based on ASCII or EBCDIC (see
65	respectively Unicode section 2.3/ISO10646 Amendment 2, and Unicode
66	Technical Report #16). With limited exceptions, it relies on the
67	system library's iconv() primitive to do charset conversion
68	(specified in SUSv2). */
69
70	#if !HAVE_ICONV
71	/* Make certain that the uses of iconv(), iconv_open(), iconv_close()
72	below, which are guarded only by if statements with compile-time
73	constant conditions, do not cause link errors. */
74	#define iconv_open(x, y) (errno = EINVAL, (iconv_t)-1)
75	#define iconv(a,b,c,d,e) (errno = EINVAL, (size_t)-1)
76	#define iconv_close(x) (void)0
77	#define ICONV_CONST
78	#endif
79
80	#if HOST_CHARSET == HOST_CHARSET_ASCII
81	#define SOURCE_CHARSET "UTF-8"
82	#define LAST_POSSIBLY_BASIC_SOURCE_CHAR 0x7e
83	#elif HOST_CHARSET == HOST_CHARSET_EBCDIC
84	#define SOURCE_CHARSET "UTF-EBCDIC"
85	#define LAST_POSSIBLY_BASIC_SOURCE_CHAR 0xFF
86	#else
87	#error "Unrecognized basic host character set"
88	#endif
89
90	#ifndef EILSEQ
91	#define EILSEQ EINVAL
92	#endif
93
94	/* This structure is used for a resizable string buffer throughout. */
95	/* Don't call it strbuf, as that conflicts with unistd.h on systems
96	such as DYNIX/ptx where unistd.h includes stropts.h. */
97	struct _cpp_strbuf
98	{
99	uchar *text;
100	size_t asize;
101	size_t len;
102	};
103
104	/* This is enough to hold any string that fits on a single 80-column
105	line, even if iconv quadruples its size (e.g. conversion from
106	ASCII to UTF-32) rounded up to a power of two. */
107	#define OUTBUF_BLOCK_SIZE 256
108
109	/* Conversions between UTF-8 and UTF-16/32 are implemented by custom
110	logic. This is because a depressing number of systems lack iconv,
111	or have have iconv libraries that do not do these conversions, so
112	we need a fallback implementation for them. To ensure the fallback
113	doesn't break due to neglect, it is used on all systems.
114
115	UTF-32 encoding is nice and simple: a four-byte binary number,
116	constrained to the range 00000000-7FFFFFFF to avoid questions of
117	signedness. We do have to cope with big- and little-endian
118	variants.
119
120	UTF-16 encoding uses two-byte binary numbers, again in big- and
121	little-endian variants, for all values in the 00000000-0000FFFF
122	range. Values in the 00010000-0010FFFF range are encoded as pairs
123	of two-byte numbers, called "surrogate pairs": given a number S in
124	this range, it is mapped to a pair (H, L) as follows:
125
126	H = (S - 0x10000) / 0x400 + 0xD800
127	L = (S - 0x10000) % 0x400 + 0xDC00
128
129	Two-byte values in the D800...DFFF range are ill-formed except as a
130	component of a surrogate pair. Even if the encoding within a
131	two-byte value is little-endian, the H member of the surrogate pair
132	comes first.
133
134	There is no way to encode values in the 00110000-7FFFFFFF range,
135	which is not currently a problem as there are no assigned code
136	points in that range; however, the author expects that it will
137	eventually become necessary to abandon UTF-16 due to this
138	limitation. Note also that, because of these pairs, UTF-16 does
139	not meet the requirements of the C standard for a wide character
140	encoding (see 3.7.3 and 6.4.4.4p11).
141
142	UTF-8 encoding looks like this:
143
144	value range encoded as
145	00000000-0000007F 0xxxxxxx
146	00000080-000007FF 110xxxxx 10xxxxxx
147	00000800-0000FFFF 1110xxxx 10xxxxxx 10xxxxxx
148	00010000-001FFFFF 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
149	00200000-03FFFFFF 111110xx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx
150	04000000-7FFFFFFF 1111110x 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx
151
152	Values in the 0000D800 ... 0000DFFF range (surrogates) are invalid,
153	which means that three-byte sequences ED xx yy, with A0 <= xx <= BF,
154	never occur. Note also that any value that can be encoded by a
155	given row of the table can also be encoded by all successive rows,
156	but this is not done; only the shortest possible encoding for any
157	given value is valid. For instance, the character 07C0 could be
158	encoded as any of DF 80, E0 9F 80, F0 80 9F 80, F8 80 80 9F 80, or
159	FC 80 80 80 9F 80. Only the first is valid.
160
161	An implementation note: the transformation from UTF-16 to UTF-8, or
162	vice versa, is easiest done by using UTF-32 as an intermediary. */
163
164	/* Internal primitives which go from an UTF-8 byte stream to native-endian
165	UTF-32 in a cppchar_t, or vice versa; this avoids an extra marshal/unmarshal
166	operation in several places below. */
167	static inline int
168	one_utf8_to_cppchar (const uchar **inbufp, size_t *inbytesleftp,
169	cppchar_t *cp)
170	{
171	static const uchar masks[6] = { 0x7F, 0x1F, 0x0F, 0x07, 0x03, 0x01 };
172	static const uchar patns[6] = { 0x00, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC };
173
174	cppchar_t c;
175	const uchar *inbuf = *inbufp;
176	size_t nbytes, i;
177
178	if (*inbytesleftp < 1)
179	return EINVAL;
180
181	c = *inbuf;
182	if (c < 0x80)
183	{
184	*cp = c;
185	*inbytesleftp -= 1;
186	*inbufp += 1;
187	return 0;
188	}
189
190	/* The number of leading 1-bits in the first byte indicates how many
191	bytes follow. */
192	for (nbytes = 2; nbytes < 7; nbytes++)
193	if ((c & ~masks[nbytes-1]) == patns[nbytes-1])
194	goto found;
195	return EILSEQ;
196	found:
197
198	if (*inbytesleftp < nbytes)
199	return EINVAL;
200
201	c = (c & masks[nbytes-1]);
202	inbuf++;
203	for (i = 1; i < nbytes; i++)
204	{
205	cppchar_t n = *inbuf++;
206	if ((n & 0xC0) != 0x80)
207	return EILSEQ;
208	c = ((c << 6) + (n & 0x3F));
209	}
210
211	/* Make sure the shortest possible encoding was used. */
212	if (c <= 0x7F && nbytes > 1) return EILSEQ;
213	if (c <= 0x7FF && nbytes > 2) return EILSEQ;
214	if (c <= 0xFFFF && nbytes > 3) return EILSEQ;
215	if (c <= 0x1FFFFF && nbytes > 4) return EILSEQ;
216	if (c <= 0x3FFFFFF && nbytes > 5) return EILSEQ;
217
218	/* Make sure the character is valid. */
219	if (c > 0x7FFFFFFF || (c >= 0xD800 && c <= 0xDFFF)) return EILSEQ;
220
221	*cp = c;
222	*inbufp = inbuf;
223	*inbytesleftp -= nbytes;
224	return 0;
225	}
226
227	static inline int
228	one_cppchar_to_utf8 (cppchar_t c, uchar **outbufp, size_t *outbytesleftp)
229	{
230	static const uchar masks[6] = { 0x00, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC };
231	static const uchar limits[6] = { 0x80, 0xE0, 0xF0, 0xF8, 0xFC, 0xFE };
232	size_t nbytes;
233	uchar buf[6], *p = &buf[6];
234	uchar *outbuf = *outbufp;
235
236	nbytes = 1;
237	if (c < 0x80)
238	*--p = c;
239	else
240	{
241	do
242	{
243	*--p = ((c & 0x3F) | 0x80);
244	c >>= 6;
245	nbytes++;
246	}
247	while (c >= 0x3F || (c & limits[nbytes-1]));
248	*--p = (c | masks[nbytes-1]);
249	}
250
251	if (*outbytesleftp < nbytes)
252	return E2BIG;
253
254	while (p < &buf[6])
255	*outbuf++ = *p++;
256	*outbytesleftp -= nbytes;
257	*outbufp = outbuf;
258	return 0;
259	}
260
261	/* The following four functions transform one character between the two
262	encodings named in the function name. All have the signature
263	int (*)(iconv_t bigend, const uchar **inbufp, size_t *inbytesleftp,
264	uchar **outbufp, size_t *outbytesleftp)
265
266	BIGEND must have the value 0 or 1, coerced to (iconv_t); it is
267	interpreted as a boolean indicating whether big-endian or
268	little-endian encoding is to be used for the member of the pair
269	that is not UTF-8.
270
271	INBUFP, INBYTESLEFTP, OUTBUFP, OUTBYTESLEFTP work exactly as they
272	do for iconv.
273
274	The return value is either 0 for success, or an errno value for
275	failure, which may be E2BIG (need more space), EILSEQ (ill-formed
276	input sequence), ir EINVAL (incomplete input sequence). */
277
278	static inline int
279	one_utf8_to_utf32 (iconv_t bigend, const uchar **inbufp, size_t *inbytesleftp,
280	uchar **outbufp, size_t *outbytesleftp)
281	{
282	uchar *outbuf;
283	cppchar_t s = 0;
284	int rval;
285
286	/* Check for space first, since we know exactly how much we need. */
287	if (*outbytesleftp < 4)
288	return E2BIG;
289
290	rval = one_utf8_to_cppchar (inbufp, inbytesleftp, cp: &s);
291	if (rval)
292	return rval;
293
294	outbuf = *outbufp;
295	outbuf[bigend ? 3 : 0] = (s & 0x000000FF);
296	outbuf[bigend ? 2 : 1] = (s & 0x0000FF00) >> 8;
297	outbuf[bigend ? 1 : 2] = (s & 0x00FF0000) >> 16;
298	outbuf[bigend ? 0 : 3] = (s & 0xFF000000) >> 24;
299
300	*outbufp += 4;
301	*outbytesleftp -= 4;
302	return 0;
303	}
304
305	static inline int
306	one_utf32_to_utf8 (iconv_t bigend, const uchar **inbufp, size_t *inbytesleftp,
307	uchar **outbufp, size_t *outbytesleftp)
308	{
309	cppchar_t s;
310	int rval;
311	const uchar *inbuf;
312
313	if (*inbytesleftp < 4)
314	return EINVAL;
315
316	inbuf = *inbufp;
317
318	s = inbuf[bigend ? 0 : 3] << 24;
319	s += inbuf[bigend ? 1 : 2] << 16;
320	s += inbuf[bigend ? 2 : 1] << 8;
321	s += inbuf[bigend ? 3 : 0];
322
323	if (s >= 0x7FFFFFFF || (s >= 0xD800 && s <= 0xDFFF))
324	return EILSEQ;
325
326	rval = one_cppchar_to_utf8 (c: s, outbufp, outbytesleftp);
327	if (rval)
328	return rval;
329
330	*inbufp += 4;
331	*inbytesleftp -= 4;
332	return 0;
333	}
334
335	static inline int
336	one_utf8_to_utf16 (iconv_t bigend, const uchar **inbufp, size_t *inbytesleftp,
337	uchar **outbufp, size_t *outbytesleftp)
338	{
339	int rval;
340	cppchar_t s = 0;
341	const uchar *save_inbuf = *inbufp;
342	size_t save_inbytesleft = *inbytesleftp;
343	uchar *outbuf = *outbufp;
344
345	rval = one_utf8_to_cppchar (inbufp, inbytesleftp, cp: &s);
346	if (rval)
347	return rval;
348
349	if (s > 0x0010FFFF)
350	{
351	*inbufp = save_inbuf;
352	*inbytesleftp = save_inbytesleft;
353	return EILSEQ;
354	}
355
356	if (s <= 0xFFFF)
357	{
358	if (*outbytesleftp < 2)
359	{
360	*inbufp = save_inbuf;
361	*inbytesleftp = save_inbytesleft;
362	return E2BIG;
363	}
364	outbuf[bigend ? 1 : 0] = (s & 0x00FF);
365	outbuf[bigend ? 0 : 1] = (s & 0xFF00) >> 8;
366
367	*outbufp += 2;
368	*outbytesleftp -= 2;
369	return 0;
370	}
371	else
372	{
373	cppchar_t hi, lo;
374
375	if (*outbytesleftp < 4)
376	{
377	*inbufp = save_inbuf;
378	*inbytesleftp = save_inbytesleft;
379	return E2BIG;
380	}
381
382	hi = (s - 0x10000) / 0x400 + 0xD800;
383	lo = (s - 0x10000) % 0x400 + 0xDC00;
384
385	/* Even if we are little-endian, put the high surrogate first.
386	??? Matches practice? */
387	outbuf[bigend ? 1 : 0] = (hi & 0x00FF);
388	outbuf[bigend ? 0 : 1] = (hi & 0xFF00) >> 8;
389	outbuf[bigend ? 3 : 2] = (lo & 0x00FF);
390	outbuf[bigend ? 2 : 3] = (lo & 0xFF00) >> 8;
391
392	*outbufp += 4;
393	*outbytesleftp -= 4;
394	return 0;
395	}
396	}
397
398	static inline int
399	one_utf16_to_utf8 (iconv_t bigend, const uchar **inbufp, size_t *inbytesleftp,
400	uchar **outbufp, size_t *outbytesleftp)
401	{
402	cppchar_t s;
403	const uchar *inbuf = *inbufp;
404	int rval;
405
406	if (*inbytesleftp < 2)
407	return EINVAL;
408	s = inbuf[bigend ? 0 : 1] << 8;
409	s += inbuf[bigend ? 1 : 0];
410
411	/* Low surrogate without immediately preceding high surrogate is invalid. */
412	if (s >= 0xDC00 && s <= 0xDFFF)
413	return EILSEQ;
414	/* High surrogate must have a following low surrogate. */
415	else if (s >= 0xD800 && s <= 0xDBFF)
416	{
417	cppchar_t hi = s, lo;
418	if (*inbytesleftp < 4)
419	return EINVAL;
420
421	lo = inbuf[bigend ? 2 : 3] << 8;
422	lo += inbuf[bigend ? 3 : 2];
423
424	if (lo < 0xDC00 || lo > 0xDFFF)
425	return EILSEQ;
426
427	s = (hi - 0xD800) * 0x400 + (lo - 0xDC00) + 0x10000;
428	}
429
430	rval = one_cppchar_to_utf8 (c: s, outbufp, outbytesleftp);
431	if (rval)
432	return rval;
433
434	/* Success - update the input pointers (one_cppchar_to_utf8 has done
435	the output pointers for us). */
436	if (s <= 0xFFFF)
437	{
438	*inbufp += 2;
439	*inbytesleftp -= 2;
440	}
441	else
442	{
443	*inbufp += 4;
444	*inbytesleftp -= 4;
445	}
446	return 0;
447	}
448
449	/* Helper routine for the next few functions. The 'const' on
450	one_conversion means that we promise not to modify what function is
451	pointed to, which lets the inliner see through it. */
452
453	static inline bool
454	conversion_loop (int (*const one_conversion)(iconv_t, const uchar **, size_t *,
455	uchar **, size_t *),
456	iconv_t cd, const uchar *from, size_t flen, struct _cpp_strbuf *to)
457	{
458	const uchar *inbuf;
459	uchar *outbuf;
460	size_t inbytesleft, outbytesleft;
461	int rval;
462
463	inbuf = from;
464	inbytesleft = flen;
465	outbuf = to->text + to->len;
466	outbytesleft = to->asize - to->len;
467
468	for (;;)
469	{
470	do
471	rval = one_conversion (cd, &inbuf, &inbytesleft,
472	&outbuf, &outbytesleft);
473	while (inbytesleft && !rval);
474
475	if (__builtin_expect (inbytesleft == 0, 1))
476	{
477	to->len = to->asize - outbytesleft;
478	return true;
479	}
480	if (rval != E2BIG)
481	{
482	errno = rval;
483	return false;
484	}
485
486	outbytesleft += OUTBUF_BLOCK_SIZE;
487	to->asize += OUTBUF_BLOCK_SIZE;
488	to->text = XRESIZEVEC (uchar, to->text, to->asize);
489	outbuf = to->text + to->asize - outbytesleft;
490	}
491	}
492
493
494	/* These functions convert entire strings between character sets.
495	They all have the signature
496
497	bool (*)(iconv_t cd, const uchar *from, size_t flen, struct _cpp_strbuf *to);
498
499	The input string FROM is converted as specified by the function
500	name plus the iconv descriptor CD (which may be fake), and the
501	result appended to TO. On any error, false is returned, otherwise true. */
502
503	/* These four use the custom conversion code above. */
504	static bool
505	convert_utf8_utf16 (iconv_t cd, const uchar *from, size_t flen,
506	struct _cpp_strbuf *to)
507	{
508	return conversion_loop (one_conversion: one_utf8_to_utf16, cd, from, flen, to);
509	}
510
511	static bool
512	convert_utf8_utf32 (iconv_t cd, const uchar *from, size_t flen,
513	struct _cpp_strbuf *to)
514	{
515	return conversion_loop (one_conversion: one_utf8_to_utf32, cd, from, flen, to);
516	}
517
518	static bool
519	convert_utf16_utf8 (iconv_t cd, const uchar *from, size_t flen,
520	struct _cpp_strbuf *to)
521	{
522	return conversion_loop (one_conversion: one_utf16_to_utf8, cd, from, flen, to);
523	}
524
525	static bool
526	convert_utf32_utf8 (iconv_t cd, const uchar *from, size_t flen,
527	struct _cpp_strbuf *to)
528	{
529	return conversion_loop (one_conversion: one_utf32_to_utf8, cd, from, flen, to);
530	}
531
532	/* Identity conversion, used when we have no alternative. */
533	static bool
534	convert_no_conversion (iconv_t cd ATTRIBUTE_UNUSED,
535	const uchar *from, size_t flen, struct _cpp_strbuf *to)
536	{
537	if (to->len + flen > to->asize)
538	{
539	to->asize = to->len + flen;
540	to->asize += to->asize / 4;
541	to->text = XRESIZEVEC (uchar, to->text, to->asize);
542	}
543	memcpy (dest: to->text + to->len, src: from, n: flen);
544	to->len += flen;
545	return true;
546	}
547
548	/* And this one uses the system iconv primitive. It's a little
549	different, since iconv's interface is a little different. */
550	#if HAVE_ICONV
551
552	#define CONVERT_ICONV_GROW_BUFFER \
553	do { \
554	outbytesleft += OUTBUF_BLOCK_SIZE; \
555	to->asize += OUTBUF_BLOCK_SIZE; \
556	to->text = XRESIZEVEC (uchar, to->text, to->asize); \
557	outbuf = (char *)to->text + to->asize - outbytesleft; \
558	} while (0)
559
560	static bool
561	convert_using_iconv (iconv_t cd, const uchar *from, size_t flen,
562	struct _cpp_strbuf *to)
563	{
564	ICONV_CONST char *inbuf;
565	char *outbuf;
566	size_t inbytesleft, outbytesleft;
567
568	/* Reset conversion descriptor and check that it is valid. */
569	if (iconv (cd, 0, 0, 0, 0) == (size_t)-1)
570	return false;
571
572	inbuf = (ICONV_CONST char *)from;
573	inbytesleft = flen;
574	outbuf = (char *)to->text + to->len;
575	outbytesleft = to->asize - to->len;
576
577	for (;;)
578	{
579	iconv (cd, &inbuf, &inbytesleft, &outbuf, &outbytesleft);
580	if (__builtin_expect (inbytesleft == 0, 1))
581	{
582	/* Close out any shift states, returning to the initial state. */
583	if (iconv (cd, 0, 0, &outbuf, &outbytesleft) == (size_t)-1)
584	{
585	if (errno != E2BIG)
586	return false;
587
588	CONVERT_ICONV_GROW_BUFFER;
589	if (iconv (cd, 0, 0, &outbuf, &outbytesleft) == (size_t)-1)
590	return false;
591	}
592
593	to->len = to->asize - outbytesleft;
594	return true;
595	}
596	if (errno != E2BIG)
597	return false;
598
599	CONVERT_ICONV_GROW_BUFFER;
600	}
601	}
602	#else
603	#define convert_using_iconv 0 /* prevent undefined symbol error below */
604	#endif
605
606	/* Arrange for the above custom conversion logic to be used automatically
607	when conversion between a suitable pair of character sets is requested. */
608
609	#define APPLY_CONVERSION(CONVERTER, FROM, FLEN, TO) \
610	CONVERTER.func (CONVERTER.cd, FROM, FLEN, TO)
611
612	struct cpp_conversion
613	{
614	const char *pair;
615	convert_f func;
616	iconv_t fake_cd;
617	};
618	static const struct cpp_conversion conversion_tab[] = {
619	{ .pair: "UTF-8/UTF-32LE", .func: convert_utf8_utf32, .fake_cd: (iconv_t)0 },
620	{ .pair: "UTF-8/UTF-32BE", .func: convert_utf8_utf32, .fake_cd: (iconv_t)1 },
621	{ .pair: "UTF-8/UTF-16LE", .func: convert_utf8_utf16, .fake_cd: (iconv_t)0 },
622	{ .pair: "UTF-8/UTF-16BE", .func: convert_utf8_utf16, .fake_cd: (iconv_t)1 },
623	{ .pair: "UTF-32LE/UTF-8", .func: convert_utf32_utf8, .fake_cd: (iconv_t)0 },
624	{ .pair: "UTF-32BE/UTF-8", .func: convert_utf32_utf8, .fake_cd: (iconv_t)1 },
625	{ .pair: "UTF-16LE/UTF-8", .func: convert_utf16_utf8, .fake_cd: (iconv_t)0 },
626	{ .pair: "UTF-16BE/UTF-8", .func: convert_utf16_utf8, .fake_cd: (iconv_t)1 },
627	};
628
629	/* Subroutine of cpp_init_iconv: initialize and return a
630	cset_converter structure for conversion from FROM to TO. If
631	iconv_open() fails, issue an error and return an identity
632	converter. Silently return an identity converter if FROM and TO
633	are identical.
634
635	PFILE is only used for generating diagnostics; setting it to NULL
636	suppresses diagnostics. */
637
638	static struct cset_converter
639	init_iconv_desc (cpp_reader *pfile, const char *to, const char *from)
640	{
641	struct cset_converter ret;
642	char *pair;
643	size_t i;
644
645	ret.to = to;
646	ret.from = from;
647
648	if (!strcasecmp (s1: to, s2: from))
649	{
650	ret.func = convert_no_conversion;
651	ret.cd = (iconv_t) -1;
652	ret.width = -1;
653	return ret;
654	}
655
656	pair = (char *) alloca(strlen(to) + strlen(from) + 2);
657
658	strcpy(dest: pair, src: from);
659	strcat(dest: pair, src: "/");
660	strcat(dest: pair, src: to);
661	for (i = 0; i < ARRAY_SIZE (conversion_tab); i++)
662	if (!strcasecmp (s1: pair, s2: conversion_tab[i].pair))
663	{
664	ret.func = conversion_tab[i].func;
665	ret.cd = conversion_tab[i].fake_cd;
666	ret.width = -1;
667	return ret;
668	}
669
670	/* No custom converter - try iconv. */
671	if (HAVE_ICONV)
672	{
673	ret.func = convert_using_iconv;
674	ret.cd = iconv_open (to, from);
675	ret.width = -1;
676
677	if (ret.cd == (iconv_t) -1)
678	{
679	if (pfile)
680	{
681	if (errno == EINVAL)
682	cpp_error (pfile, CPP_DL_ERROR, /* FIXME should be DL_SORRY */
683	msgid: "conversion from %s to %s not supported by iconv",
684	from, to);
685	else
686	cpp_errno (pfile, CPP_DL_ERROR, msgid: "iconv_open");
687	}
688	ret.func = convert_no_conversion;
689	}
690	}
691	else
692	{
693	if (pfile)
694	{
695	cpp_error (pfile, CPP_DL_ERROR, /* FIXME: should be DL_SORRY */
696	msgid: "no iconv implementation, cannot convert from %s to %s",
697	from, to);
698	}
699	ret.func = convert_no_conversion;
700	ret.cd = (iconv_t) -1;
701	ret.width = -1;
702	}
703
704	return ret;
705	}
706
707	/* If charset conversion is requested, initialize iconv(3) descriptors
708	for conversion from the source character set to the execution
709	character sets. If iconv is not present in the C library, and
710	conversion is requested, issue an error. */
711
712	void
713	cpp_init_iconv (cpp_reader *pfile)
714	{
715	const char *ncset = CPP_OPTION (pfile, narrow_charset);
716	const char *wcset = CPP_OPTION (pfile, wide_charset);
717	const char *default_wcset;
718
719	bool be = CPP_OPTION (pfile, bytes_big_endian);
720
721	if (CPP_OPTION (pfile, wchar_precision) >= 32)
722	default_wcset = be ? "UTF-32BE" : "UTF-32LE";
723	else if (CPP_OPTION (pfile, wchar_precision) >= 16)
724	default_wcset = be ? "UTF-16BE" : "UTF-16LE";
725	else
726	/* This effectively means that wide strings are not supported,
727	so don't do any conversion at all. */
728	default_wcset = SOURCE_CHARSET;
729
730	if (!ncset)
731	ncset = SOURCE_CHARSET;
732	if (!wcset)
733	wcset = default_wcset;
734
735	pfile->narrow_cset_desc = init_iconv_desc (pfile, to: ncset, SOURCE_CHARSET);
736	pfile->narrow_cset_desc.width = CPP_OPTION (pfile, char_precision);
737	pfile->utf8_cset_desc = init_iconv_desc (pfile, to: "UTF-8", SOURCE_CHARSET);
738	pfile->utf8_cset_desc.width = CPP_OPTION (pfile, char_precision);
739	pfile->char16_cset_desc = init_iconv_desc (pfile,
740	to: be ? "UTF-16BE" : "UTF-16LE",
741	SOURCE_CHARSET);
742	pfile->char16_cset_desc.width = 16;
743	pfile->char32_cset_desc = init_iconv_desc (pfile,
744	to: be ? "UTF-32BE" : "UTF-32LE",
745	SOURCE_CHARSET);
746	pfile->char32_cset_desc.width = 32;
747	pfile->wide_cset_desc = init_iconv_desc (pfile, to: wcset, SOURCE_CHARSET);
748	pfile->wide_cset_desc.width = CPP_OPTION (pfile, wchar_precision);
749	}
750
751	/* Destroy iconv(3) descriptors set up by cpp_init_iconv, if necessary. */
752	void
753	_cpp_destroy_iconv (cpp_reader *pfile)
754	{
755	if (HAVE_ICONV)
756	{
757	if (pfile->narrow_cset_desc.func == convert_using_iconv)
758	iconv_close (pfile->narrow_cset_desc.cd);
759	if (pfile->utf8_cset_desc.func == convert_using_iconv)
760	iconv_close (pfile->utf8_cset_desc.cd);
761	if (pfile->char16_cset_desc.func == convert_using_iconv)
762	iconv_close (pfile->char16_cset_desc.cd);
763	if (pfile->char32_cset_desc.func == convert_using_iconv)
764	iconv_close (pfile->char32_cset_desc.cd);
765	if (pfile->wide_cset_desc.func == convert_using_iconv)
766	iconv_close (pfile->wide_cset_desc.cd);
767	}
768	}
769
770	/* Utility routine for use by a full compiler. C is a character taken
771	from the *basic* source character set, encoded in the host's
772	execution encoding. Convert it to (the target's) execution
773	encoding, and return that value.
774
775	Issues an internal error if C's representation in the narrow
776	execution character set fails to be a single-byte value (C99
777	5.2.1p3: "The representation of each member of the source and
778	execution character sets shall fit in a byte.") May also issue an
779	internal error if C fails to be a member of the basic source
780	character set (testing this exactly is too hard, especially when
781	the host character set is EBCDIC). */
782	cppchar_t
783	cpp_host_to_exec_charset (cpp_reader *pfile, cppchar_t c)
784	{
785	uchar sbuf[1];
786	struct _cpp_strbuf tbuf;
787
788	/* This test is merely an approximation, but it suffices to catch
789	the most important thing, which is that we don't get handed a
790	character outside the unibyte range of the host character set. */
791	if (c > LAST_POSSIBLY_BASIC_SOURCE_CHAR)
792	{
793	cpp_error (pfile, CPP_DL_ICE,
794	msgid: "character 0x%lx is not in the basic source character set\n",
795	(unsigned long)c);
796	return 0;
797	}
798
799	/* Being a character in the unibyte range of the host character set,
800	we can safely splat it into a one-byte buffer and trust that that
801	is a well-formed string. */
802	sbuf[0] = c;
803
804	/* This should never need to reallocate, but just in case... */
805	tbuf.asize = 1;
806	tbuf.text = XNEWVEC (uchar, tbuf.asize);
807	tbuf.len = 0;
808
809	if (!APPLY_CONVERSION (pfile->narrow_cset_desc, sbuf, 1, &tbuf))
810	{
811	cpp_errno (pfile, CPP_DL_ICE, msgid: "converting to execution character set");
812	return 0;
813	}
814	if (tbuf.len != 1)
815	{
816	cpp_error (pfile, CPP_DL_ICE,
817	msgid: "character 0x%lx is not unibyte in execution character set",
818	(unsigned long)c);
819	return 0;
820	}
821	c = tbuf.text[0];
822	free(ptr: tbuf.text);
823	return c;
824	}
825
826
827
828	/* cpp_substring_ranges's constructor. */
829
830	cpp_substring_ranges::cpp_substring_ranges () :
831	m_ranges (NULL),
832	m_num_ranges (0),
833	m_alloc_ranges (8)
834	{
835	m_ranges = XNEWVEC (source_range, m_alloc_ranges);
836	}
837
838	/* cpp_substring_ranges's destructor. */
839
840	cpp_substring_ranges::~cpp_substring_ranges ()
841	{
842	free (ptr: m_ranges);
843	}
844
845	/* Add RANGE to the vector of source_range information. */
846
847	void
848	cpp_substring_ranges::add_range (source_range range)
849	{
850	if (m_num_ranges >= m_alloc_ranges)
851	{
852	m_alloc_ranges *= 2;
853	m_ranges
854	= (source_range *)xrealloc (m_ranges,
855	sizeof (source_range) * m_alloc_ranges);
856	}
857	m_ranges[m_num_ranges++] = range;
858	}
859
860	/* Read NUM ranges from LOC_READER, adding them to the vector of source_range
861	information. */
862
863	void
864	cpp_substring_ranges::add_n_ranges (int num,
865	cpp_string_location_reader &loc_reader)
866	{
867	for (int i = 0; i < num; i++)
868	add_range (range: loc_reader.get_next ());
869	}
870
871
872
873	/* Utility routine that computes a mask of the form 0000...111... with
874	WIDTH 1-bits. */
875	static inline size_t
876	width_to_mask (size_t width)
877	{
878	width = MIN (width, BITS_PER_CPPCHAR_T);
879	if (width >= CHAR_BIT * sizeof (size_t))
880	return ~(size_t) 0;
881	else
882	return ((size_t) 1 << width) - 1;
883	}
884
885	/* A large table of unicode character information. */
886	enum {
887	/* Valid in a C99 identifier? */
888	C99 = 1,
889	/* Valid in a C99 identifier, but not as the first character? */
890	N99 = 2,
891	/* Valid in a C++ identifier? */
892	CXX = 4,
893	/* Valid in a C11/C++11 identifier? */
894	C11 = 8,
895	/* Valid in a C11/C++11 identifier, but not as the first character? */
896	N11 = 16,
897	/* Valid in a C++23 identifier? */
898	CXX23 = 32,
899	/* Valid in a C++23 identifier, but not as the first character? */
900	NXX23 = 64,
901	/* NFC representation is not valid in an identifier? */
902	CID = 128,
903	/* Might be valid NFC form? */
904	NFC = 256,
905	/* Might be valid NFKC form? */
906	NKC = 512,
907	/* Certain preceding characters might make it not valid NFC/NKFC form? */
908	CTX = 1024
909	};
910
911	struct ucnrange {
912	/* Bitmap of flags above. */
913	unsigned short flags;
914	/* Combining class of the character. */
915	unsigned char combine;
916	/* Last character in the range described by this entry. */
917	unsigned int end;
918	};
919	#include "ucnid.h"
920
921	/* ISO 10646 defines the UCS codespace as the range 0-0x10FFFF inclusive. */
922	#define UCS_LIMIT 0x10FFFF
923
924	#include "uname2c.h"
925
926	static const char hangul_syllables[][4] = {
927	/* L */
928	"G", "GG", "N", "D", "DD", "R", "M", "B", "BB", "S", "SS", "",
929	"J", "JJ", "C", "K", "T", "P", "H",
930	/* V */
931	"A", "AE", "YA", "YAE", "EO", "E", "YEO", "YE", "O", "WA", "WAE",
932	"OE", "YO", "U", "WEO", "WE", "WI", "YU", "EU", "YI", "I",
933	/* T */
934	"", "G", "GG", "GS", "N", "NJ", "NH", "D", "L", "LG", "LM", "LB",
935	"LS", "LT", "LP", "LH", "M", "B", "BS", "S", "SS", "NG", "J", "C",
936	"K", "T", "P", "H"
937	};
938
939	static const short hangul_count[6] = { 19, 21, 28 };
940
941	/* Used for Unicode loose matching rule UAX44-LM2 matching. */
942
943	struct uname2c_data
944	{
945	char *canon_name;
946	char prev_char;
947	};
948
949	/* Map NAME, a Unicode character name or correction/control/alternate
950	alias, to a Unicode codepoint, or return (cppchar_t) -1 if
951	not found. This uses a space optimized radix tree precomputed
952	by the makeuname2c utility, with binary format documented in its
953	source makeuname2c.cc. */
954
955	static cppchar_t
956	_cpp_uname2c (const char *name, size_t len, const unsigned char *n,
957	struct uname2c_data *data)
958	{
959	do
960	{
961	char k;
962	const char *key;
963	size_t key_len, len_adj;
964	bool has_value = *n & 0x40;
965	bool has_children, no_sibling = false;
966	cppchar_t codepoint = -1;
967	const unsigned char *child = NULL;
968	int ret;
969
970	if (*n & 0x80)
971	{
972	k = ' ' + (*n++ & 0x3f);
973	key = &k;
974	key_len = 1;
975	}
976	else
977	{
978	key_len = *n++ & 0x3f;
979	key = &uname2c_dict[*n++];
980	key += (*n++ << 8);
981	}
982	if (has_value)
983	{
984	codepoint = *n + (n[1] << 8) + ((n[2] & 0x1f) << 16);
985	has_children = n[2] & 0x80;
986	no_sibling = n[2] & 0x40;
987	n += 3;
988	}
989	else
990	has_children = true;
991	if (has_children)
992	{
993	unsigned int shift = 0;
994	size_t child_off = 0;
995
996	do
997	{
998	child_off |= (*n & 0x7f) << shift;
999	shift += 7;
1000	}
1001	while ((*n++ & 0x80) != 0);
1002	child = n + child_off;
1003	}
1004	if (__builtin_expect (data == NULL, 1))
1005	{
1006	ret = memcmp (s1: name, s2: key, n: len > key_len ? key_len : len);
1007	len_adj = key_len;
1008	}
1009	else
1010	{
1011	const char *p = name, *q = key;
1012
1013	while (1)
1014	{
1015	if ((size_t) (p - name) == len || (size_t) (q - key) == key_len)
1016	break;
1017	if (*q == ' ')
1018	{
1019	++q;
1020	continue;
1021	}
1022	if (*q == '-')
1023	{
1024	/* This is the hard case. Only medial hyphens
1025	should be removed, where medial means preceded
1026	and followed by alnum. */
1027	if (ISALNUM (q == key ? data->prev_char : q[-1]))
1028	{
1029	if (q + 1 == key + key_len)
1030	{
1031	/* We don't know what the next letter will be.
1032	It could be ISALNUM, then we are supposed
1033	to omit it, or it could be a space and then
1034	we should not omit it and need to compare it.
1035	Fortunately the only 3 names with hyphen
1036	followed by non-letter are
1037	U+0F0A TIBETAN MARK BKA- SHOG YIG MGO
1038	U+0FD0 TIBETAN MARK BKA- SHOG GI MGO RGYAN
1039	U+0FD0 TIBETAN MARK BSKA- SHOG GI MGO RGYAN
1040	Furthermore, prefixes of NR2 generated
1041	ranges all end with a hyphen, but the generated
1042	part is then followed by alpha-numeric.
1043	So, let's just assume that - at the end of
1044	key is always followed by alphanumeric and
1045	so should be omitted.
1046	makeuname2c.cc verifies that this is true. */
1047	++q;
1048	continue;
1049	}
1050	else if (ISALNUM (q[1]))
1051	{
1052	++q;
1053	continue;
1054	}
1055	}
1056	}
1057	if (*p != *q)
1058	break;
1059	++p;
1060	++q;
1061	}
1062	len_adj = p - name;
1063	/* If we don't consume the whole key, signal a mismatch,
1064	but always with ret = 1, so that we keep looking through
1065	siblings. */
1066	ret = q < key + key_len;
1067	}
1068	if (ret < 0)
1069	return -1;
1070	else if (ret == 0)
1071	{
1072	if (len < len_adj)
1073	return -1;
1074	else if (codepoint >= 0xd800
1075	&& codepoint < 0xd800 + ARRAY_SIZE (uname2c_generated))
1076	{
1077	name += len_adj;
1078	len -= len_adj;
1079	if (codepoint == 0xd800)
1080	{
1081	/* NR1 - Hangul syllables. */
1082	size_t start = 0, end, i, j;
1083	int this_len, max_len;
1084	char winner[3];
1085
1086	for (i = 0; i < 3; ++i)
1087	{
1088	end = start + hangul_count[i];
1089	max_len = -1;
1090	winner[i] = -1;
1091	for (j = start; j < end; j++)
1092	{
1093	this_len = strlen (s: hangul_syllables[j]);
1094	if (len >= (size_t) this_len
1095	&& this_len > max_len
1096	&& memcmp (s1: name, s2: hangul_syllables[j],
1097	n: this_len) == 0)
1098	{
1099	max_len = this_len;
1100	winner[i] = j - start;
1101	}
1102	}
1103	if (max_len == -1)
1104	return -1;
1105	name += max_len;
1106	len -= max_len;
1107	start = end;
1108	}
1109	if (__builtin_expect (data != NULL, 0))
1110	{
1111	memcpy (dest: data->canon_name, src: key, n: key_len);
1112	data->canon_name[key_len] = '\0';
1113	for (i = 0, start = 0; i < 3; ++i)
1114	{
1115	strcat (dest: data->canon_name,
1116	src: hangul_syllables[start + winner[i]]);
1117	start += hangul_count[i];
1118	}
1119	}
1120	return (0xac00 + 21 * 28 * winner[0]
1121	+ 28 * winner[1] + winner[2]);
1122	}
1123	else
1124	{
1125	/* NR2 - prefix followed by hexadecimal codepoint. */
1126	const cppchar_t *p;
1127	size_t i;
1128
1129	if (len < 4 || len > 5)
1130	return -1;
1131	p = uname2c_pairs + uname2c_generated[codepoint - 0xd800];
1132	codepoint = 0;
1133	for (i = 0; i < len; ++i)
1134	{
1135	codepoint <<= 4;
1136	if (!ISXDIGIT (name[i]))
1137	return -1;
1138	codepoint += hex_value (name[i]);
1139	}
1140	for (; *p; p += 2)
1141	if (codepoint < *p)
1142	return -1;
1143	else if (codepoint <= p[1])
1144	{
1145	if (__builtin_expect (data != NULL, 0))
1146	{
1147	memcpy (dest: data->canon_name, src: key, n: key_len);
1148	memcpy (dest: data->canon_name + key_len, src: name, n: len);
1149	data->canon_name[key_len + len] = '\0';
1150	}
1151	return codepoint;
1152	}
1153	return -1;
1154	}
1155	}
1156	else if (__builtin_expect (data != NULL, 0))
1157	{
1158	if (len == len_adj)
1159	{
1160	memcpy (dest: data->canon_name, src: key, n: key_len);
1161	data->canon_name[key_len] = '\0';
1162	return codepoint;
1163	}
1164	if (has_children)
1165	{
1166	struct uname2c_data save = *data;
1167	memcpy (dest: data->canon_name, src: key, n: key_len);
1168	data->canon_name += key_len;
1169	data->prev_char = key[key_len - 1];
1170	codepoint = _cpp_uname2c (name: name + len_adj, len: len - len_adj,
1171	n: child, data);
1172	if (codepoint != (cppchar_t) -1)
1173	return codepoint;
1174	*data = save;
1175	}
1176	}
1177	else if (len == len_adj)
1178	return codepoint;
1179	else if (!has_children)
1180	return -1;
1181	else
1182	{
1183	name += len_adj;
1184	len -= len_adj;
1185	n = child;
1186	continue;
1187	}
1188	}
1189	if (no_sibling || (!has_value && *n == 0xff))
1190	break;
1191	}
1192	while (1);
1193	return -1;
1194	}
1195
1196	/* Try to do a loose name lookup according to Unicode loose matching rule
1197	UAX44-LM2. First ignore medial hyphens, whitespace, underscore
1198	characters and convert to upper case. */
1199
1200	static cppchar_t
1201	_cpp_uname2c_uax44_lm2 (const char *name, size_t len, char *canon_name)
1202	{
1203	char name_after_uax44_lm2[uname2c_max_name_len];
1204	char *q = name_after_uax44_lm2;
1205	const char *p;
1206
1207	for (p = name; p < name + len; p++)
1208	if (*p == '_' || *p == ' ')
1209	continue;
1210	else if (*p == '-' && p != name && ISALNUM (p[-1]) && ISALNUM (p[1]))
1211	continue;
1212	else if (q == name_after_uax44_lm2 + uname2c_max_name_len)
1213	return -1;
1214	else if (ISLOWER (*p))
1215	*q++ = TOUPPER (*p);
1216	else
1217	*q++ = *p;
1218
1219	struct uname2c_data data;
1220	data.canon_name = canon_name;
1221	data.prev_char = ' ';
1222	/* Hangul Jungseong O- E after UAX44-LM2 should be HANGULJUNGSEONGO-E
1223	and so should match U+1180. */
1224	if (q - name_after_uax44_lm2 == sizeof ("HANGULJUNGSEONGO-E") - 1
1225	&& memcmp (s1: name_after_uax44_lm2, s2: "HANGULJUNGSEONGO-E",
1226	n: sizeof ("HANGULJUNGSEONGO-E") - 1) == 0)
1227	{
1228	name_after_uax44_lm2[sizeof ("HANGULJUNGSEONGO") - 1] = 'E';
1229	--q;
1230	}
1231	cppchar_t result
1232	= _cpp_uname2c (name: name_after_uax44_lm2, len: q - name_after_uax44_lm2,
1233	n: uname2c_tree, data: &data);
1234
1235	/* Unicode UAX44-LM2 exception:
1236	U+116C HANGUL JUNGSEONG OE
1237	U+1180 HANGUL JUNGSEONG O-E
1238	We remove all medial hyphens when we shouldn't remote the U+1180 one.
1239	The U+1180 entry sorts before U+116C lexicographilly, so we get U+1180
1240	in both cases. Thus, if result is U+1180, check if user's name doesn't
1241	have a hyphen there and adjust. */
1242	if (result == 0x1180)
1243	{
1244	while (p[-1] == ' ' || p[-1] == '_')
1245	--p;
1246	gcc_assert (TOUPPER (p[-1]) == 'E');
1247	--p;
1248	while (p[-1] == ' ' || p[-1] == '_')
1249	--p;
1250	if (p[-1] != '-')
1251	{
1252	result = 0x116c;
1253	memcpy (dest: canon_name + sizeof ("HANGUL JUNGSEONG O") - 1, src: "E", n: 2);
1254	}
1255	}
1256	return result;
1257	}
1258
1259
1260	/* Returns 1 if C is valid in an identifier, 2 if C is valid except at
1261	the start of an identifier, and 0 if C is not valid in an
1262	identifier. We assume C has already gone through the checks of
1263	_cpp_valid_ucn. Also update NST for C if returning nonzero. The
1264	algorithm is a simple binary search on the table defined in
1265	ucnid.h. */
1266
1267	static int
1268	ucn_valid_in_identifier (cpp_reader *pfile, cppchar_t c,
1269	struct normalize_state *nst)
1270	{
1271	int mn, mx, md;
1272	unsigned short valid_flags, invalid_start_flags;
1273
1274	if (c > UCS_LIMIT)
1275	return 0;
1276
1277	mn = 0;
1278	mx = ARRAY_SIZE (ucnranges) - 1;
1279	while (mx != mn)
1280	{
1281	md = (mn + mx) / 2;
1282	if (c <= ucnranges[md].end)
1283	mx = md;
1284	else
1285	mn = md + 1;
1286	}
1287
1288	/* When -pedantic, we require the character to have been listed by
1289	the standard for the current language. Otherwise, we accept the
1290	union of the acceptable sets for all supported language versions. */
1291	valid_flags = C99 | CXX | C11 | CXX23;
1292	if (CPP_PEDANTIC (pfile))
1293	{
1294	if (CPP_OPTION (pfile, xid_identifiers))
1295	valid_flags = CXX23;
1296	else if (CPP_OPTION (pfile, c11_identifiers))
1297	valid_flags = C11;
1298	else if (CPP_OPTION (pfile, c99))
1299	valid_flags = C99;
1300	}
1301	if (! (ucnranges[mn].flags & valid_flags))
1302	return 0;
1303
1304	/* Update NST. */
1305	if (ucnranges[mn].combine != 0 && ucnranges[mn].combine < nst->prev_class)
1306	nst->level = normalized_none;
1307	else if (ucnranges[mn].flags & CTX)
1308	{
1309	bool safe;
1310	cppchar_t p = nst->previous;
1311
1312	/* For Hangul, characters in the range AC00-D7A3 are NFC/NFKC,
1313	and are combined algorithmically from a sequence of the form
1314	1100-1112 1161-1175 11A8-11C2
1315	(if the third is not present, it is treated as 11A7, which is not
1316	really a valid character).
1317	Unfortunately, C99 allows (only) the NFC form, but C++ allows
1318	only the combining characters. */
1319	if (c >= 0x1161 && c <= 0x1175)
1320	safe = p < 0x1100 || p > 0x1112;
1321	else if (c >= 0x11A8 && c <= 0x11C2)
1322	safe = (p < 0xAC00 || p > 0xD7A3 || (p - 0xAC00) % 28 != 0);
1323	else
1324	safe = check_nfc (pfile, c, p);
1325	if (!safe)
1326	{
1327	if ((c >= 0x1161 && c <= 0x1175) || (c >= 0x11A8 && c <= 0x11C2))
1328	nst->level = MAX (nst->level, normalized_identifier_C);
1329	else
1330	nst->level = normalized_none;
1331	}
1332	}
1333	else if (ucnranges[mn].flags & NKC)
1334	;
1335	else if (ucnranges[mn].flags & NFC)
1336	nst->level = MAX (nst->level, normalized_C);
1337	else if (ucnranges[mn].flags & CID)
1338	nst->level = MAX (nst->level, normalized_identifier_C);
1339	else
1340	nst->level = normalized_none;
1341	if (ucnranges[mn].combine == 0)
1342	nst->previous = c;
1343	nst->prev_class = ucnranges[mn].combine;
1344
1345	if (!CPP_PEDANTIC (pfile))
1346	{
1347	/* If not -pedantic, accept as character that may
1348	begin an identifier a union of characters allowed
1349	at that position in each of the character sets. */
1350	if ((ucnranges[mn].flags & (C99 | N99)) == C99
1351	|| (ucnranges[mn].flags & CXX) != 0
1352	|| (ucnranges[mn].flags & (C11 | N11)) == C11
1353	|| (ucnranges[mn].flags & (CXX23 | NXX23)) == CXX23)
1354	return 1;
1355	return 2;
1356	}
1357
1358	if (CPP_OPTION (pfile, xid_identifiers))
1359	invalid_start_flags = NXX23;
1360	else if (CPP_OPTION (pfile, c11_identifiers))
1361	invalid_start_flags = N11;
1362	else if (CPP_OPTION (pfile, c99))
1363	invalid_start_flags = N99;
1364	else
1365	invalid_start_flags = 0;
1366
1367	/* In C99, UCN digits may not begin identifiers. In C11 and C++11,
1368	UCN combining characters may not begin identifiers. */
1369	if (ucnranges[mn].flags & invalid_start_flags)
1370	return 2;
1371
1372	return 1;
1373	}
1374
1375	/* Increment char_range->m_finish by a single character. */
1376
1377	static void
1378	extend_char_range (source_range *char_range,
1379	cpp_string_location_reader *loc_reader)
1380	{
1381	if (loc_reader)
1382	{
1383	gcc_assert (char_range);
1384	char_range->m_finish = loc_reader->get_next ().m_finish;
1385	}
1386	}
1387
1388	/* [lex.charset]: The character designated by the universal character
1389	name \UNNNNNNNN is that character whose character short name in
1390	ISO/IEC 10646 is NNNNNNNN; the character designated by the
1391	universal character name \uNNNN is that character whose character
1392	short name in ISO/IEC 10646 is 0000NNNN. If the hexadecimal value
1393	for a universal character name corresponds to a surrogate code point
1394	(in the range 0xD800-0xDFFF, inclusive), the program is ill-formed.
1395	Additionally, if the hexadecimal value for a universal-character-name
1396	outside a character or string literal corresponds to a control character
1397	(in either of the ranges 0x00-0x1F or 0x7F-0x9F, both inclusive) or to a
1398	character in the basic source character set, the program is ill-formed.
1399
1400	C99 6.4.3: A universal character name shall not specify a character
1401	whose short identifier is less than 00A0 other than 0024 ($), 0040 (@),
1402	or 0060 (`), nor one in the range D800 through DFFF inclusive.
1403
1404	If the hexadecimal value is larger than the upper bound of the UCS
1405	codespace specified in ISO/IEC 10646, a pedantic warning is issued
1406	in all versions of C and in the C++20 or later versions of C++.
1407
1408	*PSTR must be preceded by "\u" or "\U"; it is assumed that the
1409	buffer end is delimited by a non-hex digit. Returns false if the
1410	UCN has not been consumed, true otherwise.
1411
1412	The value of the UCN, whether valid or invalid, is returned in *CP.
1413	Diagnostics are emitted for invalid values. PSTR is updated to point
1414	one beyond the UCN, or to the syntactically invalid character.
1415
1416	IDENTIFIER_POS is 0 when not in an identifier, 1 for the start of
1417	an identifier, or 2 otherwise.
1418
1419	If LOC_READER is non-NULL, then position information is
1420	read from *LOC_READER and CHAR_RANGE->m_finish is updated accordingly. */
1421
1422	bool
1423	_cpp_valid_ucn (cpp_reader *pfile, const uchar **pstr,
1424	const uchar *limit, int identifier_pos,
1425	struct normalize_state *nst, cppchar_t *cp,
1426	source_range *char_range,
1427	cpp_string_location_reader *loc_reader)
1428	{
1429	cppchar_t result, c;
1430	unsigned int length;
1431	const uchar *str = *pstr;
1432	const uchar *base = str - 2;
1433	bool delimited = false, named = false;
1434
1435	if (!CPP_OPTION (pfile, cplusplus) && !CPP_OPTION (pfile, c99))
1436	cpp_error (pfile, CPP_DL_WARNING,
1437	msgid: "universal character names are only valid in C++ and C99");
1438	else if (CPP_OPTION (pfile, cpp_warn_c90_c99_compat) > 0
1439	&& !CPP_OPTION (pfile, cplusplus))
1440	cpp_error (pfile, CPP_DL_WARNING,
1441	msgid: "C99's universal character names are incompatible with C90");
1442	else if (CPP_WTRADITIONAL (pfile) && identifier_pos == 0)
1443	cpp_warning (pfile, CPP_W_TRADITIONAL,
1444	msgid: "the meaning of '\\%c' is different in traditional C",
1445	(int) str[-1]);
1446
1447	result = 0;
1448	if (str[-1] == 'u')
1449	{
1450	length = 4;
1451	if (str < limit
1452	&& *str == '{'
1453	&& (!identifier_pos
1454	|| CPP_OPTION (pfile, delimited_escape_seqs)
1455	|| !CPP_OPTION (pfile, std)))
1456	{
1457	str++;
1458	/* Magic value to indicate no digits seen. */
1459	length = 32;
1460	delimited = true;
1461	extend_char_range (char_range, loc_reader);
1462	}
1463	}
1464	else if (str[-1] == 'U')
1465	length = 8;
1466	else if (str[-1] == 'N')
1467	{
1468	length = 4;
1469	if (identifier_pos
1470	&& !CPP_OPTION (pfile, delimited_escape_seqs)
1471	&& CPP_OPTION (pfile, std))
1472	{
1473	*cp = 0;
1474	return false;
1475	}
1476	if (str == limit || *str != '{')
1477	{
1478	if (identifier_pos)
1479	{
1480	*cp = 0;
1481	return false;
1482	}
1483	cpp_error (pfile, CPP_DL_ERROR, msgid: "'\\N' not followed by '{'");
1484	}
1485	else
1486	{
1487	str++;
1488	named = true;
1489	extend_char_range (char_range, loc_reader);
1490	length = 0;
1491	const uchar *name = str;
1492	bool strict = true;
1493
1494	do
1495	{
1496	if (str == limit)
1497	break;
1498	c = *str;
1499	if (!ISIDNUM (c) && c != ' ' && c != '-')
1500	break;
1501	if (ISLOWER (c) || c == '_')
1502	strict = false;
1503	str++;
1504	extend_char_range (char_range, loc_reader);
1505	}
1506	while (1);
1507
1508	if (str < limit && *str == '}')
1509	{
1510	if (identifier_pos && name == str)
1511	{
1512	cpp_warning (pfile, CPP_W_UNICODE,
1513	msgid: "empty named universal character escape "
1514	"sequence; treating it as separate tokens");
1515	*cp = 0;
1516	return false;
1517	}
1518	if (name == str)
1519	cpp_error (pfile, CPP_DL_ERROR,
1520	msgid: "empty named universal character escape sequence");
1521	else if ((!identifier_pos || strict)
1522	&& !CPP_OPTION (pfile, delimited_escape_seqs)
1523	&& CPP_OPTION (pfile, cpp_pedantic))
1524	cpp_error (pfile, CPP_DL_PEDWARN,
1525	msgid: "named universal character escapes are only valid "
1526	"in C++23");
1527	if (name == str)
1528	result = 0x40;
1529	else
1530	{
1531	/* If the name is longer than maximum length of a Unicode
1532	name, it can't be strictly valid. */
1533	if ((size_t) (str - name) > uname2c_max_name_len || !strict)
1534	result = -1;
1535	else
1536	result = _cpp_uname2c (name: (const char *) name, len: str - name,
1537	n: uname2c_tree, NULL);
1538	if (result == (cppchar_t) -1)
1539	{
1540	bool ret = true;
1541	if (identifier_pos
1542	&& (!CPP_OPTION (pfile, delimited_escape_seqs)
1543	|| !strict))
1544	ret = cpp_warning (pfile, CPP_W_UNICODE,
1545	msgid: "\\N{%.*s} is not a valid "
1546	"universal character; treating it "
1547	"as separate tokens",
1548	(int) (str - name), name);
1549	else
1550	cpp_error (pfile, CPP_DL_ERROR,
1551	msgid: "\\N{%.*s} is not a valid universal "
1552	"character", (int) (str - name), name);
1553
1554	/* Try to do a loose name lookup according to
1555	Unicode loose matching rule UAX44-LM2. */
1556	char canon_name[uname2c_max_name_len + 1];
1557	result = _cpp_uname2c_uax44_lm2 (name: (const char *) name,
1558	len: str - name, canon_name);
1559	if (result != (cppchar_t) -1 && ret)
1560	cpp_error (pfile, CPP_DL_NOTE,
1561	msgid: "did you mean \\N{%s}?", canon_name);
1562	else
1563	result = 0xC0;
1564	if (identifier_pos
1565	&& (!CPP_OPTION (pfile, delimited_escape_seqs)
1566	|| !strict))
1567	{
1568	*cp = 0;
1569	return false;
1570	}
1571	}
1572	}
1573	str++;
1574	extend_char_range (char_range, loc_reader);
1575	}
1576	else if (identifier_pos)
1577	{
1578	cpp_warning (pfile, CPP_W_UNICODE,
1579	msgid: "'\\N{' not terminated with '}' after %.*s; "
1580	"treating it as separate tokens",
1581	(int) (str - base), base);
1582	*cp = 0;
1583	return false;
1584	}
1585	else
1586	{
1587	cpp_error (pfile, CPP_DL_ERROR,
1588	msgid: "'\\N{' not terminated with '}' after %.*s",
1589	(int) (str - base), base);
1590	result = 1;
1591	}
1592	}
1593	}
1594	else
1595	{
1596	cpp_error (pfile, CPP_DL_ICE, msgid: "In _cpp_valid_ucn but not a UCN");
1597	length = 4;
1598	}
1599
1600	if (!named)
1601	do
1602	{
1603	if (str == limit)
1604	break;
1605	c = *str;
1606	if (!ISXDIGIT (c))
1607	break;
1608	str++;
1609	extend_char_range (char_range, loc_reader);
1610	if (delimited)
1611	{
1612	if (!result)
1613	/* Accept arbitrary number of leading zeros.
1614	16 is another magic value, smaller than 32 above
1615	and bigger than 8, so that upon encountering first
1616	non-zero digit we can count 8 digits and after that
1617	or in overflow bit and ensure length doesn't decrease
1618	to 0, as delimited escape sequence doesn't have upper
1619	bound on the number of hex digits. */
1620	length = 16;
1621	else if (length == 16 - 8)
1622	{
1623	/* Make sure we detect overflows. */
1624	result |= 0x8000000;
1625	++length;
1626	}
1627	}
1628
1629	result = (result << 4) + hex_value (c);
1630	}
1631	while (--length);
1632
1633	if (delimited && str < limit && *str == '}')
1634	{
1635	if (length == 32 && identifier_pos)
1636	{
1637	cpp_warning (pfile, CPP_W_UNICODE,
1638	msgid: "empty delimited escape sequence; "
1639	"treating it as separate tokens");
1640	*cp = 0;
1641	return false;
1642	}
1643	else if (length == 32)
1644	cpp_error (pfile, CPP_DL_ERROR,
1645	msgid: "empty delimited escape sequence");
1646	else if (!CPP_OPTION (pfile, delimited_escape_seqs)
1647	&& CPP_OPTION (pfile, cpp_pedantic))
1648	cpp_error (pfile, CPP_DL_PEDWARN,
1649	msgid: "delimited escape sequences are only valid in C++23");
1650	str++;
1651	length = 0;
1652	delimited = false;
1653	extend_char_range (char_range, loc_reader);
1654	}
1655
1656	/* Partial UCNs are not valid in strings, but decompose into
1657	multiple tokens in identifiers, so we can't give a helpful
1658	error message in that case. */
1659	if (length && identifier_pos)
1660	{
1661	if (delimited)
1662	cpp_warning (pfile, CPP_W_UNICODE,
1663	msgid: "'\\u{' not terminated with '}' after %.*s; "
1664	"treating it as separate tokens",
1665	(int) (str - base), base);
1666	*cp = 0;
1667	return false;
1668	}
1669
1670	*pstr = str;
1671	if (length)
1672	{
1673	if (!delimited)
1674	cpp_error (pfile, CPP_DL_ERROR,
1675	msgid: "incomplete universal character name %.*s",
1676	(int) (str - base), base);
1677	else
1678	cpp_error (pfile, CPP_DL_ERROR,
1679	msgid: "'\\u{' not terminated with '}' after %.*s",
1680	(int) (str - base), base);
1681	result = 1;
1682	}
1683	/* The C99 standard permits $, @ and ` to be specified as UCNs. We use
1684	hex escapes so that this also works with EBCDIC hosts.
1685	C++0x permits everything below 0xa0 within literals;
1686	ucn_valid_in_identifier will complain about identifiers. */
1687	else if ((result < 0xa0
1688	&& !CPP_OPTION (pfile, cplusplus)
1689	&& (result != 0x24 && result != 0x40 && result != 0x60))
1690	|| (result & 0x80000000)
1691	|| (result >= 0xD800 && result <= 0xDFFF))
1692	{
1693	cpp_error (pfile, CPP_DL_ERROR,
1694	msgid: "%.*s is not a valid universal character",
1695	(int) (str - base), base);
1696	result = 1;
1697	}
1698	else if (identifier_pos && result == 0x24
1699	&& CPP_OPTION (pfile, dollars_in_ident))
1700	{
1701	if (CPP_OPTION (pfile, warn_dollars) && !pfile->state.skipping)
1702	{
1703	CPP_OPTION (pfile, warn_dollars) = 0;
1704	cpp_error (pfile, CPP_DL_PEDWARN, msgid: "'$' in identifier or number");
1705	}
1706	NORMALIZE_STATE_UPDATE_IDNUM (nst, result);
1707	}
1708	else if (identifier_pos)
1709	{
1710	int validity = ucn_valid_in_identifier (pfile, c: result, nst);
1711
1712	if (validity == 0)
1713	cpp_error (pfile, CPP_DL_ERROR,
1714	msgid: "universal character %.*s is not valid in an identifier",
1715	(int) (str - base), base);
1716	else if (validity == 2 && identifier_pos == 1)
1717	cpp_error (pfile, CPP_DL_ERROR,
1718	msgid: "universal character %.*s is not valid at the start of an identifier",
1719	(int) (str - base), base);
1720	}
1721	else if (result > UCS_LIMIT
1722	&& (!CPP_OPTION (pfile, cplusplus)
1723	|| CPP_OPTION (pfile, lang) > CLK_CXX17))
1724	cpp_error (pfile, CPP_DL_PEDWARN,
1725	msgid: "%.*s is outside the UCS codespace",
1726	(int) (str - base), base);
1727
1728	*cp = result;
1729	return true;
1730	}
1731
1732	/* Convert an UCN, pointed to by FROM, to UTF-8 encoding, then translate
1733	it to the execution character set and write the result into TBUF,
1734	if TBUF is non-NULL.
1735	An advanced pointer is returned. Issues all relevant diagnostics.
1736	If LOC_READER is non-NULL, then RANGES must be non-NULL and CHAR_RANGE
1737	contains the location of the character so far: location information
1738	is read from *LOC_READER, and *RANGES is updated accordingly. */
1739	static const uchar *
1740	convert_ucn (cpp_reader *pfile, const uchar *from, const uchar *limit,
1741	struct _cpp_strbuf *tbuf, struct cset_converter cvt,
1742	source_range char_range,
1743	cpp_string_location_reader *loc_reader,
1744	cpp_substring_ranges *ranges)
1745	{
1746	cppchar_t ucn;
1747	uchar buf[6];
1748	uchar *bufp = buf;
1749	size_t bytesleft = 6;
1750	int rval;
1751	struct normalize_state nst = INITIAL_NORMALIZE_STATE;
1752
1753	/* loc_reader and ranges must either be both NULL, or both be non-NULL. */
1754	gcc_assert ((loc_reader != NULL) == (ranges != NULL));
1755
1756	from++; /* Skip u/U/N. */
1757
1758	/* The u/U is part of the spelling of this character. */
1759	extend_char_range (char_range: &char_range, loc_reader);
1760
1761	_cpp_valid_ucn (pfile, pstr: &from, limit, identifier_pos: 0, nst: &nst,
1762	cp: &ucn, char_range: &char_range, loc_reader);
1763
1764	rval = one_cppchar_to_utf8 (c: ucn, outbufp: &bufp, outbytesleftp: &bytesleft);
1765	if (rval)
1766	{
1767	errno = rval;
1768	cpp_errno (pfile, CPP_DL_ERROR,
1769	msgid: "converting UCN to source character set");
1770	}
1771	else
1772	{
1773	if (tbuf)
1774	if (!APPLY_CONVERSION (cvt, buf, 6 - bytesleft, tbuf))
1775	cpp_errno (pfile, CPP_DL_ERROR,
1776	msgid: "converting UCN to execution character set");
1777
1778	if (loc_reader)
1779	{
1780	int num_encoded_bytes = 6 - bytesleft;
1781	for (int i = 0; i < num_encoded_bytes; i++)
1782	ranges->add_range (range: char_range);
1783	}
1784	}
1785
1786	return from;
1787	}
1788
1789	/* Performs a similar task as _cpp_valid_ucn, but parses UTF-8-encoded
1790	extended characters rather than UCNs. If the return value is TRUE, then a
1791	character was successfully decoded and stored in *CP; *PSTR has been
1792	updated to point one past the valid UTF-8 sequence. Diagnostics may have
1793	been emitted if the character parsed is not allowed in the current context.
1794	If the return value is FALSE, then *PSTR has not been modified and *CP may
1795	equal 0, to indicate that *PSTR does not form a valid UTF-8 sequence, or it
1796	may, when processing an identifier in C mode, equal a codepoint that was
1797	validly encoded but is not allowed to appear in an identifier. In either
1798	case, no diagnostic is emitted, and the return value of FALSE should cause
1799	a new token to be formed.
1800
1801	_cpp_valid_utf8 can be called when lexing a potential identifier, or a
1802	CPP_OTHER token or for the purposes of -Winvalid-utf8 warning in string or
1803	character literals. NST is unused when not in a potential identifier.
1804
1805	As in _cpp_valid_ucn, IDENTIFIER_POS is 0 when not in an identifier, 1 for
1806	the start of an identifier, or 2 otherwise. */
1807
1808	extern bool
1809	_cpp_valid_utf8 (cpp_reader *pfile,
1810	const uchar **pstr,
1811	const uchar *limit,
1812	int identifier_pos,
1813	struct normalize_state *nst,
1814	cppchar_t *cp)
1815	{
1816	const uchar *base = *pstr;
1817	size_t inbytesleft = limit - base;
1818	if (one_utf8_to_cppchar (inbufp: pstr, inbytesleftp: &inbytesleft, cp))
1819	{
1820	/* No diagnostic here as this byte will rather become a
1821	new token. */
1822	*cp = 0;
1823	return false;
1824	}
1825
1826	if (identifier_pos)
1827	{
1828	switch (ucn_valid_in_identifier (pfile, c: *cp, nst))
1829	{
1830
1831	case 0:
1832	/* In C++, this is an error for invalid character in an identifier
1833	because logically, the UTF-8 was converted to a UCN during
1834	translation phase 1 (even though we don't physically do it that
1835	way). In C, this byte rather becomes grammatically a separate
1836	token. */
1837
1838	if (CPP_OPTION (pfile, cplusplus))
1839	cpp_error (pfile, CPP_DL_ERROR,
1840	msgid: "extended character %.*s is not valid in an identifier",
1841	(int) (*pstr - base), base);
1842	else
1843	{
1844	*pstr = base;
1845	return false;
1846	}
1847
1848	break;
1849
1850	case 2:
1851	if (identifier_pos == 1)
1852	{
1853	/* This is treated the same way in C++ or C99 -- lexed as an
1854	identifier which is then invalid because an identifier is
1855	not allowed to start with this character. */
1856	cpp_error (pfile, CPP_DL_ERROR,
1857	msgid: "extended character %.*s is not valid at the start of an identifier",
1858	(int) (*pstr - base), base);
1859	}
1860	break;
1861	}
1862	}
1863
1864	return true;
1865	}
1866
1867	/* Return true iff BUFFER of size NUM_BYTES is validly-encoded UTF-8. */
1868
1869	extern bool
1870	cpp_valid_utf8_p (const char *buffer, size_t num_bytes)
1871	{
1872	const uchar *iter = (const uchar *)buffer;
1873	size_t bytesleft = num_bytes;
1874	while (bytesleft > 0)
1875	{
1876	/* one_utf8_to_cppchar implements 5-byte and 6 byte sequences as per
1877	RFC 2279, but this has been superceded by RFC 3629, which
1878	restricts UTF-8 to 1-byte through 4-byte sequences, and
1879	states "the octet values C0, C1, F5 to FF never appear".
1880
1881	Reject such values. */
1882	if (*iter >= 0xf4)
1883	return false;
1884
1885	cppchar_t cp;
1886	int err = one_utf8_to_cppchar (inbufp: &iter, inbytesleftp: &bytesleft, cp: &cp);
1887	if (err)
1888	return false;
1889
1890	/* Additionally, Unicode declares that all codepoints above 0010FFFF are
1891	invalid because they cannot be represented in UTF-16.
1892
1893	Reject such values.*/
1894	if (cp > UCS_LIMIT)
1895	return false;
1896	}
1897	/* No problems encountered. */
1898	return true;
1899	}
1900
1901	/* Subroutine of convert_hex and convert_oct. N is the representation
1902	in the execution character set of a numeric escape; write it into the
1903	string buffer TBUF and update the end-of-string pointer therein. WIDE
1904	is true if it's a wide string that's being assembled in TBUF. This
1905	function issues no diagnostics and never fails. */
1906	static void
1907	emit_numeric_escape (cpp_reader *pfile, cppchar_t n,
1908	struct _cpp_strbuf *tbuf, struct cset_converter cvt)
1909	{
1910	size_t width = cvt.width;
1911
1912	if (width != CPP_OPTION (pfile, char_precision))
1913	{
1914	/* We have to render this into the target byte order, which may not
1915	be our byte order. */
1916	bool bigend = CPP_OPTION (pfile, bytes_big_endian);
1917	size_t cwidth = CPP_OPTION (pfile, char_precision);
1918	size_t cmask = width_to_mask (width: cwidth);
1919	size_t nbwc = width / cwidth;
1920	size_t i;
1921	size_t off = tbuf->len;
1922	cppchar_t c;
1923
1924	if (tbuf->len + nbwc > tbuf->asize)
1925	{
1926	tbuf->asize += OUTBUF_BLOCK_SIZE;
1927	tbuf->text = XRESIZEVEC (uchar, tbuf->text, tbuf->asize);
1928	}
1929
1930	for (i = 0; i < nbwc; i++)
1931	{
1932	c = n & cmask;
1933	n >>= cwidth;
1934	tbuf->text[off + (bigend ? nbwc - i - 1 : i)] = c;
1935	}
1936	tbuf->len += nbwc;
1937	}
1938	else
1939	{
1940	/* Note: this code does not handle the case where the target
1941	and host have a different number of bits in a byte. */
1942	if (tbuf->len + 1 > tbuf->asize)
1943	{
1944	tbuf->asize += OUTBUF_BLOCK_SIZE;
1945	tbuf->text = XRESIZEVEC (uchar, tbuf->text, tbuf->asize);
1946	}
1947	tbuf->text[tbuf->len++] = n;
1948	}
1949	}
1950
1951	/* Convert a hexadecimal escape, pointed to by FROM, to the execution
1952	character set and write it into the string buffer TBUF (if non-NULL).
1953	Returns an advanced pointer, and issues diagnostics as necessary.
1954	No character set translation occurs; this routine always produces the
1955	execution-set character with numeric value equal to the given hex
1956	number. You can, e.g. generate surrogate pairs this way.
1957	If LOC_READER is non-NULL, then RANGES must be non-NULL and CHAR_RANGE
1958	contains the location of the character so far: location information
1959	is read from *LOC_READER, and *RANGES is updated accordingly. */
1960	static const uchar *
1961	convert_hex (cpp_reader *pfile, const uchar *from, const uchar *limit,
1962	struct _cpp_strbuf *tbuf, struct cset_converter cvt,
1963	source_range char_range,
1964	cpp_string_location_reader *loc_reader,
1965	cpp_substring_ranges *ranges)
1966	{
1967	cppchar_t c, n = 0, overflow = 0;
1968	int digits_found = 0;
1969	size_t width = cvt.width;
1970	size_t mask = width_to_mask (width);
1971	bool delimited = false;
1972	const uchar *base = from - 1;
1973
1974	/* loc_reader and ranges must either be both NULL, or both be non-NULL. */
1975	gcc_assert ((loc_reader != NULL) == (ranges != NULL));
1976
1977	if (CPP_WTRADITIONAL (pfile))
1978	cpp_warning (pfile, CPP_W_TRADITIONAL,
1979	msgid: "the meaning of '\\x' is different in traditional C");
1980
1981	/* Skip 'x'. */
1982	from++;
1983
1984	/* The 'x' is part of the spelling of this character. */
1985	extend_char_range (char_range: &char_range, loc_reader);
1986
1987	if (from < limit && *from == '{')
1988	{
1989	delimited = true;
1990	from++;
1991	extend_char_range (char_range: &char_range, loc_reader);
1992	}
1993
1994	while (from < limit)
1995	{
1996	c = *from;
1997	if (! hex_p (c))
1998	break;
1999	from++;
2000	extend_char_range (char_range: &char_range, loc_reader);
2001	overflow |= n ^ (n << 4 >> 4);
2002	n = (n << 4) + hex_value (c);
2003	digits_found = 1;
2004	}
2005
2006	if (delimited && from < limit && *from == '}')
2007	{
2008	from++;
2009	if (!digits_found)
2010	{
2011	cpp_error (pfile, CPP_DL_ERROR,
2012	msgid: "empty delimited escape sequence");
2013	return from;
2014	}
2015	else if (!CPP_OPTION (pfile, delimited_escape_seqs)
2016	&& CPP_OPTION (pfile, cpp_pedantic))
2017	cpp_error (pfile, CPP_DL_PEDWARN,
2018	msgid: "delimited escape sequences are only valid in C++23");
2019	delimited = false;
2020	extend_char_range (char_range: &char_range, loc_reader);
2021	}
2022
2023	if (!digits_found)
2024	{
2025	cpp_error (pfile, CPP_DL_ERROR,
2026	msgid: "\\x used with no following hex digits");
2027	return from;
2028	}
2029	else if (delimited)
2030	{
2031	cpp_error (pfile, CPP_DL_ERROR,
2032	msgid: "'\\x{' not terminated with '}' after %.*s",
2033	(int) (from - base), base);
2034	return from;
2035	}
2036
2037	if (overflow | (n != (n & mask)))
2038	{
2039	cpp_error (pfile, CPP_DL_PEDWARN,
2040	msgid: "hex escape sequence out of range");
2041	n &= mask;
2042	}
2043
2044	if (tbuf)
2045	emit_numeric_escape (pfile, n, tbuf, cvt);
2046	if (ranges)
2047	ranges->add_range (range: char_range);
2048
2049	return from;
2050	}
2051
2052	/* Convert an octal escape, pointed to by FROM, to the execution
2053	character set and write it into the string buffer TBUF. Returns an
2054	advanced pointer, and issues diagnostics as necessary.
2055	No character set translation occurs; this routine always produces the
2056	execution-set character with numeric value equal to the given octal
2057	number.
2058	If LOC_READER is non-NULL, then RANGES must be non-NULL and CHAR_RANGE
2059	contains the location of the character so far: location information
2060	is read from *LOC_READER, and *RANGES is updated accordingly. */
2061	static const uchar *
2062	convert_oct (cpp_reader *pfile, const uchar *from, const uchar *limit,
2063	struct _cpp_strbuf *tbuf, struct cset_converter cvt,
2064	source_range char_range,
2065	cpp_string_location_reader *loc_reader,
2066	cpp_substring_ranges *ranges)
2067	{
2068	size_t count = 0;
2069	cppchar_t c, n = 0, overflow = 0;
2070	size_t width = cvt.width;
2071	size_t mask = width_to_mask (width);
2072	bool delimited = false;
2073	const uchar *base = from - 1;
2074
2075	/* loc_reader and ranges must either be both NULL, or both be non-NULL. */
2076	gcc_assert ((loc_reader != NULL) == (ranges != NULL));
2077
2078	if (from < limit && *from == 'o')
2079	{
2080	from++;
2081	extend_char_range (char_range: &char_range, loc_reader);
2082	if (from == limit || *from != '{')
2083	cpp_error (pfile, CPP_DL_ERROR, msgid: "'\\o' not followed by '{'");
2084	else
2085	{
2086	from++;
2087	extend_char_range (char_range: &char_range, loc_reader);
2088	delimited = true;
2089	}
2090	}
2091
2092	while (from < limit && count++ < 3)
2093	{
2094	c = *from;
2095	if (c < '0' || c > '7')
2096	break;
2097	from++;
2098	extend_char_range (char_range: &char_range, loc_reader);
2099	if (delimited)
2100	{
2101	count = 2;
2102	overflow |= n ^ (n << 3 >> 3);
2103	}
2104	n = (n << 3) + c - '0';
2105	}
2106
2107	if (delimited)
2108	{
2109	if (from < limit && *from == '}')
2110	{
2111	from++;
2112	if (count == 1)
2113	{
2114	cpp_error (pfile, CPP_DL_ERROR,
2115	msgid: "empty delimited escape sequence");
2116	return from;
2117	}
2118	else if (!CPP_OPTION (pfile, delimited_escape_seqs)
2119	&& CPP_OPTION (pfile, cpp_pedantic))
2120	cpp_error (pfile, CPP_DL_PEDWARN,
2121	msgid: "delimited escape sequences are only valid in C++23");
2122	extend_char_range (char_range: &char_range, loc_reader);
2123	}
2124	else
2125	{
2126	cpp_error (pfile, CPP_DL_ERROR,
2127	msgid: "'\\o{' not terminated with '}' after %.*s",
2128	(int) (from - base), base);
2129	return from;
2130	}
2131	}
2132
2133	if (overflow | (n != (n & mask)))
2134	{
2135	cpp_error (pfile, CPP_DL_PEDWARN,
2136	msgid: "octal escape sequence out of range");
2137	n &= mask;
2138	}
2139
2140	if (tbuf)
2141	emit_numeric_escape (pfile, n, tbuf, cvt);
2142	if (ranges)
2143	ranges->add_range (range: char_range);
2144
2145	return from;
2146	}
2147
2148	/* Convert an escape sequence (pointed to by FROM) to its value on
2149	the target, and to the execution character set. Do not scan past
2150	LIMIT. Write the converted value into TBUF, if TBUF is non-NULL.
2151	Returns an advanced pointer. Handles all relevant diagnostics.
2152	If LOC_READER is non-NULL, then RANGES must be non-NULL: location
2153	information is read from *LOC_READER, and *RANGES is updated
2154	accordingly. */
2155	static const uchar *
2156	convert_escape (cpp_reader *pfile, const uchar *from, const uchar *limit,
2157	struct _cpp_strbuf *tbuf, struct cset_converter cvt,
2158	cpp_string_location_reader *loc_reader,
2159	cpp_substring_ranges *ranges, bool uneval)
2160	{
2161	/* Values of \a \b \e \f \n \r \t \v respectively. */
2162	#if HOST_CHARSET == HOST_CHARSET_ASCII
2163	static const uchar charconsts[] = { 7, 8, 27, 12, 10, 13, 9, 11 };
2164	#elif HOST_CHARSET == HOST_CHARSET_EBCDIC
2165	static const uchar charconsts[] = { 47, 22, 39, 12, 21, 13, 5, 11 };
2166	#else
2167	#error "unknown host character set"
2168	#endif
2169
2170	uchar c;
2171
2172	/* Record the location of the backslash. */
2173	source_range char_range;
2174	if (loc_reader)
2175	char_range = loc_reader->get_next ();
2176
2177	c = *from;
2178	switch (c)
2179	{
2180	/* UCNs, hex escapes, and octal escapes are processed separately. */
2181	case 'u': case 'U': case 'N':
2182	return convert_ucn (pfile, from, limit, tbuf, cvt,
2183	char_range, loc_reader, ranges);
2184
2185	case 'x':
2186	if (uneval && CPP_PEDANTIC (pfile))
2187	cpp_error (pfile, CPP_DL_PEDWARN,
2188	msgid: "numeric escape sequence in unevaluated string: "
2189	"'\\%c'", (int) c);
2190	return convert_hex (pfile, from, limit, tbuf, cvt,
2191	char_range, loc_reader, ranges);
2192
2193	case '0': case '1': case '2': case '3':
2194	case '4': case '5': case '6': case '7':
2195	case 'o':
2196	if (uneval && CPP_PEDANTIC (pfile))
2197	cpp_error (pfile, CPP_DL_PEDWARN,
2198	msgid: "numeric escape sequence in unevaluated string: "
2199	"'\\%c'", (int) c);
2200	return convert_oct (pfile, from, limit, tbuf, cvt,
2201	char_range, loc_reader, ranges);
2202
2203	/* Various letter escapes. Get the appropriate host-charset
2204	value into C. */
2205	case '\\': case '\'': case '"': case '?': break;
2206
2207	case '(': case '{': case '[': case '%':
2208	/* '\(', etc, can be used at the beginning of a line in a long
2209	string split onto multiple lines with \-newline, to prevent
2210	Emacs or other text editors from getting confused. '\%' can
2211	be used to prevent SCCS from mangling printf format strings. */
2212	if (CPP_PEDANTIC (pfile))
2213	goto unknown;
2214	break;
2215
2216	case 'b': c = charconsts[1]; break;
2217	case 'f': c = charconsts[3]; break;
2218	case 'n': c = charconsts[4]; break;
2219	case 'r': c = charconsts[5]; break;
2220	case 't': c = charconsts[6]; break;
2221	case 'v': c = charconsts[7]; break;
2222
2223	case 'a':
2224	if (CPP_WTRADITIONAL (pfile))
2225	cpp_warning (pfile, CPP_W_TRADITIONAL,
2226	msgid: "the meaning of '\\a' is different in traditional C");
2227	c = charconsts[0];
2228	break;
2229
2230	case 'e': case 'E':
2231	if (CPP_PEDANTIC (pfile))
2232	cpp_error (pfile, CPP_DL_PEDWARN,
2233	msgid: "non-ISO-standard escape sequence, '\\%c'", (int) c);
2234	c = charconsts[2];
2235	break;
2236
2237	default:
2238	unknown:
2239	if (ISGRAPH (c))
2240	cpp_error (pfile, CPP_DL_PEDWARN,
2241	msgid: "unknown escape sequence: '\\%c'", (int) c);
2242	else
2243	{
2244	encoding_rich_location rich_loc (pfile);
2245
2246	/* diagnostic.cc does not support "%03o". When it does, this
2247	code can use %03o directly in the diagnostic again. */
2248	char buf[32];
2249	sprintf(s: buf, format: "%03o", (int) c);
2250	cpp_error_at (pfile, CPP_DL_PEDWARN, richloc: &rich_loc,
2251	msgid: "unknown escape sequence: '\\%s'", buf);
2252	}
2253	}
2254
2255	if (tbuf)
2256	/* Now convert what we have to the execution character set. */
2257	if (!APPLY_CONVERSION (cvt, &c, 1, tbuf))
2258	cpp_errno (pfile, CPP_DL_ERROR,
2259	msgid: "converting escape sequence to execution character set");
2260
2261	if (loc_reader)
2262	{
2263	char_range.m_finish = loc_reader->get_next ().m_finish;
2264	ranges->add_range (range: char_range);
2265	}
2266
2267	return from + 1;
2268	}
2269
2270	/* TYPE is a token type. The return value is the conversion needed to
2271	convert from source to execution character set for the given type. */
2272	static struct cset_converter
2273	converter_for_type (cpp_reader *pfile, enum cpp_ttype type)
2274	{
2275	switch (type)
2276	{
2277	default:
2278	return pfile->narrow_cset_desc;
2279	case CPP_UTF8CHAR:
2280	case CPP_UTF8STRING:
2281	return pfile->utf8_cset_desc;
2282	case CPP_CHAR16:
2283	case CPP_STRING16:
2284	return pfile->char16_cset_desc;
2285	case CPP_CHAR32:
2286	case CPP_STRING32:
2287	return pfile->char32_cset_desc;
2288	case CPP_WCHAR:
2289	case CPP_WSTRING:
2290	return pfile->wide_cset_desc;
2291	}
2292	}
2293
2294	/* FROM is an array of cpp_string structures of length COUNT. These
2295	are to be converted from the source to the execution character set,
2296	escape sequences translated, and finally all are to be
2297	concatenated. WIDE indicates whether or not to produce a wide
2298	string. If TO is non-NULL, the result is written into TO.
2299	If LOC_READERS and OUT are non-NULL, then location information
2300	is read from LOC_READERS (which must be an array of length COUNT),
2301	and location information is written to *RANGES.
2302
2303	Returns true for success, false for failure. */
2304
2305	static bool
2306	cpp_interpret_string_1 (cpp_reader *pfile, const cpp_string *from, size_t count,
2307	cpp_string *to, enum cpp_ttype type,
2308	cpp_string_location_reader *loc_readers,
2309	cpp_substring_ranges *out)
2310	{
2311	struct _cpp_strbuf tbuf;
2312	const uchar *p, *base, *limit;
2313	size_t i;
2314	struct cset_converter cvt = converter_for_type (pfile, type);
2315
2316	/* loc_readers and out must either be both NULL, or both be non-NULL. */
2317	gcc_assert ((loc_readers != NULL) == (out != NULL));
2318
2319	if (to)
2320	{
2321	tbuf.asize = MAX (OUTBUF_BLOCK_SIZE, from->len);
2322	tbuf.text = XNEWVEC (uchar, tbuf.asize);
2323	tbuf.len = 0;
2324	}
2325
2326	cpp_string_location_reader *loc_reader = NULL;
2327	for (i = 0; i < count; i++)
2328	{
2329	if (loc_readers)
2330	loc_reader = &loc_readers[i];
2331
2332	p = from[i].text;
2333	if (*p == 'u')
2334	{
2335	p++;
2336	if (loc_reader)
2337	loc_reader->get_next ();
2338	if (*p == '8')
2339	{
2340	p++;
2341	if (loc_reader)
2342	loc_reader->get_next ();
2343	}
2344	}
2345	else if (*p == 'L' || *p == 'U') p++;
2346	if (*p == 'R')
2347	{
2348	const uchar *prefix;
2349
2350	/* Skip over 'R"'. */
2351	p += 2;
2352	if (loc_reader)
2353	{
2354	loc_reader->get_next ();
2355	loc_reader->get_next ();
2356	}
2357	prefix = p;
2358	while (*p != '(')
2359	{
2360	p++;
2361	if (loc_reader)
2362	loc_reader->get_next ();
2363	}
2364	p++;
2365	if (loc_reader)
2366	loc_reader->get_next ();
2367	limit = from[i].text + from[i].len;
2368	if (limit >= p + (p - prefix) + 1)
2369	limit -= (p - prefix) + 1;
2370
2371	/* Raw strings are all normal characters; these can be fed
2372	directly to convert_cset. */
2373	if (to)
2374	if (!APPLY_CONVERSION (cvt, p, limit - p, &tbuf))
2375	goto fail;
2376
2377	if (loc_reader)
2378	{
2379	/* If generating source ranges, assume we have a 1:1
2380	correspondence between bytes in the source encoding and bytes
2381	in the execution encoding (e.g. if we have a UTF-8 to UTF-8
2382	conversion), so that this run of bytes in the source file
2383	corresponds to a run of bytes in the execution string.
2384	This requirement is guaranteed by an early-reject in
2385	cpp_interpret_string_ranges. */
2386	gcc_assert (cvt.func == convert_no_conversion);
2387	out->add_n_ranges (num: limit - p, loc_reader&: *loc_reader);
2388	}
2389
2390	continue;
2391	}
2392
2393	/* If we don't now have a leading quote, something has gone wrong.
2394	This can occur if cpp_interpret_string_ranges is handling a
2395	stringified macro argument, but should not be possible otherwise. */
2396	if (*p != '"' && *p != '\'')
2397	{
2398	gcc_assert (out != NULL);
2399	cpp_error (pfile, CPP_DL_ERROR, msgid: "missing open quote");
2400	if (to)
2401	free (ptr: tbuf.text);
2402	return false;
2403	}
2404
2405	/* Skip leading quote. */
2406	p++;
2407	if (loc_reader)
2408	loc_reader->get_next ();
2409
2410	limit = from[i].text + from[i].len - 1; /* Skip trailing quote. */
2411
2412	for (;;)
2413	{
2414	base = p;
2415	while (p < limit && *p != '\\')
2416	p++;
2417	if (p > base)
2418	{
2419	/* We have a run of normal characters; these can be fed
2420	directly to convert_cset. */
2421	if (to)
2422	if (!APPLY_CONVERSION (cvt, base, p - base, &tbuf))
2423	goto fail;
2424	/* Similar to above: assumes we have a 1:1 correspondence
2425	between bytes in the source encoding and bytes in the
2426	execution encoding. */
2427	if (loc_reader)
2428	{
2429	gcc_assert (cvt.func == convert_no_conversion);
2430	out->add_n_ranges (num: p - base, loc_reader&: *loc_reader);
2431	}
2432	}
2433	if (p >= limit)
2434	break;
2435
2436	struct _cpp_strbuf *tbuf_ptr = to ? &tbuf : NULL;
2437	p = convert_escape (pfile, from: p + 1, limit, tbuf: tbuf_ptr, cvt,
2438	loc_reader, ranges: out, uneval: type == CPP_UNEVAL_STRING);
2439	}
2440	}
2441
2442	if (to)
2443	{
2444	/* NUL-terminate the 'to' buffer and translate it to a cpp_string
2445	structure. */
2446	emit_numeric_escape (pfile, n: 0, tbuf: &tbuf, cvt);
2447	tbuf.text = XRESIZEVEC (uchar, tbuf.text, tbuf.len);
2448	to->text = tbuf.text;
2449	to->len = tbuf.len;
2450	}
2451	/* Use the location of the trailing quote as the location of the
2452	NUL-terminator. */
2453	if (loc_reader)
2454	{
2455	source_range range = loc_reader->get_next ();
2456	out->add_range (range);
2457	}
2458
2459	return true;
2460
2461	fail:
2462	cpp_errno (pfile, CPP_DL_ERROR, msgid: "converting to execution character set");
2463	if (to)
2464	free (ptr: tbuf.text);
2465	return false;
2466	}
2467
2468	/* FROM is an array of cpp_string structures of length COUNT. These
2469	are to be converted from the source to the execution character set,
2470	escape sequences translated, and finally all are to be
2471	concatenated. WIDE indicates whether or not to produce a wide
2472	string. The result is written into TO. Returns true for success,
2473	false for failure. */
2474	bool
2475	cpp_interpret_string (cpp_reader *pfile, const cpp_string *from, size_t count,
2476	cpp_string *to, enum cpp_ttype type)
2477	{
2478	return cpp_interpret_string_1 (pfile, from, count, to, type, NULL, NULL);
2479	}
2480
2481	/* A "do nothing" diagnostic-handling callback for use by
2482	cpp_interpret_string_ranges, so that it can temporarily suppress
2483	diagnostic-handling. */
2484
2485	static bool
2486	noop_diagnostic_cb (cpp_reader *, enum cpp_diagnostic_level,
2487	enum cpp_warning_reason, rich_location *,
2488	const char *, va_list *)
2489	{
2490	/* no-op. */
2491	return true;
2492	}
2493
2494	/* This function mimics the behavior of cpp_interpret_string, but
2495	rather than generating a string in the execution character set,
2496	*OUT is written to with the source code ranges of the characters
2497	in such a string.
2498	FROM and LOC_READERS should both be arrays of length COUNT.
2499	Returns NULL for success, or an error message for failure. */
2500
2501	const char *
2502	cpp_interpret_string_ranges (cpp_reader *pfile, const cpp_string *from,
2503	cpp_string_location_reader *loc_readers,
2504	size_t count,
2505	cpp_substring_ranges *out,
2506	enum cpp_ttype type)
2507	{
2508	/* There are a couple of cases in the range-handling in
2509	cpp_interpret_string_1 that rely on there being a 1:1 correspondence
2510	between bytes in the source encoding and bytes in the execution
2511	encoding, so that each byte in the execution string can correspond
2512	to the location of a byte in the source string.
2513
2514	This holds for the typical case of a UTF-8 to UTF-8 conversion.
2515	Enforce this requirement by only attempting to track substring
2516	locations if we have source encoding == execution encoding.
2517
2518	This is a stronger condition than we need, since we could e.g.
2519	have ASCII to EBCDIC (with 1 byte per character before and after),
2520	but it seems to be a reasonable restriction. */
2521	struct cset_converter cvt = converter_for_type (pfile, type);
2522	if (cvt.func != convert_no_conversion)
2523	return "execution character set != source character set";
2524
2525	/* For on-demand strings we have already lexed the strings, so there
2526	should be no diagnostics. However, if we have bogus source location
2527	data (or stringified macro arguments), the attempt to lex the
2528	strings could fail with an diagnostic. Temporarily install an
2529	diagnostic-handler to catch the diagnostic, so that it can lead to this call
2530	failing, rather than being emitted as a user-visible diagnostic.
2531	If an diagnostic does occur, we should see it via the return value of
2532	cpp_interpret_string_1. */
2533	bool (*saved_diagnostic_handler) (cpp_reader *, enum cpp_diagnostic_level,
2534	enum cpp_warning_reason, rich_location *,
2535	const char *, va_list *)
2536	ATTRIBUTE_FPTR_PRINTF(5,0);
2537
2538	saved_diagnostic_handler = pfile->cb.diagnostic;
2539	pfile->cb.diagnostic = noop_diagnostic_cb;
2540
2541	bool result = cpp_interpret_string_1 (pfile, from, count, NULL, type,
2542	loc_readers, out);
2543
2544	/* Restore the saved diagnostic-handler. */
2545	pfile->cb.diagnostic = saved_diagnostic_handler;
2546
2547	if (!result)
2548	return "cpp_interpret_string_1 failed";
2549
2550	/* Success. */
2551	return NULL;
2552	}
2553
2554	/* Subroutine of do_line and do_linemarker. Convert escape sequences
2555	in a string, but do not perform character set conversion. */
2556	bool
2557	cpp_interpret_string_notranslate (cpp_reader *pfile, const cpp_string *from,
2558	size_t count, cpp_string *to,
2559	enum cpp_ttype type)
2560	{
2561	struct cset_converter save_narrow_cset_desc = pfile->narrow_cset_desc;
2562	bool retval;
2563
2564	pfile->narrow_cset_desc.func = convert_no_conversion;
2565	pfile->narrow_cset_desc.cd = (iconv_t) -1;
2566	pfile->narrow_cset_desc.width = CPP_OPTION (pfile, char_precision);
2567
2568	retval = cpp_interpret_string (pfile, from, count, to,
2569	type: type == CPP_UNEVAL_STRING
2570	? CPP_UNEVAL_STRING : CPP_STRING);
2571
2572	pfile->narrow_cset_desc = save_narrow_cset_desc;
2573	return retval;
2574	}
2575
2576
2577	/* Subroutine of cpp_interpret_charconst which performs the conversion
2578	to a number, for narrow strings. STR is the string structure returned
2579	by cpp_interpret_string. PCHARS_SEEN and UNSIGNEDP are as for
2580	cpp_interpret_charconst. TYPE is the token type. */
2581	static cppchar_t
2582	narrow_str_to_charconst (cpp_reader *pfile, cpp_string str,
2583	unsigned int *pchars_seen, int *unsignedp,
2584	enum cpp_ttype type)
2585	{
2586	size_t width = CPP_OPTION (pfile, char_precision);
2587	size_t max_chars = CPP_OPTION (pfile, int_precision) / width;
2588	size_t mask = width_to_mask (width);
2589	size_t i;
2590	cppchar_t result, c;
2591	bool unsigned_p;
2592
2593	/* The value of a multi-character character constant, or a
2594	single-character character constant whose representation in the
2595	execution character set is more than one byte long, is
2596	implementation defined. This implementation defines it to be the
2597	number formed by interpreting the byte sequence in memory as a
2598	big-endian binary number. If overflow occurs, the high bytes are
2599	lost, and a warning is issued.
2600
2601	We don't want to process the NUL terminator handed back by
2602	cpp_interpret_string. */
2603	result = 0;
2604	for (i = 0; i < str.len - 1; i++)
2605	{
2606	c = str.text[i] & mask;
2607	if (width < BITS_PER_CPPCHAR_T)
2608	result = (result << width) | c;
2609	else
2610	result = c;
2611	}
2612
2613	if (type == CPP_UTF8CHAR)
2614	max_chars = 1;
2615	if (i > max_chars)
2616	{
2617	i = max_chars;
2618	cpp_error (pfile, type == CPP_UTF8CHAR ? CPP_DL_ERROR : CPP_DL_WARNING,
2619	msgid: "character constant too long for its type");
2620	}
2621	else if (i > 1 && CPP_OPTION (pfile, warn_multichar))
2622	cpp_warning (pfile, CPP_W_MULTICHAR, msgid: "multi-character character constant");
2623
2624	/* Multichar constants are of type int and therefore signed. */
2625	if (i > 1)
2626	unsigned_p = 0;
2627	else if (type == CPP_UTF8CHAR)
2628	unsigned_p = CPP_OPTION (pfile, unsigned_utf8char);
2629	else
2630	unsigned_p = CPP_OPTION (pfile, unsigned_char);
2631
2632	/* Truncate the constant to its natural width, and simultaneously
2633	sign- or zero-extend to the full width of cppchar_t.
2634	For single-character constants, the value is WIDTH bits wide.
2635	For multi-character constants, the value is INT_PRECISION bits wide. */
2636	if (i > 1)
2637	width = CPP_OPTION (pfile, int_precision);
2638	if (width < BITS_PER_CPPCHAR_T)
2639	{
2640	mask = ((cppchar_t) 1 << width) - 1;
2641	if (unsigned_p || !(result & (1 << (width - 1))))
2642	result &= mask;
2643	else
2644	result |= ~mask;
2645	}
2646	*pchars_seen = i;
2647	*unsignedp = unsigned_p;
2648	return result;
2649	}
2650
2651	/* Subroutine of cpp_interpret_charconst which performs the conversion
2652	to a number, for wide strings. STR is the string structure returned
2653	by cpp_interpret_string. PCHARS_SEEN and UNSIGNEDP are as for
2654	cpp_interpret_charconst. TYPE is the token type. */
2655	static cppchar_t
2656	wide_str_to_charconst (cpp_reader *pfile, cpp_string str,
2657	unsigned int *pchars_seen, int *unsignedp,
2658	enum cpp_ttype type)
2659	{
2660	bool bigend = CPP_OPTION (pfile, bytes_big_endian);
2661	size_t width = converter_for_type (pfile, type).width;
2662	size_t cwidth = CPP_OPTION (pfile, char_precision);
2663	size_t mask = width_to_mask (width);
2664	size_t cmask = width_to_mask (width: cwidth);
2665	size_t nbwc = width / cwidth;
2666	size_t off, i;
2667	cppchar_t result = 0, c;
2668
2669	if (str.len <= nbwc)
2670	{
2671	/* Error recovery, if no errors have been diagnosed previously,
2672	there should be at least two wide characters. Empty literals
2673	are diagnosed earlier and we can get just the zero terminator
2674	only if there were errors diagnosed during conversion. */
2675	*pchars_seen = 0;
2676	*unsignedp = 0;
2677	return 0;
2678	}
2679
2680	/* This is finicky because the string is in the target's byte order,
2681	which may not be our byte order. Only the last character, ignoring
2682	the NUL terminator, is relevant. */
2683	off = str.len - (nbwc * 2);
2684	result = 0;
2685	for (i = 0; i < nbwc; i++)
2686	{
2687	c = bigend ? str.text[off + i] : str.text[off + nbwc - i - 1];
2688	result = (result << cwidth) | (c & cmask);
2689	}
2690
2691	/* Wide character constants have type wchar_t, and a single
2692	character exactly fills a wchar_t, so a multi-character wide
2693	character constant is guaranteed to overflow. */
2694	if (str.len > nbwc * 2)
2695	cpp_error (pfile, (CPP_OPTION (pfile, cplusplus)
2696	&& (type == CPP_CHAR16
2697	|| type == CPP_CHAR32
2698	/* In C++23 this is error even for L'ab'. */
2699	|| (type == CPP_WCHAR
2700	&& CPP_OPTION (pfile, size_t_literals))))
2701	? CPP_DL_ERROR : CPP_DL_WARNING,
2702	msgid: "character constant too long for its type");
2703
2704	/* Truncate the constant to its natural width, and simultaneously
2705	sign- or zero-extend to the full width of cppchar_t. */
2706	if (width < BITS_PER_CPPCHAR_T)
2707	{
2708	if (type == CPP_CHAR16 || type == CPP_CHAR32
2709	|| CPP_OPTION (pfile, unsigned_wchar)
2710	|| !(result & (1 << (width - 1))))
2711	result &= mask;
2712	else
2713	result |= ~mask;
2714	}
2715
2716	if (type == CPP_CHAR16 || type == CPP_CHAR32
2717	|| CPP_OPTION (pfile, unsigned_wchar))
2718	*unsignedp = 1;
2719	else
2720	*unsignedp = 0;
2721
2722	*pchars_seen = 1;
2723	return result;
2724	}
2725
2726	/* Interpret a (possibly wide) character constant in TOKEN.
2727	PCHARS_SEEN points to a variable that is filled in with the number
2728	of characters seen, and UNSIGNEDP to a variable that indicates
2729	whether the result has signed type. */
2730	cppchar_t
2731	cpp_interpret_charconst (cpp_reader *pfile, const cpp_token *token,
2732	unsigned int *pchars_seen, int *unsignedp)
2733	{
2734	cpp_string str = { .len: 0, .text: 0 };
2735	bool wide = (token->type != CPP_CHAR && token->type != CPP_UTF8CHAR);
2736	int u8 = 2 * int(token->type == CPP_UTF8CHAR);
2737	cppchar_t result;
2738
2739	/* An empty constant will appear as L'', u'', U'', u8'', or '' */
2740	if (token->val.str.len == (size_t) (2 + wide + u8))
2741	{
2742	cpp_error (pfile, CPP_DL_ERROR, msgid: "empty character constant");
2743	*pchars_seen = 0;
2744	*unsignedp = 0;
2745	return 0;
2746	}
2747	else if (!cpp_interpret_string (pfile, from: &token->val.str, count: 1, to: &str,
2748	type: token->type))
2749	{
2750	*pchars_seen = 0;
2751	*unsignedp = 0;
2752	return 0;
2753	}
2754
2755	if (wide)
2756	result = wide_str_to_charconst (pfile, str, pchars_seen, unsignedp,
2757	type: token->type);
2758	else
2759	result = narrow_str_to_charconst (pfile, str, pchars_seen, unsignedp,
2760	type: token->type);
2761
2762	if (str.text != token->val.str.text)
2763	free (ptr: (void *)str.text);
2764
2765	return result;
2766	}
2767
2768	/* Convert an identifier denoted by ID and LEN, which might contain
2769	UCN escapes or UTF-8 multibyte chars, to the source character set,
2770	either UTF-8 or UTF-EBCDIC. Assumes that the identifier is actually
2771	a valid identifier. */
2772	cpp_hashnode *
2773	_cpp_interpret_identifier (cpp_reader *pfile, const uchar *id, size_t len)
2774	{
2775	/* It turns out that a UCN escape always turns into fewer characters
2776	than the escape itself, so we can allocate a temporary in advance. */
2777	uchar * buf = (uchar *) alloca (len + 1);
2778	uchar * bufp = buf;
2779	size_t idp;
2780
2781	for (idp = 0; idp < len; idp++)
2782	if (id[idp] != '\\')
2783	*bufp++ = id[idp];
2784	else
2785	{
2786	unsigned length = id[idp + 1] == 'u' ? 4 : 8;
2787	cppchar_t value = 0;
2788	size_t bufleft = len - (bufp - buf);
2789	int rval;
2790	bool delimited = false;
2791
2792	idp += 2;
2793	if (id[idp - 1] == 'N' && id[idp] == '{')
2794	{
2795	idp++;
2796	const uchar *name = &id[idp];
2797	while (idp < len
2798	&& (ISIDNUM (id[idp]) || id[idp] == ' ' || id[idp] == '-'))
2799	idp++;
2800	if (id[idp] == '}')
2801	{
2802	value = _cpp_uname2c (name: (const char *) name, len: &id[idp] - name,
2803	n: uname2c_tree, NULL);
2804	if (value == (cppchar_t) -1)
2805	value = 1;
2806	}
2807	else
2808	idp--;
2809	}
2810	else
2811	{
2812	if (length == 4 && id[idp] == '{')
2813	{
2814	delimited = true;
2815	idp++;
2816	}
2817	while (length && idp < len && ISXDIGIT (id[idp]))
2818	{
2819	value = (value << 4) + hex_value (id[idp]);
2820	idp++;
2821	if (!delimited)
2822	length--;
2823	}
2824	if (!delimited || id[idp] != '}')
2825	idp--;
2826	}
2827
2828	/* Special case for EBCDIC: if the identifier contains
2829	a '$' specified using a UCN, translate it to EBCDIC. */
2830	if (value == 0x24)
2831	{
2832	*bufp++ = '$';
2833	continue;
2834	}
2835
2836	rval = one_cppchar_to_utf8 (c: value, outbufp: &bufp, outbytesleftp: &bufleft);
2837	if (rval)
2838	{
2839	errno = rval;
2840	cpp_errno (pfile, CPP_DL_ERROR,
2841	msgid: "converting UCN to source character set");
2842	break;
2843	}
2844	}
2845
2846	return CPP_HASHNODE (ht_lookup (pfile->hash_table,
2847	buf, bufp - buf, HT_ALLOC));
2848	}
2849
2850
2851	/* Utility to strip a UTF-8 byte order marking from the beginning
2852	of a buffer. Returns the number of bytes to skip, which currently
2853	will be either 0 or 3. */
2854	int
2855	cpp_check_utf8_bom (const char *data, size_t data_length)
2856	{
2857
2858	#if HOST_CHARSET == HOST_CHARSET_ASCII
2859	const unsigned char *udata = (const unsigned char *) data;
2860	if (data_length >= 3 && udata[0] == 0xef && udata[1] == 0xbb
2861	&& udata[2] == 0xbf)
2862	return 3;
2863	#endif
2864
2865	return 0;
2866	}
2867
2868
2869	/* Convert an input buffer (containing the complete contents of one
2870	source file) from INPUT_CHARSET to the source character set. INPUT
2871	points to the input buffer, SIZE is its allocated size, and LEN is
2872	the length of the meaningful data within the buffer. The
2873	translated buffer is returned, *ST_SIZE is set to the length of
2874	the meaningful data within the translated buffer, and *BUFFER_START
2875	is set to the start of the returned buffer. *BUFFER_START may
2876	differ from the return value in the case of a BOM or other ignored
2877	marker information.
2878
2879	INPUT is expected to have been allocated with xmalloc. This
2880	function will either set *BUFFER_START to INPUT, or free it and set
2881	*BUFFER_START to a pointer to another xmalloc-allocated block of
2882	memory.
2883
2884	PFILE is only used to generate diagnostics; setting it to NULL suppresses
2885	diagnostics, and causes a return of NULL if there was any error instead. */
2886
2887	uchar *
2888	_cpp_convert_input (cpp_reader *pfile, const char *input_charset,
2889	uchar *input, size_t size, size_t len,
2890	const unsigned char **buffer_start, off_t *st_size)
2891	{
2892	struct cset_converter input_cset;
2893	struct _cpp_strbuf to;
2894	unsigned char *buffer;
2895
2896	input_cset = init_iconv_desc (pfile, SOURCE_CHARSET, from: input_charset);
2897	if (input_cset.func == convert_no_conversion)
2898	{
2899	to.text = input;
2900	to.asize = size;
2901	to.len = len;
2902	}
2903	else
2904	{
2905	to.asize = MAX (65536, len);
2906	to.text = XNEWVEC (uchar, to.asize);
2907	to.len = 0;
2908
2909	const bool ok = APPLY_CONVERSION (input_cset, input, len, &to);
2910	free (ptr: input);
2911
2912	/* Clean up the mess. */
2913	if (input_cset.func == convert_using_iconv)
2914	iconv_close (input_cset.cd);
2915
2916	/* Handle conversion failure. */
2917	if (!ok)
2918	{
2919	if (!pfile)
2920	{
2921	XDELETEVEC (to.text);
2922	*buffer_start = NULL;
2923	*st_size = 0;
2924	return NULL;
2925	}
2926	cpp_error (pfile, CPP_DL_ERROR, msgid: "failure to convert %s to %s",
2927	input_charset, SOURCE_CHARSET);
2928	}
2929	}
2930
2931	/* Resize buffer if we allocated substantially too much, or if we
2932	haven't enough space for the \n-terminator or following
2933	15 bytes of padding (used to quiet warnings from valgrind or
2934	Address Sanitizer, when the optimized lexer accesses aligned
2935	16-byte memory chunks, including the bytes after the malloced,
2936	area, and stops lexing on '\n'). */
2937	if (to.len + 4096 < to.asize || to.len + 16 > to.asize)
2938	to.text = XRESIZEVEC (uchar, to.text, to.len + 16);
2939
2940	memset (s: to.text + to.len, c: '\0', n: 16);
2941
2942	/* If the file is using old-school Mac line endings (\r only),
2943	terminate with another \r, not an \n, so that we do not mistake
2944	the \r\n sequence for a single DOS line ending and erroneously
2945	issue the "No newline at end of file" diagnostic. */
2946	if (to.len && to.text[to.len - 1] == '\r')
2947	to.text[to.len] = '\r';
2948	else
2949	to.text[to.len] = '\n';
2950
2951	buffer = to.text;
2952	*st_size = to.len;
2953
2954	/* Ignore a UTF-8 BOM if we see one and the source charset is UTF-8. Note
2955	that glib'c UTF-8 iconv() provider (as of glibc 2.7) does not ignore a
2956	BOM -- however, even if it did, we would still need this code due
2957	to the 'convert_no_conversion' case. */
2958	const int bom_len = cpp_check_utf8_bom (data: (const char *) to.text, data_length: to.len);
2959	*st_size -= bom_len;
2960	buffer += bom_len;
2961
2962	*buffer_start = to.text;
2963	return buffer;
2964	}
2965
2966	/* Decide on the default encoding to assume for input files. */
2967	const char *
2968	_cpp_default_encoding (void)
2969	{
2970	const char *current_encoding = NULL;
2971
2972	/* We disable this because the default codeset is 7-bit ASCII on
2973	most platforms, and this causes conversion failures on every
2974	file in GCC that happens to have one of the upper 128 characters
2975	in it -- most likely, as part of the name of a contributor.
2976	We should definitely recognize in-band markers of file encoding,
2977	like:
2978	- the appropriate Unicode byte-order mark (FE FF) to recognize
2979	UTF16 and UCS4 (in both big-endian and little-endian flavors)
2980	and UTF8
2981	- a "#i", "#d", "/ *", "//", " #p" or "#p" (for #pragma) to
2982	distinguish ASCII and EBCDIC.
2983	- now we can parse something like "#pragma GCC encoding <xyz>
2984	on the first line, or even Emacs/VIM's mode line tags (there's
2985	a problem here in that VIM uses the last line, and Emacs has
2986	its more elaborate "local variables" convention).
2987	- investigate whether Java has another common convention, which
2988	would be friendly to support.
2989	(Zack Weinberg and Paolo Bonzini, May 20th 2004) */
2990	#if defined (HAVE_LOCALE_H) && defined (HAVE_LANGINFO_CODESET) && 0
2991	setlocale (LC_CTYPE, "");
2992	current_encoding = nl_langinfo (CODESET);
2993	#endif
2994	if (current_encoding == NULL || *current_encoding == '\0')
2995	current_encoding = SOURCE_CHARSET;
2996
2997	return current_encoding;
2998	}
2999
3000	/* Check if the configured input charset requires no conversion, other than
3001	possibly stripping a UTF-8 BOM. */
3002	bool cpp_input_conversion_is_trivial (const char *input_charset)
3003	{
3004	return !strcasecmp (s1: input_charset, SOURCE_CHARSET);
3005	}
3006
3007	/* Implementation of class cpp_string_location_reader. */
3008
3009	/* Constructor for cpp_string_location_reader. */
3010
3011	cpp_string_location_reader::
3012	cpp_string_location_reader (location_t src_loc,
3013	line_maps *line_table)
3014	{
3015	src_loc = get_range_from_loc (set: line_table, loc: src_loc).m_start;
3016
3017	/* SRC_LOC might be a macro location. It only makes sense to do
3018	column-by-column calculations on ordinary maps, so get the
3019	corresponding location in an ordinary map. */
3020	m_loc
3021	= linemap_resolve_location (line_table, loc: src_loc,
3022	lrk: LRK_SPELLING_LOCATION, NULL);
3023
3024	const line_map_ordinary *map
3025	= linemap_check_ordinary (map: linemap_lookup (line_table, m_loc));
3026	m_offset_per_column = (1 << map->m_range_bits);
3027	}
3028
3029	/* Get the range of the next source byte. */
3030
3031	source_range
3032	cpp_string_location_reader::get_next ()
3033	{
3034	source_range result;
3035	result.m_start = m_loc;
3036	result.m_finish = m_loc;
3037	if (m_loc <= LINE_MAP_MAX_LOCATION_WITH_COLS)
3038	m_loc += m_offset_per_column;
3039	return result;
3040	}
3041
3042	cpp_display_width_computation::
3043	cpp_display_width_computation (const char *data, int data_length,
3044	const cpp_char_column_policy &policy) :
3045	m_begin (data),
3046	m_next (m_begin),
3047	m_bytes_left (data_length),
3048	m_policy (policy),
3049	m_display_cols (0)
3050	{
3051	gcc_assert (policy.m_tabstop > 0);
3052	gcc_assert (policy.m_width_cb);
3053	}
3054
3055
3056	/* The main implementation function for class cpp_display_width_computation.
3057	m_next points on entry to the start of the UTF-8 encoding of the next
3058	character, and is updated to point just after the last byte of the encoding.
3059	m_bytes_left contains on entry the remaining size of the buffer into which
3060	m_next points, and this is also updated accordingly. If m_next does not
3061	point to a valid UTF-8-encoded sequence, then it will be treated as a single
3062	byte with display width 1. m_cur_display_col is the current display column,
3063	relative to which tab stops should be expanded. Returns the display width of
3064	the codepoint just processed.
3065	If OUT is non-NULL, it is populated. */
3066
3067	int
3068	cpp_display_width_computation::process_next_codepoint (cpp_decoded_char *out)
3069	{
3070	cppchar_t c;
3071	int next_width;
3072
3073	if (out)
3074	out->m_start_byte = m_next;
3075
3076	if (*m_next == '\t')
3077	{
3078	++m_next;
3079	--m_bytes_left;
3080	next_width = m_policy.m_tabstop - (m_display_cols % m_policy.m_tabstop);
3081	if (out)
3082	{
3083	out->m_ch = '\t';
3084	out->m_valid_ch = true;
3085	}
3086	}
3087	else if (one_utf8_to_cppchar (inbufp: (const uchar **) &m_next, inbytesleftp: &m_bytes_left, cp: &c)
3088	!= 0)
3089	{
3090	/* Input is not convertible to UTF-8. This could be fine, e.g. in a
3091	string literal, so don't complain. Just treat it as if it has a width
3092	of one. */
3093	++m_next;
3094	--m_bytes_left;
3095	next_width = m_policy.m_undecoded_byte_width;
3096	if (out)
3097	out->m_valid_ch = false;
3098	}
3099	else
3100	{
3101	/* one_utf8_to_cppchar() has updated m_next and m_bytes_left for us. */
3102	next_width = m_policy.m_width_cb (c);
3103	if (out)
3104	{
3105	out->m_ch = c;
3106	out->m_valid_ch = true;
3107	}
3108	}
3109
3110	if (out)
3111	out->m_next_byte = m_next;
3112
3113	m_display_cols += next_width;
3114	return next_width;
3115	}
3116
3117	/* Utility to advance the byte stream by the minimum amount needed to consume
3118	N display columns. Returns the number of display columns that were
3119	actually skipped. This could be less than N, if there was not enough data,
3120	or more than N, if the last character to be skipped had a sufficiently large
3121	display width. */
3122	int
3123	cpp_display_width_computation::advance_display_cols (int n)
3124	{
3125	const int start = m_display_cols;
3126	const int target = start + n;
3127	while (m_display_cols < target && !done ())
3128	process_next_codepoint (NULL);
3129	return m_display_cols - start;
3130	}
3131
3132	/* For the string of length DATA_LENGTH bytes that begins at DATA, compute
3133	how many display columns are occupied by the first COLUMN bytes. COLUMN
3134	may exceed DATA_LENGTH, in which case the phantom bytes at the end are
3135	treated as if they have display width 1. Tabs are expanded to the next tab
3136	stop, relative to the start of DATA, and non-printable-ASCII characters
3137	will be escaped as per POLICY. */
3138
3139	int
3140	cpp_byte_column_to_display_column (const char *data, int data_length,
3141	int column,
3142	const cpp_char_column_policy &policy)
3143	{
3144	const int offset = MAX (0, column - data_length);
3145	cpp_display_width_computation dw (data, column - offset, policy);
3146	while (!dw.done ())
3147	dw.process_next_codepoint (NULL);
3148	return dw.display_cols_processed () + offset;
3149	}
3150
3151	/* For the string of length DATA_LENGTH bytes that begins at DATA, compute
3152	the least number of bytes that will result in at least DISPLAY_COL display
3153	columns. The return value may exceed DATA_LENGTH if the entire string does
3154	not occupy enough display columns. Non-printable-ASCII characters
3155	will be escaped as per POLICY. */
3156
3157	int
3158	cpp_display_column_to_byte_column (const char *data, int data_length,
3159	int display_col,
3160	const cpp_char_column_policy &policy)
3161	{
3162	cpp_display_width_computation dw (data, data_length, policy);
3163	const int avail_display = dw.advance_display_cols (n: display_col);
3164	return dw.bytes_processed () + MAX (0, display_col - avail_display);
3165	}
3166
3167	template <typename PropertyType>
3168	PropertyType
3169	get_cppchar_property (cppchar_t c,
3170	const cppchar_t *range_ends,
3171	const PropertyType *range_values,
3172	size_t num_ranges,
3173	PropertyType default_value)
3174	{
3175	if (__builtin_expect (c <= range_ends[0], true))
3176	return range_values[0];
3177
3178	/* Binary search the tables. */
3179	int begin = 1;
3180	static const int end = num_ranges;
3181	int len = end - begin;
3182	do
3183	{
3184	int half = len/2;
3185	int middle = begin + half;
3186	if (c > range_ends[middle])
3187	{
3188	begin = middle + 1;
3189	len -= half + 1;
3190	}
3191	else
3192	len = half;
3193	} while (len);
3194
3195	if (__builtin_expect (begin != end, true))
3196	return range_values[begin];
3197
3198	return default_value;
3199	}
3200
3201	/* Our own version of wcwidth(). We don't use the actual wcwidth() in glibc,
3202	because that will inspect the user's locale, and in particular in an ASCII
3203	locale, it will not return anything useful for extended characters. But GCC
3204	in other respects (see e.g. _cpp_default_encoding()) behaves as if
3205	everything is UTF-8. We also make some tweaks that are useful for the way
3206	GCC needs to use this data, e.g. tabs and other control characters should be
3207	treated as having width 1. The lookup tables are generated from
3208	contrib/unicode/gen_wcwidth.py and were made by simply calling glibc
3209	wcwidth() on all codepoints, then applying the small tweaks. These tables
3210	are not highly optimized, but for the present purpose of outputting
3211	diagnostics, they are sufficient. */
3212
3213	#include "generated_cpp_wcwidth.h"
3214
3215	int
3216	cpp_wcwidth (cppchar_t c)
3217	{
3218	const size_t num_ranges
3219	= sizeof wcwidth_range_ends / sizeof (*wcwidth_range_ends);
3220	return get_cppchar_property<unsigned char > (c,
3221	range_ends: &wcwidth_range_ends[0],
3222	range_values: &wcwidth_widths[0],
3223	num_ranges,
3224	default_value: 1);
3225	}
3226
3227	#include "combining-chars.inc"
3228
3229	bool
3230	cpp_is_combining_char (cppchar_t c)
3231	{
3232	const size_t num_ranges
3233	= sizeof combining_range_ends / sizeof (*combining_range_ends);
3234	return get_cppchar_property<bool> (c,
3235	range_ends: &combining_range_ends[0],
3236	range_values: &is_combining[0],
3237	num_ranges,
3238	default_value: false);
3239	}
3240
3241	#include "printable-chars.inc"
3242
3243	bool
3244	cpp_is_printable_char (cppchar_t c)
3245	{
3246	const size_t num_ranges
3247	= sizeof printable_range_ends / sizeof (*printable_range_ends);
3248	return get_cppchar_property<bool> (c,
3249	range_ends: &printable_range_ends[0],
3250	range_values: &is_printable[0],
3251	num_ranges,
3252	default_value: false);
3253	}
3254