1	//===--- LiteralSupport.cpp - Code to parse and process literals ----------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This file implements the NumericLiteralParser, CharLiteralParser, and
10	// StringLiteralParser interfaces.
11	//
12	//===----------------------------------------------------------------------===//
13
14	#include "clang/Lex/LiteralSupport.h"
15	#include "clang/Basic/CharInfo.h"
16	#include "clang/Basic/LangOptions.h"
17	#include "clang/Basic/SourceLocation.h"
18	#include "clang/Basic/TargetInfo.h"
19	#include "clang/Lex/LexDiagnostic.h"
20	#include "clang/Lex/Lexer.h"
21	#include "clang/Lex/Preprocessor.h"
22	#include "clang/Lex/Token.h"
23	#include "llvm/ADT/APInt.h"
24	#include "llvm/ADT/SmallVector.h"
25	#include "llvm/ADT/StringExtras.h"
26	#include "llvm/ADT/StringSwitch.h"
27	#include "llvm/Support/ConvertUTF.h"
28	#include "llvm/Support/Error.h"
29	#include "llvm/Support/ErrorHandling.h"
30	#include "llvm/Support/Unicode.h"
31	#include <algorithm>
32	#include <cassert>
33	#include <cstddef>
34	#include <cstdint>
35	#include <cstring>
36	#include <string>
37
38	using namespace clang;
39
40	static unsigned getCharWidth(tok::TokenKind kind, const TargetInfo &Target) {
41	switch (kind) {
42	default: llvm_unreachable("Unknown token type!");
43	case tok::char_constant:
44	case tok::string_literal:
45	case tok::utf8_char_constant:
46	case tok::utf8_string_literal:
47	return Target.getCharWidth();
48	case tok::wide_char_constant:
49	case tok::wide_string_literal:
50	return Target.getWCharWidth();
51	case tok::utf16_char_constant:
52	case tok::utf16_string_literal:
53	return Target.getChar16Width();
54	case tok::utf32_char_constant:
55	case tok::utf32_string_literal:
56	return Target.getChar32Width();
57	}
58	}
59
60	static unsigned getEncodingPrefixLen(tok::TokenKind kind) {
61	switch (kind) {
62	default:
63	llvm_unreachable("Unknown token type!");
64	case tok::char_constant:
65	case tok::string_literal:
66	return 0;
67	case tok::utf8_char_constant:
68	case tok::utf8_string_literal:
69	return 2;
70	case tok::wide_char_constant:
71	case tok::wide_string_literal:
72	case tok::utf16_char_constant:
73	case tok::utf16_string_literal:
74	case tok::utf32_char_constant:
75	case tok::utf32_string_literal:
76	return 1;
77	}
78	}
79
80	static CharSourceRange MakeCharSourceRange(const LangOptions &Features,
81	FullSourceLoc TokLoc,
82	const char *TokBegin,
83	const char *TokRangeBegin,
84	const char *TokRangeEnd) {
85	SourceLocation Begin =
86	Lexer::AdvanceToTokenCharacter(TokStart: TokLoc, Characters: TokRangeBegin - TokBegin,
87	SM: TokLoc.getManager(), LangOpts: Features);
88	SourceLocation End =
89	Lexer::AdvanceToTokenCharacter(TokStart: Begin, Characters: TokRangeEnd - TokRangeBegin,
90	SM: TokLoc.getManager(), LangOpts: Features);
91	return CharSourceRange::getCharRange(B: Begin, E: End);
92	}
93
94	/// Produce a diagnostic highlighting some portion of a literal.
95	///
96	/// Emits the diagnostic \p DiagID, highlighting the range of characters from
97	/// \p TokRangeBegin (inclusive) to \p TokRangeEnd (exclusive), which must be
98	/// a substring of a spelling buffer for the token beginning at \p TokBegin.
99	static DiagnosticBuilder Diag(DiagnosticsEngine *Diags,
100	const LangOptions &Features, FullSourceLoc TokLoc,
101	const char *TokBegin, const char *TokRangeBegin,
102	const char *TokRangeEnd, unsigned DiagID) {
103	SourceLocation Begin =
104	Lexer::AdvanceToTokenCharacter(TokStart: TokLoc, Characters: TokRangeBegin - TokBegin,
105	SM: TokLoc.getManager(), LangOpts: Features);
106	return Diags->Report(Loc: Begin, DiagID) <<
107	MakeCharSourceRange(Features, TokLoc, TokBegin, TokRangeBegin, TokRangeEnd);
108	}
109
110	static bool IsEscapeValidInUnevaluatedStringLiteral(char Escape) {
111	switch (Escape) {
112	case '\'':
113	case '"':
114	case '?':
115	case '\\':
116	case 'a':
117	case 'b':
118	case 'f':
119	case 'n':
120	case 'r':
121	case 't':
122	case 'v':
123	return true;
124	}
125	return false;
126	}
127
128	/// ProcessCharEscape - Parse a standard C escape sequence, which can occur in
129	/// either a character or a string literal.
130	static unsigned ProcessCharEscape(const char *ThisTokBegin,
131	const char *&ThisTokBuf,
132	const char *ThisTokEnd, bool &HadError,
133	FullSourceLoc Loc, unsigned CharWidth,
134	DiagnosticsEngine *Diags,
135	const LangOptions &Features,
136	StringLiteralEvalMethod EvalMethod) {
137	const char *EscapeBegin = ThisTokBuf;
138	bool Delimited = false;
139	bool EndDelimiterFound = false;
140
141	// Skip the '\' char.
142	++ThisTokBuf;
143
144	// We know that this character can't be off the end of the buffer, because
145	// that would have been \", which would not have been the end of string.
146	unsigned ResultChar = *ThisTokBuf++;
147	char Escape = ResultChar;
148	switch (ResultChar) {
149	// These map to themselves.
150	case '\\': case '\'': case '"': case '?': break;
151
152	// These have fixed mappings.
153	case 'a':
154	// TODO: K&R: the meaning of '\\a' is different in traditional C
155	ResultChar = 7;
156	break;
157	case 'b':
158	ResultChar = 8;
159	break;
160	case 'e':
161	if (Diags)
162	Diag(Diags, Features, Loc, ThisTokBegin, EscapeBegin, ThisTokBuf,
163	diag::ext_nonstandard_escape) << "e";
164	ResultChar = 27;
165	break;
166	case 'E':
167	if (Diags)
168	Diag(Diags, Features, Loc, ThisTokBegin, EscapeBegin, ThisTokBuf,
169	diag::ext_nonstandard_escape) << "E";
170	ResultChar = 27;
171	break;
172	case 'f':
173	ResultChar = 12;
174	break;
175	case 'n':
176	ResultChar = 10;
177	break;
178	case 'r':
179	ResultChar = 13;
180	break;
181	case 't':
182	ResultChar = 9;
183	break;
184	case 'v':
185	ResultChar = 11;
186	break;
187	case 'x': { // Hex escape.
188	ResultChar = 0;
189	if (ThisTokBuf != ThisTokEnd && *ThisTokBuf == '{') {
190	Delimited = true;
191	ThisTokBuf++;
192	if (*ThisTokBuf == '}') {
193	Diag(Diags, Features, Loc, ThisTokBegin, EscapeBegin, ThisTokBuf,
194	diag::err_delimited_escape_empty);
195	return ResultChar;
196	}
197	} else if (ThisTokBuf == ThisTokEnd || !isHexDigit(c: *ThisTokBuf)) {
198	if (Diags)
199	Diag(Diags, Features, Loc, ThisTokBegin, EscapeBegin, ThisTokBuf,
200	diag::err_hex_escape_no_digits) << "x";
201	return ResultChar;
202	}
203
204	// Hex escapes are a maximal series of hex digits.
205	bool Overflow = false;
206	for (; ThisTokBuf != ThisTokEnd; ++ThisTokBuf) {
207	if (Delimited && *ThisTokBuf == '}') {
208	ThisTokBuf++;
209	EndDelimiterFound = true;
210	break;
211	}
212	int CharVal = llvm::hexDigitValue(C: *ThisTokBuf);
213	if (CharVal == -1) {
214	// Non delimited hex escape sequences stop at the first non-hex digit.
215	if (!Delimited)
216	break;
217	HadError = true;
218	if (Diags)
219	Diag(Diags, Features, Loc, ThisTokBegin, EscapeBegin, ThisTokBuf,
220	diag::err_delimited_escape_invalid)
221	<< StringRef(ThisTokBuf, 1);
222	continue;
223	}
224	// About to shift out a digit?
225	if (ResultChar & 0xF0000000)
226	Overflow = true;
227	ResultChar <<= 4;
228	ResultChar |= CharVal;
229	}
230	// See if any bits will be truncated when evaluated as a character.
231	if (CharWidth != 32 && (ResultChar >> CharWidth) != 0) {
232	Overflow = true;
233	ResultChar &= ~0U >> (32-CharWidth);
234	}
235
236	// Check for overflow.
237	if (!HadError && Overflow) { // Too many digits to fit in
238	HadError = true;
239	if (Diags)
240	Diag(Diags, Features, Loc, ThisTokBegin, EscapeBegin, ThisTokBuf,
241	diag::err_escape_too_large)
242	<< 0;
243	}
244	break;
245	}
246	case '0': case '1': case '2': case '3':
247	case '4': case '5': case '6': case '7': {
248	// Octal escapes.
249	--ThisTokBuf;
250	ResultChar = 0;
251
252	// Octal escapes are a series of octal digits with maximum length 3.
253	// "\0123" is a two digit sequence equal to "\012" "3".
254	unsigned NumDigits = 0;
255	do {
256	ResultChar <<= 3;
257	ResultChar |= *ThisTokBuf++ - '0';
258	++NumDigits;
259	} while (ThisTokBuf != ThisTokEnd && NumDigits < 3 &&
260	ThisTokBuf[0] >= '0' && ThisTokBuf[0] <= '7');
261
262	// Check for overflow. Reject '\777', but not L'\777'.
263	if (CharWidth != 32 && (ResultChar >> CharWidth) != 0) {
264	if (Diags)
265	Diag(Diags, Features, Loc, ThisTokBegin, EscapeBegin, ThisTokBuf,
266	diag::err_escape_too_large) << 1;
267	ResultChar &= ~0U >> (32-CharWidth);
268	}
269	break;
270	}
271	case 'o': {
272	bool Overflow = false;
273	if (ThisTokBuf == ThisTokEnd || *ThisTokBuf != '{') {
274	HadError = true;
275	if (Diags)
276	Diag(Diags, Features, Loc, ThisTokBegin, EscapeBegin, ThisTokBuf,
277	diag::err_delimited_escape_missing_brace)
278	<< "o";
279
280	break;
281	}
282	ResultChar = 0;
283	Delimited = true;
284	++ThisTokBuf;
285	if (*ThisTokBuf == '}') {
286	Diag(Diags, Features, Loc, ThisTokBegin, EscapeBegin, ThisTokBuf,
287	diag::err_delimited_escape_empty);
288	return ResultChar;
289	}
290
291	while (ThisTokBuf != ThisTokEnd) {
292	if (*ThisTokBuf == '}') {
293	EndDelimiterFound = true;
294	ThisTokBuf++;
295	break;
296	}
297	if (*ThisTokBuf < '0' || *ThisTokBuf > '7') {
298	HadError = true;
299	if (Diags)
300	Diag(Diags, Features, Loc, ThisTokBegin, EscapeBegin, ThisTokBuf,
301	diag::err_delimited_escape_invalid)
302	<< StringRef(ThisTokBuf, 1);
303	ThisTokBuf++;
304	continue;
305	}
306	// Check if one of the top three bits is set before shifting them out.
307	if (ResultChar & 0xE0000000)
308	Overflow = true;
309
310	ResultChar <<= 3;
311	ResultChar |= *ThisTokBuf++ - '0';
312	}
313	// Check for overflow. Reject '\777', but not L'\777'.
314	if (!HadError &&
315	(Overflow || (CharWidth != 32 && (ResultChar >> CharWidth) != 0))) {
316	HadError = true;
317	if (Diags)
318	Diag(Diags, Features, Loc, ThisTokBegin, EscapeBegin, ThisTokBuf,
319	diag::err_escape_too_large)
320	<< 1;
321	ResultChar &= ~0U >> (32 - CharWidth);
322	}
323	break;
324	}
325	// Otherwise, these are not valid escapes.
326	case '(': case '{': case '[': case '%':
327	// GCC accepts these as extensions. We warn about them as such though.
328	if (Diags)
329	Diag(Diags, Features, Loc, ThisTokBegin, EscapeBegin, ThisTokBuf,
330	diag::ext_nonstandard_escape)
331	<< std::string(1, ResultChar);
332	break;
333	default:
334	if (!Diags)
335	break;
336
337	if (isPrintable(c: ResultChar))
338	Diag(Diags, Features, Loc, ThisTokBegin, EscapeBegin, ThisTokBuf,
339	diag::ext_unknown_escape)
340	<< std::string(1, ResultChar);
341	else
342	Diag(Diags, Features, Loc, ThisTokBegin, EscapeBegin, ThisTokBuf,
343	diag::ext_unknown_escape)
344	<< "x" + llvm::utohexstr(X: ResultChar);
345	break;
346	}
347
348	if (Delimited && Diags) {
349	if (!EndDelimiterFound)
350	Diag(Diags, Features, Loc, ThisTokBegin, EscapeBegin, ThisTokBuf,
351	diag::err_expected)
352	<< tok::r_brace;
353	else if (!HadError) {
354	Diag(Diags, Features, Loc, ThisTokBegin, EscapeBegin, ThisTokBuf,
355	Features.CPlusPlus23 ? diag::warn_cxx23_delimited_escape_sequence
356	: diag::ext_delimited_escape_sequence)
357	<< /*delimited*/ 0 << (Features.CPlusPlus ? 1 : 0);
358	}
359	}
360
361	if (EvalMethod == StringLiteralEvalMethod::Unevaluated &&
362	!IsEscapeValidInUnevaluatedStringLiteral(Escape)) {
363	Diag(Diags, Features, Loc, ThisTokBegin, EscapeBegin, ThisTokBuf,
364	diag::err_unevaluated_string_invalid_escape_sequence)
365	<< StringRef(EscapeBegin, ThisTokBuf - EscapeBegin);
366	HadError = true;
367	}
368
369	return ResultChar;
370	}
371
372	static void appendCodePoint(unsigned Codepoint,
373	llvm::SmallVectorImpl<char> &Str) {
374	char ResultBuf[4];
375	char *ResultPtr = ResultBuf;
376	if (llvm::ConvertCodePointToUTF8(Source: Codepoint, ResultPtr))
377	Str.append(in_start: ResultBuf, in_end: ResultPtr);
378	}
379
380	void clang::expandUCNs(SmallVectorImpl<char> &Buf, StringRef Input) {
381	for (StringRef::iterator I = Input.begin(), E = Input.end(); I != E; ++I) {
382	if (*I != '\\') {
383	Buf.push_back(Elt: *I);
384	continue;
385	}
386
387	++I;
388	char Kind = *I;
389	++I;
390
391	assert(Kind == 'u' || Kind == 'U' || Kind == 'N');
392	uint32_t CodePoint = 0;
393
394	if (Kind == 'u' && *I == '{') {
395	for (++I; *I != '}'; ++I) {
396	unsigned Value = llvm::hexDigitValue(C: *I);
397	assert(Value != -1U);
398	CodePoint <<= 4;
399	CodePoint += Value;
400	}
401	appendCodePoint(Codepoint: CodePoint, Str&: Buf);
402	continue;
403	}
404
405	if (Kind == 'N') {
406	assert(*I == '{');
407	++I;
408	auto Delim = std::find(first: I, last: Input.end(), val: '}');
409	assert(Delim != Input.end());
410	StringRef Name(I, std::distance(first: I, last: Delim));
411	std::optional<llvm::sys::unicode::LooseMatchingResult> Res =
412	llvm::sys::unicode::nameToCodepointLooseMatching(Name);
413	assert(Res && "could not find a codepoint that was previously found");
414	CodePoint = Res->CodePoint;
415	assert(CodePoint != 0xFFFFFFFF);
416	appendCodePoint(Codepoint: CodePoint, Str&: Buf);
417	I = Delim;
418	continue;
419	}
420
421	unsigned NumHexDigits;
422	if (Kind == 'u')
423	NumHexDigits = 4;
424	else
425	NumHexDigits = 8;
426
427	assert(I + NumHexDigits <= E);
428
429	for (; NumHexDigits != 0; ++I, --NumHexDigits) {
430	unsigned Value = llvm::hexDigitValue(C: *I);
431	assert(Value != -1U);
432
433	CodePoint <<= 4;
434	CodePoint += Value;
435	}
436
437	appendCodePoint(Codepoint: CodePoint, Str&: Buf);
438	--I;
439	}
440	}
441
442	bool clang::isFunctionLocalStringLiteralMacro(tok::TokenKind K,
443	const LangOptions &LO) {
444	return LO.MicrosoftExt &&
445	(K == tok::kw___FUNCTION__ || K == tok::kw_L__FUNCTION__ ||
446	K == tok::kw___FUNCSIG__ || K == tok::kw_L__FUNCSIG__ ||
447	K == tok::kw___FUNCDNAME__);
448	}
449
450	bool clang::tokenIsLikeStringLiteral(const Token &Tok, const LangOptions &LO) {
451	return tok::isStringLiteral(K: Tok.getKind()) ||
452	isFunctionLocalStringLiteralMacro(K: Tok.getKind(), LO);
453	}
454
455	static bool ProcessNumericUCNEscape(const char *ThisTokBegin,
456	const char *&ThisTokBuf,
457	const char *ThisTokEnd, uint32_t &UcnVal,
458	unsigned short &UcnLen, bool &Delimited,
459	FullSourceLoc Loc, DiagnosticsEngine *Diags,
460	const LangOptions &Features,
461	bool in_char_string_literal = false) {
462	const char *UcnBegin = ThisTokBuf;
463	bool HasError = false;
464	bool EndDelimiterFound = false;
465
466	// Skip the '\u' char's.
467	ThisTokBuf += 2;
468	Delimited = false;
469	if (UcnBegin[1] == 'u' && in_char_string_literal &&
470	ThisTokBuf != ThisTokEnd && *ThisTokBuf == '{') {
471	Delimited = true;
472	ThisTokBuf++;
473	} else if (ThisTokBuf == ThisTokEnd || !isHexDigit(c: *ThisTokBuf)) {
474	if (Diags)
475	Diag(Diags, Features, Loc, ThisTokBegin, UcnBegin, ThisTokBuf,
476	diag::err_hex_escape_no_digits)
477	<< StringRef(&ThisTokBuf[-1], 1);
478	return false;
479	}
480	UcnLen = (ThisTokBuf[-1] == 'u' ? 4 : 8);
481
482	bool Overflow = false;
483	unsigned short Count = 0;
484	for (; ThisTokBuf != ThisTokEnd && (Delimited || Count != UcnLen);
485	++ThisTokBuf) {
486	if (Delimited && *ThisTokBuf == '}') {
487	++ThisTokBuf;
488	EndDelimiterFound = true;
489	break;
490	}
491	int CharVal = llvm::hexDigitValue(C: *ThisTokBuf);
492	if (CharVal == -1) {
493	HasError = true;
494	if (!Delimited)
495	break;
496	if (Diags) {
497	Diag(Diags, Features, Loc, ThisTokBegin, UcnBegin, ThisTokBuf,
498	diag::err_delimited_escape_invalid)
499	<< StringRef(ThisTokBuf, 1);
500	}
501	Count++;
502	continue;
503	}
504	if (UcnVal & 0xF0000000) {
505	Overflow = true;
506	continue;
507	}
508	UcnVal <<= 4;
509	UcnVal |= CharVal;
510	Count++;
511	}
512
513	if (Overflow) {
514	if (Diags)
515	Diag(Diags, Features, Loc, ThisTokBegin, UcnBegin, ThisTokBuf,
516	diag::err_escape_too_large)
517	<< 0;
518	return false;
519	}
520
521	if (Delimited && !EndDelimiterFound) {
522	if (Diags) {
523	Diag(Diags, Features, Loc, ThisTokBegin, UcnBegin, ThisTokBuf,
524	diag::err_expected)
525	<< tok::r_brace;
526	}
527	return false;
528	}
529
530	// If we didn't consume the proper number of digits, there is a problem.
531	if (Count == 0 || (!Delimited && Count != UcnLen)) {
532	if (Diags)
533	Diag(Diags, Features, Loc, ThisTokBegin, UcnBegin, ThisTokBuf,
534	Delimited ? diag::err_delimited_escape_empty
535	: diag::err_ucn_escape_incomplete);
536	return false;
537	}
538	return !HasError;
539	}
540
541	static void DiagnoseInvalidUnicodeCharacterName(
542	DiagnosticsEngine *Diags, const LangOptions &Features, FullSourceLoc Loc,
543	const char *TokBegin, const char *TokRangeBegin, const char *TokRangeEnd,
544	llvm::StringRef Name) {
545
546	Diag(Diags, Features, Loc, TokBegin, TokRangeBegin, TokRangeEnd,
547	diag::err_invalid_ucn_name)
548	<< Name;
549
550	namespace u = llvm::sys::unicode;
551
552	std::optional<u::LooseMatchingResult> Res =
553	u::nameToCodepointLooseMatching(Name);
554	if (Res) {
555	Diag(Diags, Features, Loc, TokBegin, TokRangeBegin, TokRangeEnd,
556	diag::note_invalid_ucn_name_loose_matching)
557	<< FixItHint::CreateReplacement(
558	RemoveRange: MakeCharSourceRange(Features, TokLoc: Loc, TokBegin, TokRangeBegin,
559	TokRangeEnd),
560	Code: Res->Name);
561	return;
562	}
563
564	unsigned Distance = 0;
565	SmallVector<u::MatchForCodepointName> Matches =
566	u::nearestMatchesForCodepointName(Pattern: Name, MaxMatchesCount: 5);
567	assert(!Matches.empty() && "No unicode characters found");
568
569	for (const auto &Match : Matches) {
570	if (Distance == 0)
571	Distance = Match.Distance;
572	if (std::max(a: Distance, b: Match.Distance) -
573	std::min(a: Distance, b: Match.Distance) >
574	3)
575	break;
576	Distance = Match.Distance;
577
578	std::string Str;
579	llvm::UTF32 V = Match.Value;
580	bool Converted =
581	llvm::convertUTF32ToUTF8String(Src: llvm::ArrayRef<llvm::UTF32>(&V, 1), Out&: Str);
582	(void)Converted;
583	assert(Converted && "Found a match wich is not a unicode character");
584
585	Diag(Diags, Features, Loc, TokBegin, TokRangeBegin, TokRangeEnd,
586	diag::note_invalid_ucn_name_candidate)
587	<< Match.Name << llvm::utohexstr(X: Match.Value)
588	<< Str // FIXME: Fix the rendering of non printable characters
589	<< FixItHint::CreateReplacement(
590	RemoveRange: MakeCharSourceRange(Features, TokLoc: Loc, TokBegin, TokRangeBegin,
591	TokRangeEnd),
592	Code: Match.Name);
593	}
594	}
595
596	static bool ProcessNamedUCNEscape(const char *ThisTokBegin,
597	const char *&ThisTokBuf,
598	const char *ThisTokEnd, uint32_t &UcnVal,
599	unsigned short &UcnLen, FullSourceLoc Loc,
600	DiagnosticsEngine *Diags,
601	const LangOptions &Features) {
602	const char *UcnBegin = ThisTokBuf;
603	assert(UcnBegin[0] == '\\' && UcnBegin[1] == 'N');
604	ThisTokBuf += 2;
605	if (ThisTokBuf == ThisTokEnd || *ThisTokBuf != '{') {
606	if (Diags) {
607	Diag(Diags, Features, Loc, ThisTokBegin, UcnBegin, ThisTokBuf,
608	diag::err_delimited_escape_missing_brace)
609	<< StringRef(&ThisTokBuf[-1], 1);
610	}
611	return false;
612	}
613	ThisTokBuf++;
614	const char *ClosingBrace = std::find_if(first: ThisTokBuf, last: ThisTokEnd, pred: [](char C) {
615	return C == '}' || isVerticalWhitespace(c: C);
616	});
617	bool Incomplete = ClosingBrace == ThisTokEnd;
618	bool Empty = ClosingBrace == ThisTokBuf;
619	if (Incomplete || Empty) {
620	if (Diags) {
621	Diag(Diags, Features, Loc, ThisTokBegin, UcnBegin, ThisTokBuf,
622	Incomplete ? diag::err_ucn_escape_incomplete
623	: diag::err_delimited_escape_empty)
624	<< StringRef(&UcnBegin[1], 1);
625	}
626	ThisTokBuf = ClosingBrace == ThisTokEnd ? ClosingBrace : ClosingBrace + 1;
627	return false;
628	}
629	StringRef Name(ThisTokBuf, ClosingBrace - ThisTokBuf);
630	ThisTokBuf = ClosingBrace + 1;
631	std::optional<char32_t> Res = llvm::sys::unicode::nameToCodepointStrict(Name);
632	if (!Res) {
633	if (Diags)
634	DiagnoseInvalidUnicodeCharacterName(Diags, Features, Loc, TokBegin: ThisTokBegin,
635	TokRangeBegin: &UcnBegin[3], TokRangeEnd: ClosingBrace, Name);
636	return false;
637	}
638	UcnVal = *Res;
639	UcnLen = UcnVal > 0xFFFF ? 8 : 4;
640	return true;
641	}
642
643	/// ProcessUCNEscape - Read the Universal Character Name, check constraints and
644	/// return the UTF32.
645	static bool ProcessUCNEscape(const char *ThisTokBegin, const char *&ThisTokBuf,
646	const char *ThisTokEnd, uint32_t &UcnVal,
647	unsigned short &UcnLen, FullSourceLoc Loc,
648	DiagnosticsEngine *Diags,
649	const LangOptions &Features,
650	bool in_char_string_literal = false) {
651
652	bool HasError;
653	const char *UcnBegin = ThisTokBuf;
654	bool IsDelimitedEscapeSequence = false;
655	bool IsNamedEscapeSequence = false;
656	if (ThisTokBuf[1] == 'N') {
657	IsNamedEscapeSequence = true;
658	HasError = !ProcessNamedUCNEscape(ThisTokBegin, ThisTokBuf, ThisTokEnd,
659	UcnVal, UcnLen, Loc, Diags, Features);
660	} else {
661	HasError =
662	!ProcessNumericUCNEscape(ThisTokBegin, ThisTokBuf, ThisTokEnd, UcnVal,
663	UcnLen, Delimited&: IsDelimitedEscapeSequence, Loc, Diags,
664	Features, in_char_string_literal);
665	}
666	if (HasError)
667	return false;
668
669	// Check UCN constraints (C99 6.4.3p2) [C++11 lex.charset p2]
670	if ((0xD800 <= UcnVal && UcnVal <= 0xDFFF) || // surrogate codepoints
671	UcnVal > 0x10FFFF) { // maximum legal UTF32 value
672	if (Diags)
673	Diag(Diags, Features, Loc, ThisTokBegin, UcnBegin, ThisTokBuf,
674	diag::err_ucn_escape_invalid);
675	return false;
676	}
677
678	// C23 and C++11 allow UCNs that refer to control characters
679	// and basic source characters inside character and string literals
680	if (UcnVal < 0xa0 &&
681	// $, @, ` are allowed in all language modes
682	(UcnVal != 0x24 && UcnVal != 0x40 && UcnVal != 0x60)) {
683	bool IsError =
684	(!(Features.CPlusPlus11 || Features.C23) || !in_char_string_literal);
685	if (Diags) {
686	char BasicSCSChar = UcnVal;
687	if (UcnVal >= 0x20 && UcnVal < 0x7f)
688	Diag(Diags, Features, Loc, ThisTokBegin, UcnBegin, ThisTokBuf,
689	IsError ? diag::err_ucn_escape_basic_scs
690	: Features.CPlusPlus
691	? diag::warn_cxx98_compat_literal_ucn_escape_basic_scs
692	: diag::warn_c23_compat_literal_ucn_escape_basic_scs)
693	<< StringRef(&BasicSCSChar, 1);
694	else
695	Diag(Diags, Features, Loc, ThisTokBegin, UcnBegin, ThisTokBuf,
696	IsError ? diag::err_ucn_control_character
697	: Features.CPlusPlus
698	? diag::warn_cxx98_compat_literal_ucn_control_character
699	: diag::warn_c23_compat_literal_ucn_control_character);
700	}
701	if (IsError)
702	return false;
703	}
704
705	if (!Features.CPlusPlus && !Features.C99 && Diags)
706	Diag(Diags, Features, Loc, ThisTokBegin, UcnBegin, ThisTokBuf,
707	diag::warn_ucn_not_valid_in_c89_literal);
708
709	if ((IsDelimitedEscapeSequence || IsNamedEscapeSequence) && Diags)
710	Diag(Diags, Features, Loc, ThisTokBegin, UcnBegin, ThisTokBuf,
711	Features.CPlusPlus23 ? diag::warn_cxx23_delimited_escape_sequence
712	: diag::ext_delimited_escape_sequence)
713	<< (IsNamedEscapeSequence ? 1 : 0) << (Features.CPlusPlus ? 1 : 0);
714
715	return true;
716	}
717
718	/// MeasureUCNEscape - Determine the number of bytes within the resulting string
719	/// which this UCN will occupy.
720	static int MeasureUCNEscape(const char *ThisTokBegin, const char *&ThisTokBuf,
721	const char *ThisTokEnd, unsigned CharByteWidth,
722	const LangOptions &Features, bool &HadError) {
723	// UTF-32: 4 bytes per escape.
724	if (CharByteWidth == 4)
725	return 4;
726
727	uint32_t UcnVal = 0;
728	unsigned short UcnLen = 0;
729	FullSourceLoc Loc;
730
731	if (!ProcessUCNEscape(ThisTokBegin, ThisTokBuf, ThisTokEnd, UcnVal,
732	UcnLen, Loc, Diags: nullptr, Features, in_char_string_literal: true)) {
733	HadError = true;
734	return 0;
735	}
736
737	// UTF-16: 2 bytes for BMP, 4 bytes otherwise.
738	if (CharByteWidth == 2)
739	return UcnVal <= 0xFFFF ? 2 : 4;
740
741	// UTF-8.
742	if (UcnVal < 0x80)
743	return 1;
744	if (UcnVal < 0x800)
745	return 2;
746	if (UcnVal < 0x10000)
747	return 3;
748	return 4;
749	}
750
751	/// EncodeUCNEscape - Read the Universal Character Name, check constraints and
752	/// convert the UTF32 to UTF8 or UTF16. This is a subroutine of
753	/// StringLiteralParser. When we decide to implement UCN's for identifiers,
754	/// we will likely rework our support for UCN's.
755	static void EncodeUCNEscape(const char *ThisTokBegin, const char *&ThisTokBuf,
756	const char *ThisTokEnd,
757	char *&ResultBuf, bool &HadError,
758	FullSourceLoc Loc, unsigned CharByteWidth,
759	DiagnosticsEngine *Diags,
760	const LangOptions &Features) {
761	typedef uint32_t UTF32;
762	UTF32 UcnVal = 0;
763	unsigned short UcnLen = 0;
764	if (!ProcessUCNEscape(ThisTokBegin, ThisTokBuf, ThisTokEnd, UcnVal, UcnLen,
765	Loc, Diags, Features, in_char_string_literal: true)) {
766	HadError = true;
767	return;
768	}
769
770	assert((CharByteWidth == 1 || CharByteWidth == 2 || CharByteWidth == 4) &&
771	"only character widths of 1, 2, or 4 bytes supported");
772
773	(void)UcnLen;
774	assert((UcnLen== 4 || UcnLen== 8) && "only ucn length of 4 or 8 supported");
775
776	if (CharByteWidth == 4) {
777	// FIXME: Make the type of the result buffer correct instead of
778	// using reinterpret_cast.
779	llvm::UTF32 *ResultPtr = reinterpret_cast<llvm::UTF32*>(ResultBuf);
780	*ResultPtr = UcnVal;
781	ResultBuf += 4;
782	return;
783	}
784
785	if (CharByteWidth == 2) {
786	// FIXME: Make the type of the result buffer correct instead of
787	// using reinterpret_cast.
788	llvm::UTF16 *ResultPtr = reinterpret_cast<llvm::UTF16*>(ResultBuf);
789
790	if (UcnVal <= (UTF32)0xFFFF) {
791	*ResultPtr = UcnVal;
792	ResultBuf += 2;
793	return;
794	}
795
796	// Convert to UTF16.
797	UcnVal -= 0x10000;
798	*ResultPtr = 0xD800 + (UcnVal >> 10);
799	*(ResultPtr+1) = 0xDC00 + (UcnVal & 0x3FF);
800	ResultBuf += 4;
801	return;
802	}
803
804	assert(CharByteWidth == 1 && "UTF-8 encoding is only for 1 byte characters");
805
806	// Now that we've parsed/checked the UCN, we convert from UTF32->UTF8.
807	// The conversion below was inspired by:
808	// http://www.unicode.org/Public/PROGRAMS/CVTUTF/ConvertUTF.c
809	// First, we determine how many bytes the result will require.
810	typedef uint8_t UTF8;
811
812	unsigned short bytesToWrite = 0;
813	if (UcnVal < (UTF32)0x80)
814	bytesToWrite = 1;
815	else if (UcnVal < (UTF32)0x800)
816	bytesToWrite = 2;
817	else if (UcnVal < (UTF32)0x10000)
818	bytesToWrite = 3;
819	else
820	bytesToWrite = 4;
821
822	const unsigned byteMask = 0xBF;
823	const unsigned byteMark = 0x80;
824
825	// Once the bits are split out into bytes of UTF8, this is a mask OR-ed
826	// into the first byte, depending on how many bytes follow.
827	static const UTF8 firstByteMark[5] = {
828	0x00, 0x00, 0xC0, 0xE0, 0xF0
829	};
830	// Finally, we write the bytes into ResultBuf.
831	ResultBuf += bytesToWrite;
832	switch (bytesToWrite) { // note: everything falls through.
833	case 4:
834	*--ResultBuf = (UTF8)((UcnVal | byteMark) & byteMask); UcnVal >>= 6;
835	[[fallthrough]];
836	case 3:
837	*--ResultBuf = (UTF8)((UcnVal | byteMark) & byteMask); UcnVal >>= 6;
838	[[fallthrough]];
839	case 2:
840	*--ResultBuf = (UTF8)((UcnVal | byteMark) & byteMask); UcnVal >>= 6;
841	[[fallthrough]];
842	case 1:
843	*--ResultBuf = (UTF8) (UcnVal | firstByteMark[bytesToWrite]);
844	}
845	// Update the buffer.
846	ResultBuf += bytesToWrite;
847	}
848
849	/// integer-constant: [C99 6.4.4.1]
850	/// decimal-constant integer-suffix
851	/// octal-constant integer-suffix
852	/// hexadecimal-constant integer-suffix
853	/// binary-literal integer-suffix [GNU, C++1y]
854	/// user-defined-integer-literal: [C++11 lex.ext]
855	/// decimal-literal ud-suffix
856	/// octal-literal ud-suffix
857	/// hexadecimal-literal ud-suffix
858	/// binary-literal ud-suffix [GNU, C++1y]
859	/// decimal-constant:
860	/// nonzero-digit
861	/// decimal-constant digit
862	/// octal-constant:
863	/// 0
864	/// octal-constant octal-digit
865	/// hexadecimal-constant:
866	/// hexadecimal-prefix hexadecimal-digit
867	/// hexadecimal-constant hexadecimal-digit
868	/// hexadecimal-prefix: one of
869	/// 0x 0X
870	/// binary-literal:
871	/// 0b binary-digit
872	/// 0B binary-digit
873	/// binary-literal binary-digit
874	/// integer-suffix:
875	/// unsigned-suffix [long-suffix]
876	/// unsigned-suffix [long-long-suffix]
877	/// long-suffix [unsigned-suffix]
878	/// long-long-suffix [unsigned-sufix]
879	/// nonzero-digit:
880	/// 1 2 3 4 5 6 7 8 9
881	/// octal-digit:
882	/// 0 1 2 3 4 5 6 7
883	/// hexadecimal-digit:
884	/// 0 1 2 3 4 5 6 7 8 9
885	/// a b c d e f
886	/// A B C D E F
887	/// binary-digit:
888	/// 0
889	/// 1
890	/// unsigned-suffix: one of
891	/// u U
892	/// long-suffix: one of
893	/// l L
894	/// long-long-suffix: one of
895	/// ll LL
896	///
897	/// floating-constant: [C99 6.4.4.2]
898	/// TODO: add rules...
899	///
900	NumericLiteralParser::NumericLiteralParser(StringRef TokSpelling,
901	SourceLocation TokLoc,
902	const SourceManager &SM,
903	const LangOptions &LangOpts,
904	const TargetInfo &Target,
905	DiagnosticsEngine &Diags)
906	: SM(SM), LangOpts(LangOpts), Diags(Diags),
907	ThisTokBegin(TokSpelling.begin()), ThisTokEnd(TokSpelling.end()) {
908
909	s = DigitsBegin = ThisTokBegin;
910	saw_exponent = false;
911	saw_period = false;
912	saw_ud_suffix = false;
913	saw_fixed_point_suffix = false;
914	isLong = false;
915	isUnsigned = false;
916	isLongLong = false;
917	isSizeT = false;
918	isHalf = false;
919	isFloat = false;
920	isImaginary = false;
921	isFloat16 = false;
922	isFloat128 = false;
923	MicrosoftInteger = 0;
924	isFract = false;
925	isAccum = false;
926	hadError = false;
927	isBitInt = false;
928
929	// This routine assumes that the range begin/end matches the regex for integer
930	// and FP constants (specifically, the 'pp-number' regex), and assumes that
931	// the byte at "*end" is both valid and not part of the regex. Because of
932	// this, it doesn't have to check for 'overscan' in various places.
933	// Note: For HLSL, the end token is allowed to be '.' which would be in the
934	// 'pp-number' regex. This is required to support vector swizzles on numeric
935	// constants (i.e. 1.xx or 1.5f.rrr).
936	if (isPreprocessingNumberBody(c: *ThisTokEnd) &&
937	!(LangOpts.HLSL && *ThisTokEnd == '.')) {
938	Diags.Report(TokLoc, diag::err_lexing_numeric);
939	hadError = true;
940	return;
941	}
942
943	if (*s == '0') { // parse radix
944	ParseNumberStartingWithZero(TokLoc);
945	if (hadError)
946	return;
947	} else { // the first digit is non-zero
948	radix = 10;
949	s = SkipDigits(ptr: s);
950	if (s == ThisTokEnd) {
951	// Done.
952	} else {
953	ParseDecimalOrOctalCommon(TokLoc);
954	if (hadError)
955	return;
956	}
957	}
958
959	SuffixBegin = s;
960	checkSeparator(TokLoc, Pos: s, IsAfterDigits: CSK_AfterDigits);
961
962	// Initial scan to lookahead for fixed point suffix.
963	if (LangOpts.FixedPoint) {
964	for (const char *c = s; c != ThisTokEnd; ++c) {
965	if (*c == 'r' || *c == 'k' || *c == 'R' || *c == 'K') {
966	saw_fixed_point_suffix = true;
967	break;
968	}
969	}
970	}
971
972	// Parse the suffix. At this point we can classify whether we have an FP or
973	// integer constant.
974	bool isFixedPointConstant = isFixedPointLiteral();
975	bool isFPConstant = isFloatingLiteral();
976	bool HasSize = false;
977	bool DoubleUnderscore = false;
978
979	// Loop over all of the characters of the suffix. If we see something bad,
980	// we break out of the loop.
981	for (; s != ThisTokEnd; ++s) {
982	switch (*s) {
983	case 'R':
984	case 'r':
985	if (!LangOpts.FixedPoint)
986	break;
987	if (isFract || isAccum) break;
988	if (!(saw_period || saw_exponent)) break;
989	isFract = true;
990	continue;
991	case 'K':
992	case 'k':
993	if (!LangOpts.FixedPoint)
994	break;
995	if (isFract || isAccum) break;
996	if (!(saw_period || saw_exponent)) break;
997	isAccum = true;
998	continue;
999	case 'h': // FP Suffix for "half".
1000	case 'H':
1001	// OpenCL Extension v1.2 s9.5 - h or H suffix for half type.
1002	if (!(LangOpts.Half || LangOpts.FixedPoint))
1003	break;
1004	if (isIntegerLiteral()) break; // Error for integer constant.
1005	if (HasSize)
1006	break;
1007	HasSize = true;
1008	isHalf = true;
1009	continue; // Success.
1010	case 'f': // FP Suffix for "float"
1011	case 'F':
1012	if (!isFPConstant) break; // Error for integer constant.
1013	if (HasSize)
1014	break;
1015	HasSize = true;
1016
1017	// CUDA host and device may have different _Float16 support, therefore
1018	// allows f16 literals to avoid false alarm.
1019	// When we compile for OpenMP target offloading on NVPTX, f16 suffix
1020	// should also be supported.
1021	// ToDo: more precise check for CUDA.
1022	// TODO: AMDGPU might also support it in the future.
1023	if ((Target.hasFloat16Type() || LangOpts.CUDA ||
1024	(LangOpts.OpenMPIsTargetDevice && Target.getTriple().isNVPTX())) &&
1025	s + 2 < ThisTokEnd && s[1] == '1' && s[2] == '6') {
1026	s += 2; // success, eat up 2 characters.
1027	isFloat16 = true;
1028	continue;
1029	}
1030
1031	isFloat = true;
1032	continue; // Success.
1033	case 'q': // FP Suffix for "__float128"
1034	case 'Q':
1035	if (!isFPConstant) break; // Error for integer constant.
1036	if (HasSize)
1037	break;
1038	HasSize = true;
1039	isFloat128 = true;
1040	continue; // Success.
1041	case 'u':
1042	case 'U':
1043	if (isFPConstant) break; // Error for floating constant.
1044	if (isUnsigned) break; // Cannot be repeated.
1045	isUnsigned = true;
1046	continue; // Success.
1047	case 'l':
1048	case 'L':
1049	if (HasSize)
1050	break;
1051	HasSize = true;
1052
1053	// Check for long long. The L's need to be adjacent and the same case.
1054	if (s[1] == s[0]) {
1055	assert(s + 1 < ThisTokEnd && "didn't maximally munch?");
1056	if (isFPConstant) break; // long long invalid for floats.
1057	isLongLong = true;
1058	++s; // Eat both of them.
1059	} else {
1060	isLong = true;
1061	}
1062	continue; // Success.
1063	case 'z':
1064	case 'Z':
1065	if (isFPConstant)
1066	break; // Invalid for floats.
1067	if (HasSize)
1068	break;
1069	HasSize = true;
1070	isSizeT = true;
1071	continue;
1072	case 'i':
1073	case 'I':
1074	if (LangOpts.MicrosoftExt && !isFPConstant) {
1075	// Allow i8, i16, i32, and i64. First, look ahead and check if
1076	// suffixes are Microsoft integers and not the imaginary unit.
1077	uint8_t Bits = 0;
1078	size_t ToSkip = 0;
1079	switch (s[1]) {
1080	case '8': // i8 suffix
1081	Bits = 8;
1082	ToSkip = 2;
1083	break;
1084	case '1':
1085	if (s[2] == '6') { // i16 suffix
1086	Bits = 16;
1087	ToSkip = 3;
1088	}
1089	break;
1090	case '3':
1091	if (s[2] == '2') { // i32 suffix
1092	Bits = 32;
1093	ToSkip = 3;
1094	}
1095	break;
1096	case '6':
1097	if (s[2] == '4') { // i64 suffix
1098	Bits = 64;
1099	ToSkip = 3;
1100	}
1101	break;
1102	default:
1103	break;
1104	}
1105	if (Bits) {
1106	if (HasSize)
1107	break;
1108	HasSize = true;
1109	MicrosoftInteger = Bits;
1110	s += ToSkip;
1111	assert(s <= ThisTokEnd && "didn't maximally munch?");
1112	break;
1113	}
1114	}
1115	[[fallthrough]];
1116	case 'j':
1117	case 'J':
1118	if (isImaginary) break; // Cannot be repeated.
1119	isImaginary = true;
1120	continue; // Success.
1121	case '_':
1122	if (isFPConstant)
1123	break; // Invalid for floats
1124	if (HasSize)
1125	break;
1126	if (DoubleUnderscore)
1127	break; // Cannot be repeated.
1128	if (LangOpts.CPlusPlus && s + 2 < ThisTokEnd &&
1129	s[1] == '_') { // s + 2 < ThisTokEnd to ensure some character exists
1130	// after __
1131	DoubleUnderscore = true;
1132	s += 2; // Skip both '_'
1133	if (s + 1 < ThisTokEnd &&
1134	(*s == 'u' || *s == 'U')) { // Ensure some character after 'u'/'U'
1135	isUnsigned = true;
1136	++s;
1137	}
1138	if (s + 1 < ThisTokEnd &&
1139	((*s == 'w' && *(++s) == 'b') || (*s == 'W' && *(++s) == 'B'))) {
1140	isBitInt = true;
1141	HasSize = true;
1142	continue;
1143	}
1144	}
1145	break;
1146	case 'w':
1147	case 'W':
1148	if (isFPConstant)
1149	break; // Invalid for floats.
1150	if (HasSize)
1151	break; // Invalid if we already have a size for the literal.
1152
1153	// wb and WB are allowed, but a mixture of cases like Wb or wB is not. We
1154	// explicitly do not support the suffix in C++ as an extension because a
1155	// library-based UDL that resolves to a library type may be more
1156	// appropriate there. The same rules apply for __wb/__WB.
1157	if ((!LangOpts.CPlusPlus || DoubleUnderscore) && s + 1 < ThisTokEnd &&
1158	((s[0] == 'w' && s[1] == 'b') || (s[0] == 'W' && s[1] == 'B'))) {
1159	isBitInt = true;
1160	HasSize = true;
1161	++s; // Skip both characters (2nd char skipped on continue).
1162	continue; // Success.
1163	}
1164	}
1165	// If we reached here, there was an error or a ud-suffix.
1166	break;
1167	}
1168
1169	// "i", "if", and "il" are user-defined suffixes in C++1y.
1170	if (s != ThisTokEnd || isImaginary) {
1171	// FIXME: Don't bother expanding UCNs if !tok.hasUCN().
1172	expandUCNs(Buf&: UDSuffixBuf, Input: StringRef(SuffixBegin, ThisTokEnd - SuffixBegin));
1173	if (isValidUDSuffix(LangOpts, Suffix: UDSuffixBuf)) {
1174	if (!isImaginary) {
1175	// Any suffix pieces we might have parsed are actually part of the
1176	// ud-suffix.
1177	isLong = false;
1178	isUnsigned = false;
1179	isLongLong = false;
1180	isSizeT = false;
1181	isFloat = false;
1182	isFloat16 = false;
1183	isHalf = false;
1184	isImaginary = false;
1185	isBitInt = false;
1186	MicrosoftInteger = 0;
1187	saw_fixed_point_suffix = false;
1188	isFract = false;
1189	isAccum = false;
1190	}
1191
1192	saw_ud_suffix = true;
1193	return;
1194	}
1195
1196	if (s != ThisTokEnd) {
1197	// Report an error if there are any.
1198	Diags.Report(Lexer::AdvanceToTokenCharacter(
1199	TokStart: TokLoc, Characters: SuffixBegin - ThisTokBegin, SM, LangOpts),
1200	diag::err_invalid_suffix_constant)
1201	<< StringRef(SuffixBegin, ThisTokEnd - SuffixBegin)
1202	<< (isFixedPointConstant ? 2 : isFPConstant);
1203	hadError = true;
1204	}
1205	}
1206
1207	if (!hadError && saw_fixed_point_suffix) {
1208	assert(isFract || isAccum);
1209	}
1210	}
1211
1212	/// ParseDecimalOrOctalCommon - This method is called for decimal or octal
1213	/// numbers. It issues an error for illegal digits, and handles floating point
1214	/// parsing. If it detects a floating point number, the radix is set to 10.
1215	void NumericLiteralParser::ParseDecimalOrOctalCommon(SourceLocation TokLoc){
1216	assert((radix == 8 || radix == 10) && "Unexpected radix");
1217
1218	// If we have a hex digit other than 'e' (which denotes a FP exponent) then
1219	// the code is using an incorrect base.
1220	if (isHexDigit(c: *s) && *s != 'e' && *s != 'E' &&
1221	!isValidUDSuffix(LangOpts, Suffix: StringRef(s, ThisTokEnd - s))) {
1222	Diags.Report(
1223	Lexer::AdvanceToTokenCharacter(TokStart: TokLoc, Characters: s - ThisTokBegin, SM, LangOpts),
1224	diag::err_invalid_digit)
1225	<< StringRef(s, 1) << (radix == 8 ? 1 : 0);
1226	hadError = true;
1227	return;
1228	}
1229
1230	if (*s == '.') {
1231	checkSeparator(TokLoc, Pos: s, IsAfterDigits: CSK_AfterDigits);
1232	s++;
1233	radix = 10;
1234	saw_period = true;
1235	checkSeparator(TokLoc, Pos: s, IsAfterDigits: CSK_BeforeDigits);
1236	s = SkipDigits(ptr: s); // Skip suffix.
1237	}
1238	if (*s == 'e' || *s == 'E') { // exponent
1239	checkSeparator(TokLoc, Pos: s, IsAfterDigits: CSK_AfterDigits);
1240	const char *Exponent = s;
1241	s++;
1242	radix = 10;
1243	saw_exponent = true;
1244	if (s != ThisTokEnd && (*s == '+' || *s == '-')) s++; // sign
1245	const char *first_non_digit = SkipDigits(ptr: s);
1246	if (containsDigits(Start: s, End: first_non_digit)) {
1247	checkSeparator(TokLoc, Pos: s, IsAfterDigits: CSK_BeforeDigits);
1248	s = first_non_digit;
1249	} else {
1250	if (!hadError) {
1251	Diags.Report(Lexer::AdvanceToTokenCharacter(
1252	TokStart: TokLoc, Characters: Exponent - ThisTokBegin, SM, LangOpts),
1253	diag::err_exponent_has_no_digits);
1254	hadError = true;
1255	}
1256	return;
1257	}
1258	}
1259	}
1260
1261	/// Determine whether a suffix is a valid ud-suffix. We avoid treating reserved
1262	/// suffixes as ud-suffixes, because the diagnostic experience is better if we
1263	/// treat it as an invalid suffix.
1264	bool NumericLiteralParser::isValidUDSuffix(const LangOptions &LangOpts,
1265	StringRef Suffix) {
1266	if (!LangOpts.CPlusPlus11 || Suffix.empty())
1267	return false;
1268
1269	// By C++11 [lex.ext]p10, ud-suffixes starting with an '_' are always valid.
1270	// Suffixes starting with '__' (double underscore) are for use by
1271	// the implementation.
1272	if (Suffix.starts_with(Prefix: "_") && !Suffix.starts_with(Prefix: "__"))
1273	return true;
1274
1275	// In C++11, there are no library suffixes.
1276	if (!LangOpts.CPlusPlus14)
1277	return false;
1278
1279	// In C++14, "s", "h", "min", "ms", "us", and "ns" are used in the library.
1280	// Per tweaked N3660, "il", "i", and "if" are also used in the library.
1281	// In C++2a "d" and "y" are used in the library.
1282	return llvm::StringSwitch<bool>(Suffix)
1283	.Cases(S0: "h", S1: "min", S2: "s", Value: true)
1284	.Cases(S0: "ms", S1: "us", S2: "ns", Value: true)
1285	.Cases(S0: "il", S1: "i", S2: "if", Value: true)
1286	.Cases(S0: "d", S1: "y", Value: LangOpts.CPlusPlus20)
1287	.Default(Value: false);
1288	}
1289
1290	void NumericLiteralParser::checkSeparator(SourceLocation TokLoc,
1291	const char *Pos,
1292	CheckSeparatorKind IsAfterDigits) {
1293	if (IsAfterDigits == CSK_AfterDigits) {
1294	if (Pos == ThisTokBegin)
1295	return;
1296	--Pos;
1297	} else if (Pos == ThisTokEnd)
1298	return;
1299
1300	if (isDigitSeparator(C: *Pos)) {
1301	Diags.Report(Lexer::AdvanceToTokenCharacter(TokStart: TokLoc, Characters: Pos - ThisTokBegin, SM,
1302	LangOpts),
1303	diag::err_digit_separator_not_between_digits)
1304	<< IsAfterDigits;
1305	hadError = true;
1306	}
1307	}
1308
1309	/// ParseNumberStartingWithZero - This method is called when the first character
1310	/// of the number is found to be a zero. This means it is either an octal
1311	/// number (like '04') or a hex number ('0x123a') a binary number ('0b1010') or
1312	/// a floating point number (01239.123e4). Eat the prefix, determining the
1313	/// radix etc.
1314	void NumericLiteralParser::ParseNumberStartingWithZero(SourceLocation TokLoc) {
1315	assert(s[0] == '0' && "Invalid method call");
1316	s++;
1317
1318	int c1 = s[0];
1319
1320	// Handle a hex number like 0x1234.
1321	if ((c1 == 'x' || c1 == 'X') && (isHexDigit(c: s[1]) || s[1] == '.')) {
1322	s++;
1323	assert(s < ThisTokEnd && "didn't maximally munch?");
1324	radix = 16;
1325	DigitsBegin = s;
1326	s = SkipHexDigits(ptr: s);
1327	bool HasSignificandDigits = containsDigits(Start: DigitsBegin, End: s);
1328	if (s == ThisTokEnd) {
1329	// Done.
1330	} else if (*s == '.') {
1331	s++;
1332	saw_period = true;
1333	const char *floatDigitsBegin = s;
1334	s = SkipHexDigits(ptr: s);
1335	if (containsDigits(Start: floatDigitsBegin, End: s))
1336	HasSignificandDigits = true;
1337	if (HasSignificandDigits)
1338	checkSeparator(TokLoc, Pos: floatDigitsBegin, IsAfterDigits: CSK_BeforeDigits);
1339	}
1340
1341	if (!HasSignificandDigits) {
1342	Diags.Report(Lexer::AdvanceToTokenCharacter(TokStart: TokLoc, Characters: s - ThisTokBegin, SM,
1343	LangOpts),
1344	diag::err_hex_constant_requires)
1345	<< LangOpts.CPlusPlus << 1;
1346	hadError = true;
1347	return;
1348	}
1349
1350	// A binary exponent can appear with or with a '.'. If dotted, the
1351	// binary exponent is required.
1352	if (*s == 'p' || *s == 'P') {
1353	checkSeparator(TokLoc, Pos: s, IsAfterDigits: CSK_AfterDigits);
1354	const char *Exponent = s;
1355	s++;
1356	saw_exponent = true;
1357	if (s != ThisTokEnd && (*s == '+' || *s == '-')) s++; // sign
1358	const char *first_non_digit = SkipDigits(ptr: s);
1359	if (!containsDigits(Start: s, End: first_non_digit)) {
1360	if (!hadError) {
1361	Diags.Report(Lexer::AdvanceToTokenCharacter(
1362	TokStart: TokLoc, Characters: Exponent - ThisTokBegin, SM, LangOpts),
1363	diag::err_exponent_has_no_digits);
1364	hadError = true;
1365	}
1366	return;
1367	}
1368	checkSeparator(TokLoc, Pos: s, IsAfterDigits: CSK_BeforeDigits);
1369	s = first_non_digit;
1370
1371	if (!LangOpts.HexFloats)
1372	Diags.Report(TokLoc, LangOpts.CPlusPlus
1373	? diag::ext_hex_literal_invalid
1374	: diag::ext_hex_constant_invalid);
1375	else if (LangOpts.CPlusPlus17)
1376	Diags.Report(TokLoc, diag::warn_cxx17_hex_literal);
1377	} else if (saw_period) {
1378	Diags.Report(Lexer::AdvanceToTokenCharacter(TokLoc, s - ThisTokBegin, SM,
1379	LangOpts),
1380	diag::err_hex_constant_requires)
1381	<< LangOpts.CPlusPlus << 0;
1382	hadError = true;
1383	}
1384	return;
1385	}
1386
1387	// Handle simple binary numbers 0b01010
1388	if ((c1 == 'b' || c1 == 'B') && (s[1] == '0' || s[1] == '1')) {
1389	// 0b101010 is a C++14 and C23 extension.
1390	unsigned DiagId;
1391	if (LangOpts.CPlusPlus14)
1392	DiagId = diag::warn_cxx11_compat_binary_literal;
1393	else if (LangOpts.C23)
1394	DiagId = diag::warn_c23_compat_binary_literal;
1395	else if (LangOpts.CPlusPlus)
1396	DiagId = diag::ext_binary_literal_cxx14;
1397	else
1398	DiagId = diag::ext_binary_literal;
1399	Diags.Report(Loc: TokLoc, DiagID: DiagId);
1400	++s;
1401	assert(s < ThisTokEnd && "didn't maximally munch?");
1402	radix = 2;
1403	DigitsBegin = s;
1404	s = SkipBinaryDigits(ptr: s);
1405	if (s == ThisTokEnd) {
1406	// Done.
1407	} else if (isHexDigit(c: *s) &&
1408	!isValidUDSuffix(LangOpts, Suffix: StringRef(s, ThisTokEnd - s))) {
1409	Diags.Report(Lexer::AdvanceToTokenCharacter(TokLoc, s - ThisTokBegin, SM,
1410	LangOpts),
1411	diag::err_invalid_digit)
1412	<< StringRef(s, 1) << 2;
1413	hadError = true;
1414	}
1415	// Other suffixes will be diagnosed by the caller.
1416	return;
1417	}
1418
1419	// For now, the radix is set to 8. If we discover that we have a
1420	// floating point constant, the radix will change to 10. Octal floating
1421	// point constants are not permitted (only decimal and hexadecimal).
1422	radix = 8;
1423	const char *PossibleNewDigitStart = s;
1424	s = SkipOctalDigits(ptr: s);
1425	// When the value is 0 followed by a suffix (like 0wb), we want to leave 0
1426	// as the start of the digits. So if skipping octal digits does not skip
1427	// anything, we leave the digit start where it was.
1428	if (s != PossibleNewDigitStart)
1429	DigitsBegin = PossibleNewDigitStart;
1430
1431	if (s == ThisTokEnd)
1432	return; // Done, simple octal number like 01234
1433
1434	// If we have some other non-octal digit that *is* a decimal digit, see if
1435	// this is part of a floating point number like 094.123 or 09e1.
1436	if (isDigit(c: *s)) {
1437	const char *EndDecimal = SkipDigits(ptr: s);
1438	if (EndDecimal[0] == '.' || EndDecimal[0] == 'e' || EndDecimal[0] == 'E') {
1439	s = EndDecimal;
1440	radix = 10;
1441	}
1442	}
1443
1444	ParseDecimalOrOctalCommon(TokLoc);
1445	}
1446
1447	static bool alwaysFitsInto64Bits(unsigned Radix, unsigned NumDigits) {
1448	switch (Radix) {
1449	case 2:
1450	return NumDigits <= 64;
1451	case 8:
1452	return NumDigits <= 64 / 3; // Digits are groups of 3 bits.
1453	case 10:
1454	return NumDigits <= 19; // floor(log10(2^64))
1455	case 16:
1456	return NumDigits <= 64 / 4; // Digits are groups of 4 bits.
1457	default:
1458	llvm_unreachable("impossible Radix");
1459	}
1460	}
1461
1462	/// GetIntegerValue - Convert this numeric literal value to an APInt that
1463	/// matches Val's input width. If there is an overflow, set Val to the low bits
1464	/// of the result and return true. Otherwise, return false.
1465	bool NumericLiteralParser::GetIntegerValue(llvm::APInt &Val) {
1466	// Fast path: Compute a conservative bound on the maximum number of
1467	// bits per digit in this radix. If we can't possibly overflow a
1468	// uint64 based on that bound then do the simple conversion to
1469	// integer. This avoids the expensive overflow checking below, and
1470	// handles the common cases that matter (small decimal integers and
1471	// hex/octal values which don't overflow).
1472	const unsigned NumDigits = SuffixBegin - DigitsBegin;
1473	if (alwaysFitsInto64Bits(Radix: radix, NumDigits)) {
1474	uint64_t N = 0;
1475	for (const char *Ptr = DigitsBegin; Ptr != SuffixBegin; ++Ptr)
1476	if (!isDigitSeparator(C: *Ptr))
1477	N = N * radix + llvm::hexDigitValue(C: *Ptr);
1478
1479	// This will truncate the value to Val's input width. Simply check
1480	// for overflow by comparing.
1481	Val = N;
1482	return Val.getZExtValue() != N;
1483	}
1484
1485	Val = 0;
1486	const char *Ptr = DigitsBegin;
1487
1488	llvm::APInt RadixVal(Val.getBitWidth(), radix);
1489	llvm::APInt CharVal(Val.getBitWidth(), 0);
1490	llvm::APInt OldVal = Val;
1491
1492	bool OverflowOccurred = false;
1493	while (Ptr < SuffixBegin) {
1494	if (isDigitSeparator(C: *Ptr)) {
1495	++Ptr;
1496	continue;
1497	}
1498
1499	unsigned C = llvm::hexDigitValue(C: *Ptr++);
1500
1501	// If this letter is out of bound for this radix, reject it.
1502	assert(C < radix && "NumericLiteralParser ctor should have rejected this");
1503
1504	CharVal = C;
1505
1506	// Add the digit to the value in the appropriate radix. If adding in digits
1507	// made the value smaller, then this overflowed.
1508	OldVal = Val;
1509
1510	// Multiply by radix, did overflow occur on the multiply?
1511	Val *= RadixVal;
1512	OverflowOccurred |= Val.udiv(RHS: RadixVal) != OldVal;
1513
1514	// Add value, did overflow occur on the value?
1515	// (a + b) ult b <=> overflow
1516	Val += CharVal;
1517	OverflowOccurred |= Val.ult(RHS: CharVal);
1518	}
1519	return OverflowOccurred;
1520	}
1521
1522	llvm::APFloat::opStatus
1523	NumericLiteralParser::GetFloatValue(llvm::APFloat &Result) {
1524	using llvm::APFloat;
1525
1526	unsigned n = std::min(a: SuffixBegin - ThisTokBegin, b: ThisTokEnd - ThisTokBegin);
1527
1528	llvm::SmallString<16> Buffer;
1529	StringRef Str(ThisTokBegin, n);
1530	if (Str.contains(C: '\'')) {
1531	Buffer.reserve(N: n);
1532	std::remove_copy_if(first: Str.begin(), last: Str.end(), result: std::back_inserter(x&: Buffer),
1533	pred: &isDigitSeparator);
1534	Str = Buffer;
1535	}
1536
1537	auto StatusOrErr =
1538	Result.convertFromString(Str, APFloat::rmNearestTiesToEven);
1539	assert(StatusOrErr && "Invalid floating point representation");
1540	return !errorToBool(Err: StatusOrErr.takeError()) ? *StatusOrErr
1541	: APFloat::opInvalidOp;
1542	}
1543
1544	static inline bool IsExponentPart(char c, bool isHex) {
1545	if (isHex)
1546	return c == 'p' || c == 'P';
1547	return c == 'e' || c == 'E';
1548	}
1549
1550	bool NumericLiteralParser::GetFixedPointValue(llvm::APInt &StoreVal, unsigned Scale) {
1551	assert(radix == 16 || radix == 10);
1552
1553	// Find how many digits are needed to store the whole literal.
1554	unsigned NumDigits = SuffixBegin - DigitsBegin;
1555	if (saw_period) --NumDigits;
1556
1557	// Initial scan of the exponent if it exists
1558	bool ExpOverflowOccurred = false;
1559	bool NegativeExponent = false;
1560	const char *ExponentBegin;
1561	uint64_t Exponent = 0;
1562	int64_t BaseShift = 0;
1563	if (saw_exponent) {
1564	const char *Ptr = DigitsBegin;
1565
1566	while (!IsExponentPart(c: *Ptr, isHex: radix == 16))
1567	++Ptr;
1568	ExponentBegin = Ptr;
1569	++Ptr;
1570	NegativeExponent = *Ptr == '-';
1571	if (NegativeExponent) ++Ptr;
1572
1573	unsigned NumExpDigits = SuffixBegin - Ptr;
1574	if (alwaysFitsInto64Bits(Radix: radix, NumDigits: NumExpDigits)) {
1575	llvm::StringRef ExpStr(Ptr, NumExpDigits);
1576	llvm::APInt ExpInt(/*numBits=*/64, ExpStr, /*radix=*/10);
1577	Exponent = ExpInt.getZExtValue();
1578	} else {
1579	ExpOverflowOccurred = true;
1580	}
1581
1582	if (NegativeExponent) BaseShift -= Exponent;
1583	else BaseShift += Exponent;
1584	}
1585
1586	// Number of bits needed for decimal literal is
1587	// ceil(NumDigits * log2(10)) Integral part
1588	// + Scale Fractional part
1589	// + ceil(Exponent * log2(10)) Exponent
1590	// --------------------------------------------------
1591	// ceil((NumDigits + Exponent) * log2(10)) + Scale
1592	//
1593	// But for simplicity in handling integers, we can round up log2(10) to 4,
1594	// making:
1595	// 4 * (NumDigits + Exponent) + Scale
1596	//
1597	// Number of digits needed for hexadecimal literal is
1598	// 4 * NumDigits Integral part
1599	// + Scale Fractional part
1600	// + Exponent Exponent
1601	// --------------------------------------------------
1602	// (4 * NumDigits) + Scale + Exponent
1603	uint64_t NumBitsNeeded;
1604	if (radix == 10)
1605	NumBitsNeeded = 4 * (NumDigits + Exponent) + Scale;
1606	else
1607	NumBitsNeeded = 4 * NumDigits + Exponent + Scale;
1608
1609	if (NumBitsNeeded > std::numeric_limits<unsigned>::max())
1610	ExpOverflowOccurred = true;
1611	llvm::APInt Val(static_cast<unsigned>(NumBitsNeeded), 0, /*isSigned=*/false);
1612
1613	bool FoundDecimal = false;
1614
1615	int64_t FractBaseShift = 0;
1616	const char *End = saw_exponent ? ExponentBegin : SuffixBegin;
1617	for (const char *Ptr = DigitsBegin; Ptr < End; ++Ptr) {
1618	if (*Ptr == '.') {
1619	FoundDecimal = true;
1620	continue;
1621	}
1622
1623	// Normal reading of an integer
1624	unsigned C = llvm::hexDigitValue(C: *Ptr);
1625	assert(C < radix && "NumericLiteralParser ctor should have rejected this");
1626
1627	Val *= radix;
1628	Val += C;
1629
1630	if (FoundDecimal)
1631	// Keep track of how much we will need to adjust this value by from the
1632	// number of digits past the radix point.
1633	--FractBaseShift;
1634	}
1635
1636	// For a radix of 16, we will be multiplying by 2 instead of 16.
1637	if (radix == 16) FractBaseShift *= 4;
1638	BaseShift += FractBaseShift;
1639
1640	Val <<= Scale;
1641
1642	uint64_t Base = (radix == 16) ? 2 : 10;
1643	if (BaseShift > 0) {
1644	for (int64_t i = 0; i < BaseShift; ++i) {
1645	Val *= Base;
1646	}
1647	} else if (BaseShift < 0) {
1648	for (int64_t i = BaseShift; i < 0 && !Val.isZero(); ++i)
1649	Val = Val.udiv(RHS: Base);
1650	}
1651
1652	bool IntOverflowOccurred = false;
1653	auto MaxVal = llvm::APInt::getMaxValue(numBits: StoreVal.getBitWidth());
1654	if (Val.getBitWidth() > StoreVal.getBitWidth()) {
1655	IntOverflowOccurred |= Val.ugt(RHS: MaxVal.zext(width: Val.getBitWidth()));
1656	StoreVal = Val.trunc(width: StoreVal.getBitWidth());
1657	} else if (Val.getBitWidth() < StoreVal.getBitWidth()) {
1658	IntOverflowOccurred |= Val.zext(width: MaxVal.getBitWidth()).ugt(RHS: MaxVal);
1659	StoreVal = Val.zext(width: StoreVal.getBitWidth());
1660	} else {
1661	StoreVal = Val;
1662	}
1663
1664	return IntOverflowOccurred || ExpOverflowOccurred;
1665	}
1666
1667	/// \verbatim
1668	/// user-defined-character-literal: [C++11 lex.ext]
1669	/// character-literal ud-suffix
1670	/// ud-suffix:
1671	/// identifier
1672	/// character-literal: [C++11 lex.ccon]
1673	/// ' c-char-sequence '
1674	/// u' c-char-sequence '
1675	/// U' c-char-sequence '
1676	/// L' c-char-sequence '
1677	/// u8' c-char-sequence ' [C++1z lex.ccon]
1678	/// c-char-sequence:
1679	/// c-char
1680	/// c-char-sequence c-char
1681	/// c-char:
1682	/// any member of the source character set except the single-quote ',
1683	/// backslash \, or new-line character
1684	/// escape-sequence
1685	/// universal-character-name
1686	/// escape-sequence:
1687	/// simple-escape-sequence
1688	/// octal-escape-sequence
1689	/// hexadecimal-escape-sequence
1690	/// simple-escape-sequence:
1691	/// one of \' \" \? \\ \a \b \f \n \r \t \v
1692	/// octal-escape-sequence:
1693	/// \ octal-digit
1694	/// \ octal-digit octal-digit
1695	/// \ octal-digit octal-digit octal-digit
1696	/// hexadecimal-escape-sequence:
1697	/// \x hexadecimal-digit
1698	/// hexadecimal-escape-sequence hexadecimal-digit
1699	/// universal-character-name: [C++11 lex.charset]
1700	/// \u hex-quad
1701	/// \U hex-quad hex-quad
1702	/// hex-quad:
1703	/// hex-digit hex-digit hex-digit hex-digit
1704	/// \endverbatim
1705	///
1706	CharLiteralParser::CharLiteralParser(const char *begin, const char *end,
1707	SourceLocation Loc, Preprocessor &PP,
1708	tok::TokenKind kind) {
1709	// At this point we know that the character matches the regex "(L|u|U)?'.*'".
1710	HadError = false;
1711
1712	Kind = kind;
1713
1714	const char *TokBegin = begin;
1715
1716	// Skip over wide character determinant.
1717	if (Kind != tok::char_constant)
1718	++begin;
1719	if (Kind == tok::utf8_char_constant)
1720	++begin;
1721
1722	// Skip over the entry quote.
1723	if (begin[0] != '\'') {
1724	PP.Diag(Loc, diag::err_lexing_char);
1725	HadError = true;
1726	return;
1727	}
1728
1729	++begin;
1730
1731	// Remove an optional ud-suffix.
1732	if (end[-1] != '\'') {
1733	const char *UDSuffixEnd = end;
1734	do {
1735	--end;
1736	} while (end[-1] != '\'');
1737	// FIXME: Don't bother with this if !tok.hasUCN().
1738	expandUCNs(Buf&: UDSuffixBuf, Input: StringRef(end, UDSuffixEnd - end));
1739	UDSuffixOffset = end - TokBegin;
1740	}
1741
1742	// Trim the ending quote.
1743	assert(end != begin && "Invalid token lexed");
1744	--end;
1745
1746	// FIXME: The "Value" is an uint64_t so we can handle char literals of
1747	// up to 64-bits.
1748	// FIXME: This extensively assumes that 'char' is 8-bits.
1749	assert(PP.getTargetInfo().getCharWidth() == 8 &&
1750	"Assumes char is 8 bits");
1751	assert(PP.getTargetInfo().getIntWidth() <= 64 &&
1752	(PP.getTargetInfo().getIntWidth() & 7) == 0 &&
1753	"Assumes sizeof(int) on target is <= 64 and a multiple of char");
1754	assert(PP.getTargetInfo().getWCharWidth() <= 64 &&
1755	"Assumes sizeof(wchar) on target is <= 64");
1756
1757	SmallVector<uint32_t, 4> codepoint_buffer;
1758	codepoint_buffer.resize(N: end - begin);
1759	uint32_t *buffer_begin = &codepoint_buffer.front();
1760	uint32_t *buffer_end = buffer_begin + codepoint_buffer.size();
1761
1762	// Unicode escapes representing characters that cannot be correctly
1763	// represented in a single code unit are disallowed in character literals
1764	// by this implementation.
1765	uint32_t largest_character_for_kind;
1766	if (tok::wide_char_constant == Kind) {
1767	largest_character_for_kind =
1768	0xFFFFFFFFu >> (32-PP.getTargetInfo().getWCharWidth());
1769	} else if (tok::utf8_char_constant == Kind) {
1770	largest_character_for_kind = 0x7F;
1771	} else if (tok::utf16_char_constant == Kind) {
1772	largest_character_for_kind = 0xFFFF;
1773	} else if (tok::utf32_char_constant == Kind) {
1774	largest_character_for_kind = 0x10FFFF;
1775	} else {
1776	largest_character_for_kind = 0x7Fu;
1777	}
1778
1779	while (begin != end) {
1780	// Is this a span of non-escape characters?
1781	if (begin[0] != '\\') {
1782	char const *start = begin;
1783	do {
1784	++begin;
1785	} while (begin != end && *begin != '\\');
1786
1787	char const *tmp_in_start = start;
1788	uint32_t *tmp_out_start = buffer_begin;
1789	llvm::ConversionResult res =
1790	llvm::ConvertUTF8toUTF32(sourceStart: reinterpret_cast<llvm::UTF8 const **>(&start),
1791	sourceEnd: reinterpret_cast<llvm::UTF8 const *>(begin),
1792	targetStart: &buffer_begin, targetEnd: buffer_end, flags: llvm::strictConversion);
1793	if (res != llvm::conversionOK) {
1794	// If we see bad encoding for unprefixed character literals, warn and
1795	// simply copy the byte values, for compatibility with gcc and
1796	// older versions of clang.
1797	bool NoErrorOnBadEncoding = isOrdinary();
1798	unsigned Msg = diag::err_bad_character_encoding;
1799	if (NoErrorOnBadEncoding)
1800	Msg = diag::warn_bad_character_encoding;
1801	PP.Diag(Loc, DiagID: Msg);
1802	if (NoErrorOnBadEncoding) {
1803	start = tmp_in_start;
1804	buffer_begin = tmp_out_start;
1805	for (; start != begin; ++start, ++buffer_begin)
1806	*buffer_begin = static_cast<uint8_t>(*start);
1807	} else {
1808	HadError = true;
1809	}
1810	} else {
1811	for (; tmp_out_start < buffer_begin; ++tmp_out_start) {
1812	if (*tmp_out_start > largest_character_for_kind) {
1813	HadError = true;
1814	PP.Diag(Loc, diag::err_character_too_large);
1815	}
1816	}
1817	}
1818
1819	continue;
1820	}
1821	// Is this a Universal Character Name escape?
1822	if (begin[1] == 'u' || begin[1] == 'U' || begin[1] == 'N') {
1823	unsigned short UcnLen = 0;
1824	if (!ProcessUCNEscape(ThisTokBegin: TokBegin, ThisTokBuf&: begin, ThisTokEnd: end, UcnVal&: *buffer_begin, UcnLen,
1825	Loc: FullSourceLoc(Loc, PP.getSourceManager()),
1826	Diags: &PP.getDiagnostics(), Features: PP.getLangOpts(), in_char_string_literal: true)) {
1827	HadError = true;
1828	} else if (*buffer_begin > largest_character_for_kind) {
1829	HadError = true;
1830	PP.Diag(Loc, diag::err_character_too_large);
1831	}
1832
1833	++buffer_begin;
1834	continue;
1835	}
1836	unsigned CharWidth = getCharWidth(kind: Kind, Target: PP.getTargetInfo());
1837	uint64_t result =
1838	ProcessCharEscape(ThisTokBegin: TokBegin, ThisTokBuf&: begin, ThisTokEnd: end, HadError,
1839	Loc: FullSourceLoc(Loc, PP.getSourceManager()), CharWidth,
1840	Diags: &PP.getDiagnostics(), Features: PP.getLangOpts(),
1841	EvalMethod: StringLiteralEvalMethod::Evaluated);
1842	*buffer_begin++ = result;
1843	}
1844
1845	unsigned NumCharsSoFar = buffer_begin - &codepoint_buffer.front();
1846
1847	if (NumCharsSoFar > 1) {
1848	if (isOrdinary() && NumCharsSoFar == 4)
1849	PP.Diag(Loc, diag::warn_four_char_character_literal);
1850	else if (isOrdinary())
1851	PP.Diag(Loc, diag::warn_multichar_character_literal);
1852	else {
1853	PP.Diag(Loc, diag::err_multichar_character_literal) << (isWide() ? 0 : 1);
1854	HadError = true;
1855	}
1856	IsMultiChar = true;
1857	} else {
1858	IsMultiChar = false;
1859	}
1860
1861	llvm::APInt LitVal(PP.getTargetInfo().getIntWidth(), 0);
1862
1863	// Narrow character literals act as though their value is concatenated
1864	// in this implementation, but warn on overflow.
1865	bool multi_char_too_long = false;
1866	if (isOrdinary() && isMultiChar()) {
1867	LitVal = 0;
1868	for (size_t i = 0; i < NumCharsSoFar; ++i) {
1869	// check for enough leading zeros to shift into
1870	multi_char_too_long |= (LitVal.countl_zero() < 8);
1871	LitVal <<= 8;
1872	LitVal = LitVal + (codepoint_buffer[i] & 0xFF);
1873	}
1874	} else if (NumCharsSoFar > 0) {
1875	// otherwise just take the last character
1876	LitVal = buffer_begin[-1];
1877	}
1878
1879	if (!HadError && multi_char_too_long) {
1880	PP.Diag(Loc, diag::warn_char_constant_too_large);
1881	}
1882
1883	// Transfer the value from APInt to uint64_t
1884	Value = LitVal.getZExtValue();
1885
1886	// If this is a single narrow character, sign extend it (e.g. '\xFF' is "-1")
1887	// if 'char' is signed for this target (C99 6.4.4.4p10). Note that multiple
1888	// character constants are not sign extended in the this implementation:
1889	// '\xFF\xFF' = 65536 and '\x0\xFF' = 255, which matches GCC.
1890	if (isOrdinary() && NumCharsSoFar == 1 && (Value & 128) &&
1891	PP.getLangOpts().CharIsSigned)
1892	Value = (signed char)Value;
1893	}
1894
1895	/// \verbatim
1896	/// string-literal: [C++0x lex.string]
1897	/// encoding-prefix " [s-char-sequence] "
1898	/// encoding-prefix R raw-string
1899	/// encoding-prefix:
1900	/// u8
1901	/// u
1902	/// U
1903	/// L
1904	/// s-char-sequence:
1905	/// s-char
1906	/// s-char-sequence s-char
1907	/// s-char:
1908	/// any member of the source character set except the double-quote ",
1909	/// backslash \, or new-line character
1910	/// escape-sequence
1911	/// universal-character-name
1912	/// raw-string:
1913	/// " d-char-sequence ( r-char-sequence ) d-char-sequence "
1914	/// r-char-sequence:
1915	/// r-char
1916	/// r-char-sequence r-char
1917	/// r-char:
1918	/// any member of the source character set, except a right parenthesis )
1919	/// followed by the initial d-char-sequence (which may be empty)
1920	/// followed by a double quote ".
1921	/// d-char-sequence:
1922	/// d-char
1923	/// d-char-sequence d-char
1924	/// d-char:
1925	/// any member of the basic source character set except:
1926	/// space, the left parenthesis (, the right parenthesis ),
1927	/// the backslash \, and the control characters representing horizontal
1928	/// tab, vertical tab, form feed, and newline.
1929	/// escape-sequence: [C++0x lex.ccon]
1930	/// simple-escape-sequence
1931	/// octal-escape-sequence
1932	/// hexadecimal-escape-sequence
1933	/// simple-escape-sequence:
1934	/// one of \' \" \? \\ \a \b \f \n \r \t \v
1935	/// octal-escape-sequence:
1936	/// \ octal-digit
1937	/// \ octal-digit octal-digit
1938	/// \ octal-digit octal-digit octal-digit
1939	/// hexadecimal-escape-sequence:
1940	/// \x hexadecimal-digit
1941	/// hexadecimal-escape-sequence hexadecimal-digit
1942	/// universal-character-name:
1943	/// \u hex-quad
1944	/// \U hex-quad hex-quad
1945	/// hex-quad:
1946	/// hex-digit hex-digit hex-digit hex-digit
1947	/// \endverbatim
1948	///
1949	StringLiteralParser::StringLiteralParser(ArrayRef<Token> StringToks,
1950	Preprocessor &PP,
1951	StringLiteralEvalMethod EvalMethod)
1952	: SM(PP.getSourceManager()), Features(PP.getLangOpts()),
1953	Target(PP.getTargetInfo()), Diags(&PP.getDiagnostics()),
1954	MaxTokenLength(0), SizeBound(0), CharByteWidth(0), Kind(tok::unknown),
1955	ResultPtr(ResultBuf.data()), EvalMethod(EvalMethod), hadError(false),
1956	Pascal(false) {
1957	init(StringToks);
1958	}
1959
1960	void StringLiteralParser::init(ArrayRef<Token> StringToks){
1961	// The literal token may have come from an invalid source location (e.g. due
1962	// to a PCH error), in which case the token length will be 0.
1963	if (StringToks.empty() || StringToks[0].getLength() < 2)
1964	return DiagnoseLexingError(Loc: SourceLocation());
1965
1966	// Scan all of the string portions, remember the max individual token length,
1967	// computing a bound on the concatenated string length, and see whether any
1968	// piece is a wide-string. If any of the string portions is a wide-string
1969	// literal, the result is a wide-string literal [C99 6.4.5p4].
1970	assert(!StringToks.empty() && "expected at least one token");
1971	MaxTokenLength = StringToks[0].getLength();
1972	assert(StringToks[0].getLength() >= 2 && "literal token is invalid!");
1973	SizeBound = StringToks[0].getLength() - 2; // -2 for "".
1974	hadError = false;
1975
1976	// Determines the kind of string from the prefix
1977	Kind = tok::string_literal;
1978
1979	/// (C99 5.1.1.2p1). The common case is only one string fragment.
1980	for (const Token &Tok : StringToks) {
1981	if (Tok.getLength() < 2)
1982	return DiagnoseLexingError(Loc: Tok.getLocation());
1983
1984	// The string could be shorter than this if it needs cleaning, but this is a
1985	// reasonable bound, which is all we need.
1986	assert(Tok.getLength() >= 2 && "literal token is invalid!");
1987	SizeBound += Tok.getLength() - 2; // -2 for "".
1988
1989	// Remember maximum string piece length.
1990	if (Tok.getLength() > MaxTokenLength)
1991	MaxTokenLength = Tok.getLength();
1992
1993	// Remember if we see any wide or utf-8/16/32 strings.
1994	// Also check for illegal concatenations.
1995	if (isUnevaluated() && Tok.getKind() != tok::string_literal) {
1996	if (Diags) {
1997	SourceLocation PrefixEndLoc = Lexer::AdvanceToTokenCharacter(
1998	TokStart: Tok.getLocation(), Characters: getEncodingPrefixLen(kind: Tok.getKind()), SM,
1999	LangOpts: Features);
2000	CharSourceRange Range =
2001	CharSourceRange::getCharRange(R: {Tok.getLocation(), PrefixEndLoc});
2002	StringRef Prefix(SM.getCharacterData(SL: Tok.getLocation()),
2003	getEncodingPrefixLen(kind: Tok.getKind()));
2004	Diags->Report(Tok.getLocation(),
2005	Features.CPlusPlus26
2006	? diag::err_unevaluated_string_prefix
2007	: diag::warn_unevaluated_string_prefix)
2008	<< Prefix << Features.CPlusPlus << FixItHint::CreateRemoval(Range);
2009	}
2010	if (Features.CPlusPlus26)
2011	hadError = true;
2012	} else if (Tok.isNot(K: Kind) && Tok.isNot(K: tok::string_literal)) {
2013	if (isOrdinary()) {
2014	Kind = Tok.getKind();
2015	} else {
2016	if (Diags)
2017	Diags->Report(Tok.getLocation(), diag::err_unsupported_string_concat);
2018	hadError = true;
2019	}
2020	}
2021	}
2022
2023	// Include space for the null terminator.
2024	++SizeBound;
2025
2026	// TODO: K&R warning: "traditional C rejects string constant concatenation"
2027
2028	// Get the width in bytes of char/wchar_t/char16_t/char32_t
2029	CharByteWidth = getCharWidth(kind: Kind, Target);
2030	assert((CharByteWidth & 7) == 0 && "Assumes character size is byte multiple");
2031	CharByteWidth /= 8;
2032
2033	// The output buffer size needs to be large enough to hold wide characters.
2034	// This is a worst-case assumption which basically corresponds to L"" "long".
2035	SizeBound *= CharByteWidth;
2036
2037	// Size the temporary buffer to hold the result string data.
2038	ResultBuf.resize(N: SizeBound);
2039
2040	// Likewise, but for each string piece.
2041	SmallString<512> TokenBuf;
2042	TokenBuf.resize(N: MaxTokenLength);
2043
2044	// Loop over all the strings, getting their spelling, and expanding them to
2045	// wide strings as appropriate.
2046	ResultPtr = &ResultBuf[0]; // Next byte to fill in.
2047
2048	Pascal = false;
2049
2050	SourceLocation UDSuffixTokLoc;
2051
2052	for (unsigned i = 0, e = StringToks.size(); i != e; ++i) {
2053	const char *ThisTokBuf = &TokenBuf[0];
2054	// Get the spelling of the token, which eliminates trigraphs, etc. We know
2055	// that ThisTokBuf points to a buffer that is big enough for the whole token
2056	// and 'spelled' tokens can only shrink.
2057	bool StringInvalid = false;
2058	unsigned ThisTokLen =
2059	Lexer::getSpelling(Tok: StringToks[i], Buffer&: ThisTokBuf, SourceMgr: SM, LangOpts: Features,
2060	Invalid: &StringInvalid);
2061	if (StringInvalid)
2062	return DiagnoseLexingError(Loc: StringToks[i].getLocation());
2063
2064	const char *ThisTokBegin = ThisTokBuf;
2065	const char *ThisTokEnd = ThisTokBuf+ThisTokLen;
2066
2067	// Remove an optional ud-suffix.
2068	if (ThisTokEnd[-1] != '"') {
2069	const char *UDSuffixEnd = ThisTokEnd;
2070	do {
2071	--ThisTokEnd;
2072	} while (ThisTokEnd[-1] != '"');
2073
2074	StringRef UDSuffix(ThisTokEnd, UDSuffixEnd - ThisTokEnd);
2075
2076	if (UDSuffixBuf.empty()) {
2077	if (StringToks[i].hasUCN())
2078	expandUCNs(Buf&: UDSuffixBuf, Input: UDSuffix);
2079	else
2080	UDSuffixBuf.assign(RHS: UDSuffix);
2081	UDSuffixToken = i;
2082	UDSuffixOffset = ThisTokEnd - ThisTokBuf;
2083	UDSuffixTokLoc = StringToks[i].getLocation();
2084	} else {
2085	SmallString<32> ExpandedUDSuffix;
2086	if (StringToks[i].hasUCN()) {
2087	expandUCNs(Buf&: ExpandedUDSuffix, Input: UDSuffix);
2088	UDSuffix = ExpandedUDSuffix;
2089	}
2090
2091	// C++11 [lex.ext]p8: At the end of phase 6, if a string literal is the
2092	// result of a concatenation involving at least one user-defined-string-
2093	// literal, all the participating user-defined-string-literals shall
2094	// have the same ud-suffix.
2095	bool UnevaluatedStringHasUDL = isUnevaluated() && !UDSuffix.empty();
2096	if (UDSuffixBuf != UDSuffix || UnevaluatedStringHasUDL) {
2097	if (Diags) {
2098	SourceLocation TokLoc = StringToks[i].getLocation();
2099	if (UnevaluatedStringHasUDL) {
2100	Diags->Report(TokLoc, diag::err_unevaluated_string_udl)
2101	<< SourceRange(TokLoc, TokLoc);
2102	} else {
2103	Diags->Report(TokLoc, diag::err_string_concat_mixed_suffix)
2104	<< UDSuffixBuf << UDSuffix
2105	<< SourceRange(UDSuffixTokLoc, UDSuffixTokLoc);
2106	}
2107	}
2108	hadError = true;
2109	}
2110	}
2111	}
2112
2113	// Strip the end quote.
2114	--ThisTokEnd;
2115
2116	// TODO: Input character set mapping support.
2117
2118	// Skip marker for wide or unicode strings.
2119	if (ThisTokBuf[0] == 'L' || ThisTokBuf[0] == 'u' || ThisTokBuf[0] == 'U') {
2120	++ThisTokBuf;
2121	// Skip 8 of u8 marker for utf8 strings.
2122	if (ThisTokBuf[0] == '8')
2123	++ThisTokBuf;
2124	}
2125
2126	// Check for raw string
2127	if (ThisTokBuf[0] == 'R') {
2128	if (ThisTokBuf[1] != '"') {
2129	// The file may have come from PCH and then changed after loading the
2130	// PCH; Fail gracefully.
2131	return DiagnoseLexingError(Loc: StringToks[i].getLocation());
2132	}
2133	ThisTokBuf += 2; // skip R"
2134
2135	// C++11 [lex.string]p2: A `d-char-sequence` shall consist of at most 16
2136	// characters.
2137	constexpr unsigned MaxRawStrDelimLen = 16;
2138
2139	const char *Prefix = ThisTokBuf;
2140	while (static_cast<unsigned>(ThisTokBuf - Prefix) < MaxRawStrDelimLen &&
2141	ThisTokBuf[0] != '(')
2142	++ThisTokBuf;
2143	if (ThisTokBuf[0] != '(')
2144	return DiagnoseLexingError(Loc: StringToks[i].getLocation());
2145	++ThisTokBuf; // skip '('
2146
2147	// Remove same number of characters from the end
2148	ThisTokEnd -= ThisTokBuf - Prefix;
2149	if (ThisTokEnd < ThisTokBuf)
2150	return DiagnoseLexingError(Loc: StringToks[i].getLocation());
2151
2152	// C++14 [lex.string]p4: A source-file new-line in a raw string literal
2153	// results in a new-line in the resulting execution string-literal.
2154	StringRef RemainingTokenSpan(ThisTokBuf, ThisTokEnd - ThisTokBuf);
2155	while (!RemainingTokenSpan.empty()) {
2156	// Split the string literal on \r\n boundaries.
2157	size_t CRLFPos = RemainingTokenSpan.find(Str: "\r\n");
2158	StringRef BeforeCRLF = RemainingTokenSpan.substr(Start: 0, N: CRLFPos);
2159	StringRef AfterCRLF = RemainingTokenSpan.substr(Start: CRLFPos);
2160
2161	// Copy everything before the \r\n sequence into the string literal.
2162	if (CopyStringFragment(Tok: StringToks[i], TokBegin: ThisTokBegin, Fragment: BeforeCRLF))
2163	hadError = true;
2164
2165	// Point into the \n inside the \r\n sequence and operate on the
2166	// remaining portion of the literal.
2167	RemainingTokenSpan = AfterCRLF.substr(Start: 1);
2168	}
2169	} else {
2170	if (ThisTokBuf[0] != '"') {
2171	// The file may have come from PCH and then changed after loading the
2172	// PCH; Fail gracefully.
2173	return DiagnoseLexingError(Loc: StringToks[i].getLocation());
2174	}
2175	++ThisTokBuf; // skip "
2176
2177	// Check if this is a pascal string
2178	if (!isUnevaluated() && Features.PascalStrings &&
2179	ThisTokBuf + 1 != ThisTokEnd && ThisTokBuf[0] == '\\' &&
2180	ThisTokBuf[1] == 'p') {
2181
2182	// If the \p sequence is found in the first token, we have a pascal string
2183	// Otherwise, if we already have a pascal string, ignore the first \p
2184	if (i == 0) {
2185	++ThisTokBuf;
2186	Pascal = true;
2187	} else if (Pascal)
2188	ThisTokBuf += 2;
2189	}
2190
2191	while (ThisTokBuf != ThisTokEnd) {
2192	// Is this a span of non-escape characters?
2193	if (ThisTokBuf[0] != '\\') {
2194	const char *InStart = ThisTokBuf;
2195	do {
2196	++ThisTokBuf;
2197	} while (ThisTokBuf != ThisTokEnd && ThisTokBuf[0] != '\\');
2198
2199	// Copy the character span over.
2200	if (CopyStringFragment(Tok: StringToks[i], TokBegin: ThisTokBegin,
2201	Fragment: StringRef(InStart, ThisTokBuf - InStart)))
2202	hadError = true;
2203	continue;
2204	}
2205	// Is this a Universal Character Name escape?
2206	if (ThisTokBuf[1] == 'u' || ThisTokBuf[1] == 'U' ||
2207	ThisTokBuf[1] == 'N') {
2208	EncodeUCNEscape(ThisTokBegin, ThisTokBuf, ThisTokEnd,
2209	ResultBuf&: ResultPtr, HadError&: hadError,
2210	Loc: FullSourceLoc(StringToks[i].getLocation(), SM),
2211	CharByteWidth, Diags, Features);
2212	continue;
2213	}
2214	// Otherwise, this is a non-UCN escape character. Process it.
2215	unsigned ResultChar =
2216	ProcessCharEscape(ThisTokBegin, ThisTokBuf, ThisTokEnd, HadError&: hadError,
2217	Loc: FullSourceLoc(StringToks[i].getLocation(), SM),
2218	CharWidth: CharByteWidth * 8, Diags, Features, EvalMethod);
2219
2220	if (CharByteWidth == 4) {
2221	// FIXME: Make the type of the result buffer correct instead of
2222	// using reinterpret_cast.
2223	llvm::UTF32 *ResultWidePtr = reinterpret_cast<llvm::UTF32*>(ResultPtr);
2224	*ResultWidePtr = ResultChar;
2225	ResultPtr += 4;
2226	} else if (CharByteWidth == 2) {
2227	// FIXME: Make the type of the result buffer correct instead of
2228	// using reinterpret_cast.
2229	llvm::UTF16 *ResultWidePtr = reinterpret_cast<llvm::UTF16*>(ResultPtr);
2230	*ResultWidePtr = ResultChar & 0xFFFF;
2231	ResultPtr += 2;
2232	} else {
2233	assert(CharByteWidth == 1 && "Unexpected char width");
2234	*ResultPtr++ = ResultChar & 0xFF;
2235	}
2236	}
2237	}
2238	}
2239
2240	assert((!Pascal || !isUnevaluated()) &&
2241	"Pascal string in unevaluated context");
2242	if (Pascal) {
2243	if (CharByteWidth == 4) {
2244	// FIXME: Make the type of the result buffer correct instead of
2245	// using reinterpret_cast.
2246	llvm::UTF32 *ResultWidePtr = reinterpret_cast<llvm::UTF32*>(ResultBuf.data());
2247	ResultWidePtr[0] = GetNumStringChars() - 1;
2248	} else if (CharByteWidth == 2) {
2249	// FIXME: Make the type of the result buffer correct instead of
2250	// using reinterpret_cast.
2251	llvm::UTF16 *ResultWidePtr = reinterpret_cast<llvm::UTF16*>(ResultBuf.data());
2252	ResultWidePtr[0] = GetNumStringChars() - 1;
2253	} else {
2254	assert(CharByteWidth == 1 && "Unexpected char width");
2255	ResultBuf[0] = GetNumStringChars() - 1;
2256	}
2257
2258	// Verify that pascal strings aren't too large.
2259	if (GetStringLength() > 256) {
2260	if (Diags)
2261	Diags->Report(StringToks.front().getLocation(),
2262	diag::err_pascal_string_too_long)
2263	<< SourceRange(StringToks.front().getLocation(),
2264	StringToks.back().getLocation());
2265	hadError = true;
2266	return;
2267	}
2268	} else if (Diags) {
2269	// Complain if this string literal has too many characters.
2270	unsigned MaxChars = Features.CPlusPlus? 65536 : Features.C99 ? 4095 : 509;
2271
2272	if (GetNumStringChars() > MaxChars)
2273	Diags->Report(StringToks.front().getLocation(),
2274	diag::ext_string_too_long)
2275	<< GetNumStringChars() << MaxChars
2276	<< (Features.CPlusPlus ? 2 : Features.C99 ? 1 : 0)
2277	<< SourceRange(StringToks.front().getLocation(),
2278	StringToks.back().getLocation());
2279	}
2280	}
2281
2282	static const char *resyncUTF8(const char *Err, const char *End) {
2283	if (Err == End)
2284	return End;
2285	End = Err + std::min<unsigned>(a: llvm::getNumBytesForUTF8(firstByte: *Err), b: End-Err);
2286	while (++Err != End && (*Err & 0xC0) == 0x80)
2287	;
2288	return Err;
2289	}
2290
2291	/// This function copies from Fragment, which is a sequence of bytes
2292	/// within Tok's contents (which begin at TokBegin) into ResultPtr.
2293	/// Performs widening for multi-byte characters.
2294	bool StringLiteralParser::CopyStringFragment(const Token &Tok,
2295	const char *TokBegin,
2296	StringRef Fragment) {
2297	const llvm::UTF8 *ErrorPtrTmp;
2298	if (ConvertUTF8toWide(WideCharWidth: CharByteWidth, Source: Fragment, ResultPtr, ErrorPtr&: ErrorPtrTmp))
2299	return false;
2300
2301	// If we see bad encoding for unprefixed string literals, warn and
2302	// simply copy the byte values, for compatibility with gcc and older
2303	// versions of clang.
2304	bool NoErrorOnBadEncoding = isOrdinary();
2305	if (NoErrorOnBadEncoding) {
2306	memcpy(dest: ResultPtr, src: Fragment.data(), n: Fragment.size());
2307	ResultPtr += Fragment.size();
2308	}
2309
2310	if (Diags) {
2311	const char *ErrorPtr = reinterpret_cast<const char *>(ErrorPtrTmp);
2312
2313	FullSourceLoc SourceLoc(Tok.getLocation(), SM);
2314	const DiagnosticBuilder &Builder =
2315	Diag(Diags, Features, SourceLoc, TokBegin,
2316	ErrorPtr, resyncUTF8(ErrorPtr, Fragment.end()),
2317	NoErrorOnBadEncoding ? diag::warn_bad_string_encoding
2318	: diag::err_bad_string_encoding);
2319
2320	const char *NextStart = resyncUTF8(Err: ErrorPtr, End: Fragment.end());
2321	StringRef NextFragment(NextStart, Fragment.end()-NextStart);
2322
2323	// Decode into a dummy buffer.
2324	SmallString<512> Dummy;
2325	Dummy.reserve(N: Fragment.size() * CharByteWidth);
2326	char *Ptr = Dummy.data();
2327
2328	while (!ConvertUTF8toWide(WideCharWidth: CharByteWidth, Source: NextFragment, ResultPtr&: Ptr, ErrorPtr&: ErrorPtrTmp)) {
2329	const char *ErrorPtr = reinterpret_cast<const char *>(ErrorPtrTmp);
2330	NextStart = resyncUTF8(Err: ErrorPtr, End: Fragment.end());
2331	Builder << MakeCharSourceRange(Features, TokLoc: SourceLoc, TokBegin,
2332	TokRangeBegin: ErrorPtr, TokRangeEnd: NextStart);
2333	NextFragment = StringRef(NextStart, Fragment.end()-NextStart);
2334	}
2335	}
2336	return !NoErrorOnBadEncoding;
2337	}
2338
2339	void StringLiteralParser::DiagnoseLexingError(SourceLocation Loc) {
2340	hadError = true;
2341	if (Diags)
2342	Diags->Report(Loc, diag::err_lexing_string);
2343	}
2344
2345	/// getOffsetOfStringByte - This function returns the offset of the
2346	/// specified byte of the string data represented by Token. This handles
2347	/// advancing over escape sequences in the string.
2348	unsigned StringLiteralParser::getOffsetOfStringByte(const Token &Tok,
2349	unsigned ByteNo) const {
2350	// Get the spelling of the token.
2351	SmallString<32> SpellingBuffer;
2352	SpellingBuffer.resize(N: Tok.getLength());
2353
2354	bool StringInvalid = false;
2355	const char *SpellingPtr = &SpellingBuffer[0];
2356	unsigned TokLen = Lexer::getSpelling(Tok, Buffer&: SpellingPtr, SourceMgr: SM, LangOpts: Features,
2357	Invalid: &StringInvalid);
2358	if (StringInvalid)
2359	return 0;
2360
2361	const char *SpellingStart = SpellingPtr;
2362	const char *SpellingEnd = SpellingPtr+TokLen;
2363
2364	// Handle UTF-8 strings just like narrow strings.
2365	if (SpellingPtr[0] == 'u' && SpellingPtr[1] == '8')
2366	SpellingPtr += 2;
2367
2368	assert(SpellingPtr[0] != 'L' && SpellingPtr[0] != 'u' &&
2369	SpellingPtr[0] != 'U' && "Doesn't handle wide or utf strings yet");
2370
2371	// For raw string literals, this is easy.
2372	if (SpellingPtr[0] == 'R') {
2373	assert(SpellingPtr[1] == '"' && "Should be a raw string literal!");
2374	// Skip 'R"'.
2375	SpellingPtr += 2;
2376	while (*SpellingPtr != '(') {
2377	++SpellingPtr;
2378	assert(SpellingPtr < SpellingEnd && "Missing ( for raw string literal");
2379	}
2380	// Skip '('.
2381	++SpellingPtr;
2382	return SpellingPtr - SpellingStart + ByteNo;
2383	}
2384
2385	// Skip over the leading quote
2386	assert(SpellingPtr[0] == '"' && "Should be a string literal!");
2387	++SpellingPtr;
2388
2389	// Skip over bytes until we find the offset we're looking for.
2390	while (ByteNo) {
2391	assert(SpellingPtr < SpellingEnd && "Didn't find byte offset!");
2392
2393	// Step over non-escapes simply.
2394	if (*SpellingPtr != '\\') {
2395	++SpellingPtr;
2396	--ByteNo;
2397	continue;
2398	}
2399
2400	// Otherwise, this is an escape character. Advance over it.
2401	bool HadError = false;
2402	if (SpellingPtr[1] == 'u' || SpellingPtr[1] == 'U' ||
2403	SpellingPtr[1] == 'N') {
2404	const char *EscapePtr = SpellingPtr;
2405	unsigned Len = MeasureUCNEscape(ThisTokBegin: SpellingStart, ThisTokBuf&: SpellingPtr, ThisTokEnd: SpellingEnd,
2406	CharByteWidth: 1, Features, HadError);
2407	if (Len > ByteNo) {
2408	// ByteNo is somewhere within the escape sequence.
2409	SpellingPtr = EscapePtr;
2410	break;
2411	}
2412	ByteNo -= Len;
2413	} else {
2414	ProcessCharEscape(ThisTokBegin: SpellingStart, ThisTokBuf&: SpellingPtr, ThisTokEnd: SpellingEnd, HadError,
2415	Loc: FullSourceLoc(Tok.getLocation(), SM), CharWidth: CharByteWidth * 8,
2416	Diags, Features, EvalMethod: StringLiteralEvalMethod::Evaluated);
2417	--ByteNo;
2418	}
2419	assert(!HadError && "This method isn't valid on erroneous strings");
2420	}
2421
2422	return SpellingPtr-SpellingStart;
2423	}
2424
2425	/// Determine whether a suffix is a valid ud-suffix. We avoid treating reserved
2426	/// suffixes as ud-suffixes, because the diagnostic experience is better if we
2427	/// treat it as an invalid suffix.
2428	bool StringLiteralParser::isValidUDSuffix(const LangOptions &LangOpts,
2429	StringRef Suffix) {
2430	return NumericLiteralParser::isValidUDSuffix(LangOpts, Suffix) ||
2431	Suffix == "sv";
2432	}
2433