1	//
2	// Copyright (C) 2014-2016 LunarG, Inc.
3	// Copyright (C) 2015-2020 Google, Inc.
4	// Copyright (C) 2017, 2022-2024 Arm Limited.
5	// Modifications Copyright (C) 2020 Advanced Micro Devices, Inc. All rights reserved.
6	//
7	// All rights reserved.
8	//
9	// Redistribution and use in source and binary forms, with or without
10	// modification, are permitted provided that the following conditions
11	// are met:
12	//
13	// Redistributions of source code must retain the above copyright
14	// notice, this list of conditions and the following disclaimer.
15	//
16	// Redistributions in binary form must reproduce the above
17	// copyright notice, this list of conditions and the following
18	// disclaimer in the documentation and/or other materials provided
19	// with the distribution.
20	//
21	// Neither the name of 3Dlabs Inc. Ltd. nor the names of its
22	// contributors may be used to endorse or promote products derived
23	// from this software without specific prior written permission.
24	//
25	// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
26	// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
27	// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
28	// FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
29	// COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
30	// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
31	// BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
32	// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
33	// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
34	// LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
35	// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
36	// POSSIBILITY OF SUCH DAMAGE.
37
38	//
39	// Visit the nodes in the glslang intermediate tree representation to
40	// translate them to SPIR-V.
41	//
42
43	#include "spirv.hpp"
44	#include "GlslangToSpv.h"
45	#include "SpvBuilder.h"
46	#include "SpvTools.h"
47	namespace spv {
48	#include "GLSL.std.450.h"
49	#include "GLSL.ext.KHR.h"
50	#include "GLSL.ext.EXT.h"
51	#include "GLSL.ext.AMD.h"
52	#include "GLSL.ext.NV.h"
53	#include "GLSL.ext.ARM.h"
54	#include "GLSL.ext.QCOM.h"
55	#include "NonSemanticDebugPrintf.h"
56	}
57
58	// Glslang includes
59	#include "../glslang/MachineIndependent/localintermediate.h"
60	#include "../glslang/MachineIndependent/SymbolTable.h"
61	#include "../glslang/Include/Common.h"
62
63	// Build-time generated includes
64	#include "glslang/build_info.h"
65
66	#include <fstream>
67	#include <iomanip>
68	#include <list>
69	#include <map>
70	#include <optional>
71	#include <stack>
72	#include <string>
73	#include <vector>
74
75	namespace {
76
77	namespace {
78	class SpecConstantOpModeGuard {
79	public:
80	SpecConstantOpModeGuard(spv::Builder* builder)
81	: builder_(builder) {
82	previous_flag_ = builder->isInSpecConstCodeGenMode();
83	}
84	~SpecConstantOpModeGuard() {
85	previous_flag_ ? builder_->setToSpecConstCodeGenMode()
86	: builder_->setToNormalCodeGenMode();
87	}
88	void turnOnSpecConstantOpMode() {
89	builder_->setToSpecConstCodeGenMode();
90	}
91
92	private:
93	spv::Builder* builder_;
94	bool previous_flag_;
95	};
96
97	struct OpDecorations {
98	public:
99	OpDecorations(spv::Decoration precision, spv::Decoration noContraction, spv::Decoration nonUniform) :
100	precision(precision)
101	,
102	noContraction(noContraction),
103	nonUniform(nonUniform)
104	{ }
105
106	spv::Decoration precision;
107
108	void addNoContraction(spv::Builder& builder, spv::Id t) { builder.addDecoration(t, noContraction); }
109	void addNonUniform(spv::Builder& builder, spv::Id t) { builder.addDecoration(t, nonUniform); }
110	protected:
111	spv::Decoration noContraction;
112	spv::Decoration nonUniform;
113	};
114
115	} // namespace
116
117	using namespace QtShaderTools;
118
119	//
120	// The main holder of information for translating glslang to SPIR-V.
121	//
122	// Derives from the AST walking base class.
123	//
124	class TGlslangToSpvTraverser : public glslang::TIntermTraverser {
125	public:
126	TGlslangToSpvTraverser(unsigned int spvVersion, const glslang::TIntermediate*, spv::SpvBuildLogger* logger,
127	glslang::SpvOptions& options);
128	virtual ~TGlslangToSpvTraverser() { }
129
130	bool visitAggregate(glslang::TVisit, glslang::TIntermAggregate*);
131	bool visitBinary(glslang::TVisit, glslang::TIntermBinary*);
132	void visitConstantUnion(glslang::TIntermConstantUnion*);
133	bool visitSelection(glslang::TVisit, glslang::TIntermSelection*);
134	bool visitSwitch(glslang::TVisit, glslang::TIntermSwitch*);
135	void visitSymbol(glslang::TIntermSymbol* symbol);
136	bool visitUnary(glslang::TVisit, glslang::TIntermUnary*);
137	bool visitLoop(glslang::TVisit, glslang::TIntermLoop*);
138	bool visitBranch(glslang::TVisit visit, glslang::TIntermBranch*);
139
140	void finishSpv(bool compileOnly);
141	void dumpSpv(std::vector<unsigned int>& out);
142
143	protected:
144	TGlslangToSpvTraverser(TGlslangToSpvTraverser&);
145	TGlslangToSpvTraverser& operator=(TGlslangToSpvTraverser&);
146
147	spv::Decoration TranslateInterpolationDecoration(const glslang::TQualifier& qualifier);
148	spv::Decoration TranslateAuxiliaryStorageDecoration(const glslang::TQualifier& qualifier);
149	spv::Decoration TranslateNonUniformDecoration(const glslang::TQualifier& qualifier);
150	spv::Decoration TranslateNonUniformDecoration(const spv::Builder::AccessChain::CoherentFlags& coherentFlags);
151	spv::Builder::AccessChain::CoherentFlags TranslateCoherent(const glslang::TType& type);
152	spv::MemoryAccessMask TranslateMemoryAccess(const spv::Builder::AccessChain::CoherentFlags &coherentFlags);
153	spv::ImageOperandsMask TranslateImageOperands(const spv::Builder::AccessChain::CoherentFlags &coherentFlags);
154	spv::Scope TranslateMemoryScope(const spv::Builder::AccessChain::CoherentFlags &coherentFlags);
155	spv::BuiltIn TranslateBuiltInDecoration(glslang::TBuiltInVariable, bool memberDeclaration);
156	spv::ImageFormat TranslateImageFormat(const glslang::TType& type);
157	spv::SelectionControlMask TranslateSelectionControl(const glslang::TIntermSelection&) const;
158	spv::SelectionControlMask TranslateSwitchControl(const glslang::TIntermSwitch&) const;
159	spv::LoopControlMask TranslateLoopControl(const glslang::TIntermLoop&, std::vector<unsigned int>& operands) const;
160	spv::StorageClass TranslateStorageClass(const glslang::TType&);
161	void TranslateLiterals(const glslang::TVector<const glslang::TIntermConstantUnion*>&, std::vector<unsigned>&) const;
162	void addIndirectionIndexCapabilities(const glslang::TType& baseType, const glslang::TType& indexType);
163	spv::Id createSpvVariable(const glslang::TIntermSymbol*, spv::Id forcedType);
164	spv::Id getSampledType(const glslang::TSampler&);
165	spv::Id getInvertedSwizzleType(const glslang::TIntermTyped&);
166	spv::Id createInvertedSwizzle(spv::Decoration precision, const glslang::TIntermTyped&, spv::Id parentResult);
167	void convertSwizzle(const glslang::TIntermAggregate&, std::vector<unsigned>& swizzle);
168	spv::Id convertGlslangToSpvType(const glslang::TType& type, bool forwardReferenceOnly = false);
169	spv::Id convertGlslangToSpvType(const glslang::TType& type, glslang::TLayoutPacking, const glslang::TQualifier&,
170	bool lastBufferBlockMember, bool forwardReferenceOnly = false);
171	void applySpirvDecorate(const glslang::TType& type, spv::Id id, std::optional<int> member);
172	bool filterMember(const glslang::TType& member);
173	spv::Id convertGlslangStructToSpvType(const glslang::TType&, const glslang::TTypeList* glslangStruct,
174	glslang::TLayoutPacking, const glslang::TQualifier&);
175	spv::LinkageType convertGlslangLinkageToSpv(glslang::TLinkType glslangLinkType);
176	void decorateStructType(const glslang::TType&, const glslang::TTypeList* glslangStruct, glslang::TLayoutPacking,
177	const glslang::TQualifier&, spv::Id, const std::vector<spv::Id>& spvMembers);
178	spv::Id makeArraySizeId(const glslang::TArraySizes&, int dim, bool allowZero = false);
179	spv::Id accessChainLoad(const glslang::TType& type);
180	void accessChainStore(const glslang::TType& type, spv::Id rvalue);
181	void multiTypeStore(const glslang::TType&, spv::Id rValue);
182	spv::Id convertLoadedBoolInUniformToUint(const glslang::TType& type, spv::Id nominalTypeId, spv::Id loadedId);
183	glslang::TLayoutPacking getExplicitLayout(const glslang::TType& type) const;
184	int getArrayStride(const glslang::TType& arrayType, glslang::TLayoutPacking, glslang::TLayoutMatrix);
185	int getMatrixStride(const glslang::TType& matrixType, glslang::TLayoutPacking, glslang::TLayoutMatrix);
186	void updateMemberOffset(const glslang::TType& structType, const glslang::TType& memberType, int& currentOffset,
187	int& nextOffset, glslang::TLayoutPacking, glslang::TLayoutMatrix);
188	void declareUseOfStructMember(const glslang::TTypeList& members, int glslangMember);
189
190	bool isShaderEntryPoint(const glslang::TIntermAggregate* node);
191	bool writableParam(glslang::TStorageQualifier) const;
192	bool originalParam(glslang::TStorageQualifier, const glslang::TType&, bool implicitThisParam);
193	void makeFunctions(const glslang::TIntermSequence&);
194	void makeGlobalInitializers(const glslang::TIntermSequence&);
195	void collectRayTracingLinkerObjects();
196	void visitFunctions(const glslang::TIntermSequence&);
197	void handleFunctionEntry(const glslang::TIntermAggregate* node);
198	void translateArguments(const glslang::TIntermAggregate& node, std::vector<spv::Id>& arguments,
199	spv::Builder::AccessChain::CoherentFlags &lvalueCoherentFlags);
200	void translateArguments(glslang::TIntermUnary& node, std::vector<spv::Id>& arguments);
201	spv::Id createImageTextureFunctionCall(glslang::TIntermOperator* node);
202	spv::Id handleUserFunctionCall(const glslang::TIntermAggregate*);
203
204	spv::Id createBinaryOperation(glslang::TOperator op, OpDecorations&, spv::Id typeId, spv::Id left, spv::Id right,
205	glslang::TBasicType typeProxy, bool reduceComparison = true);
206	spv::Id createBinaryMatrixOperation(spv::Op, OpDecorations&, spv::Id typeId, spv::Id left, spv::Id right);
207	spv::Id createUnaryOperation(glslang::TOperator op, OpDecorations&, spv::Id typeId, spv::Id operand,
208	glslang::TBasicType typeProxy,
209	const spv::Builder::AccessChain::CoherentFlags &lvalueCoherentFlags,
210	const glslang::TType &opType);
211	spv::Id createUnaryMatrixOperation(spv::Op op, OpDecorations&, spv::Id typeId, spv::Id operand,
212	glslang::TBasicType typeProxy);
213	spv::Id createConversion(glslang::TOperator op, OpDecorations&, spv::Id destTypeId, spv::Id operand,
214	glslang::TBasicType typeProxy);
215	spv::Id createIntWidthConversion(glslang::TOperator op, spv::Id operand, int vectorSize, spv::Id destType);
216	spv::Id makeSmearedConstant(spv::Id constant, int vectorSize);
217	spv::Id createAtomicOperation(glslang::TOperator op, spv::Decoration precision, spv::Id typeId,
218	std::vector<spv::Id>& operands, glslang::TBasicType typeProxy,
219	const spv::Builder::AccessChain::CoherentFlags &lvalueCoherentFlags,
220	const glslang::TType &opType);
221	spv::Id createInvocationsOperation(glslang::TOperator op, spv::Id typeId, std::vector<spv::Id>& operands,
222	glslang::TBasicType typeProxy);
223	spv::Id CreateInvocationsVectorOperation(spv::Op op, spv::GroupOperation groupOperation,
224	spv::Id typeId, std::vector<spv::Id>& operands);
225	spv::Id createSubgroupOperation(glslang::TOperator op, spv::Id typeId, std::vector<spv::Id>& operands,
226	glslang::TBasicType typeProxy);
227	spv::Id createMiscOperation(glslang::TOperator op, spv::Decoration precision, spv::Id typeId,
228	std::vector<spv::Id>& operands, glslang::TBasicType typeProxy);
229	spv::Id createNoArgOperation(glslang::TOperator op, spv::Decoration precision, spv::Id typeId);
230	spv::Id getSymbolId(const glslang::TIntermSymbol* node);
231	void addMeshNVDecoration(spv::Id id, int member, const glslang::TQualifier & qualifier);
232	bool hasQCOMImageProceessingDecoration(spv::Id id, spv::Decoration decor);
233	void addImageProcessingQCOMDecoration(spv::Id id, spv::Decoration decor);
234	void addImageProcessing2QCOMDecoration(spv::Id id, bool isForGather);
235	spv::Id createSpvConstant(const glslang::TIntermTyped&);
236	spv::Id createSpvConstantFromConstUnionArray(const glslang::TType& type, const glslang::TConstUnionArray&,
237	int& nextConst, bool specConstant);
238	bool isTrivialLeaf(const glslang::TIntermTyped* node);
239	bool isTrivial(const glslang::TIntermTyped* node);
240	spv::Id createShortCircuit(glslang::TOperator, glslang::TIntermTyped& left, glslang::TIntermTyped& right);
241	spv::Id getExtBuiltins(const char* name);
242	std::pair<spv::Id, spv::Id> getForcedType(glslang::TBuiltInVariable builtIn, const glslang::TType&);
243	spv::Id translateForcedType(spv::Id object);
244	spv::Id createCompositeConstruct(spv::Id typeId, std::vector<spv::Id> constituents);
245
246	glslang::SpvOptions& options;
247	spv::Function* shaderEntry;
248	spv::Function* currentFunction;
249	spv::Instruction* entryPoint;
250	int sequenceDepth;
251
252	spv::SpvBuildLogger* logger;
253
254	// There is a 1:1 mapping between a spv builder and a module; this is thread safe
255	spv::Builder builder;
256	bool inEntryPoint;
257	bool entryPointTerminated;
258	bool linkageOnly; // true when visiting the set of objects in the AST present only for
259	// establishing interface, whether or not they were statically used
260	std::set<spv::Id> iOSet; // all input/output variables from either static use or declaration of interface
261	const glslang::TIntermediate* glslangIntermediate;
262	bool nanMinMaxClamp; // true if use NMin/NMax/NClamp instead of FMin/FMax/FClamp
263	spv::Id stdBuiltins;
264	spv::Id nonSemanticDebugPrintf;
265	std::unordered_map<std::string, spv::Id> extBuiltinMap;
266
267	std::unordered_map<long long, spv::Id> symbolValues;
268	std::unordered_map<uint32_t, spv::Id> builtInVariableIds;
269	std::unordered_set<long long> rValueParameters; // set of formal function parameters passed as rValues,
270	// rather than a pointer
271	std::unordered_map<std::string, spv::Function*> functionMap;
272	std::unordered_map<const glslang::TTypeList*, spv::Id> structMap[glslang::ElpCount][glslang::ElmCount];
273	// for mapping glslang block indices to spv indices (e.g., due to hidden members):
274	std::unordered_map<long long, std::vector<int>> memberRemapper;
275	// for mapping glslang symbol struct to symbol Id
276	std::unordered_map<const glslang::TTypeList*, long long> glslangTypeToIdMap;
277	std::stack<bool> breakForLoop; // false means break for switch
278	std::unordered_map<std::string, const glslang::TIntermSymbol*> counterOriginator;
279	// Map pointee types for EbtReference to their forward pointers
280	std::map<const glslang::TType *, spv::Id> forwardPointers;
281	// Type forcing, for when SPIR-V wants a different type than the AST,
282	// requiring local translation to and from SPIR-V type on every access.
283	// Maps <builtin-variable-id -> AST-required-type-id>
284	std::unordered_map<spv::Id, spv::Id> forceType;
285	// Used by Task shader while generating opearnds for OpEmitMeshTasksEXT
286	spv::Id taskPayloadID;
287	// Used later for generating OpTraceKHR/OpExecuteCallableKHR/OpHitObjectRecordHit*/OpHitObjectGetShaderBindingTableData
288	std::unordered_map<unsigned int, glslang::TIntermSymbol *> locationToSymbol[4];
289	std::unordered_map<spv::Id, std::vector<spv::Decoration> > idToQCOMDecorations;
290	};
291
292	//
293	// Helper functions for translating glslang representations to SPIR-V enumerants.
294	//
295
296	// Translate glslang profile to SPIR-V source language.
297	spv::SourceLanguage TranslateSourceLanguage(glslang::EShSource source, EProfile profile)
298	{
299	switch (source) {
300	case glslang::EShSourceGlsl:
301	switch (profile) {
302	case ENoProfile:
303	case ECoreProfile:
304	case ECompatibilityProfile:
305	return spv::SourceLanguageGLSL;
306	case EEsProfile:
307	return spv::SourceLanguageESSL;
308	default:
309	return spv::SourceLanguageUnknown;
310	}
311	case glslang::EShSourceHlsl:
312	return spv::SourceLanguageHLSL;
313	default:
314	return spv::SourceLanguageUnknown;
315	}
316	}
317
318	// Translate glslang language (stage) to SPIR-V execution model.
319	spv::ExecutionModel TranslateExecutionModel(EShLanguage stage, bool isMeshShaderEXT = false)
320	{
321	switch (stage) {
322	case EShLangVertex: return spv::ExecutionModelVertex;
323	case EShLangFragment: return spv::ExecutionModelFragment;
324	case EShLangCompute: return spv::ExecutionModelGLCompute;
325	case EShLangTessControl: return spv::ExecutionModelTessellationControl;
326	case EShLangTessEvaluation: return spv::ExecutionModelTessellationEvaluation;
327	case EShLangGeometry: return spv::ExecutionModelGeometry;
328	case EShLangRayGen: return spv::ExecutionModelRayGenerationKHR;
329	case EShLangIntersect: return spv::ExecutionModelIntersectionKHR;
330	case EShLangAnyHit: return spv::ExecutionModelAnyHitKHR;
331	case EShLangClosestHit: return spv::ExecutionModelClosestHitKHR;
332	case EShLangMiss: return spv::ExecutionModelMissKHR;
333	case EShLangCallable: return spv::ExecutionModelCallableKHR;
334	case EShLangTask: return (isMeshShaderEXT)? spv::ExecutionModelTaskEXT : spv::ExecutionModelTaskNV;
335	case EShLangMesh: return (isMeshShaderEXT)? spv::ExecutionModelMeshEXT: spv::ExecutionModelMeshNV;
336	default:
337	assert(0);
338	return spv::ExecutionModelFragment;
339	}
340	}
341
342	// Translate glslang sampler type to SPIR-V dimensionality.
343	spv::Dim TranslateDimensionality(const glslang::TSampler& sampler)
344	{
345	switch (sampler.dim) {
346	case glslang::Esd1D: return spv::Dim1D;
347	case glslang::Esd2D: return spv::Dim2D;
348	case glslang::Esd3D: return spv::Dim3D;
349	case glslang::EsdCube: return spv::DimCube;
350	case glslang::EsdRect: return spv::DimRect;
351	case glslang::EsdBuffer: return spv::DimBuffer;
352	case glslang::EsdSubpass: return spv::DimSubpassData;
353	case glslang::EsdAttachmentEXT: return spv::DimTileImageDataEXT;
354	default:
355	assert(0);
356	return spv::Dim2D;
357	}
358	}
359
360	// Translate glslang precision to SPIR-V precision decorations.
361	spv::Decoration TranslatePrecisionDecoration(glslang::TPrecisionQualifier glslangPrecision)
362	{
363	switch (glslangPrecision) {
364	case glslang::EpqLow: return spv::DecorationRelaxedPrecision;
365	case glslang::EpqMedium: return spv::DecorationRelaxedPrecision;
366	default:
367	return spv::NoPrecision;
368	}
369	}
370
371	// Translate glslang type to SPIR-V precision decorations.
372	spv::Decoration TranslatePrecisionDecoration(const glslang::TType& type)
373	{
374	return TranslatePrecisionDecoration(glslangPrecision: type.getQualifier().precision);
375	}
376
377	// Translate glslang type to SPIR-V block decorations.
378	spv::Decoration TranslateBlockDecoration(const glslang::TStorageQualifier storage, bool useStorageBuffer)
379	{
380	switch (storage) {
381	case glslang::EvqUniform: return spv::DecorationBlock;
382	case glslang::EvqBuffer: return useStorageBuffer ? spv::DecorationBlock : spv::DecorationBufferBlock;
383	case glslang::EvqVaryingIn: return spv::DecorationBlock;
384	case glslang::EvqVaryingOut: return spv::DecorationBlock;
385	case glslang::EvqShared: return spv::DecorationBlock;
386	case glslang::EvqPayload: return spv::DecorationBlock;
387	case glslang::EvqPayloadIn: return spv::DecorationBlock;
388	case glslang::EvqHitAttr: return spv::DecorationBlock;
389	case glslang::EvqCallableData: return spv::DecorationBlock;
390	case glslang::EvqCallableDataIn: return spv::DecorationBlock;
391	case glslang::EvqHitObjectAttrNV: return spv::DecorationBlock;
392	default:
393	assert(0);
394	break;
395	}
396
397	return spv::DecorationMax;
398	}
399
400	// Translate glslang type to SPIR-V memory decorations.
401	void TranslateMemoryDecoration(const glslang::TQualifier& qualifier, std::vector<spv::Decoration>& memory,
402	bool useVulkanMemoryModel)
403	{
404	if (!useVulkanMemoryModel) {
405	if (qualifier.isVolatile()) {
406	memory.push_back(x: spv::DecorationVolatile);
407	memory.push_back(x: spv::DecorationCoherent);
408	} else if (qualifier.isCoherent()) {
409	memory.push_back(x: spv::DecorationCoherent);
410	}
411	}
412	if (qualifier.isRestrict())
413	memory.push_back(x: spv::DecorationRestrict);
414	if (qualifier.isReadOnly())
415	memory.push_back(x: spv::DecorationNonWritable);
416	if (qualifier.isWriteOnly())
417	memory.push_back(x: spv::DecorationNonReadable);
418	}
419
420	// Translate glslang type to SPIR-V layout decorations.
421	spv::Decoration TranslateLayoutDecoration(const glslang::TType& type, glslang::TLayoutMatrix matrixLayout)
422	{
423	if (type.isMatrix()) {
424	switch (matrixLayout) {
425	case glslang::ElmRowMajor:
426	return spv::DecorationRowMajor;
427	case glslang::ElmColumnMajor:
428	return spv::DecorationColMajor;
429	default:
430	// opaque layouts don't need a majorness
431	return spv::DecorationMax;
432	}
433	} else {
434	switch (type.getBasicType()) {
435	default:
436	return spv::DecorationMax;
437	break;
438	case glslang::EbtBlock:
439	switch (type.getQualifier().storage) {
440	case glslang::EvqShared:
441	case glslang::EvqUniform:
442	case glslang::EvqBuffer:
443	switch (type.getQualifier().layoutPacking) {
444	case glslang::ElpShared: return spv::DecorationGLSLShared;
445	case glslang::ElpPacked: return spv::DecorationGLSLPacked;
446	default:
447	return spv::DecorationMax;
448	}
449	case glslang::EvqVaryingIn:
450	case glslang::EvqVaryingOut:
451	if (type.getQualifier().isTaskMemory()) {
452	switch (type.getQualifier().layoutPacking) {
453	case glslang::ElpShared: return spv::DecorationGLSLShared;
454	case glslang::ElpPacked: return spv::DecorationGLSLPacked;
455	default: break;
456	}
457	} else {
458	assert(type.getQualifier().layoutPacking == glslang::ElpNone);
459	}
460	return spv::DecorationMax;
461	case glslang::EvqPayload:
462	case glslang::EvqPayloadIn:
463	case glslang::EvqHitAttr:
464	case glslang::EvqCallableData:
465	case glslang::EvqCallableDataIn:
466	case glslang::EvqHitObjectAttrNV:
467	return spv::DecorationMax;
468	default:
469	assert(0);
470	return spv::DecorationMax;
471	}
472	}
473	}
474	}
475
476	// Translate glslang type to SPIR-V interpolation decorations.
477	// Returns spv::DecorationMax when no decoration
478	// should be applied.
479	spv::Decoration TGlslangToSpvTraverser::TranslateInterpolationDecoration(const glslang::TQualifier& qualifier)
480	{
481	if (qualifier.smooth)
482	// Smooth decoration doesn't exist in SPIR-V 1.0
483	return spv::DecorationMax;
484	else if (qualifier.isNonPerspective())
485	return spv::DecorationNoPerspective;
486	else if (qualifier.flat)
487	return spv::DecorationFlat;
488	else if (qualifier.isExplicitInterpolation()) {
489	builder.addExtension(ext: spv::E_SPV_AMD_shader_explicit_vertex_parameter);
490	return spv::DecorationExplicitInterpAMD;
491	}
492	else
493	return spv::DecorationMax;
494	}
495
496	// Translate glslang type to SPIR-V auxiliary storage decorations.
497	// Returns spv::DecorationMax when no decoration
498	// should be applied.
499	spv::Decoration TGlslangToSpvTraverser::TranslateAuxiliaryStorageDecoration(const glslang::TQualifier& qualifier)
500	{
501	if (qualifier.centroid)
502	return spv::DecorationCentroid;
503	else if (qualifier.patch)
504	return spv::DecorationPatch;
505	else if (qualifier.sample) {
506	builder.addCapability(cap: spv::CapabilitySampleRateShading);
507	return spv::DecorationSample;
508	}
509
510	return spv::DecorationMax;
511	}
512
513	// If glslang type is invariant, return SPIR-V invariant decoration.
514	spv::Decoration TranslateInvariantDecoration(const glslang::TQualifier& qualifier)
515	{
516	if (qualifier.invariant)
517	return spv::DecorationInvariant;
518	else
519	return spv::DecorationMax;
520	}
521
522	// If glslang type is noContraction, return SPIR-V NoContraction decoration.
523	spv::Decoration TranslateNoContractionDecoration(const glslang::TQualifier& qualifier)
524	{
525	if (qualifier.isNoContraction())
526	return spv::DecorationNoContraction;
527	else
528	return spv::DecorationMax;
529	}
530
531	// If glslang type is nonUniform, return SPIR-V NonUniform decoration.
532	spv::Decoration TGlslangToSpvTraverser::TranslateNonUniformDecoration(const glslang::TQualifier& qualifier)
533	{
534	if (qualifier.isNonUniform()) {
535	builder.addIncorporatedExtension(ext: "SPV_EXT_descriptor_indexing", incorporatedVersion: spv::Spv_1_5);
536	builder.addCapability(cap: spv::CapabilityShaderNonUniformEXT);
537	return spv::DecorationNonUniformEXT;
538	} else
539	return spv::DecorationMax;
540	}
541
542	// If lvalue flags contains nonUniform, return SPIR-V NonUniform decoration.
543	spv::Decoration TGlslangToSpvTraverser::TranslateNonUniformDecoration(
544	const spv::Builder::AccessChain::CoherentFlags& coherentFlags)
545	{
546	if (coherentFlags.isNonUniform()) {
547	builder.addIncorporatedExtension(ext: "SPV_EXT_descriptor_indexing", incorporatedVersion: spv::Spv_1_5);
548	builder.addCapability(cap: spv::CapabilityShaderNonUniformEXT);
549	return spv::DecorationNonUniformEXT;
550	} else
551	return spv::DecorationMax;
552	}
553
554	spv::MemoryAccessMask TGlslangToSpvTraverser::TranslateMemoryAccess(
555	const spv::Builder::AccessChain::CoherentFlags &coherentFlags)
556	{
557	spv::MemoryAccessMask mask = spv::MemoryAccessMaskNone;
558
559	if (!glslangIntermediate->usingVulkanMemoryModel() || coherentFlags.isImage)
560	return mask;
561
562	if (coherentFlags.isVolatile() || coherentFlags.anyCoherent()) {
563	mask = mask | spv::MemoryAccessMakePointerAvailableKHRMask |
564	spv::MemoryAccessMakePointerVisibleKHRMask;
565	}
566
567	if (coherentFlags.nonprivate) {
568	mask = mask | spv::MemoryAccessNonPrivatePointerKHRMask;
569	}
570	if (coherentFlags.volatil) {
571	mask = mask | spv::MemoryAccessVolatileMask;
572	}
573	if (mask != spv::MemoryAccessMaskNone) {
574	builder.addCapability(cap: spv::CapabilityVulkanMemoryModelKHR);
575	}
576
577	return mask;
578	}
579
580	spv::ImageOperandsMask TGlslangToSpvTraverser::TranslateImageOperands(
581	const spv::Builder::AccessChain::CoherentFlags &coherentFlags)
582	{
583	spv::ImageOperandsMask mask = spv::ImageOperandsMaskNone;
584
585	if (!glslangIntermediate->usingVulkanMemoryModel())
586	return mask;
587
588	if (coherentFlags.volatil ||
589	coherentFlags.anyCoherent()) {
590	mask = mask | spv::ImageOperandsMakeTexelAvailableKHRMask |
591	spv::ImageOperandsMakeTexelVisibleKHRMask;
592	}
593	if (coherentFlags.nonprivate) {
594	mask = mask | spv::ImageOperandsNonPrivateTexelKHRMask;
595	}
596	if (coherentFlags.volatil) {
597	mask = mask | spv::ImageOperandsVolatileTexelKHRMask;
598	}
599	if (mask != spv::ImageOperandsMaskNone) {
600	builder.addCapability(cap: spv::CapabilityVulkanMemoryModelKHR);
601	}
602
603	return mask;
604	}
605
606	spv::Builder::AccessChain::CoherentFlags TGlslangToSpvTraverser::TranslateCoherent(const glslang::TType& type)
607	{
608	spv::Builder::AccessChain::CoherentFlags flags = {};
609	flags.coherent = type.getQualifier().coherent;
610	flags.devicecoherent = type.getQualifier().devicecoherent;
611	flags.queuefamilycoherent = type.getQualifier().queuefamilycoherent;
612	// shared variables are implicitly workgroupcoherent in GLSL.
613	flags.workgroupcoherent = type.getQualifier().workgroupcoherent ||
614	type.getQualifier().storage == glslang::EvqShared;
615	flags.subgroupcoherent = type.getQualifier().subgroupcoherent;
616	flags.shadercallcoherent = type.getQualifier().shadercallcoherent;
617	flags.volatil = type.getQualifier().volatil;
618	// *coherent variables are implicitly nonprivate in GLSL
619	flags.nonprivate = type.getQualifier().nonprivate ||
620	flags.anyCoherent() ||
621	flags.volatil;
622	flags.isImage = type.getBasicType() == glslang::EbtSampler;
623	flags.nonUniform = type.getQualifier().nonUniform;
624	return flags;
625	}
626
627	spv::Scope TGlslangToSpvTraverser::TranslateMemoryScope(
628	const spv::Builder::AccessChain::CoherentFlags &coherentFlags)
629	{
630	spv::Scope scope = spv::ScopeMax;
631
632	if (coherentFlags.volatil || coherentFlags.coherent) {
633	// coherent defaults to Device scope in the old model, QueueFamilyKHR scope in the new model
634	scope = glslangIntermediate->usingVulkanMemoryModel() ? spv::ScopeQueueFamilyKHR : spv::ScopeDevice;
635	} else if (coherentFlags.devicecoherent) {
636	scope = spv::ScopeDevice;
637	} else if (coherentFlags.queuefamilycoherent) {
638	scope = spv::ScopeQueueFamilyKHR;
639	} else if (coherentFlags.workgroupcoherent) {
640	scope = spv::ScopeWorkgroup;
641	} else if (coherentFlags.subgroupcoherent) {
642	scope = spv::ScopeSubgroup;
643	} else if (coherentFlags.shadercallcoherent) {
644	scope = spv::ScopeShaderCallKHR;
645	}
646	if (glslangIntermediate->usingVulkanMemoryModel() && scope == spv::ScopeDevice) {
647	builder.addCapability(cap: spv::CapabilityVulkanMemoryModelDeviceScopeKHR);
648	}
649
650	return scope;
651	}
652
653	// Translate a glslang built-in variable to a SPIR-V built in decoration. Also generate
654	// associated capabilities when required. For some built-in variables, a capability
655	// is generated only when using the variable in an executable instruction, but not when
656	// just declaring a struct member variable with it. This is true for PointSize,
657	// ClipDistance, and CullDistance.
658	spv::BuiltIn TGlslangToSpvTraverser::TranslateBuiltInDecoration(glslang::TBuiltInVariable builtIn,
659	bool memberDeclaration)
660	{
661	switch (builtIn) {
662	case glslang::EbvPointSize:
663	// Defer adding the capability until the built-in is actually used.
664	if (! memberDeclaration) {
665	switch (glslangIntermediate->getStage()) {
666	case EShLangGeometry:
667	builder.addCapability(cap: spv::CapabilityGeometryPointSize);
668	break;
669	case EShLangTessControl:
670	case EShLangTessEvaluation:
671	builder.addCapability(cap: spv::CapabilityTessellationPointSize);
672	break;
673	default:
674	break;
675	}
676	}
677	return spv::BuiltInPointSize;
678
679	case glslang::EbvPosition: return spv::BuiltInPosition;
680	case glslang::EbvVertexId: return spv::BuiltInVertexId;
681	case glslang::EbvInstanceId: return spv::BuiltInInstanceId;
682	case glslang::EbvVertexIndex: return spv::BuiltInVertexIndex;
683	case glslang::EbvInstanceIndex: return spv::BuiltInInstanceIndex;
684
685	case glslang::EbvFragCoord: return spv::BuiltInFragCoord;
686	case glslang::EbvPointCoord: return spv::BuiltInPointCoord;
687	case glslang::EbvFace: return spv::BuiltInFrontFacing;
688	case glslang::EbvFragDepth: return spv::BuiltInFragDepth;
689
690	case glslang::EbvNumWorkGroups: return spv::BuiltInNumWorkgroups;
691	case glslang::EbvWorkGroupSize: return spv::BuiltInWorkgroupSize;
692	case glslang::EbvWorkGroupId: return spv::BuiltInWorkgroupId;
693	case glslang::EbvLocalInvocationId: return spv::BuiltInLocalInvocationId;
694	case glslang::EbvLocalInvocationIndex: return spv::BuiltInLocalInvocationIndex;
695	case glslang::EbvGlobalInvocationId: return spv::BuiltInGlobalInvocationId;
696
697	// These *Distance capabilities logically belong here, but if the member is declared and
698	// then never used, consumers of SPIR-V prefer the capability not be declared.
699	// They are now generated when used, rather than here when declared.
700	// Potentially, the specification should be more clear what the minimum
701	// use needed is to trigger the capability.
702	//
703	case glslang::EbvClipDistance:
704	if (!memberDeclaration)
705	builder.addCapability(cap: spv::CapabilityClipDistance);
706	return spv::BuiltInClipDistance;
707
708	case glslang::EbvCullDistance:
709	if (!memberDeclaration)
710	builder.addCapability(cap: spv::CapabilityCullDistance);
711	return spv::BuiltInCullDistance;
712
713	case glslang::EbvViewportIndex:
714	if (glslangIntermediate->getStage() == EShLangGeometry ||
715	glslangIntermediate->getStage() == EShLangFragment) {
716	builder.addCapability(cap: spv::CapabilityMultiViewport);
717	}
718	if (glslangIntermediate->getStage() == EShLangVertex ||
719	glslangIntermediate->getStage() == EShLangTessControl ||
720	glslangIntermediate->getStage() == EShLangTessEvaluation) {
721
722	if (builder.getSpvVersion() < spv::Spv_1_5) {
723	builder.addIncorporatedExtension(ext: spv::E_SPV_EXT_shader_viewport_index_layer, incorporatedVersion: spv::Spv_1_5);
724	builder.addCapability(cap: spv::CapabilityShaderViewportIndexLayerEXT);
725	}
726	else
727	builder.addCapability(cap: spv::CapabilityShaderViewportIndex);
728	}
729	return spv::BuiltInViewportIndex;
730
731	case glslang::EbvSampleId:
732	builder.addCapability(cap: spv::CapabilitySampleRateShading);
733	return spv::BuiltInSampleId;
734
735	case glslang::EbvSamplePosition:
736	builder.addCapability(cap: spv::CapabilitySampleRateShading);
737	return spv::BuiltInSamplePosition;
738
739	case glslang::EbvSampleMask:
740	return spv::BuiltInSampleMask;
741
742	case glslang::EbvLayer:
743	if (glslangIntermediate->getStage() == EShLangMesh) {
744	return spv::BuiltInLayer;
745	}
746	if (glslangIntermediate->getStage() == EShLangGeometry ||
747	glslangIntermediate->getStage() == EShLangFragment) {
748	builder.addCapability(cap: spv::CapabilityGeometry);
749	}
750	if (glslangIntermediate->getStage() == EShLangVertex ||
751	glslangIntermediate->getStage() == EShLangTessControl ||
752	glslangIntermediate->getStage() == EShLangTessEvaluation) {
753
754	if (builder.getSpvVersion() < spv::Spv_1_5) {
755	builder.addIncorporatedExtension(ext: spv::E_SPV_EXT_shader_viewport_index_layer, incorporatedVersion: spv::Spv_1_5);
756	builder.addCapability(cap: spv::CapabilityShaderViewportIndexLayerEXT);
757	} else
758	builder.addCapability(cap: spv::CapabilityShaderLayer);
759	}
760	return spv::BuiltInLayer;
761
762	case glslang::EbvBaseVertex:
763	builder.addIncorporatedExtension(ext: spv::E_SPV_KHR_shader_draw_parameters, incorporatedVersion: spv::Spv_1_3);
764	builder.addCapability(cap: spv::CapabilityDrawParameters);
765	return spv::BuiltInBaseVertex;
766
767	case glslang::EbvBaseInstance:
768	builder.addIncorporatedExtension(ext: spv::E_SPV_KHR_shader_draw_parameters, incorporatedVersion: spv::Spv_1_3);
769	builder.addCapability(cap: spv::CapabilityDrawParameters);
770	return spv::BuiltInBaseInstance;
771
772	case glslang::EbvDrawId:
773	builder.addIncorporatedExtension(ext: spv::E_SPV_KHR_shader_draw_parameters, incorporatedVersion: spv::Spv_1_3);
774	builder.addCapability(cap: spv::CapabilityDrawParameters);
775	return spv::BuiltInDrawIndex;
776
777	case glslang::EbvPrimitiveId:
778	if (glslangIntermediate->getStage() == EShLangFragment)
779	builder.addCapability(cap: spv::CapabilityGeometry);
780	return spv::BuiltInPrimitiveId;
781
782	case glslang::EbvFragStencilRef:
783	builder.addExtension(ext: spv::E_SPV_EXT_shader_stencil_export);
784	builder.addCapability(cap: spv::CapabilityStencilExportEXT);
785	return spv::BuiltInFragStencilRefEXT;
786
787	case glslang::EbvShadingRateKHR:
788	builder.addExtension(ext: spv::E_SPV_KHR_fragment_shading_rate);
789	builder.addCapability(cap: spv::CapabilityFragmentShadingRateKHR);
790	return spv::BuiltInShadingRateKHR;
791
792	case glslang::EbvPrimitiveShadingRateKHR:
793	builder.addExtension(ext: spv::E_SPV_KHR_fragment_shading_rate);
794	builder.addCapability(cap: spv::CapabilityFragmentShadingRateKHR);
795	return spv::BuiltInPrimitiveShadingRateKHR;
796
797	case glslang::EbvInvocationId: return spv::BuiltInInvocationId;
798	case glslang::EbvTessLevelInner: return spv::BuiltInTessLevelInner;
799	case glslang::EbvTessLevelOuter: return spv::BuiltInTessLevelOuter;
800	case glslang::EbvTessCoord: return spv::BuiltInTessCoord;
801	case glslang::EbvPatchVertices: return spv::BuiltInPatchVertices;
802	case glslang::EbvHelperInvocation: return spv::BuiltInHelperInvocation;
803
804	case glslang::EbvSubGroupSize:
805	builder.addExtension(ext: spv::E_SPV_KHR_shader_ballot);
806	builder.addCapability(cap: spv::CapabilitySubgroupBallotKHR);
807	return spv::BuiltInSubgroupSize;
808
809	case glslang::EbvSubGroupInvocation:
810	builder.addExtension(ext: spv::E_SPV_KHR_shader_ballot);
811	builder.addCapability(cap: spv::CapabilitySubgroupBallotKHR);
812	return spv::BuiltInSubgroupLocalInvocationId;
813
814	case glslang::EbvSubGroupEqMask:
815	builder.addExtension(ext: spv::E_SPV_KHR_shader_ballot);
816	builder.addCapability(cap: spv::CapabilitySubgroupBallotKHR);
817	return spv::BuiltInSubgroupEqMask;
818
819	case glslang::EbvSubGroupGeMask:
820	builder.addExtension(ext: spv::E_SPV_KHR_shader_ballot);
821	builder.addCapability(cap: spv::CapabilitySubgroupBallotKHR);
822	return spv::BuiltInSubgroupGeMask;
823
824	case glslang::EbvSubGroupGtMask:
825	builder.addExtension(ext: spv::E_SPV_KHR_shader_ballot);
826	builder.addCapability(cap: spv::CapabilitySubgroupBallotKHR);
827	return spv::BuiltInSubgroupGtMask;
828
829	case glslang::EbvSubGroupLeMask:
830	builder.addExtension(ext: spv::E_SPV_KHR_shader_ballot);
831	builder.addCapability(cap: spv::CapabilitySubgroupBallotKHR);
832	return spv::BuiltInSubgroupLeMask;
833
834	case glslang::EbvSubGroupLtMask:
835	builder.addExtension(ext: spv::E_SPV_KHR_shader_ballot);
836	builder.addCapability(cap: spv::CapabilitySubgroupBallotKHR);
837	return spv::BuiltInSubgroupLtMask;
838
839	case glslang::EbvNumSubgroups:
840	builder.addCapability(cap: spv::CapabilityGroupNonUniform);
841	return spv::BuiltInNumSubgroups;
842
843	case glslang::EbvSubgroupID:
844	builder.addCapability(cap: spv::CapabilityGroupNonUniform);
845	return spv::BuiltInSubgroupId;
846
847	case glslang::EbvSubgroupSize2:
848	builder.addCapability(cap: spv::CapabilityGroupNonUniform);
849	return spv::BuiltInSubgroupSize;
850
851	case glslang::EbvSubgroupInvocation2:
852	builder.addCapability(cap: spv::CapabilityGroupNonUniform);
853	return spv::BuiltInSubgroupLocalInvocationId;
854
855	case glslang::EbvSubgroupEqMask2:
856	builder.addCapability(cap: spv::CapabilityGroupNonUniform);
857	builder.addCapability(cap: spv::CapabilityGroupNonUniformBallot);
858	return spv::BuiltInSubgroupEqMask;
859
860	case glslang::EbvSubgroupGeMask2:
861	builder.addCapability(cap: spv::CapabilityGroupNonUniform);
862	builder.addCapability(cap: spv::CapabilityGroupNonUniformBallot);
863	return spv::BuiltInSubgroupGeMask;
864
865	case glslang::EbvSubgroupGtMask2:
866	builder.addCapability(cap: spv::CapabilityGroupNonUniform);
867	builder.addCapability(cap: spv::CapabilityGroupNonUniformBallot);
868	return spv::BuiltInSubgroupGtMask;
869
870	case glslang::EbvSubgroupLeMask2:
871	builder.addCapability(cap: spv::CapabilityGroupNonUniform);
872	builder.addCapability(cap: spv::CapabilityGroupNonUniformBallot);
873	return spv::BuiltInSubgroupLeMask;
874
875	case glslang::EbvSubgroupLtMask2:
876	builder.addCapability(cap: spv::CapabilityGroupNonUniform);
877	builder.addCapability(cap: spv::CapabilityGroupNonUniformBallot);
878	return spv::BuiltInSubgroupLtMask;
879
880	case glslang::EbvBaryCoordNoPersp:
881	builder.addExtension(ext: spv::E_SPV_AMD_shader_explicit_vertex_parameter);
882	return spv::BuiltInBaryCoordNoPerspAMD;
883
884	case glslang::EbvBaryCoordNoPerspCentroid:
885	builder.addExtension(ext: spv::E_SPV_AMD_shader_explicit_vertex_parameter);
886	return spv::BuiltInBaryCoordNoPerspCentroidAMD;
887
888	case glslang::EbvBaryCoordNoPerspSample:
889	builder.addExtension(ext: spv::E_SPV_AMD_shader_explicit_vertex_parameter);
890	return spv::BuiltInBaryCoordNoPerspSampleAMD;
891
892	case glslang::EbvBaryCoordSmooth:
893	builder.addExtension(ext: spv::E_SPV_AMD_shader_explicit_vertex_parameter);
894	return spv::BuiltInBaryCoordSmoothAMD;
895
896	case glslang::EbvBaryCoordSmoothCentroid:
897	builder.addExtension(ext: spv::E_SPV_AMD_shader_explicit_vertex_parameter);
898	return spv::BuiltInBaryCoordSmoothCentroidAMD;
899
900	case glslang::EbvBaryCoordSmoothSample:
901	builder.addExtension(ext: spv::E_SPV_AMD_shader_explicit_vertex_parameter);
902	return spv::BuiltInBaryCoordSmoothSampleAMD;
903
904	case glslang::EbvBaryCoordPullModel:
905	builder.addExtension(ext: spv::E_SPV_AMD_shader_explicit_vertex_parameter);
906	return spv::BuiltInBaryCoordPullModelAMD;
907
908	case glslang::EbvDeviceIndex:
909	builder.addIncorporatedExtension(ext: spv::E_SPV_KHR_device_group, incorporatedVersion: spv::Spv_1_3);
910	builder.addCapability(cap: spv::CapabilityDeviceGroup);
911	return spv::BuiltInDeviceIndex;
912
913	case glslang::EbvViewIndex:
914	builder.addIncorporatedExtension(ext: spv::E_SPV_KHR_multiview, incorporatedVersion: spv::Spv_1_3);
915	builder.addCapability(cap: spv::CapabilityMultiView);
916	return spv::BuiltInViewIndex;
917
918	case glslang::EbvFragSizeEXT:
919	builder.addExtension(ext: spv::E_SPV_EXT_fragment_invocation_density);
920	builder.addCapability(cap: spv::CapabilityFragmentDensityEXT);
921	return spv::BuiltInFragSizeEXT;
922
923	case glslang::EbvFragInvocationCountEXT:
924	builder.addExtension(ext: spv::E_SPV_EXT_fragment_invocation_density);
925	builder.addCapability(cap: spv::CapabilityFragmentDensityEXT);
926	return spv::BuiltInFragInvocationCountEXT;
927
928	case glslang::EbvViewportMaskNV:
929	if (!memberDeclaration) {
930	builder.addExtension(ext: spv::E_SPV_NV_viewport_array2);
931	builder.addCapability(cap: spv::CapabilityShaderViewportMaskNV);
932	}
933	return spv::BuiltInViewportMaskNV;
934	case glslang::EbvSecondaryPositionNV:
935	if (!memberDeclaration) {
936	builder.addExtension(ext: spv::E_SPV_NV_stereo_view_rendering);
937	builder.addCapability(cap: spv::CapabilityShaderStereoViewNV);
938	}
939	return spv::BuiltInSecondaryPositionNV;
940	case glslang::EbvSecondaryViewportMaskNV:
941	if (!memberDeclaration) {
942	builder.addExtension(ext: spv::E_SPV_NV_stereo_view_rendering);
943	builder.addCapability(cap: spv::CapabilityShaderStereoViewNV);
944	}
945	return spv::BuiltInSecondaryViewportMaskNV;
946	case glslang::EbvPositionPerViewNV:
947	if (!memberDeclaration) {
948	builder.addExtension(ext: spv::E_SPV_NVX_multiview_per_view_attributes);
949	builder.addCapability(cap: spv::CapabilityPerViewAttributesNV);
950	}
951	return spv::BuiltInPositionPerViewNV;
952	case glslang::EbvViewportMaskPerViewNV:
953	if (!memberDeclaration) {
954	builder.addExtension(ext: spv::E_SPV_NVX_multiview_per_view_attributes);
955	builder.addCapability(cap: spv::CapabilityPerViewAttributesNV);
956	}
957	return spv::BuiltInViewportMaskPerViewNV;
958	case glslang::EbvFragFullyCoveredNV:
959	builder.addExtension(ext: spv::E_SPV_EXT_fragment_fully_covered);
960	builder.addCapability(cap: spv::CapabilityFragmentFullyCoveredEXT);
961	return spv::BuiltInFullyCoveredEXT;
962	case glslang::EbvFragmentSizeNV:
963	builder.addExtension(ext: spv::E_SPV_NV_shading_rate);
964	builder.addCapability(cap: spv::CapabilityShadingRateNV);
965	return spv::BuiltInFragmentSizeNV;
966	case glslang::EbvInvocationsPerPixelNV:
967	builder.addExtension(ext: spv::E_SPV_NV_shading_rate);
968	builder.addCapability(cap: spv::CapabilityShadingRateNV);
969	return spv::BuiltInInvocationsPerPixelNV;
970
971	// ray tracing
972	case glslang::EbvLaunchId:
973	return spv::BuiltInLaunchIdKHR;
974	case glslang::EbvLaunchSize:
975	return spv::BuiltInLaunchSizeKHR;
976	case glslang::EbvWorldRayOrigin:
977	return spv::BuiltInWorldRayOriginKHR;
978	case glslang::EbvWorldRayDirection:
979	return spv::BuiltInWorldRayDirectionKHR;
980	case glslang::EbvObjectRayOrigin:
981	return spv::BuiltInObjectRayOriginKHR;
982	case glslang::EbvObjectRayDirection:
983	return spv::BuiltInObjectRayDirectionKHR;
984	case glslang::EbvRayTmin:
985	return spv::BuiltInRayTminKHR;
986	case glslang::EbvRayTmax:
987	return spv::BuiltInRayTmaxKHR;
988	case glslang::EbvCullMask:
989	return spv::BuiltInCullMaskKHR;
990	case glslang::EbvPositionFetch:
991	return spv::BuiltInHitTriangleVertexPositionsKHR;
992	case glslang::EbvInstanceCustomIndex:
993	return spv::BuiltInInstanceCustomIndexKHR;
994	case glslang::EbvHitT:
995	{
996	// this is a GLSL alias of RayTmax
997	// in SPV_NV_ray_tracing it has a dedicated builtin
998	// but in SPV_KHR_ray_tracing it gets mapped to RayTmax
999	auto& extensions = glslangIntermediate->getRequestedExtensions();
1000	if (extensions.find(x: "GL_NV_ray_tracing") != extensions.end()) {
1001	return spv::BuiltInHitTNV;
1002	} else {
1003	return spv::BuiltInRayTmaxKHR;
1004	}
1005	}
1006	case glslang::EbvHitKind:
1007	return spv::BuiltInHitKindKHR;
1008	case glslang::EbvObjectToWorld:
1009	case glslang::EbvObjectToWorld3x4:
1010	return spv::BuiltInObjectToWorldKHR;
1011	case glslang::EbvWorldToObject:
1012	case glslang::EbvWorldToObject3x4:
1013	return spv::BuiltInWorldToObjectKHR;
1014	case glslang::EbvIncomingRayFlags:
1015	return spv::BuiltInIncomingRayFlagsKHR;
1016	case glslang::EbvGeometryIndex:
1017	return spv::BuiltInRayGeometryIndexKHR;
1018	case glslang::EbvCurrentRayTimeNV:
1019	builder.addExtension(ext: spv::E_SPV_NV_ray_tracing_motion_blur);
1020	builder.addCapability(cap: spv::CapabilityRayTracingMotionBlurNV);
1021	return spv::BuiltInCurrentRayTimeNV;
1022	case glslang::EbvMicroTrianglePositionNV:
1023	builder.addCapability(cap: spv::CapabilityRayTracingDisplacementMicromapNV);
1024	builder.addExtension(ext: "SPV_NV_displacement_micromap");
1025	return spv::BuiltInHitMicroTriangleVertexPositionsNV;
1026	case glslang::EbvMicroTriangleBaryNV:
1027	builder.addCapability(cap: spv::CapabilityRayTracingDisplacementMicromapNV);
1028	builder.addExtension(ext: "SPV_NV_displacement_micromap");
1029	return spv::BuiltInHitMicroTriangleVertexBarycentricsNV;
1030	case glslang::EbvHitKindFrontFacingMicroTriangleNV:
1031	builder.addCapability(cap: spv::CapabilityRayTracingDisplacementMicromapNV);
1032	builder.addExtension(ext: "SPV_NV_displacement_micromap");
1033	return spv::BuiltInHitKindFrontFacingMicroTriangleNV;
1034	case glslang::EbvHitKindBackFacingMicroTriangleNV:
1035	builder.addCapability(cap: spv::CapabilityRayTracingDisplacementMicromapNV);
1036	builder.addExtension(ext: "SPV_NV_displacement_micromap");
1037	return spv::BuiltInHitKindBackFacingMicroTriangleNV;
1038
1039	// barycentrics
1040	case glslang::EbvBaryCoordNV:
1041	builder.addExtension(ext: spv::E_SPV_NV_fragment_shader_barycentric);
1042	builder.addCapability(cap: spv::CapabilityFragmentBarycentricNV);
1043	return spv::BuiltInBaryCoordNV;
1044	case glslang::EbvBaryCoordNoPerspNV:
1045	builder.addExtension(ext: spv::E_SPV_NV_fragment_shader_barycentric);
1046	builder.addCapability(cap: spv::CapabilityFragmentBarycentricNV);
1047	return spv::BuiltInBaryCoordNoPerspNV;
1048
1049	case glslang::EbvBaryCoordEXT:
1050	builder.addExtension(ext: spv::E_SPV_KHR_fragment_shader_barycentric);
1051	builder.addCapability(cap: spv::CapabilityFragmentBarycentricKHR);
1052	return spv::BuiltInBaryCoordKHR;
1053	case glslang::EbvBaryCoordNoPerspEXT:
1054	builder.addExtension(ext: spv::E_SPV_KHR_fragment_shader_barycentric);
1055	builder.addCapability(cap: spv::CapabilityFragmentBarycentricKHR);
1056	return spv::BuiltInBaryCoordNoPerspKHR;
1057
1058	// mesh shaders
1059	case glslang::EbvTaskCountNV:
1060	return spv::BuiltInTaskCountNV;
1061	case glslang::EbvPrimitiveCountNV:
1062	return spv::BuiltInPrimitiveCountNV;
1063	case glslang::EbvPrimitiveIndicesNV:
1064	return spv::BuiltInPrimitiveIndicesNV;
1065	case glslang::EbvClipDistancePerViewNV:
1066	return spv::BuiltInClipDistancePerViewNV;
1067	case glslang::EbvCullDistancePerViewNV:
1068	return spv::BuiltInCullDistancePerViewNV;
1069	case glslang::EbvLayerPerViewNV:
1070	return spv::BuiltInLayerPerViewNV;
1071	case glslang::EbvMeshViewCountNV:
1072	return spv::BuiltInMeshViewCountNV;
1073	case glslang::EbvMeshViewIndicesNV:
1074	return spv::BuiltInMeshViewIndicesNV;
1075
1076	// SPV_EXT_mesh_shader
1077	case glslang::EbvPrimitivePointIndicesEXT:
1078	return spv::BuiltInPrimitivePointIndicesEXT;
1079	case glslang::EbvPrimitiveLineIndicesEXT:
1080	return spv::BuiltInPrimitiveLineIndicesEXT;
1081	case glslang::EbvPrimitiveTriangleIndicesEXT:
1082	return spv::BuiltInPrimitiveTriangleIndicesEXT;
1083	case glslang::EbvCullPrimitiveEXT:
1084	return spv::BuiltInCullPrimitiveEXT;
1085
1086	// sm builtins
1087	case glslang::EbvWarpsPerSM:
1088	builder.addExtension(ext: spv::E_SPV_NV_shader_sm_builtins);
1089	builder.addCapability(cap: spv::CapabilityShaderSMBuiltinsNV);
1090	return spv::BuiltInWarpsPerSMNV;
1091	case glslang::EbvSMCount:
1092	builder.addExtension(ext: spv::E_SPV_NV_shader_sm_builtins);
1093	builder.addCapability(cap: spv::CapabilityShaderSMBuiltinsNV);
1094	return spv::BuiltInSMCountNV;
1095	case glslang::EbvWarpID:
1096	builder.addExtension(ext: spv::E_SPV_NV_shader_sm_builtins);
1097	builder.addCapability(cap: spv::CapabilityShaderSMBuiltinsNV);
1098	return spv::BuiltInWarpIDNV;
1099	case glslang::EbvSMID:
1100	builder.addExtension(ext: spv::E_SPV_NV_shader_sm_builtins);
1101	builder.addCapability(cap: spv::CapabilityShaderSMBuiltinsNV);
1102	return spv::BuiltInSMIDNV;
1103
1104	// ARM builtins
1105	case glslang::EbvCoreCountARM:
1106	builder.addExtension(ext: spv::E_SPV_ARM_core_builtins);
1107	builder.addCapability(cap: spv::CapabilityCoreBuiltinsARM);
1108	return spv::BuiltInCoreCountARM;
1109	case glslang::EbvCoreIDARM:
1110	builder.addExtension(ext: spv::E_SPV_ARM_core_builtins);
1111	builder.addCapability(cap: spv::CapabilityCoreBuiltinsARM);
1112	return spv::BuiltInCoreIDARM;
1113	case glslang::EbvCoreMaxIDARM:
1114	builder.addExtension(ext: spv::E_SPV_ARM_core_builtins);
1115	builder.addCapability(cap: spv::CapabilityCoreBuiltinsARM);
1116	return spv::BuiltInCoreMaxIDARM;
1117	case glslang::EbvWarpIDARM:
1118	builder.addExtension(ext: spv::E_SPV_ARM_core_builtins);
1119	builder.addCapability(cap: spv::CapabilityCoreBuiltinsARM);
1120	return spv::BuiltInWarpIDARM;
1121	case glslang::EbvWarpMaxIDARM:
1122	builder.addExtension(ext: spv::E_SPV_ARM_core_builtins);
1123	builder.addCapability(cap: spv::CapabilityCoreBuiltinsARM);
1124	return spv::BuiltInWarpMaxIDARM;
1125
1126	default:
1127	return spv::BuiltInMax;
1128	}
1129	}
1130
1131	// Translate glslang image layout format to SPIR-V image format.
1132	spv::ImageFormat TGlslangToSpvTraverser::TranslateImageFormat(const glslang::TType& type)
1133	{
1134	assert(type.getBasicType() == glslang::EbtSampler);
1135
1136	// Check for capabilities
1137	switch (type.getQualifier().getFormat()) {
1138	case glslang::ElfRg32f:
1139	case glslang::ElfRg16f:
1140	case glslang::ElfR11fG11fB10f:
1141	case glslang::ElfR16f:
1142	case glslang::ElfRgba16:
1143	case glslang::ElfRgb10A2:
1144	case glslang::ElfRg16:
1145	case glslang::ElfRg8:
1146	case glslang::ElfR16:
1147	case glslang::ElfR8:
1148	case glslang::ElfRgba16Snorm:
1149	case glslang::ElfRg16Snorm:
1150	case glslang::ElfRg8Snorm:
1151	case glslang::ElfR16Snorm:
1152	case glslang::ElfR8Snorm:
1153
1154	case glslang::ElfRg32i:
1155	case glslang::ElfRg16i:
1156	case glslang::ElfRg8i:
1157	case glslang::ElfR16i:
1158	case glslang::ElfR8i:
1159
1160	case glslang::ElfRgb10a2ui:
1161	case glslang::ElfRg32ui:
1162	case glslang::ElfRg16ui:
1163	case glslang::ElfRg8ui:
1164	case glslang::ElfR16ui:
1165	case glslang::ElfR8ui:
1166	builder.addCapability(cap: spv::CapabilityStorageImageExtendedFormats);
1167	break;
1168
1169	case glslang::ElfR64ui:
1170	case glslang::ElfR64i:
1171	builder.addExtension(ext: spv::E_SPV_EXT_shader_image_int64);
1172	builder.addCapability(cap: spv::CapabilityInt64ImageEXT);
1173	break;
1174	default:
1175	break;
1176	}
1177
1178	// do the translation
1179	switch (type.getQualifier().getFormat()) {
1180	case glslang::ElfNone: return spv::ImageFormatUnknown;
1181	case glslang::ElfRgba32f: return spv::ImageFormatRgba32f;
1182	case glslang::ElfRgba16f: return spv::ImageFormatRgba16f;
1183	case glslang::ElfR32f: return spv::ImageFormatR32f;
1184	case glslang::ElfRgba8: return spv::ImageFormatRgba8;
1185	case glslang::ElfRgba8Snorm: return spv::ImageFormatRgba8Snorm;
1186	case glslang::ElfRg32f: return spv::ImageFormatRg32f;
1187	case glslang::ElfRg16f: return spv::ImageFormatRg16f;
1188	case glslang::ElfR11fG11fB10f: return spv::ImageFormatR11fG11fB10f;
1189	case glslang::ElfR16f: return spv::ImageFormatR16f;
1190	case glslang::ElfRgba16: return spv::ImageFormatRgba16;
1191	case glslang::ElfRgb10A2: return spv::ImageFormatRgb10A2;
1192	case glslang::ElfRg16: return spv::ImageFormatRg16;
1193	case glslang::ElfRg8: return spv::ImageFormatRg8;
1194	case glslang::ElfR16: return spv::ImageFormatR16;
1195	case glslang::ElfR8: return spv::ImageFormatR8;
1196	case glslang::ElfRgba16Snorm: return spv::ImageFormatRgba16Snorm;
1197	case glslang::ElfRg16Snorm: return spv::ImageFormatRg16Snorm;
1198	case glslang::ElfRg8Snorm: return spv::ImageFormatRg8Snorm;
1199	case glslang::ElfR16Snorm: return spv::ImageFormatR16Snorm;
1200	case glslang::ElfR8Snorm: return spv::ImageFormatR8Snorm;
1201	case glslang::ElfRgba32i: return spv::ImageFormatRgba32i;
1202	case glslang::ElfRgba16i: return spv::ImageFormatRgba16i;
1203	case glslang::ElfRgba8i: return spv::ImageFormatRgba8i;
1204	case glslang::ElfR32i: return spv::ImageFormatR32i;
1205	case glslang::ElfRg32i: return spv::ImageFormatRg32i;
1206	case glslang::ElfRg16i: return spv::ImageFormatRg16i;
1207	case glslang::ElfRg8i: return spv::ImageFormatRg8i;
1208	case glslang::ElfR16i: return spv::ImageFormatR16i;
1209	case glslang::ElfR8i: return spv::ImageFormatR8i;
1210	case glslang::ElfRgba32ui: return spv::ImageFormatRgba32ui;
1211	case glslang::ElfRgba16ui: return spv::ImageFormatRgba16ui;
1212	case glslang::ElfRgba8ui: return spv::ImageFormatRgba8ui;
1213	case glslang::ElfR32ui: return spv::ImageFormatR32ui;
1214	case glslang::ElfRg32ui: return spv::ImageFormatRg32ui;
1215	case glslang::ElfRg16ui: return spv::ImageFormatRg16ui;
1216	case glslang::ElfRgb10a2ui: return spv::ImageFormatRgb10a2ui;
1217	case glslang::ElfRg8ui: return spv::ImageFormatRg8ui;
1218	case glslang::ElfR16ui: return spv::ImageFormatR16ui;
1219	case glslang::ElfR8ui: return spv::ImageFormatR8ui;
1220	case glslang::ElfR64ui: return spv::ImageFormatR64ui;
1221	case glslang::ElfR64i: return spv::ImageFormatR64i;
1222	default: return spv::ImageFormatMax;
1223	}
1224	}
1225
1226	spv::SelectionControlMask TGlslangToSpvTraverser::TranslateSelectionControl(
1227	const glslang::TIntermSelection& selectionNode) const
1228	{
1229	if (selectionNode.getFlatten())
1230	return spv::SelectionControlFlattenMask;
1231	if (selectionNode.getDontFlatten())
1232	return spv::SelectionControlDontFlattenMask;
1233	return spv::SelectionControlMaskNone;
1234	}
1235
1236	spv::SelectionControlMask TGlslangToSpvTraverser::TranslateSwitchControl(const glslang::TIntermSwitch& switchNode)
1237	const
1238	{
1239	if (switchNode.getFlatten())
1240	return spv::SelectionControlFlattenMask;
1241	if (switchNode.getDontFlatten())
1242	return spv::SelectionControlDontFlattenMask;
1243	return spv::SelectionControlMaskNone;
1244	}
1245
1246	// return a non-0 dependency if the dependency argument must be set
1247	spv::LoopControlMask TGlslangToSpvTraverser::TranslateLoopControl(const glslang::TIntermLoop& loopNode,
1248	std::vector<unsigned int>& operands) const
1249	{
1250	spv::LoopControlMask control = spv::LoopControlMaskNone;
1251
1252	if (loopNode.getDontUnroll())
1253	control = control | spv::LoopControlDontUnrollMask;
1254	if (loopNode.getUnroll())
1255	control = control | spv::LoopControlUnrollMask;
1256	if (unsigned(loopNode.getLoopDependency()) == glslang::TIntermLoop::dependencyInfinite)
1257	control = control | spv::LoopControlDependencyInfiniteMask;
1258	else if (loopNode.getLoopDependency() > 0) {
1259	control = control | spv::LoopControlDependencyLengthMask;
1260	operands.push_back(x: (unsigned int)loopNode.getLoopDependency());
1261	}
1262	if (glslangIntermediate->getSpv().spv >= glslang::EShTargetSpv_1_4) {
1263	if (loopNode.getMinIterations() > 0) {
1264	control = control | spv::LoopControlMinIterationsMask;
1265	operands.push_back(x: loopNode.getMinIterations());
1266	}
1267	if (loopNode.getMaxIterations() < glslang::TIntermLoop::iterationsInfinite) {
1268	control = control | spv::LoopControlMaxIterationsMask;
1269	operands.push_back(x: loopNode.getMaxIterations());
1270	}
1271	if (loopNode.getIterationMultiple() > 1) {
1272	control = control | spv::LoopControlIterationMultipleMask;
1273	operands.push_back(x: loopNode.getIterationMultiple());
1274	}
1275	if (loopNode.getPeelCount() > 0) {
1276	control = control | spv::LoopControlPeelCountMask;
1277	operands.push_back(x: loopNode.getPeelCount());
1278	}
1279	if (loopNode.getPartialCount() > 0) {
1280	control = control | spv::LoopControlPartialCountMask;
1281	operands.push_back(x: loopNode.getPartialCount());
1282	}
1283	}
1284
1285	return control;
1286	}
1287
1288	// Translate glslang type to SPIR-V storage class.
1289	spv::StorageClass TGlslangToSpvTraverser::TranslateStorageClass(const glslang::TType& type)
1290	{
1291	if (type.getBasicType() == glslang::EbtRayQuery || type.getBasicType() == glslang::EbtHitObjectNV)
1292	return spv::StorageClassPrivate;
1293	if (type.getQualifier().isSpirvByReference()) {
1294	if (type.getQualifier().isParamInput() || type.getQualifier().isParamOutput())
1295	return spv::StorageClassFunction;
1296	}
1297	if (type.getQualifier().isPipeInput())
1298	return spv::StorageClassInput;
1299	if (type.getQualifier().isPipeOutput())
1300	return spv::StorageClassOutput;
1301	if (type.getQualifier().storage == glslang::EvqTileImageEXT || type.isAttachmentEXT()) {
1302	builder.addExtension(ext: spv::E_SPV_EXT_shader_tile_image);
1303	builder.addCapability(cap: spv::CapabilityTileImageColorReadAccessEXT);
1304	return spv::StorageClassTileImageEXT;
1305	}
1306
1307	if (glslangIntermediate->getSource() != glslang::EShSourceHlsl ||
1308	type.getQualifier().storage == glslang::EvqUniform) {
1309	if (type.isAtomic())
1310	return spv::StorageClassAtomicCounter;
1311	if (type.containsOpaque() && !glslangIntermediate->getBindlessMode())
1312	return spv::StorageClassUniformConstant;
1313	}
1314
1315	if (type.getQualifier().isUniformOrBuffer() &&
1316	type.getQualifier().isShaderRecord()) {
1317	return spv::StorageClassShaderRecordBufferKHR;
1318	}
1319
1320	if (glslangIntermediate->usingStorageBuffer() && type.getQualifier().storage == glslang::EvqBuffer) {
1321	builder.addIncorporatedExtension(ext: spv::E_SPV_KHR_storage_buffer_storage_class, incorporatedVersion: spv::Spv_1_3);
1322	return spv::StorageClassStorageBuffer;
1323	}
1324
1325	if (type.getQualifier().isUniformOrBuffer()) {
1326	if (type.getQualifier().isPushConstant())
1327	return spv::StorageClassPushConstant;
1328	if (type.getBasicType() == glslang::EbtBlock)
1329	return spv::StorageClassUniform;
1330	return spv::StorageClassUniformConstant;
1331	}
1332
1333	if (type.getQualifier().storage == glslang::EvqShared && type.getBasicType() == glslang::EbtBlock) {
1334	builder.addExtension(ext: spv::E_SPV_KHR_workgroup_memory_explicit_layout);
1335	builder.addCapability(cap: spv::CapabilityWorkgroupMemoryExplicitLayoutKHR);
1336	return spv::StorageClassWorkgroup;
1337	}
1338
1339	switch (type.getQualifier().storage) {
1340	case glslang::EvqGlobal: return spv::StorageClassPrivate;
1341	case glslang::EvqConstReadOnly: return spv::StorageClassFunction;
1342	case glslang::EvqTemporary: return spv::StorageClassFunction;
1343	case glslang::EvqShared: return spv::StorageClassWorkgroup;
1344	case glslang::EvqPayload: return spv::StorageClassRayPayloadKHR;
1345	case glslang::EvqPayloadIn: return spv::StorageClassIncomingRayPayloadKHR;
1346	case glslang::EvqHitAttr: return spv::StorageClassHitAttributeKHR;
1347	case glslang::EvqCallableData: return spv::StorageClassCallableDataKHR;
1348	case glslang::EvqCallableDataIn: return spv::StorageClassIncomingCallableDataKHR;
1349	case glslang::EvqtaskPayloadSharedEXT : return spv::StorageClassTaskPayloadWorkgroupEXT;
1350	case glslang::EvqHitObjectAttrNV: return spv::StorageClassHitObjectAttributeNV;
1351	case glslang::EvqSpirvStorageClass: return static_cast<spv::StorageClass>(type.getQualifier().spirvStorageClass);
1352	default:
1353	assert(0);
1354	break;
1355	}
1356
1357	return spv::StorageClassFunction;
1358	}
1359
1360	// Translate glslang constants to SPIR-V literals
1361	void TGlslangToSpvTraverser::TranslateLiterals(const glslang::TVector<const glslang::TIntermConstantUnion*>& constants,
1362	std::vector<unsigned>& literals) const
1363	{
1364	for (auto constant : constants) {
1365	if (constant->getBasicType() == glslang::EbtFloat) {
1366	float floatValue = static_cast<float>(constant->getConstArray()[0].getDConst());
1367	unsigned literal;
1368	static_assert(sizeof(literal) == sizeof(floatValue), "sizeof(unsigned) != sizeof(float)");
1369	memcpy(dest: &literal, src: &floatValue, n: sizeof(literal));
1370	literals.push_back(x: literal);
1371	} else if (constant->getBasicType() == glslang::EbtInt) {
1372	unsigned literal = constant->getConstArray()[0].getIConst();
1373	literals.push_back(x: literal);
1374	} else if (constant->getBasicType() == glslang::EbtUint) {
1375	unsigned literal = constant->getConstArray()[0].getUConst();
1376	literals.push_back(x: literal);
1377	} else if (constant->getBasicType() == glslang::EbtBool) {
1378	unsigned literal = constant->getConstArray()[0].getBConst();
1379	literals.push_back(x: literal);
1380	} else if (constant->getBasicType() == glslang::EbtString) {
1381	auto str = constant->getConstArray()[0].getSConst()->c_str();
1382	unsigned literal = 0;
1383	char* literalPtr = reinterpret_cast<char*>(&literal);
1384	unsigned charCount = 0;
1385	char ch = 0;
1386	do {
1387	ch = *(str++);
1388	*(literalPtr++) = ch;
1389	++charCount;
1390	if (charCount == 4) {
1391	literals.push_back(x: literal);
1392	literalPtr = reinterpret_cast<char*>(&literal);
1393	charCount = 0;
1394	}
1395	} while (ch != 0);
1396
1397	// Partial literal is padded with 0
1398	if (charCount > 0) {
1399	for (; charCount < 4; ++charCount)
1400	*(literalPtr++) = 0;
1401	literals.push_back(x: literal);
1402	}
1403	} else
1404	assert(0); // Unexpected type
1405	}
1406	}
1407
1408	// Add capabilities pertaining to how an array is indexed.
1409	void TGlslangToSpvTraverser::addIndirectionIndexCapabilities(const glslang::TType& baseType,
1410	const glslang::TType& indexType)
1411	{
1412	if (indexType.getQualifier().isNonUniform()) {
1413	// deal with an asserted non-uniform index
1414	// SPV_EXT_descriptor_indexing already added in TranslateNonUniformDecoration
1415	if (baseType.getBasicType() == glslang::EbtSampler) {
1416	if (baseType.getQualifier().hasAttachment())
1417	builder.addCapability(cap: spv::CapabilityInputAttachmentArrayNonUniformIndexingEXT);
1418	else if (baseType.isImage() && baseType.getSampler().isBuffer())
1419	builder.addCapability(cap: spv::CapabilityStorageTexelBufferArrayNonUniformIndexingEXT);
1420	else if (baseType.isTexture() && baseType.getSampler().isBuffer())
1421	builder.addCapability(cap: spv::CapabilityUniformTexelBufferArrayNonUniformIndexingEXT);
1422	else if (baseType.isImage())
1423	builder.addCapability(cap: spv::CapabilityStorageImageArrayNonUniformIndexingEXT);
1424	else if (baseType.isTexture())
1425	builder.addCapability(cap: spv::CapabilitySampledImageArrayNonUniformIndexingEXT);
1426	} else if (baseType.getBasicType() == glslang::EbtBlock) {
1427	if (baseType.getQualifier().storage == glslang::EvqBuffer)
1428	builder.addCapability(cap: spv::CapabilityStorageBufferArrayNonUniformIndexingEXT);
1429	else if (baseType.getQualifier().storage == glslang::EvqUniform)
1430	builder.addCapability(cap: spv::CapabilityUniformBufferArrayNonUniformIndexingEXT);
1431	}
1432	} else {
1433	// assume a dynamically uniform index
1434	if (baseType.getBasicType() == glslang::EbtSampler) {
1435	if (baseType.getQualifier().hasAttachment()) {
1436	builder.addIncorporatedExtension(ext: "SPV_EXT_descriptor_indexing", incorporatedVersion: spv::Spv_1_5);
1437	builder.addCapability(cap: spv::CapabilityInputAttachmentArrayDynamicIndexingEXT);
1438	} else if (baseType.isImage() && baseType.getSampler().isBuffer()) {
1439	builder.addIncorporatedExtension(ext: "SPV_EXT_descriptor_indexing", incorporatedVersion: spv::Spv_1_5);
1440	builder.addCapability(cap: spv::CapabilityStorageTexelBufferArrayDynamicIndexingEXT);
1441	} else if (baseType.isTexture() && baseType.getSampler().isBuffer()) {
1442	builder.addIncorporatedExtension(ext: "SPV_EXT_descriptor_indexing", incorporatedVersion: spv::Spv_1_5);
1443	builder.addCapability(cap: spv::CapabilityUniformTexelBufferArrayDynamicIndexingEXT);
1444	}
1445	}
1446	}
1447	}
1448
1449	// Return whether or not the given type is something that should be tied to a
1450	// descriptor set.
1451	bool IsDescriptorResource(const glslang::TType& type)
1452	{
1453	// uniform and buffer blocks are included, unless it is a push_constant
1454	if (type.getBasicType() == glslang::EbtBlock)
1455	return type.getQualifier().isUniformOrBuffer() &&
1456	! type.getQualifier().isShaderRecord() &&
1457	! type.getQualifier().isPushConstant();
1458
1459	// non block...
1460	// basically samplerXXX/subpass/sampler/texture are all included
1461	// if they are the global-scope-class, not the function parameter
1462	// (or local, if they ever exist) class.
1463	if (type.getBasicType() == glslang::EbtSampler ||
1464	type.getBasicType() == glslang::EbtAccStruct)
1465	return type.getQualifier().isUniformOrBuffer();
1466
1467	// None of the above.
1468	return false;
1469	}
1470
1471	void InheritQualifiers(glslang::TQualifier& child, const glslang::TQualifier& parent)
1472	{
1473	if (child.layoutMatrix == glslang::ElmNone)
1474	child.layoutMatrix = parent.layoutMatrix;
1475
1476	if (parent.invariant)
1477	child.invariant = true;
1478	if (parent.flat)
1479	child.flat = true;
1480	if (parent.centroid)
1481	child.centroid = true;
1482	if (parent.nopersp)
1483	child.nopersp = true;
1484	if (parent.explicitInterp)
1485	child.explicitInterp = true;
1486	if (parent.perPrimitiveNV)
1487	child.perPrimitiveNV = true;
1488	if (parent.perViewNV)
1489	child.perViewNV = true;
1490	if (parent.perTaskNV)
1491	child.perTaskNV = true;
1492	if (parent.storage == glslang::EvqtaskPayloadSharedEXT)
1493	child.storage = glslang::EvqtaskPayloadSharedEXT;
1494	if (parent.patch)
1495	child.patch = true;
1496	if (parent.sample)
1497	child.sample = true;
1498	if (parent.coherent)
1499	child.coherent = true;
1500	if (parent.devicecoherent)
1501	child.devicecoherent = true;
1502	if (parent.queuefamilycoherent)
1503	child.queuefamilycoherent = true;
1504	if (parent.workgroupcoherent)
1505	child.workgroupcoherent = true;
1506	if (parent.subgroupcoherent)
1507	child.subgroupcoherent = true;
1508	if (parent.shadercallcoherent)
1509	child.shadercallcoherent = true;
1510	if (parent.nonprivate)
1511	child.nonprivate = true;
1512	if (parent.volatil)
1513	child.volatil = true;
1514	if (parent.restrict)
1515	child.restrict = true;
1516	if (parent.readonly)
1517	child.readonly = true;
1518	if (parent.writeonly)
1519	child.writeonly = true;
1520	if (parent.nonUniform)
1521	child.nonUniform = true;
1522	}
1523
1524	bool HasNonLayoutQualifiers(const glslang::TType& type, const glslang::TQualifier& qualifier)
1525	{
1526	// This should list qualifiers that simultaneous satisfy:
1527	// - struct members might inherit from a struct declaration
1528	// (note that non-block structs don't explicitly inherit,
1529	// only implicitly, meaning no decoration involved)
1530	// - affect decorations on the struct members
1531	// (note smooth does not, and expecting something like volatile
1532	// to effect the whole object)
1533	// - are not part of the offset/st430/etc or row/column-major layout
1534	return qualifier.invariant || (qualifier.hasLocation() && type.getBasicType() == glslang::EbtBlock);
1535	}
1536
1537	//
1538	// Implement the TGlslangToSpvTraverser class.
1539	//
1540
1541	TGlslangToSpvTraverser::TGlslangToSpvTraverser(unsigned int spvVersion,
1542	const glslang::TIntermediate* glslangIntermediate,
1543	spv::SpvBuildLogger* buildLogger, glslang::SpvOptions& options) :
1544	TIntermTraverser(true, false, true),
1545	options(options),
1546	shaderEntry(nullptr), currentFunction(nullptr),
1547	sequenceDepth(0), logger(buildLogger),
1548	builder(spvVersion, (glslang::GetKhronosToolId() << 16) | glslang::GetSpirvGeneratorVersion(), logger),
1549	inEntryPoint(false), entryPointTerminated(false), linkageOnly(false),
1550	glslangIntermediate(glslangIntermediate),
1551	nanMinMaxClamp(glslangIntermediate->getNanMinMaxClamp()),
1552	nonSemanticDebugPrintf(0),
1553	taskPayloadID(0)
1554	{
1555	bool isMeshShaderExt = (glslangIntermediate->getRequestedExtensions().find(x: glslang::E_GL_EXT_mesh_shader) !=
1556	glslangIntermediate->getRequestedExtensions().end());
1557	spv::ExecutionModel executionModel = TranslateExecutionModel(stage: glslangIntermediate->getStage(), isMeshShaderEXT: isMeshShaderExt);
1558
1559	builder.clearAccessChain();
1560	builder.setSource(lang: TranslateSourceLanguage(source: glslangIntermediate->getSource(), profile: glslangIntermediate->getProfile()),
1561	version: glslangIntermediate->getVersion());
1562
1563	if (options.emitNonSemanticShaderDebugSource)
1564	this->options.emitNonSemanticShaderDebugInfo = true;
1565	if (options.emitNonSemanticShaderDebugInfo)
1566	this->options.generateDebugInfo = true;
1567
1568	if (this->options.generateDebugInfo) {
1569	if (this->options.emitNonSemanticShaderDebugInfo) {
1570	builder.setEmitNonSemanticShaderDebugInfo(this->options.emitNonSemanticShaderDebugSource);
1571	}
1572	else {
1573	builder.setEmitSpirvDebugInfo();
1574	}
1575	builder.setDebugSourceFile(glslangIntermediate->getSourceFile());
1576
1577	// Set the source shader's text. If for SPV version 1.0, include
1578	// a preamble in comments stating the OpModuleProcessed instructions.
1579	// Otherwise, emit those as actual instructions.
1580	std::string text;
1581	const std::vector<std::string>& processes = glslangIntermediate->getProcesses();
1582	for (int p = 0; p < (int)processes.size(); ++p) {
1583	if (glslangIntermediate->getSpv().spv < glslang::EShTargetSpv_1_1) {
1584	text.append(s: "// OpModuleProcessed ");
1585	text.append(str: processes[p]);
1586	text.append(s: "\n");
1587	} else
1588	builder.addModuleProcessed(p: processes[p]);
1589	}
1590	if (glslangIntermediate->getSpv().spv < glslang::EShTargetSpv_1_1 && (int)processes.size() > 0)
1591	text.append(s: "#line 1\n");
1592	text.append(str: glslangIntermediate->getSourceText());
1593	builder.setSourceText(text);
1594	// Pass name and text for all included files
1595	const std::map<std::string, std::string>& include_txt = glslangIntermediate->getIncludeText();
1596	for (auto iItr = include_txt.begin(); iItr != include_txt.end(); ++iItr)
1597	builder.addInclude(name: iItr->first, text: iItr->second);
1598	}
1599
1600	stdBuiltins = builder.import("GLSL.std.450");
1601
1602	spv::AddressingModel addressingModel = spv::AddressingModelLogical;
1603	spv::MemoryModel memoryModel = spv::MemoryModelGLSL450;
1604
1605	if (glslangIntermediate->usingPhysicalStorageBuffer()) {
1606	addressingModel = spv::AddressingModelPhysicalStorageBuffer64EXT;
1607	builder.addIncorporatedExtension(ext: spv::E_SPV_KHR_physical_storage_buffer, incorporatedVersion: spv::Spv_1_5);
1608	builder.addCapability(cap: spv::CapabilityPhysicalStorageBufferAddressesEXT);
1609	}
1610	if (glslangIntermediate->usingVulkanMemoryModel()) {
1611	memoryModel = spv::MemoryModelVulkanKHR;
1612	builder.addCapability(cap: spv::CapabilityVulkanMemoryModelKHR);
1613	builder.addIncorporatedExtension(ext: spv::E_SPV_KHR_vulkan_memory_model, incorporatedVersion: spv::Spv_1_5);
1614	}
1615	builder.setMemoryModel(addr: addressingModel, mem: memoryModel);
1616
1617	if (glslangIntermediate->usingVariablePointers()) {
1618	builder.addCapability(cap: spv::CapabilityVariablePointers);
1619	}
1620
1621	// If not linking, there is no entry point
1622	if (!options.compileOnly) {
1623	shaderEntry = builder.makeEntryPoint(glslangIntermediate->getEntryPointName().c_str());
1624	entryPoint =
1625	builder.addEntryPoint(executionModel, shaderEntry, name: glslangIntermediate->getEntryPointName().c_str());
1626	}
1627
1628	// Add the source extensions
1629	const auto& sourceExtensions = glslangIntermediate->getRequestedExtensions();
1630	for (auto it = sourceExtensions.begin(); it != sourceExtensions.end(); ++it)
1631	builder.addSourceExtension(ext: it->c_str());
1632
1633	// Add the top-level modes for this shader.
1634
1635	if (glslangIntermediate->getXfbMode()) {
1636	builder.addCapability(cap: spv::CapabilityTransformFeedback);
1637	builder.addExecutionMode(shaderEntry, mode: spv::ExecutionModeXfb);
1638	}
1639
1640	if (glslangIntermediate->getLayoutPrimitiveCulling()) {
1641	builder.addCapability(cap: spv::CapabilityRayTraversalPrimitiveCullingKHR);
1642	}
1643
1644	if (glslangIntermediate->getSubgroupUniformControlFlow()) {
1645	builder.addExtension(ext: spv::E_SPV_KHR_subgroup_uniform_control_flow);
1646	builder.addExecutionMode(shaderEntry, mode: spv::ExecutionModeSubgroupUniformControlFlowKHR);
1647	}
1648	if (glslangIntermediate->getMaximallyReconverges()) {
1649	builder.addExtension(ext: spv::E_SPV_KHR_maximal_reconvergence);
1650	builder.addExecutionMode(shaderEntry, mode: spv::ExecutionModeMaximallyReconvergesKHR);
1651	}
1652
1653	if (glslangIntermediate->getQuadDerivMode())
1654	{
1655	builder.addCapability(cap: spv::CapabilityQuadControlKHR);
1656	builder.addExtension(ext: spv::E_SPV_KHR_quad_control);
1657	builder.addExecutionMode(shaderEntry, mode: spv::ExecutionModeQuadDerivativesKHR);
1658	}
1659
1660	if (glslangIntermediate->getReqFullQuadsMode())
1661	{
1662	builder.addCapability(cap: spv::CapabilityQuadControlKHR);
1663	builder.addExtension(ext: spv::E_SPV_KHR_quad_control);
1664	builder.addExecutionMode(shaderEntry, mode: spv::ExecutionModeRequireFullQuadsKHR);
1665	}
1666
1667	unsigned int mode;
1668	switch (glslangIntermediate->getStage()) {
1669	case EShLangVertex:
1670	builder.addCapability(cap: spv::CapabilityShader);
1671	break;
1672
1673	case EShLangFragment:
1674	builder.addCapability(cap: spv::CapabilityShader);
1675	if (glslangIntermediate->getPixelCenterInteger())
1676	builder.addExecutionMode(shaderEntry, mode: spv::ExecutionModePixelCenterInteger);
1677
1678	if (glslangIntermediate->getOriginUpperLeft())
1679	builder.addExecutionMode(shaderEntry, mode: spv::ExecutionModeOriginUpperLeft);
1680	else
1681	builder.addExecutionMode(shaderEntry, mode: spv::ExecutionModeOriginLowerLeft);
1682
1683	if (glslangIntermediate->getEarlyFragmentTests())
1684	builder.addExecutionMode(shaderEntry, mode: spv::ExecutionModeEarlyFragmentTests);
1685
1686	if (glslangIntermediate->getEarlyAndLateFragmentTestsAMD())
1687	{
1688	builder.addExecutionMode(shaderEntry, mode: spv::ExecutionModeEarlyAndLateFragmentTestsAMD);
1689	builder.addExtension(ext: spv::E_SPV_AMD_shader_early_and_late_fragment_tests);
1690	}
1691
1692	if (glslangIntermediate->getPostDepthCoverage()) {
1693	builder.addCapability(cap: spv::CapabilitySampleMaskPostDepthCoverage);
1694	builder.addExecutionMode(shaderEntry, mode: spv::ExecutionModePostDepthCoverage);
1695	builder.addExtension(ext: spv::E_SPV_KHR_post_depth_coverage);
1696	}
1697
1698	if (glslangIntermediate->getNonCoherentColorAttachmentReadEXT()) {
1699	builder.addCapability(cap: spv::CapabilityTileImageColorReadAccessEXT);
1700	builder.addExecutionMode(shaderEntry, mode: spv::ExecutionModeNonCoherentColorAttachmentReadEXT);
1701	builder.addExtension(ext: spv::E_SPV_EXT_shader_tile_image);
1702	}
1703
1704	if (glslangIntermediate->getNonCoherentDepthAttachmentReadEXT()) {
1705	builder.addCapability(cap: spv::CapabilityTileImageDepthReadAccessEXT);
1706	builder.addExecutionMode(shaderEntry, mode: spv::ExecutionModeNonCoherentDepthAttachmentReadEXT);
1707	builder.addExtension(ext: spv::E_SPV_EXT_shader_tile_image);
1708	}
1709
1710	if (glslangIntermediate->getNonCoherentStencilAttachmentReadEXT()) {
1711	builder.addCapability(cap: spv::CapabilityTileImageStencilReadAccessEXT);
1712	builder.addExecutionMode(shaderEntry, mode: spv::ExecutionModeNonCoherentStencilAttachmentReadEXT);
1713	builder.addExtension(ext: spv::E_SPV_EXT_shader_tile_image);
1714	}
1715
1716	if (glslangIntermediate->isDepthReplacing())
1717	builder.addExecutionMode(shaderEntry, mode: spv::ExecutionModeDepthReplacing);
1718
1719	if (glslangIntermediate->isStencilReplacing())
1720	builder.addExecutionMode(shaderEntry, mode: spv::ExecutionModeStencilRefReplacingEXT);
1721
1722	switch(glslangIntermediate->getDepth()) {
1723	case glslang::EldGreater: mode = spv::ExecutionModeDepthGreater; break;
1724	case glslang::EldLess: mode = spv::ExecutionModeDepthLess; break;
1725	case glslang::EldUnchanged: mode = spv::ExecutionModeDepthUnchanged; break;
1726	default: mode = spv::ExecutionModeMax; break;
1727	}
1728
1729	if (mode != spv::ExecutionModeMax)
1730	builder.addExecutionMode(shaderEntry, mode: (spv::ExecutionMode)mode);
1731
1732	switch (glslangIntermediate->getStencil()) {
1733	case glslang::ElsRefUnchangedFrontAMD: mode = spv::ExecutionModeStencilRefUnchangedFrontAMD; break;
1734	case glslang::ElsRefGreaterFrontAMD: mode = spv::ExecutionModeStencilRefGreaterFrontAMD; break;
1735	case glslang::ElsRefLessFrontAMD: mode = spv::ExecutionModeStencilRefLessFrontAMD; break;
1736	case glslang::ElsRefUnchangedBackAMD: mode = spv::ExecutionModeStencilRefUnchangedBackAMD; break;
1737	case glslang::ElsRefGreaterBackAMD: mode = spv::ExecutionModeStencilRefGreaterBackAMD; break;
1738	case glslang::ElsRefLessBackAMD: mode = spv::ExecutionModeStencilRefLessBackAMD; break;
1739	default: mode = spv::ExecutionModeMax; break;
1740	}
1741
1742	if (mode != spv::ExecutionModeMax)
1743	builder.addExecutionMode(shaderEntry, mode: (spv::ExecutionMode)mode);
1744	switch (glslangIntermediate->getInterlockOrdering()) {
1745	case glslang::EioPixelInterlockOrdered: mode = spv::ExecutionModePixelInterlockOrderedEXT;
1746	break;
1747	case glslang::EioPixelInterlockUnordered: mode = spv::ExecutionModePixelInterlockUnorderedEXT;
1748	break;
1749	case glslang::EioSampleInterlockOrdered: mode = spv::ExecutionModeSampleInterlockOrderedEXT;
1750	break;
1751	case glslang::EioSampleInterlockUnordered: mode = spv::ExecutionModeSampleInterlockUnorderedEXT;
1752	break;
1753	case glslang::EioShadingRateInterlockOrdered: mode = spv::ExecutionModeShadingRateInterlockOrderedEXT;
1754	break;
1755	case glslang::EioShadingRateInterlockUnordered: mode = spv::ExecutionModeShadingRateInterlockUnorderedEXT;
1756	break;
1757	default: mode = spv::ExecutionModeMax;
1758	break;
1759	}
1760	if (mode != spv::ExecutionModeMax) {
1761	builder.addExecutionMode(shaderEntry, mode: (spv::ExecutionMode)mode);
1762	if (mode == spv::ExecutionModeShadingRateInterlockOrderedEXT ||
1763	mode == spv::ExecutionModeShadingRateInterlockUnorderedEXT) {
1764	builder.addCapability(cap: spv::CapabilityFragmentShaderShadingRateInterlockEXT);
1765	} else if (mode == spv::ExecutionModePixelInterlockOrderedEXT ||
1766	mode == spv::ExecutionModePixelInterlockUnorderedEXT) {
1767	builder.addCapability(cap: spv::CapabilityFragmentShaderPixelInterlockEXT);
1768	} else {
1769	builder.addCapability(cap: spv::CapabilityFragmentShaderSampleInterlockEXT);
1770	}
1771	builder.addExtension(ext: spv::E_SPV_EXT_fragment_shader_interlock);
1772	}
1773	break;
1774
1775	case EShLangCompute: {
1776	builder.addCapability(cap: spv::CapabilityShader);
1777	bool needSizeId = false;
1778	for (int dim = 0; dim < 3; ++dim) {
1779	if ((glslangIntermediate->getLocalSizeSpecId(dim) != glslang::TQualifier::layoutNotSet)) {
1780	needSizeId = true;
1781	break;
1782	}
1783	}
1784	if (glslangIntermediate->getSpv().spv >= glslang::EShTargetSpv_1_6 && needSizeId) {
1785	std::vector<spv::Id> dimConstId;
1786	for (int dim = 0; dim < 3; ++dim) {
1787	bool specConst = (glslangIntermediate->getLocalSizeSpecId(dim) != glslang::TQualifier::layoutNotSet);
1788	dimConstId.push_back(x: builder.makeUintConstant(u: glslangIntermediate->getLocalSize(dim), specConstant: specConst));
1789	if (specConst) {
1790	builder.addDecoration(dimConstId.back(), spv::DecorationSpecId,
1791	num: glslangIntermediate->getLocalSizeSpecId(dim));
1792	needSizeId = true;
1793	}
1794	}
1795	builder.addExecutionModeId(shaderEntry, mode: spv::ExecutionModeLocalSizeId, operandIds: dimConstId);
1796	} else {
1797	builder.addExecutionMode(shaderEntry, mode: spv::ExecutionModeLocalSize, value1: glslangIntermediate->getLocalSize(dim: 0),
1798	value2: glslangIntermediate->getLocalSize(dim: 1),
1799	value3: glslangIntermediate->getLocalSize(dim: 2));
1800	}
1801	if (glslangIntermediate->getLayoutDerivativeModeNone() == glslang::LayoutDerivativeGroupQuads) {
1802	builder.addCapability(cap: spv::CapabilityComputeDerivativeGroupQuadsNV);
1803	builder.addExecutionMode(shaderEntry, mode: spv::ExecutionModeDerivativeGroupQuadsNV);
1804	builder.addExtension(ext: spv::E_SPV_NV_compute_shader_derivatives);
1805	} else if (glslangIntermediate->getLayoutDerivativeModeNone() == glslang::LayoutDerivativeGroupLinear) {
1806	builder.addCapability(cap: spv::CapabilityComputeDerivativeGroupLinearNV);
1807	builder.addExecutionMode(shaderEntry, mode: spv::ExecutionModeDerivativeGroupLinearNV);
1808	builder.addExtension(ext: spv::E_SPV_NV_compute_shader_derivatives);
1809	}
1810	break;
1811	}
1812	case EShLangTessEvaluation:
1813	case EShLangTessControl:
1814	builder.addCapability(cap: spv::CapabilityTessellation);
1815
1816	glslang::TLayoutGeometry primitive;
1817
1818	if (glslangIntermediate->getStage() == EShLangTessControl) {
1819	builder.addExecutionMode(shaderEntry, mode: spv::ExecutionModeOutputVertices,
1820	value1: glslangIntermediate->getVertices());
1821	primitive = glslangIntermediate->getOutputPrimitive();
1822	} else {
1823	primitive = glslangIntermediate->getInputPrimitive();
1824	}
1825
1826	switch (primitive) {
1827	case glslang::ElgTriangles: mode = spv::ExecutionModeTriangles; break;
1828	case glslang::ElgQuads: mode = spv::ExecutionModeQuads; break;
1829	case glslang::ElgIsolines: mode = spv::ExecutionModeIsolines; break;
1830	default: mode = spv::ExecutionModeMax; break;
1831	}
1832	if (mode != spv::ExecutionModeMax)
1833	builder.addExecutionMode(shaderEntry, mode: (spv::ExecutionMode)mode);
1834
1835	switch (glslangIntermediate->getVertexSpacing()) {
1836	case glslang::EvsEqual: mode = spv::ExecutionModeSpacingEqual; break;
1837	case glslang::EvsFractionalEven: mode = spv::ExecutionModeSpacingFractionalEven; break;
1838	case glslang::EvsFractionalOdd: mode = spv::ExecutionModeSpacingFractionalOdd; break;
1839	default: mode = spv::ExecutionModeMax; break;
1840	}
1841	if (mode != spv::ExecutionModeMax)
1842	builder.addExecutionMode(shaderEntry, mode: (spv::ExecutionMode)mode);
1843
1844	switch (glslangIntermediate->getVertexOrder()) {
1845	case glslang::EvoCw: mode = spv::ExecutionModeVertexOrderCw; break;
1846	case glslang::EvoCcw: mode = spv::ExecutionModeVertexOrderCcw; break;
1847	default: mode = spv::ExecutionModeMax; break;
1848	}
1849	if (mode != spv::ExecutionModeMax)
1850	builder.addExecutionMode(shaderEntry, mode: (spv::ExecutionMode)mode);
1851
1852	if (glslangIntermediate->getPointMode())
1853	builder.addExecutionMode(shaderEntry, mode: spv::ExecutionModePointMode);
1854	break;
1855
1856	case EShLangGeometry:
1857	builder.addCapability(cap: spv::CapabilityGeometry);
1858	switch (glslangIntermediate->getInputPrimitive()) {
1859	case glslang::ElgPoints: mode = spv::ExecutionModeInputPoints; break;
1860	case glslang::ElgLines: mode = spv::ExecutionModeInputLines; break;
1861	case glslang::ElgLinesAdjacency: mode = spv::ExecutionModeInputLinesAdjacency; break;
1862	case glslang::ElgTriangles: mode = spv::ExecutionModeTriangles; break;
1863	case glslang::ElgTrianglesAdjacency: mode = spv::ExecutionModeInputTrianglesAdjacency; break;
1864	default: mode = spv::ExecutionModeMax; break;
1865	}
1866	if (mode != spv::ExecutionModeMax)
1867	builder.addExecutionMode(shaderEntry, mode: (spv::ExecutionMode)mode);
1868
1869	builder.addExecutionMode(shaderEntry, mode: spv::ExecutionModeInvocations, value1: glslangIntermediate->getInvocations());
1870
1871	switch (glslangIntermediate->getOutputPrimitive()) {
1872	case glslang::ElgPoints: mode = spv::ExecutionModeOutputPoints; break;
1873	case glslang::ElgLineStrip: mode = spv::ExecutionModeOutputLineStrip; break;
1874	case glslang::ElgTriangleStrip: mode = spv::ExecutionModeOutputTriangleStrip; break;
1875	default: mode = spv::ExecutionModeMax; break;
1876	}
1877	if (mode != spv::ExecutionModeMax)
1878	builder.addExecutionMode(shaderEntry, mode: (spv::ExecutionMode)mode);
1879	builder.addExecutionMode(shaderEntry, mode: spv::ExecutionModeOutputVertices, value1: glslangIntermediate->getVertices());
1880	break;
1881
1882	case EShLangRayGen:
1883	case EShLangIntersect:
1884	case EShLangAnyHit:
1885	case EShLangClosestHit:
1886	case EShLangMiss:
1887	case EShLangCallable:
1888	{
1889	auto& extensions = glslangIntermediate->getRequestedExtensions();
1890	if (extensions.find(x: "GL_NV_ray_tracing") == extensions.end()) {
1891	builder.addCapability(cap: spv::CapabilityRayTracingKHR);
1892	builder.addExtension(ext: "SPV_KHR_ray_tracing");
1893	}
1894	else {
1895	builder.addCapability(cap: spv::CapabilityRayTracingNV);
1896	builder.addExtension(ext: "SPV_NV_ray_tracing");
1897	}
1898	if (glslangIntermediate->getStage() != EShLangRayGen && glslangIntermediate->getStage() != EShLangCallable) {
1899	if (extensions.find(x: "GL_EXT_ray_cull_mask") != extensions.end()) {
1900	builder.addCapability(cap: spv::CapabilityRayCullMaskKHR);
1901	builder.addExtension(ext: "SPV_KHR_ray_cull_mask");
1902	}
1903	if (extensions.find(x: "GL_EXT_ray_tracing_position_fetch") != extensions.end()) {
1904	builder.addCapability(cap: spv::CapabilityRayTracingPositionFetchKHR);
1905	builder.addExtension(ext: "SPV_KHR_ray_tracing_position_fetch");
1906	}
1907	}
1908	break;
1909	}
1910	case EShLangTask:
1911	case EShLangMesh:
1912	if(isMeshShaderExt) {
1913	builder.addCapability(cap: spv::CapabilityMeshShadingEXT);
1914	builder.addExtension(ext: spv::E_SPV_EXT_mesh_shader);
1915	} else {
1916	builder.addCapability(cap: spv::CapabilityMeshShadingNV);
1917	builder.addExtension(ext: spv::E_SPV_NV_mesh_shader);
1918	}
1919	if (glslangIntermediate->getSpv().spv >= glslang::EShTargetSpv_1_6) {
1920	std::vector<spv::Id> dimConstId;
1921	for (int dim = 0; dim < 3; ++dim) {
1922	bool specConst = (glslangIntermediate->getLocalSizeSpecId(dim) != glslang::TQualifier::layoutNotSet);
1923	dimConstId.push_back(x: builder.makeUintConstant(u: glslangIntermediate->getLocalSize(dim), specConstant: specConst));
1924	if (specConst) {
1925	builder.addDecoration(dimConstId.back(), spv::DecorationSpecId,
1926	num: glslangIntermediate->getLocalSizeSpecId(dim));
1927	}
1928	}
1929	builder.addExecutionModeId(shaderEntry, mode: spv::ExecutionModeLocalSizeId, operandIds: dimConstId);
1930	} else {
1931	builder.addExecutionMode(shaderEntry, mode: spv::ExecutionModeLocalSize, value1: glslangIntermediate->getLocalSize(dim: 0),
1932	value2: glslangIntermediate->getLocalSize(dim: 1),
1933	value3: glslangIntermediate->getLocalSize(dim: 2));
1934	}
1935	if (glslangIntermediate->getStage() == EShLangMesh) {
1936	builder.addExecutionMode(shaderEntry, mode: spv::ExecutionModeOutputVertices,
1937	value1: glslangIntermediate->getVertices());
1938	builder.addExecutionMode(shaderEntry, mode: spv::ExecutionModeOutputPrimitivesNV,
1939	value1: glslangIntermediate->getPrimitives());
1940
1941	switch (glslangIntermediate->getOutputPrimitive()) {
1942	case glslang::ElgPoints: mode = spv::ExecutionModeOutputPoints; break;
1943	case glslang::ElgLines: mode = spv::ExecutionModeOutputLinesNV; break;
1944	case glslang::ElgTriangles: mode = spv::ExecutionModeOutputTrianglesNV; break;
1945	default: mode = spv::ExecutionModeMax; break;
1946	}
1947	if (mode != spv::ExecutionModeMax)
1948	builder.addExecutionMode(shaderEntry, mode: (spv::ExecutionMode)mode);
1949	}
1950	break;
1951
1952	default:
1953	break;
1954	}
1955
1956	//
1957	// Add SPIR-V requirements (GL_EXT_spirv_intrinsics)
1958	//
1959	if (glslangIntermediate->hasSpirvRequirement()) {
1960	const glslang::TSpirvRequirement& spirvRequirement = glslangIntermediate->getSpirvRequirement();
1961
1962	// Add SPIR-V extension requirement
1963	for (auto& extension : spirvRequirement.extensions)
1964	builder.addExtension(ext: extension.c_str());
1965
1966	// Add SPIR-V capability requirement
1967	for (auto capability : spirvRequirement.capabilities)
1968	builder.addCapability(cap: static_cast<spv::Capability>(capability));
1969	}
1970
1971	//
1972	// Add SPIR-V execution mode qualifiers (GL_EXT_spirv_intrinsics)
1973	//
1974	if (glslangIntermediate->hasSpirvExecutionMode()) {
1975	const glslang::TSpirvExecutionMode spirvExecutionMode = glslangIntermediate->getSpirvExecutionMode();
1976
1977	// Add spirv_execution_mode
1978	for (auto& mode : spirvExecutionMode.modes) {
1979	if (!mode.second.empty()) {
1980	std::vector<unsigned> literals;
1981	TranslateLiterals(constants: mode.second, literals);
1982	builder.addExecutionMode(shaderEntry, mode: static_cast<spv::ExecutionMode>(mode.first), literals);
1983	} else
1984	builder.addExecutionMode(shaderEntry, mode: static_cast<spv::ExecutionMode>(mode.first));
1985	}
1986
1987	// Add spirv_execution_mode_id
1988	for (auto& modeId : spirvExecutionMode.modeIds) {
1989	std::vector<spv::Id> operandIds;
1990	assert(!modeId.second.empty());
1991	for (auto extraOperand : modeId.second) {
1992	if (extraOperand->getType().getQualifier().isSpecConstant())
1993	operandIds.push_back(x: getSymbolId(node: extraOperand->getAsSymbolNode()));
1994	else
1995	operandIds.push_back(x: createSpvConstant(*extraOperand));
1996	}
1997	builder.addExecutionModeId(shaderEntry, mode: static_cast<spv::ExecutionMode>(modeId.first), operandIds);
1998	}
1999	}
2000	}
2001
2002	// Finish creating SPV, after the traversal is complete.
2003	void TGlslangToSpvTraverser::finishSpv(bool compileOnly)
2004	{
2005	// If not linking, an entry point is not expected
2006	if (!compileOnly) {
2007	// Finish the entry point function
2008	if (!entryPointTerminated) {
2009	builder.setBuildPoint(shaderEntry->getLastBlock());
2010	builder.leaveFunction();
2011	}
2012
2013	// finish off the entry-point SPV instruction by adding the Input/Output <id>
2014	entryPoint->reserveOperands(count: iOSet.size());
2015	for (auto id : iOSet)
2016	entryPoint->addIdOperand(id);
2017	}
2018
2019	// Add capabilities, extensions, remove unneeded decorations, etc.,
2020	// based on the resulting SPIR-V.
2021	// Note: WebGPU code generation must have the opportunity to aggressively
2022	// prune unreachable merge blocks and continue targets.
2023	builder.postProcess(compileOnly);
2024	}
2025
2026	// Write the SPV into 'out'.
2027	void TGlslangToSpvTraverser::dumpSpv(std::vector<unsigned int>& out)
2028	{
2029	builder.dump(out);
2030	}
2031
2032	//
2033	// Implement the traversal functions.
2034	//
2035	// Return true from interior nodes to have the external traversal
2036	// continue on to children. Return false if children were
2037	// already processed.
2038	//
2039
2040	//
2041	// Symbols can turn into
2042	// - uniform/input reads
2043	// - output writes
2044	// - complex lvalue base setups: foo.bar[3].... , where we see foo and start up an access chain
2045	// - something simple that degenerates into the last bullet
2046	//
2047	void TGlslangToSpvTraverser::visitSymbol(glslang::TIntermSymbol* symbol)
2048	{
2049	// We update the line information even though no code might be generated here
2050	// This is helpful to yield correct lines for control flow instructions
2051	if (!linkageOnly) {
2052	builder.setDebugSourceLocation(line: symbol->getLoc().line, filename: symbol->getLoc().getFilename());
2053	}
2054
2055	SpecConstantOpModeGuard spec_constant_op_mode_setter(&builder);
2056	if (symbol->getType().isStruct())
2057	glslangTypeToIdMap[symbol->getType().getStruct()] = symbol->getId();
2058
2059	if (symbol->getType().getQualifier().isSpecConstant())
2060	spec_constant_op_mode_setter.turnOnSpecConstantOpMode();
2061	#ifdef ENABLE_HLSL
2062	// Skip symbol handling if it is string-typed
2063	if (symbol->getBasicType() == glslang::EbtString)
2064	return;
2065	#endif
2066
2067	// getSymbolId() will set up all the IO decorations on the first call.
2068	// Formal function parameters were mapped during makeFunctions().
2069	spv::Id id = getSymbolId(node: symbol);
2070
2071	if (symbol->getType().getQualifier().isTaskPayload())
2072	taskPayloadID = id; // cache the taskPayloadID to be used it as operand for OpEmitMeshTasksEXT
2073
2074	if (builder.isPointer(resultId: id)) {
2075	if (!symbol->getType().getQualifier().isParamInput() &&
2076	!symbol->getType().getQualifier().isParamOutput()) {
2077	// Include all "static use" and "linkage only" interface variables on the OpEntryPoint instruction
2078	// Consider adding to the OpEntryPoint interface list.
2079	// Only looking at structures if they have at least one member.
2080	if (!symbol->getType().isStruct() || symbol->getType().getStruct()->size() > 0) {
2081	spv::StorageClass sc = builder.getStorageClass(resultId: id);
2082	// Before SPIR-V 1.4, we only want to include Input and Output.
2083	// Starting with SPIR-V 1.4, we want all globals.
2084	if ((glslangIntermediate->getSpv().spv >= glslang::EShTargetSpv_1_4 && builder.isGlobalVariable(resultId: id)) ||
2085	(sc == spv::StorageClassInput || sc == spv::StorageClassOutput)) {
2086	iOSet.insert(x: id);
2087	}
2088	}
2089	}
2090
2091	// If the SPIR-V type is required to be different than the AST type
2092	// (for ex SubgroupMasks or 3x4 ObjectToWorld/WorldToObject matrices),
2093	// translate now from the SPIR-V type to the AST type, for the consuming
2094	// operation.
2095	// Note this turns it from an l-value to an r-value.
2096	// Currently, all symbols needing this are inputs; avoid the map lookup when non-input.
2097	if (symbol->getType().getQualifier().storage == glslang::EvqVaryingIn)
2098	id = translateForcedType(object: id);
2099	}
2100
2101	// Only process non-linkage-only nodes for generating actual static uses
2102	if (! linkageOnly || symbol->getQualifier().isSpecConstant()) {
2103	// Prepare to generate code for the access
2104
2105	// L-value chains will be computed left to right. We're on the symbol now,
2106	// which is the left-most part of the access chain, so now is "clear" time,
2107	// followed by setting the base.
2108	builder.clearAccessChain();
2109
2110	// For now, we consider all user variables as being in memory, so they are pointers,
2111	// except for
2112	// A) R-Value arguments to a function, which are an intermediate object.
2113	// See comments in handleUserFunctionCall().
2114	// B) Specialization constants (normal constants don't even come in as a variable),
2115	// These are also pure R-values.
2116	// C) R-Values from type translation, see above call to translateForcedType()
2117	glslang::TQualifier qualifier = symbol->getQualifier();
2118	if (qualifier.isSpecConstant() || rValueParameters.find(x: symbol->getId()) != rValueParameters.end() ||
2119	!builder.isPointerType(typeId: builder.getTypeId(resultId: id)))
2120	builder.setAccessChainRValue(id);
2121	else
2122	builder.setAccessChainLValue(id);
2123	}
2124
2125	#ifdef ENABLE_HLSL
2126	// Process linkage-only nodes for any special additional interface work.
2127	if (linkageOnly) {
2128	if (glslangIntermediate->getHlslFunctionality1()) {
2129	// Map implicit counter buffers to their originating buffers, which should have been
2130	// seen by now, given earlier pruning of unused counters, and preservation of order
2131	// of declaration.
2132	if (symbol->getType().getQualifier().isUniformOrBuffer()) {
2133	if (!glslangIntermediate->hasCounterBufferName(symbol->getName())) {
2134	// Save possible originating buffers for counter buffers, keyed by
2135	// making the potential counter-buffer name.
2136	std::string keyName = symbol->getName().c_str();
2137	keyName = glslangIntermediate->addCounterBufferName(keyName);
2138	counterOriginator[keyName] = symbol;
2139	} else {
2140	// Handle a counter buffer, by finding the saved originating buffer.
2141	std::string keyName = symbol->getName().c_str();
2142	auto it = counterOriginator.find(keyName);
2143	if (it != counterOriginator.end()) {
2144	id = getSymbolId(it->second);
2145	if (id != spv::NoResult) {
2146	spv::Id counterId = getSymbolId(symbol);
2147	if (counterId != spv::NoResult) {
2148	builder.addExtension("SPV_GOOGLE_hlsl_functionality1");
2149	builder.addDecorationId(id, spv::DecorationHlslCounterBufferGOOGLE, counterId);
2150	}
2151	}
2152	}
2153	}
2154	}
2155	}
2156	}
2157	#endif
2158	}
2159
2160	bool TGlslangToSpvTraverser::visitBinary(glslang::TVisit /* visit */, glslang::TIntermBinary* node)
2161	{
2162	builder.setDebugSourceLocation(line: node->getLoc().line, filename: node->getLoc().getFilename());
2163	if (node->getLeft()->getAsSymbolNode() != nullptr && node->getLeft()->getType().isStruct()) {
2164	glslangTypeToIdMap[node->getLeft()->getType().getStruct()] = node->getLeft()->getAsSymbolNode()->getId();
2165	}
2166	if (node->getRight()->getAsSymbolNode() != nullptr && node->getRight()->getType().isStruct()) {
2167	glslangTypeToIdMap[node->getRight()->getType().getStruct()] = node->getRight()->getAsSymbolNode()->getId();
2168	}
2169
2170	SpecConstantOpModeGuard spec_constant_op_mode_setter(&builder);
2171	if (node->getType().getQualifier().isSpecConstant())
2172	spec_constant_op_mode_setter.turnOnSpecConstantOpMode();
2173
2174	// First, handle special cases
2175	switch (node->getOp()) {
2176	case glslang::EOpAssign:
2177	case glslang::EOpAddAssign:
2178	case glslang::EOpSubAssign:
2179	case glslang::EOpMulAssign:
2180	case glslang::EOpVectorTimesMatrixAssign:
2181	case glslang::EOpVectorTimesScalarAssign:
2182	case glslang::EOpMatrixTimesScalarAssign:
2183	case glslang::EOpMatrixTimesMatrixAssign:
2184	case glslang::EOpDivAssign:
2185	case glslang::EOpModAssign:
2186	case glslang::EOpAndAssign:
2187	case glslang::EOpInclusiveOrAssign:
2188	case glslang::EOpExclusiveOrAssign:
2189	case glslang::EOpLeftShiftAssign:
2190	case glslang::EOpRightShiftAssign:
2191	// A bin-op assign "a += b" means the same thing as "a = a + b"
2192	// where a is evaluated before b. For a simple assignment, GLSL
2193	// says to evaluate the left before the right. So, always, left
2194	// node then right node.
2195	{
2196	// get the left l-value, save it away
2197	builder.clearAccessChain();
2198	node->getLeft()->traverse(this);
2199	spv::Builder::AccessChain lValue = builder.getAccessChain();
2200
2201	// evaluate the right
2202	builder.clearAccessChain();
2203	node->getRight()->traverse(this);
2204	spv::Id rValue = accessChainLoad(type: node->getRight()->getType());
2205
2206	// reset line number for assignment
2207	builder.setDebugSourceLocation(line: node->getLoc().line, filename: node->getLoc().getFilename());
2208
2209	if (node->getOp() != glslang::EOpAssign) {
2210	// the left is also an r-value
2211	builder.setAccessChain(lValue);
2212	spv::Id leftRValue = accessChainLoad(type: node->getLeft()->getType());
2213
2214	// do the operation
2215	spv::Builder::AccessChain::CoherentFlags coherentFlags = TranslateCoherent(type: node->getLeft()->getType());
2216	coherentFlags |= TranslateCoherent(type: node->getRight()->getType());
2217	OpDecorations decorations = { TranslatePrecisionDecoration(glslangPrecision: node->getOperationPrecision()),
2218	TranslateNoContractionDecoration(qualifier: node->getType().getQualifier()),
2219	TranslateNonUniformDecoration(coherentFlags) };
2220	rValue = createBinaryOperation(op: node->getOp(), decorations,
2221	typeId: convertGlslangToSpvType(type: node->getType()), left: leftRValue, right: rValue,
2222	typeProxy: node->getType().getBasicType());
2223
2224	// these all need their counterparts in createBinaryOperation()
2225	assert(rValue != spv::NoResult);
2226	}
2227
2228	// store the result
2229	builder.setAccessChain(lValue);
2230	multiTypeStore(node->getLeft()->getType(), rValue);
2231
2232	// assignments are expressions having an rValue after they are evaluated...
2233	builder.clearAccessChain();
2234	builder.setAccessChainRValue(rValue);
2235	}
2236	return false;
2237	case glslang::EOpIndexDirect:
2238	case glslang::EOpIndexDirectStruct:
2239	{
2240	// Structure, array, matrix, or vector indirection with statically known index.
2241	// Get the left part of the access chain.
2242	node->getLeft()->traverse(this);
2243
2244	// Add the next element in the chain
2245
2246	const int glslangIndex = node->getRight()->getAsConstantUnion()->getConstArray()[0].getIConst();
2247	if (! node->getLeft()->getType().isArray() &&
2248	node->getLeft()->getType().isVector() &&
2249	node->getOp() == glslang::EOpIndexDirect) {
2250	// Swizzle is uniform so propagate uniform into access chain
2251	spv::Builder::AccessChain::CoherentFlags coherentFlags = TranslateCoherent(type: node->getLeft()->getType());
2252	coherentFlags.nonUniform = 0;
2253	// This is essentially a hard-coded vector swizzle of size 1,
2254	// so short circuit the access-chain stuff with a swizzle.
2255	std::vector<unsigned> swizzle;
2256	swizzle.push_back(x: glslangIndex);
2257	int dummySize;
2258	builder.accessChainPushSwizzle(swizzle, preSwizzleBaseType: convertGlslangToSpvType(type: node->getLeft()->getType()),
2259	coherentFlags,
2260	alignment: glslangIntermediate->getBaseAlignmentScalar(
2261	node->getLeft()->getType(), size&: dummySize));
2262	} else {
2263
2264	// Load through a block reference is performed with a dot operator that
2265	// is mapped to EOpIndexDirectStruct. When we get to the actual reference,
2266	// do a load and reset the access chain.
2267	if (node->getLeft()->isReference() &&
2268	!node->getLeft()->getType().isArray() &&
2269	node->getOp() == glslang::EOpIndexDirectStruct)
2270	{
2271	spv::Id left = accessChainLoad(type: node->getLeft()->getType());
2272	builder.clearAccessChain();
2273	builder.setAccessChainLValue(left);
2274	}
2275
2276	int spvIndex = glslangIndex;
2277	if (node->getLeft()->getBasicType() == glslang::EbtBlock &&
2278	node->getOp() == glslang::EOpIndexDirectStruct)
2279	{
2280	// This may be, e.g., an anonymous block-member selection, which generally need
2281	// index remapping due to hidden members in anonymous blocks.
2282	long long glslangId = glslangTypeToIdMap[node->getLeft()->getType().getStruct()];
2283	if (memberRemapper.find(x: glslangId) != memberRemapper.end()) {
2284	std::vector<int>& remapper = memberRemapper[glslangId];
2285	assert(remapper.size() > 0);
2286	spvIndex = remapper[glslangIndex];
2287	}
2288	}
2289
2290	// Struct reference propagates uniform lvalue
2291	spv::Builder::AccessChain::CoherentFlags coherentFlags =
2292	TranslateCoherent(type: node->getLeft()->getType());
2293	coherentFlags.nonUniform = 0;
2294
2295	// normal case for indexing array or structure or block
2296	builder.accessChainPush(offset: builder.makeIntConstant(i: spvIndex),
2297	coherentFlags,
2298	alignment: node->getLeft()->getType().getBufferReferenceAlignment());
2299
2300	// Add capabilities here for accessing PointSize and clip/cull distance.
2301	// We have deferred generation of associated capabilities until now.
2302	if (node->getLeft()->getType().isStruct() && ! node->getLeft()->getType().isArray())
2303	declareUseOfStructMember(members: *(node->getLeft()->getType().getStruct()), glslangMember: glslangIndex);
2304	}
2305	}
2306	return false;
2307	case glslang::EOpIndexIndirect:
2308	{
2309	// Array, matrix, or vector indirection with variable index.
2310	// Will use native SPIR-V access-chain for and array indirection;
2311	// matrices are arrays of vectors, so will also work for a matrix.
2312	// Will use the access chain's 'component' for variable index into a vector.
2313
2314	// This adapter is building access chains left to right.
2315	// Set up the access chain to the left.
2316	node->getLeft()->traverse(this);
2317
2318	// save it so that computing the right side doesn't trash it
2319	spv::Builder::AccessChain partial = builder.getAccessChain();
2320
2321	// compute the next index in the chain
2322	builder.clearAccessChain();
2323	node->getRight()->traverse(this);
2324	spv::Id index = accessChainLoad(type: node->getRight()->getType());
2325
2326	addIndirectionIndexCapabilities(baseType: node->getLeft()->getType(), indexType: node->getRight()->getType());
2327
2328	// restore the saved access chain
2329	builder.setAccessChain(partial);
2330
2331	// Only if index is nonUniform should we propagate nonUniform into access chain
2332	spv::Builder::AccessChain::CoherentFlags index_flags = TranslateCoherent(type: node->getRight()->getType());
2333	spv::Builder::AccessChain::CoherentFlags coherent_flags = TranslateCoherent(type: node->getLeft()->getType());
2334	coherent_flags.nonUniform = index_flags.nonUniform;
2335
2336	if (! node->getLeft()->getType().isArray() && node->getLeft()->getType().isVector()) {
2337	int dummySize;
2338	builder.accessChainPushComponent(
2339	component: index, preSwizzleBaseType: convertGlslangToSpvType(type: node->getLeft()->getType()), coherentFlags: coherent_flags,
2340	alignment: glslangIntermediate->getBaseAlignmentScalar(node->getLeft()->getType(),
2341	size&: dummySize));
2342	} else
2343	builder.accessChainPush(offset: index, coherentFlags: coherent_flags,
2344	alignment: node->getLeft()->getType().getBufferReferenceAlignment());
2345	}
2346	return false;
2347	case glslang::EOpVectorSwizzle:
2348	{
2349	node->getLeft()->traverse(this);
2350	std::vector<unsigned> swizzle;
2351	convertSwizzle(*node->getRight()->getAsAggregate(), swizzle);
2352	int dummySize;
2353	builder.accessChainPushSwizzle(swizzle, preSwizzleBaseType: convertGlslangToSpvType(type: node->getLeft()->getType()),
2354	coherentFlags: TranslateCoherent(type: node->getLeft()->getType()),
2355	alignment: glslangIntermediate->getBaseAlignmentScalar(node->getLeft()->getType(),
2356	size&: dummySize));
2357	}
2358	return false;
2359	case glslang::EOpMatrixSwizzle:
2360	logger->missingFunctionality(f: "matrix swizzle");
2361	return true;
2362	case glslang::EOpLogicalOr:
2363	case glslang::EOpLogicalAnd:
2364	{
2365
2366	// These may require short circuiting, but can sometimes be done as straight
2367	// binary operations. The right operand must be short circuited if it has
2368	// side effects, and should probably be if it is complex.
2369	if (isTrivial(node: node->getRight()->getAsTyped()))
2370	break; // handle below as a normal binary operation
2371	// otherwise, we need to do dynamic short circuiting on the right operand
2372	spv::Id result = createShortCircuit(node->getOp(), left&: *node->getLeft()->getAsTyped(),
2373	right&: *node->getRight()->getAsTyped());
2374	builder.clearAccessChain();
2375	builder.setAccessChainRValue(result);
2376	}
2377	return false;
2378	default:
2379	break;
2380	}
2381
2382	// Assume generic binary op...
2383
2384	// get right operand
2385	builder.clearAccessChain();
2386	node->getLeft()->traverse(this);
2387	spv::Id left = accessChainLoad(type: node->getLeft()->getType());
2388
2389	// get left operand
2390	builder.clearAccessChain();
2391	node->getRight()->traverse(this);
2392	spv::Id right = accessChainLoad(type: node->getRight()->getType());
2393
2394	// get result
2395	OpDecorations decorations = { TranslatePrecisionDecoration(glslangPrecision: node->getOperationPrecision()),
2396	TranslateNoContractionDecoration(qualifier: node->getType().getQualifier()),
2397	TranslateNonUniformDecoration(qualifier: node->getType().getQualifier()) };
2398	spv::Id result = createBinaryOperation(op: node->getOp(), decorations,
2399	typeId: convertGlslangToSpvType(type: node->getType()), left, right,
2400	typeProxy: node->getLeft()->getType().getBasicType());
2401
2402	builder.clearAccessChain();
2403	if (! result) {
2404	logger->missingFunctionality(f: "unknown glslang binary operation");
2405	return true; // pick up a child as the place-holder result
2406	} else {
2407	builder.setAccessChainRValue(result);
2408	return false;
2409	}
2410	}
2411
2412	spv::Id TGlslangToSpvTraverser::convertLoadedBoolInUniformToUint(const glslang::TType& type,
2413	spv::Id nominalTypeId,
2414	spv::Id loadedId)
2415	{
2416	if (builder.isScalarType(typeId: nominalTypeId)) {
2417	// Conversion for bool
2418	spv::Id boolType = builder.makeBoolType();
2419	if (nominalTypeId != boolType)
2420	return builder.createBinOp(spv::OpINotEqual, typeId: boolType, operand1: loadedId, operand2: builder.makeUintConstant(u: 0));
2421	} else if (builder.isVectorType(typeId: nominalTypeId)) {
2422	// Conversion for bvec
2423	int vecSize = builder.getNumTypeComponents(typeId: nominalTypeId);
2424	spv::Id bvecType = builder.makeVectorType(component: builder.makeBoolType(), size: vecSize);
2425	if (nominalTypeId != bvecType)
2426	loadedId = builder.createBinOp(spv::OpINotEqual, typeId: bvecType, operand1: loadedId,
2427	operand2: makeSmearedConstant(constant: builder.makeUintConstant(u: 0), vectorSize: vecSize));
2428	} else if (builder.isArrayType(typeId: nominalTypeId)) {
2429	// Conversion for bool array
2430	spv::Id boolArrayTypeId = convertGlslangToSpvType(type);
2431	if (nominalTypeId != boolArrayTypeId)
2432	{
2433	// Use OpCopyLogical from SPIR-V 1.4 if available.
2434	if (glslangIntermediate->getSpv().spv >= glslang::EShTargetSpv_1_4)
2435	return builder.createUnaryOp(spv::OpCopyLogical, typeId: boolArrayTypeId, operand: loadedId);
2436
2437	glslang::TType glslangElementType(type, 0);
2438	spv::Id elementNominalTypeId = builder.getContainedTypeId(typeId: nominalTypeId);
2439	std::vector<spv::Id> constituents;
2440	for (int index = 0; index < type.getOuterArraySize(); ++index) {
2441	// get the element
2442	spv::Id elementValue = builder.createCompositeExtract(composite: loadedId, typeId: elementNominalTypeId, index);
2443
2444	// recursively convert it
2445	spv::Id elementConvertedValue = convertLoadedBoolInUniformToUint(type: glslangElementType, nominalTypeId: elementNominalTypeId, loadedId: elementValue);
2446	constituents.push_back(x: elementConvertedValue);
2447	}
2448	return builder.createCompositeConstruct(typeId: boolArrayTypeId, constituents);
2449	}
2450	}
2451
2452	return loadedId;
2453	}
2454
2455	// Figure out what, if any, type changes are needed when accessing a specific built-in.
2456	// Returns <the type SPIR-V requires for declarion, the type to translate to on use>.
2457	// Also see comment for 'forceType', regarding tracking SPIR-V-required types.
2458	std::pair<spv::Id, spv::Id> TGlslangToSpvTraverser::getForcedType(glslang::TBuiltInVariable glslangBuiltIn,
2459	const glslang::TType& glslangType)
2460	{
2461	switch(glslangBuiltIn)
2462	{
2463	case glslang::EbvSubGroupEqMask:
2464	case glslang::EbvSubGroupGeMask:
2465	case glslang::EbvSubGroupGtMask:
2466	case glslang::EbvSubGroupLeMask:
2467	case glslang::EbvSubGroupLtMask: {
2468	// these require changing a 64-bit scaler -> a vector of 32-bit components
2469	if (glslangType.isVector())
2470	break;
2471	spv::Id ivec4_type = builder.makeVectorType(component: builder.makeUintType(width: 32), size: 4);
2472	spv::Id uint64_type = builder.makeUintType(width: 64);
2473	std::pair<spv::Id, spv::Id> ret(ivec4_type, uint64_type);
2474	return ret;
2475	}
2476	// There are no SPIR-V builtins defined for these and map onto original non-transposed
2477	// builtins. During visitBinary we insert a transpose
2478	case glslang::EbvWorldToObject3x4:
2479	case glslang::EbvObjectToWorld3x4: {
2480	spv::Id mat43 = builder.makeMatrixType(component: builder.makeFloatType(width: 32), cols: 4, rows: 3);
2481	spv::Id mat34 = builder.makeMatrixType(component: builder.makeFloatType(width: 32), cols: 3, rows: 4);
2482	std::pair<spv::Id, spv::Id> ret(mat43, mat34);
2483	return ret;
2484	}
2485	default:
2486	break;
2487	}
2488
2489	std::pair<spv::Id, spv::Id> ret(spv::NoType, spv::NoType);
2490	return ret;
2491	}
2492
2493	// For an object previously identified (see getForcedType() and forceType)
2494	// as needing type translations, do the translation needed for a load, turning
2495	// an L-value into in R-value.
2496	spv::Id TGlslangToSpvTraverser::translateForcedType(spv::Id object)
2497	{
2498	const auto forceIt = forceType.find(x: object);
2499	if (forceIt == forceType.end())
2500	return object;
2501
2502	spv::Id desiredTypeId = forceIt->second;
2503	spv::Id objectTypeId = builder.getTypeId(resultId: object);
2504	assert(builder.isPointerType(objectTypeId));
2505	objectTypeId = builder.getContainedTypeId(typeId: objectTypeId);
2506	if (builder.isVectorType(typeId: objectTypeId) &&
2507	builder.getScalarTypeWidth(typeId: builder.getContainedTypeId(typeId: objectTypeId)) == 32) {
2508	if (builder.getScalarTypeWidth(typeId: desiredTypeId) == 64) {
2509	// handle 32-bit v.xy* -> 64-bit
2510	builder.clearAccessChain();
2511	builder.setAccessChainLValue(object);
2512	object = builder.accessChainLoad(precision: spv::NoPrecision, l_nonUniform: spv::DecorationMax, r_nonUniform: spv::DecorationMax, ResultType: objectTypeId);
2513	std::vector<spv::Id> components;
2514	components.push_back(x: builder.createCompositeExtract(composite: object, typeId: builder.getContainedTypeId(typeId: objectTypeId), index: 0));
2515	components.push_back(x: builder.createCompositeExtract(composite: object, typeId: builder.getContainedTypeId(typeId: objectTypeId), index: 1));
2516
2517	spv::Id vecType = builder.makeVectorType(component: builder.getContainedTypeId(typeId: objectTypeId), size: 2);
2518	return builder.createUnaryOp(spv::OpBitcast, typeId: desiredTypeId,
2519	operand: builder.createCompositeConstruct(typeId: vecType, constituents: components));
2520	} else {
2521	logger->missingFunctionality(f: "forcing 32-bit vector type to non 64-bit scalar");
2522	}
2523	} else if (builder.isMatrixType(typeId: objectTypeId)) {
2524	// There are no SPIR-V builtins defined for 3x4 variants of ObjectToWorld/WorldToObject
2525	// and we insert a transpose after loading the original non-transposed builtins
2526	builder.clearAccessChain();
2527	builder.setAccessChainLValue(object);
2528	object = builder.accessChainLoad(precision: spv::NoPrecision, l_nonUniform: spv::DecorationMax, r_nonUniform: spv::DecorationMax, ResultType: objectTypeId);
2529	return builder.createUnaryOp(spv::OpTranspose, typeId: desiredTypeId, operand: object);
2530
2531	} else {
2532	logger->missingFunctionality(f: "forcing non 32-bit vector type");
2533	}
2534
2535	return object;
2536	}
2537
2538	bool TGlslangToSpvTraverser::visitUnary(glslang::TVisit /* visit */, glslang::TIntermUnary* node)
2539	{
2540	builder.setDebugSourceLocation(line: node->getLoc().line, filename: node->getLoc().getFilename());
2541
2542	SpecConstantOpModeGuard spec_constant_op_mode_setter(&builder);
2543	if (node->getType().getQualifier().isSpecConstant())
2544	spec_constant_op_mode_setter.turnOnSpecConstantOpMode();
2545
2546	spv::Id result = spv::NoResult;
2547
2548	// try texturing first
2549	result = createImageTextureFunctionCall(node);
2550	if (result != spv::NoResult) {
2551	builder.clearAccessChain();
2552	builder.setAccessChainRValue(result);
2553
2554	return false; // done with this node
2555	}
2556
2557	// Non-texturing.
2558
2559	if (node->getOp() == glslang::EOpArrayLength) {
2560	// Quite special; won't want to evaluate the operand.
2561
2562	// Currently, the front-end does not allow .length() on an array until it is sized,
2563	// except for the last block membeor of an SSBO.
2564	// TODO: If this changes, link-time sized arrays might show up here, and need their
2565	// size extracted.
2566
2567	// Normal .length() would have been constant folded by the front-end.
2568	// So, this has to be block.lastMember.length().
2569	// SPV wants "block" and member number as the operands, go get them.
2570
2571	spv::Id length;
2572	if (node->getOperand()->getType().isCoopMat()) {
2573	spv::Id typeId = convertGlslangToSpvType(type: node->getOperand()->getType());
2574	assert(builder.isCooperativeMatrixType(typeId));
2575
2576	if (node->getOperand()->getType().isCoopMatKHR()) {
2577	length = builder.createCooperativeMatrixLengthKHR(type: typeId);
2578	} else {
2579	spec_constant_op_mode_setter.turnOnSpecConstantOpMode();
2580	length = builder.createCooperativeMatrixLengthNV(type: typeId);
2581	}
2582	} else {
2583	glslang::TIntermTyped* block = node->getOperand()->getAsBinaryNode()->getLeft();
2584	block->traverse(this);
2585	unsigned int member = node->getOperand()->getAsBinaryNode()->getRight()->getAsConstantUnion()
2586	->getConstArray()[0].getUConst();
2587	length = builder.createArrayLength(base: builder.accessChainGetLValue(), member);
2588	}
2589
2590	// GLSL semantics say the result of .length() is an int, while SPIR-V says
2591	// signedness must be 0. So, convert from SPIR-V unsigned back to GLSL's
2592	// AST expectation of a signed result.
2593	if (glslangIntermediate->getSource() == glslang::EShSourceGlsl) {
2594	if (builder.isInSpecConstCodeGenMode()) {
2595	length = builder.createBinOp(spv::OpIAdd, typeId: builder.makeIntType(width: 32), operand1: length, operand2: builder.makeIntConstant(i: 0));
2596	} else {
2597	length = builder.createUnaryOp(spv::OpBitcast, typeId: builder.makeIntType(width: 32), operand: length);
2598	}
2599	}
2600
2601	builder.clearAccessChain();
2602	builder.setAccessChainRValue(length);
2603
2604	return false;
2605	}
2606
2607	// Force variable declaration - Debug Mode Only
2608	if (node->getOp() == glslang::EOpDeclare) {
2609	builder.clearAccessChain();
2610	node->getOperand()->traverse(this);
2611	builder.clearAccessChain();
2612	return false;
2613	}
2614
2615	// Start by evaluating the operand
2616
2617	// Does it need a swizzle inversion? If so, evaluation is inverted;
2618	// operate first on the swizzle base, then apply the swizzle.
2619	spv::Id invertedType = spv::NoType;
2620	auto resultType = [&invertedType, &node, this](){ return invertedType != spv::NoType ?
2621	invertedType : convertGlslangToSpvType(type: node->getType()); };
2622	if (node->getOp() == glslang::EOpInterpolateAtCentroid)
2623	invertedType = getInvertedSwizzleType(*node->getOperand());
2624
2625	builder.clearAccessChain();
2626	TIntermNode *operandNode;
2627	if (invertedType != spv::NoType)
2628	operandNode = node->getOperand()->getAsBinaryNode()->getLeft();
2629	else
2630	operandNode = node->getOperand();
2631
2632	operandNode->traverse(this);
2633
2634	spv::Id operand = spv::NoResult;
2635
2636	spv::Builder::AccessChain::CoherentFlags lvalueCoherentFlags;
2637
2638	const auto hitObjectOpsWithLvalue = [](glslang::TOperator op) {
2639	switch(op) {
2640	case glslang::EOpReorderThreadNV:
2641	case glslang::EOpHitObjectGetCurrentTimeNV:
2642	case glslang::EOpHitObjectGetHitKindNV:
2643	case glslang::EOpHitObjectGetPrimitiveIndexNV:
2644	case glslang::EOpHitObjectGetGeometryIndexNV:
2645	case glslang::EOpHitObjectGetInstanceIdNV:
2646	case glslang::EOpHitObjectGetInstanceCustomIndexNV:
2647	case glslang::EOpHitObjectGetObjectRayDirectionNV:
2648	case glslang::EOpHitObjectGetObjectRayOriginNV:
2649	case glslang::EOpHitObjectGetWorldRayDirectionNV:
2650	case glslang::EOpHitObjectGetWorldRayOriginNV:
2651	case glslang::EOpHitObjectGetWorldToObjectNV:
2652	case glslang::EOpHitObjectGetObjectToWorldNV:
2653	case glslang::EOpHitObjectGetRayTMaxNV:
2654	case glslang::EOpHitObjectGetRayTMinNV:
2655	case glslang::EOpHitObjectIsEmptyNV:
2656	case glslang::EOpHitObjectIsHitNV:
2657	case glslang::EOpHitObjectIsMissNV:
2658	case glslang::EOpHitObjectRecordEmptyNV:
2659	case glslang::EOpHitObjectGetShaderBindingTableRecordIndexNV:
2660	case glslang::EOpHitObjectGetShaderRecordBufferHandleNV:
2661	return true;
2662	default:
2663	return false;
2664	}
2665	};
2666
2667	if (node->getOp() == glslang::EOpAtomicCounterIncrement ||
2668	node->getOp() == glslang::EOpAtomicCounterDecrement ||
2669	node->getOp() == glslang::EOpAtomicCounter ||
2670	(node->getOp() == glslang::EOpInterpolateAtCentroid &&
2671	glslangIntermediate->getSource() != glslang::EShSourceHlsl) ||
2672	node->getOp() == glslang::EOpRayQueryProceed ||
2673	node->getOp() == glslang::EOpRayQueryGetRayTMin ||
2674	node->getOp() == glslang::EOpRayQueryGetRayFlags ||
2675	node->getOp() == glslang::EOpRayQueryGetWorldRayOrigin ||
2676	node->getOp() == glslang::EOpRayQueryGetWorldRayDirection ||
2677	node->getOp() == glslang::EOpRayQueryGetIntersectionCandidateAABBOpaque ||
2678	node->getOp() == glslang::EOpRayQueryTerminate ||
2679	node->getOp() == glslang::EOpRayQueryConfirmIntersection ||
2680	(node->getOp() == glslang::EOpSpirvInst && operandNode->getAsTyped()->getQualifier().isSpirvByReference()) ||
2681	hitObjectOpsWithLvalue(node->getOp())) {
2682	operand = builder.accessChainGetLValue(); // Special case l-value operands
2683	lvalueCoherentFlags = builder.getAccessChain().coherentFlags;
2684	lvalueCoherentFlags |= TranslateCoherent(type: operandNode->getAsTyped()->getType());
2685	} else if (operandNode->getAsTyped()->getQualifier().isSpirvLiteral()) {
2686	// Will be translated to a literal value, make a placeholder here
2687	operand = spv::NoResult;
2688	} else {
2689	operand = accessChainLoad(type: node->getOperand()->getType());
2690	}
2691
2692	OpDecorations decorations = { TranslatePrecisionDecoration(glslangPrecision: node->getOperationPrecision()),
2693	TranslateNoContractionDecoration(qualifier: node->getType().getQualifier()),
2694	TranslateNonUniformDecoration(qualifier: node->getType().getQualifier()) };
2695
2696	// it could be a conversion
2697	if (! result)
2698	result = createConversion(op: node->getOp(), decorations, destTypeId: resultType(), operand,
2699	typeProxy: node->getOperand()->getBasicType());
2700
2701	// if not, then possibly an operation
2702	if (! result)
2703	result = createUnaryOperation(op: node->getOp(), decorations, typeId: resultType(), operand,
2704	typeProxy: node->getOperand()->getBasicType(), lvalueCoherentFlags, opType: node->getType());
2705
2706	// it could be attached to a SPIR-V intruction
2707	if (!result) {
2708	if (node->getOp() == glslang::EOpSpirvInst) {
2709	const auto& spirvInst = node->getSpirvInstruction();
2710	if (spirvInst.set == "") {
2711	spv::IdImmediate idImmOp = {true, operand};
2712	if (operandNode->getAsTyped()->getQualifier().isSpirvLiteral()) {
2713	// Translate the constant to a literal value
2714	std::vector<unsigned> literals;
2715	glslang::TVector<const glslang::TIntermConstantUnion*> constants;
2716	constants.push_back(x: operandNode->getAsConstantUnion());
2717	TranslateLiterals(constants, literals);
2718	idImmOp = {false, literals[0]};
2719	}
2720
2721	if (node->getBasicType() == glslang::EbtVoid)
2722	builder.createNoResultOp(static_cast<spv::Op>(spirvInst.id), operands: {idImmOp});
2723	else
2724	result = builder.createOp(static_cast<spv::Op>(spirvInst.id), typeId: resultType(), operands: {idImmOp});
2725	} else {
2726	result = builder.createBuiltinCall(
2727	resultType: resultType(), builtins: spirvInst.set == "GLSL.std.450" ? stdBuiltins : getExtBuiltins(name: spirvInst.set.c_str()),
2728	entryPoint: spirvInst.id, args: {operand});
2729	}
2730
2731	if (node->getBasicType() == glslang::EbtVoid)
2732	return false; // done with this node
2733	}
2734	}
2735
2736	if (result) {
2737	if (invertedType) {
2738	result = createInvertedSwizzle(precision: decorations.precision, *node->getOperand(), parentResult: result);
2739	decorations.addNonUniform(builder, t: result);
2740	}
2741
2742	builder.clearAccessChain();
2743	builder.setAccessChainRValue(result);
2744
2745	return false; // done with this node
2746	}
2747
2748	// it must be a special case, check...
2749	switch (node->getOp()) {
2750	case glslang::EOpPostIncrement:
2751	case glslang::EOpPostDecrement:
2752	case glslang::EOpPreIncrement:
2753	case glslang::EOpPreDecrement:
2754	{
2755	// we need the integer value "1" or the floating point "1.0" to add/subtract
2756	spv::Id one = 0;
2757	if (node->getBasicType() == glslang::EbtFloat)
2758	one = builder.makeFloatConstant(f: 1.0F);
2759	else if (node->getBasicType() == glslang::EbtDouble)
2760	one = builder.makeDoubleConstant(d: 1.0);
2761	else if (node->getBasicType() == glslang::EbtFloat16)
2762	one = builder.makeFloat16Constant(f16: 1.0F);
2763	else if (node->getBasicType() == glslang::EbtInt8 || node->getBasicType() == glslang::EbtUint8)
2764	one = builder.makeInt8Constant(i: 1);
2765	else if (node->getBasicType() == glslang::EbtInt16 || node->getBasicType() == glslang::EbtUint16)
2766	one = builder.makeInt16Constant(i: 1);
2767	else if (node->getBasicType() == glslang::EbtInt64 || node->getBasicType() == glslang::EbtUint64)
2768	one = builder.makeInt64Constant(i: 1);
2769	else
2770	one = builder.makeIntConstant(i: 1);
2771	glslang::TOperator op;
2772	if (node->getOp() == glslang::EOpPreIncrement ||
2773	node->getOp() == glslang::EOpPostIncrement)
2774	op = glslang::EOpAdd;
2775	else
2776	op = glslang::EOpSub;
2777
2778	spv::Id result = createBinaryOperation(op, decorations,
2779	typeId: convertGlslangToSpvType(type: node->getType()), left: operand, right: one,
2780	typeProxy: node->getType().getBasicType());
2781	assert(result != spv::NoResult);
2782
2783	// The result of operation is always stored, but conditionally the
2784	// consumed result. The consumed result is always an r-value.
2785	builder.accessChainStore(rvalue: result,
2786	nonUniform: TranslateNonUniformDecoration(coherentFlags: builder.getAccessChain().coherentFlags));
2787	builder.clearAccessChain();
2788	if (node->getOp() == glslang::EOpPreIncrement ||
2789	node->getOp() == glslang::EOpPreDecrement)
2790	builder.setAccessChainRValue(result);
2791	else
2792	builder.setAccessChainRValue(operand);
2793	}
2794
2795	return false;
2796
2797	case glslang::EOpAssumeEXT:
2798	builder.addCapability(cap: spv::CapabilityExpectAssumeKHR);
2799	builder.addExtension(ext: spv::E_SPV_KHR_expect_assume);
2800	builder.createNoResultOp(spv::OpAssumeTrueKHR, operand);
2801	return false;
2802	case glslang::EOpEmitStreamVertex:
2803	builder.createNoResultOp(spv::OpEmitStreamVertex, operand);
2804	return false;
2805	case glslang::EOpEndStreamPrimitive:
2806	builder.createNoResultOp(spv::OpEndStreamPrimitive, operand);
2807	return false;
2808	case glslang::EOpRayQueryTerminate:
2809	builder.createNoResultOp(spv::OpRayQueryTerminateKHR, operand);
2810	return false;
2811	case glslang::EOpRayQueryConfirmIntersection:
2812	builder.createNoResultOp(spv::OpRayQueryConfirmIntersectionKHR, operand);
2813	return false;
2814	case glslang::EOpReorderThreadNV:
2815	builder.createNoResultOp(spv::OpReorderThreadWithHitObjectNV, operand);
2816	return false;
2817	case glslang::EOpHitObjectRecordEmptyNV:
2818	builder.createNoResultOp(spv::OpHitObjectRecordEmptyNV, operand);
2819	return false;
2820
2821	default:
2822	logger->missingFunctionality(f: "unknown glslang unary");
2823	return true; // pick up operand as placeholder result
2824	}
2825	}
2826
2827	// Construct a composite object, recursively copying members if their types don't match
2828	spv::Id TGlslangToSpvTraverser::createCompositeConstruct(spv::Id resultTypeId, std::vector<spv::Id> constituents)
2829	{
2830	for (int c = 0; c < (int)constituents.size(); ++c) {
2831	spv::Id& constituent = constituents[c];
2832	spv::Id lType = builder.getContainedTypeId(typeId: resultTypeId, c);
2833	spv::Id rType = builder.getTypeId(resultId: constituent);
2834	if (lType != rType) {
2835	if (glslangIntermediate->getSpv().spv >= glslang::EShTargetSpv_1_4) {
2836	constituent = builder.createUnaryOp(spv::OpCopyLogical, typeId: lType, operand: constituent);
2837	} else if (builder.isStructType(typeId: rType)) {
2838	std::vector<spv::Id> rTypeConstituents;
2839	int numrTypeConstituents = builder.getNumTypeConstituents(typeId: rType);
2840	for (int i = 0; i < numrTypeConstituents; ++i) {
2841	rTypeConstituents.push_back(x: builder.createCompositeExtract(composite: constituent,
2842	typeId: builder.getContainedTypeId(typeId: rType, i), index: i));
2843	}
2844	constituents[c] = createCompositeConstruct(resultTypeId: lType, constituents: rTypeConstituents);
2845	} else {
2846	assert(builder.isArrayType(rType));
2847	std::vector<spv::Id> rTypeConstituents;
2848	int numrTypeConstituents = builder.getNumTypeConstituents(typeId: rType);
2849
2850	spv::Id elementRType = builder.getContainedTypeId(typeId: rType);
2851	for (int i = 0; i < numrTypeConstituents; ++i) {
2852	rTypeConstituents.push_back(x: builder.createCompositeExtract(composite: constituent, typeId: elementRType, index: i));
2853	}
2854	constituents[c] = createCompositeConstruct(resultTypeId: lType, constituents: rTypeConstituents);
2855	}
2856	}
2857	}
2858	return builder.createCompositeConstruct(typeId: resultTypeId, constituents);
2859	}
2860
2861	bool TGlslangToSpvTraverser::visitAggregate(glslang::TVisit visit, glslang::TIntermAggregate* node)
2862	{
2863	SpecConstantOpModeGuard spec_constant_op_mode_setter(&builder);
2864	if (node->getType().getQualifier().isSpecConstant())
2865	spec_constant_op_mode_setter.turnOnSpecConstantOpMode();
2866
2867	spv::Id result = spv::NoResult;
2868	spv::Id invertedType = spv::NoType; // to use to override the natural type of the node
2869	std::vector<spv::Builder::AccessChain> complexLvalues; // for holding swizzling l-values too complex for
2870	// SPIR-V, for an out parameter
2871	std::vector<spv::Id> temporaryLvalues; // temporaries to pass, as proxies for complexLValues
2872
2873	auto resultType = [&invertedType, &node, this](){ return invertedType != spv::NoType ?
2874	invertedType :
2875	convertGlslangToSpvType(type: node->getType()); };
2876
2877	// try texturing
2878	result = createImageTextureFunctionCall(node);
2879	if (result != spv::NoResult) {
2880	builder.clearAccessChain();
2881	builder.setAccessChainRValue(result);
2882
2883	return false;
2884	} else if (node->getOp() == glslang::EOpImageStore ||
2885	node->getOp() == glslang::EOpImageStoreLod ||
2886	node->getOp() == glslang::EOpImageAtomicStore) {
2887	// "imageStore" is a special case, which has no result
2888	return false;
2889	}
2890
2891	glslang::TOperator binOp = glslang::EOpNull;
2892	bool reduceComparison = true;
2893	bool isMatrix = false;
2894	bool noReturnValue = false;
2895	bool atomic = false;
2896
2897	spv::Builder::AccessChain::CoherentFlags lvalueCoherentFlags;
2898
2899	assert(node->getOp());
2900
2901	spv::Decoration precision = TranslatePrecisionDecoration(glslangPrecision: node->getOperationPrecision());
2902
2903	switch (node->getOp()) {
2904	case glslang::EOpScope:
2905	case glslang::EOpSequence:
2906	{
2907	if (visit == glslang::EvPreVisit) {
2908	++sequenceDepth;
2909	if (sequenceDepth == 1) {
2910	// If this is the parent node of all the functions, we want to see them
2911	// early, so all call points have actual SPIR-V functions to reference.
2912	// In all cases, still let the traverser visit the children for us.
2913	makeFunctions(node->getAsAggregate()->getSequence());
2914
2915	// Global initializers is specific to the shader entry point, which does not exist in compile-only mode
2916	if (!options.compileOnly) {
2917	// Also, we want all globals initializers to go into the beginning of the entry point, before
2918	// anything else gets there, so visit out of order, doing them all now.
2919	makeGlobalInitializers(node->getAsAggregate()->getSequence());
2920	}
2921
2922	//Pre process linker objects for ray tracing stages
2923	if (glslangIntermediate->isRayTracingStage())
2924	collectRayTracingLinkerObjects();
2925
2926	// Initializers are done, don't want to visit again, but functions and link objects need to be processed,
2927	// so do them manually.
2928	visitFunctions(node->getAsAggregate()->getSequence());
2929
2930	return false;
2931	} else {
2932	if (node->getOp() == glslang::EOpScope)
2933	builder.enterLexicalBlock(line: 0);
2934	}
2935	} else {
2936	if (sequenceDepth > 1 && node->getOp() == glslang::EOpScope)
2937	builder.leaveLexicalBlock();
2938	--sequenceDepth;
2939	}
2940
2941	return true;
2942	}
2943	case glslang::EOpLinkerObjects:
2944	{
2945	if (visit == glslang::EvPreVisit)
2946	linkageOnly = true;
2947	else
2948	linkageOnly = false;
2949
2950	return true;
2951	}
2952	case glslang::EOpComma:
2953	{
2954	// processing from left to right naturally leaves the right-most
2955	// lying around in the access chain
2956	glslang::TIntermSequence& glslangOperands = node->getSequence();
2957	for (int i = 0; i < (int)glslangOperands.size(); ++i)
2958	glslangOperands[i]->traverse(this);
2959
2960	return false;
2961	}
2962	case glslang::EOpFunction:
2963	if (visit == glslang::EvPreVisit) {
2964	if (options.generateDebugInfo) {
2965	builder.setDebugSourceLocation(line: node->getLoc().line, filename: node->getLoc().getFilename());
2966	}
2967	if (isShaderEntryPoint(node)) {
2968	inEntryPoint = true;
2969	builder.setBuildPoint(shaderEntry->getLastBlock());
2970	builder.enterFunction(function: shaderEntry);
2971	currentFunction = shaderEntry;
2972	} else {
2973	handleFunctionEntry(node);
2974	}
2975	if (options.generateDebugInfo && !options.emitNonSemanticShaderDebugInfo) {
2976	const auto& loc = node->getLoc();
2977	const char* sourceFileName = loc.getFilename();
2978	spv::Id sourceFileId = sourceFileName ? builder.getStringId(str: sourceFileName) : builder.getMainFileId();
2979	currentFunction->setDebugLineInfo(fileName: sourceFileId, line: loc.line, column: loc.column);
2980	}
2981	} else {
2982	if (inEntryPoint)
2983	entryPointTerminated = true;
2984	builder.leaveFunction();
2985	inEntryPoint = false;
2986	}
2987
2988	return true;
2989	case glslang::EOpParameters:
2990	// Parameters will have been consumed by EOpFunction processing, but not
2991	// the body, so we still visited the function node's children, making this
2992	// child redundant.
2993	return false;
2994	case glslang::EOpFunctionCall:
2995	{
2996	builder.setDebugSourceLocation(line: node->getLoc().line, filename: node->getLoc().getFilename());
2997	if (node->isUserDefined())
2998	result = handleUserFunctionCall(node);
2999	if (result) {
3000	builder.clearAccessChain();
3001	builder.setAccessChainRValue(result);
3002	} else
3003	logger->missingFunctionality(f: "missing user function; linker needs to catch that");
3004
3005	return false;
3006	}
3007	case glslang::EOpConstructMat2x2:
3008	case glslang::EOpConstructMat2x3:
3009	case glslang::EOpConstructMat2x4:
3010	case glslang::EOpConstructMat3x2:
3011	case glslang::EOpConstructMat3x3:
3012	case glslang::EOpConstructMat3x4:
3013	case glslang::EOpConstructMat4x2:
3014	case glslang::EOpConstructMat4x3:
3015	case glslang::EOpConstructMat4x4:
3016	case glslang::EOpConstructDMat2x2:
3017	case glslang::EOpConstructDMat2x3:
3018	case glslang::EOpConstructDMat2x4:
3019	case glslang::EOpConstructDMat3x2:
3020	case glslang::EOpConstructDMat3x3:
3021	case glslang::EOpConstructDMat3x4:
3022	case glslang::EOpConstructDMat4x2:
3023	case glslang::EOpConstructDMat4x3:
3024	case glslang::EOpConstructDMat4x4:
3025	case glslang::EOpConstructIMat2x2:
3026	case glslang::EOpConstructIMat2x3:
3027	case glslang::EOpConstructIMat2x4:
3028	case glslang::EOpConstructIMat3x2:
3029	case glslang::EOpConstructIMat3x3:
3030	case glslang::EOpConstructIMat3x4:
3031	case glslang::EOpConstructIMat4x2:
3032	case glslang::EOpConstructIMat4x3:
3033	case glslang::EOpConstructIMat4x4:
3034	case glslang::EOpConstructUMat2x2:
3035	case glslang::EOpConstructUMat2x3:
3036	case glslang::EOpConstructUMat2x4:
3037	case glslang::EOpConstructUMat3x2:
3038	case glslang::EOpConstructUMat3x3:
3039	case glslang::EOpConstructUMat3x4:
3040	case glslang::EOpConstructUMat4x2:
3041	case glslang::EOpConstructUMat4x3:
3042	case glslang::EOpConstructUMat4x4:
3043	case glslang::EOpConstructBMat2x2:
3044	case glslang::EOpConstructBMat2x3:
3045	case glslang::EOpConstructBMat2x4:
3046	case glslang::EOpConstructBMat3x2:
3047	case glslang::EOpConstructBMat3x3:
3048	case glslang::EOpConstructBMat3x4:
3049	case glslang::EOpConstructBMat4x2:
3050	case glslang::EOpConstructBMat4x3:
3051	case glslang::EOpConstructBMat4x4:
3052	case glslang::EOpConstructF16Mat2x2:
3053	case glslang::EOpConstructF16Mat2x3:
3054	case glslang::EOpConstructF16Mat2x4:
3055	case glslang::EOpConstructF16Mat3x2:
3056	case glslang::EOpConstructF16Mat3x3:
3057	case glslang::EOpConstructF16Mat3x4:
3058	case glslang::EOpConstructF16Mat4x2:
3059	case glslang::EOpConstructF16Mat4x3:
3060	case glslang::EOpConstructF16Mat4x4:
3061	isMatrix = true;
3062	[[fallthrough]];
3063	case glslang::EOpConstructFloat:
3064	case glslang::EOpConstructVec2:
3065	case glslang::EOpConstructVec3:
3066	case glslang::EOpConstructVec4:
3067	case glslang::EOpConstructDouble:
3068	case glslang::EOpConstructDVec2:
3069	case glslang::EOpConstructDVec3:
3070	case glslang::EOpConstructDVec4:
3071	case glslang::EOpConstructFloat16:
3072	case glslang::EOpConstructF16Vec2:
3073	case glslang::EOpConstructF16Vec3:
3074	case glslang::EOpConstructF16Vec4:
3075	case glslang::EOpConstructBool:
3076	case glslang::EOpConstructBVec2:
3077	case glslang::EOpConstructBVec3:
3078	case glslang::EOpConstructBVec4:
3079	case glslang::EOpConstructInt8:
3080	case glslang::EOpConstructI8Vec2:
3081	case glslang::EOpConstructI8Vec3:
3082	case glslang::EOpConstructI8Vec4:
3083	case glslang::EOpConstructUint8:
3084	case glslang::EOpConstructU8Vec2:
3085	case glslang::EOpConstructU8Vec3:
3086	case glslang::EOpConstructU8Vec4:
3087	case glslang::EOpConstructInt16:
3088	case glslang::EOpConstructI16Vec2:
3089	case glslang::EOpConstructI16Vec3:
3090	case glslang::EOpConstructI16Vec4:
3091	case glslang::EOpConstructUint16:
3092	case glslang::EOpConstructU16Vec2:
3093	case glslang::EOpConstructU16Vec3:
3094	case glslang::EOpConstructU16Vec4:
3095	case glslang::EOpConstructInt:
3096	case glslang::EOpConstructIVec2:
3097	case glslang::EOpConstructIVec3:
3098	case glslang::EOpConstructIVec4:
3099	case glslang::EOpConstructUint:
3100	case glslang::EOpConstructUVec2:
3101	case glslang::EOpConstructUVec3:
3102	case glslang::EOpConstructUVec4:
3103	case glslang::EOpConstructInt64:
3104	case glslang::EOpConstructI64Vec2:
3105	case glslang::EOpConstructI64Vec3:
3106	case glslang::EOpConstructI64Vec4:
3107	case glslang::EOpConstructUint64:
3108	case glslang::EOpConstructU64Vec2:
3109	case glslang::EOpConstructU64Vec3:
3110	case glslang::EOpConstructU64Vec4:
3111	case glslang::EOpConstructStruct:
3112	case glslang::EOpConstructTextureSampler:
3113	case glslang::EOpConstructReference:
3114	case glslang::EOpConstructCooperativeMatrixNV:
3115	case glslang::EOpConstructCooperativeMatrixKHR:
3116	{
3117	builder.setDebugSourceLocation(line: node->getLoc().line, filename: node->getLoc().getFilename());
3118	std::vector<spv::Id> arguments;
3119	translateArguments(node: *node, arguments, lvalueCoherentFlags);
3120	spv::Id constructed;
3121	if (node->getOp() == glslang::EOpConstructTextureSampler) {
3122	const glslang::TType& texType = node->getSequence()[0]->getAsTyped()->getType();
3123	if (glslangIntermediate->getSpv().spv >= glslang::EShTargetSpv_1_6 &&
3124	texType.getSampler().isBuffer()) {
3125	// SamplerBuffer is not supported in spirv1.6 so
3126	// `samplerBuffer(textureBuffer, sampler)` is a no-op
3127	// and textureBuffer is the result going forward
3128	constructed = arguments[0];
3129	} else
3130	constructed = builder.createOp(spv::OpSampledImage, typeId: resultType(), operands: arguments);
3131	} else if (node->getOp() == glslang::EOpConstructStruct ||
3132	node->getOp() == glslang::EOpConstructCooperativeMatrixNV ||
3133	node->getOp() == glslang::EOpConstructCooperativeMatrixKHR ||
3134	node->getType().isArray()) {
3135	std::vector<spv::Id> constituents;
3136	for (int c = 0; c < (int)arguments.size(); ++c)
3137	constituents.push_back(x: arguments[c]);
3138	constructed = createCompositeConstruct(resultTypeId: resultType(), constituents);
3139	} else if (isMatrix)
3140	constructed = builder.createMatrixConstructor(precision, sources: arguments, constructee: resultType());
3141	else
3142	constructed = builder.createConstructor(precision, sources: arguments, resultTypeId: resultType());
3143
3144	if (node->getType().getQualifier().isNonUniform()) {
3145	builder.addDecoration(constructed, spv::DecorationNonUniformEXT);
3146	}
3147
3148	builder.clearAccessChain();
3149	builder.setAccessChainRValue(constructed);
3150
3151	return false;
3152	}
3153
3154	// These six are component-wise compares with component-wise results.
3155	// Forward on to createBinaryOperation(), requesting a vector result.
3156	case glslang::EOpLessThan:
3157	case glslang::EOpGreaterThan:
3158	case glslang::EOpLessThanEqual:
3159	case glslang::EOpGreaterThanEqual:
3160	case glslang::EOpVectorEqual:
3161	case glslang::EOpVectorNotEqual:
3162	{
3163	// Map the operation to a binary
3164	binOp = node->getOp();
3165	reduceComparison = false;
3166	switch (node->getOp()) {
3167	case glslang::EOpVectorEqual: binOp = glslang::EOpVectorEqual; break;
3168	case glslang::EOpVectorNotEqual: binOp = glslang::EOpVectorNotEqual; break;
3169	default: binOp = node->getOp(); break;
3170	}
3171
3172	break;
3173	}
3174	case glslang::EOpMul:
3175	// component-wise matrix multiply
3176	binOp = glslang::EOpMul;
3177	break;
3178	case glslang::EOpOuterProduct:
3179	// two vectors multiplied to make a matrix
3180	binOp = glslang::EOpOuterProduct;
3181	break;
3182	case glslang::EOpDot:
3183	{
3184	// for scalar dot product, use multiply
3185	glslang::TIntermSequence& glslangOperands = node->getSequence();
3186	if (glslangOperands[0]->getAsTyped()->getVectorSize() == 1)
3187	binOp = glslang::EOpMul;
3188	break;
3189	}
3190	case glslang::EOpMod:
3191	// when an aggregate, this is the floating-point mod built-in function,
3192	// which can be emitted by the one in createBinaryOperation()
3193	binOp = glslang::EOpMod;
3194	break;
3195
3196	case glslang::EOpEmitVertex:
3197	case glslang::EOpEndPrimitive:
3198	case glslang::EOpBarrier:
3199	case glslang::EOpMemoryBarrier:
3200	case glslang::EOpMemoryBarrierAtomicCounter:
3201	case glslang::EOpMemoryBarrierBuffer:
3202	case glslang::EOpMemoryBarrierImage:
3203	case glslang::EOpMemoryBarrierShared:
3204	case glslang::EOpGroupMemoryBarrier:
3205	case glslang::EOpDeviceMemoryBarrier:
3206	case glslang::EOpAllMemoryBarrierWithGroupSync:
3207	case glslang::EOpDeviceMemoryBarrierWithGroupSync:
3208	case glslang::EOpWorkgroupMemoryBarrier:
3209	case glslang::EOpWorkgroupMemoryBarrierWithGroupSync:
3210	case glslang::EOpSubgroupBarrier:
3211	case glslang::EOpSubgroupMemoryBarrier:
3212	case glslang::EOpSubgroupMemoryBarrierBuffer:
3213	case glslang::EOpSubgroupMemoryBarrierImage:
3214	case glslang::EOpSubgroupMemoryBarrierShared:
3215	noReturnValue = true;
3216	// These all have 0 operands and will naturally finish up in the code below for 0 operands
3217	break;
3218
3219	case glslang::EOpAtomicAdd:
3220	case glslang::EOpAtomicSubtract:
3221	case glslang::EOpAtomicMin:
3222	case glslang::EOpAtomicMax:
3223	case glslang::EOpAtomicAnd:
3224	case glslang::EOpAtomicOr:
3225	case glslang::EOpAtomicXor:
3226	case glslang::EOpAtomicExchange:
3227	case glslang::EOpAtomicCompSwap:
3228	atomic = true;
3229	break;
3230
3231	case glslang::EOpAtomicStore:
3232	noReturnValue = true;
3233	[[fallthrough]];
3234	case glslang::EOpAtomicLoad:
3235	atomic = true;
3236	break;
3237
3238	case glslang::EOpAtomicCounterAdd:
3239	case glslang::EOpAtomicCounterSubtract:
3240	case glslang::EOpAtomicCounterMin:
3241	case glslang::EOpAtomicCounterMax:
3242	case glslang::EOpAtomicCounterAnd:
3243	case glslang::EOpAtomicCounterOr:
3244	case glslang::EOpAtomicCounterXor:
3245	case glslang::EOpAtomicCounterExchange:
3246	case glslang::EOpAtomicCounterCompSwap:
3247	builder.addExtension(ext: "SPV_KHR_shader_atomic_counter_ops");
3248	builder.addCapability(cap: spv::CapabilityAtomicStorageOps);
3249	atomic = true;
3250	break;
3251
3252	case glslang::EOpAbsDifference:
3253	case glslang::EOpAddSaturate:
3254	case glslang::EOpSubSaturate:
3255	case glslang::EOpAverage:
3256	case glslang::EOpAverageRounded:
3257	case glslang::EOpMul32x16:
3258	builder.addCapability(cap: spv::CapabilityIntegerFunctions2INTEL);
3259	builder.addExtension(ext: "SPV_INTEL_shader_integer_functions2");
3260	binOp = node->getOp();
3261	break;
3262
3263	case glslang::EOpExpectEXT:
3264	builder.addCapability(cap: spv::CapabilityExpectAssumeKHR);
3265	builder.addExtension(ext: spv::E_SPV_KHR_expect_assume);
3266	binOp = node->getOp();
3267	break;
3268
3269	case glslang::EOpIgnoreIntersectionNV:
3270	case glslang::EOpTerminateRayNV:
3271	case glslang::EOpTraceNV:
3272	case glslang::EOpTraceRayMotionNV:
3273	case glslang::EOpTraceKHR:
3274	case glslang::EOpExecuteCallableNV:
3275	case glslang::EOpExecuteCallableKHR:
3276	case glslang::EOpWritePackedPrimitiveIndices4x8NV:
3277	case glslang::EOpEmitMeshTasksEXT:
3278	case glslang::EOpSetMeshOutputsEXT:
3279	noReturnValue = true;
3280	break;
3281	case glslang::EOpRayQueryInitialize:
3282	case glslang::EOpRayQueryTerminate:
3283	case glslang::EOpRayQueryGenerateIntersection:
3284	case glslang::EOpRayQueryConfirmIntersection:
3285	builder.addExtension(ext: "SPV_KHR_ray_query");
3286	builder.addCapability(cap: spv::CapabilityRayQueryKHR);
3287	noReturnValue = true;
3288	break;
3289	case glslang::EOpRayQueryProceed:
3290	case glslang::EOpRayQueryGetIntersectionType:
3291	case glslang::EOpRayQueryGetRayTMin:
3292	case glslang::EOpRayQueryGetRayFlags:
3293	case glslang::EOpRayQueryGetIntersectionT:
3294	case glslang::EOpRayQueryGetIntersectionInstanceCustomIndex:
3295	case glslang::EOpRayQueryGetIntersectionInstanceId:
3296	case glslang::EOpRayQueryGetIntersectionInstanceShaderBindingTableRecordOffset:
3297	case glslang::EOpRayQueryGetIntersectionGeometryIndex:
3298	case glslang::EOpRayQueryGetIntersectionPrimitiveIndex:
3299	case glslang::EOpRayQueryGetIntersectionBarycentrics:
3300	case glslang::EOpRayQueryGetIntersectionFrontFace:
3301	case glslang::EOpRayQueryGetIntersectionCandidateAABBOpaque:
3302	case glslang::EOpRayQueryGetIntersectionObjectRayDirection:
3303	case glslang::EOpRayQueryGetIntersectionObjectRayOrigin:
3304	case glslang::EOpRayQueryGetWorldRayDirection:
3305	case glslang::EOpRayQueryGetWorldRayOrigin:
3306	case glslang::EOpRayQueryGetIntersectionObjectToWorld:
3307	case glslang::EOpRayQueryGetIntersectionWorldToObject:
3308	builder.addExtension(ext: "SPV_KHR_ray_query");
3309	builder.addCapability(cap: spv::CapabilityRayQueryKHR);
3310	break;
3311	case glslang::EOpCooperativeMatrixLoad:
3312	case glslang::EOpCooperativeMatrixStore:
3313	case glslang::EOpCooperativeMatrixLoadNV:
3314	case glslang::EOpCooperativeMatrixStoreNV:
3315	noReturnValue = true;
3316	break;
3317	case glslang::EOpBeginInvocationInterlock:
3318	case glslang::EOpEndInvocationInterlock:
3319	builder.addExtension(ext: spv::E_SPV_EXT_fragment_shader_interlock);
3320	noReturnValue = true;
3321	break;
3322
3323	case glslang::EOpHitObjectTraceRayNV:
3324	case glslang::EOpHitObjectTraceRayMotionNV:
3325	case glslang::EOpHitObjectGetAttributesNV:
3326	case glslang::EOpHitObjectExecuteShaderNV:
3327	case glslang::EOpHitObjectRecordEmptyNV:
3328	case glslang::EOpHitObjectRecordMissNV:
3329	case glslang::EOpHitObjectRecordMissMotionNV:
3330	case glslang::EOpHitObjectRecordHitNV:
3331	case glslang::EOpHitObjectRecordHitMotionNV:
3332	case glslang::EOpHitObjectRecordHitWithIndexNV:
3333	case glslang::EOpHitObjectRecordHitWithIndexMotionNV:
3334	case glslang::EOpReorderThreadNV:
3335	noReturnValue = true;
3336	[[fallthrough]];
3337	case glslang::EOpHitObjectIsEmptyNV:
3338	case glslang::EOpHitObjectIsMissNV:
3339	case glslang::EOpHitObjectIsHitNV:
3340	case glslang::EOpHitObjectGetRayTMinNV:
3341	case glslang::EOpHitObjectGetRayTMaxNV:
3342	case glslang::EOpHitObjectGetObjectRayOriginNV:
3343	case glslang::EOpHitObjectGetObjectRayDirectionNV:
3344	case glslang::EOpHitObjectGetWorldRayOriginNV:
3345	case glslang::EOpHitObjectGetWorldRayDirectionNV:
3346	case glslang::EOpHitObjectGetObjectToWorldNV:
3347	case glslang::EOpHitObjectGetWorldToObjectNV:
3348	case glslang::EOpHitObjectGetInstanceCustomIndexNV:
3349	case glslang::EOpHitObjectGetInstanceIdNV:
3350	case glslang::EOpHitObjectGetGeometryIndexNV:
3351	case glslang::EOpHitObjectGetPrimitiveIndexNV:
3352	case glslang::EOpHitObjectGetHitKindNV:
3353	case glslang::EOpHitObjectGetCurrentTimeNV:
3354	case glslang::EOpHitObjectGetShaderBindingTableRecordIndexNV:
3355	case glslang::EOpHitObjectGetShaderRecordBufferHandleNV:
3356	builder.addExtension(ext: spv::E_SPV_NV_shader_invocation_reorder);
3357	builder.addCapability(cap: spv::CapabilityShaderInvocationReorderNV);
3358	break;
3359	case glslang::EOpRayQueryGetIntersectionTriangleVertexPositionsEXT:
3360	builder.addExtension(ext: spv::E_SPV_KHR_ray_tracing_position_fetch);
3361	builder.addCapability(cap: spv::CapabilityRayQueryPositionFetchKHR);
3362	noReturnValue = true;
3363	break;
3364
3365	case glslang::EOpImageSampleWeightedQCOM:
3366	builder.addCapability(cap: spv::CapabilityTextureSampleWeightedQCOM);
3367	builder.addExtension(ext: spv::E_SPV_QCOM_image_processing);
3368	break;
3369	case glslang::EOpImageBoxFilterQCOM:
3370	builder.addCapability(cap: spv::CapabilityTextureBoxFilterQCOM);
3371	builder.addExtension(ext: spv::E_SPV_QCOM_image_processing);
3372	break;
3373	case glslang::EOpImageBlockMatchSADQCOM:
3374	case glslang::EOpImageBlockMatchSSDQCOM:
3375	builder.addCapability(cap: spv::CapabilityTextureBlockMatchQCOM);
3376	builder.addExtension(ext: spv::E_SPV_QCOM_image_processing);
3377	break;
3378
3379	case glslang::EOpImageBlockMatchWindowSSDQCOM:
3380	case glslang::EOpImageBlockMatchWindowSADQCOM:
3381	builder.addCapability(cap: spv::CapabilityTextureBlockMatchQCOM);
3382	builder.addExtension(ext: spv::E_SPV_QCOM_image_processing);
3383	builder.addCapability(cap: spv::CapabilityTextureBlockMatch2QCOM);
3384	builder.addExtension(ext: spv::E_SPV_QCOM_image_processing2);
3385	break;
3386
3387	case glslang::EOpImageBlockMatchGatherSSDQCOM:
3388	case glslang::EOpImageBlockMatchGatherSADQCOM:
3389	builder.addCapability(cap: spv::CapabilityTextureBlockMatchQCOM);
3390	builder.addExtension(ext: spv::E_SPV_QCOM_image_processing);
3391	builder.addCapability(cap: spv::CapabilityTextureBlockMatch2QCOM);
3392	builder.addExtension(ext: spv::E_SPV_QCOM_image_processing2);
3393	break;
3394
3395	case glslang::EOpFetchMicroTriangleVertexPositionNV:
3396	case glslang::EOpFetchMicroTriangleVertexBarycentricNV:
3397	builder.addExtension(ext: spv::E_SPV_NV_displacement_micromap);
3398	builder.addCapability(cap: spv::CapabilityDisplacementMicromapNV);
3399	break;
3400
3401	case glslang::EOpDebugPrintf:
3402	noReturnValue = true;
3403	break;
3404
3405	default:
3406	break;
3407	}
3408
3409	//
3410	// See if it maps to a regular operation.
3411	//
3412	if (binOp != glslang::EOpNull) {
3413	glslang::TIntermTyped* left = node->getSequence()[0]->getAsTyped();
3414	glslang::TIntermTyped* right = node->getSequence()[1]->getAsTyped();
3415	assert(left && right);
3416
3417	builder.clearAccessChain();
3418	left->traverse(this);
3419	spv::Id leftId = accessChainLoad(type: left->getType());
3420
3421	builder.clearAccessChain();
3422	right->traverse(this);
3423	spv::Id rightId = accessChainLoad(type: right->getType());
3424
3425	builder.setDebugSourceLocation(line: node->getLoc().line, filename: node->getLoc().getFilename());
3426	OpDecorations decorations = { precision,
3427	TranslateNoContractionDecoration(qualifier: node->getType().getQualifier()),
3428	TranslateNonUniformDecoration(qualifier: node->getType().getQualifier()) };
3429	result = createBinaryOperation(op: binOp, decorations,
3430	typeId: resultType(), left: leftId, right: rightId,
3431	typeProxy: left->getType().getBasicType(), reduceComparison);
3432
3433	// code above should only make binOp that exists in createBinaryOperation
3434	assert(result != spv::NoResult);
3435	builder.clearAccessChain();
3436	builder.setAccessChainRValue(result);
3437
3438	return false;
3439	}
3440
3441	//
3442	// Create the list of operands.
3443	//
3444	glslang::TIntermSequence& glslangOperands = node->getSequence();
3445	std::vector<spv::Id> operands;
3446	std::vector<spv::IdImmediate> memoryAccessOperands;
3447	for (int arg = 0; arg < (int)glslangOperands.size(); ++arg) {
3448	// special case l-value operands; there are just a few
3449	bool lvalue = false;
3450	switch (node->getOp()) {
3451	case glslang::EOpModf:
3452	if (arg == 1)
3453	lvalue = true;
3454	break;
3455
3456
3457
3458	case glslang::EOpHitObjectRecordHitNV:
3459	case glslang::EOpHitObjectRecordHitMotionNV:
3460	case glslang::EOpHitObjectRecordHitWithIndexNV:
3461	case glslang::EOpHitObjectRecordHitWithIndexMotionNV:
3462	case glslang::EOpHitObjectTraceRayNV:
3463	case glslang::EOpHitObjectTraceRayMotionNV:
3464	case glslang::EOpHitObjectExecuteShaderNV:
3465	case glslang::EOpHitObjectRecordMissNV:
3466	case glslang::EOpHitObjectRecordMissMotionNV:
3467	case glslang::EOpHitObjectGetAttributesNV:
3468	if (arg == 0)
3469	lvalue = true;
3470	break;
3471
3472	case glslang::EOpRayQueryInitialize:
3473	case glslang::EOpRayQueryTerminate:
3474	case glslang::EOpRayQueryConfirmIntersection:
3475	case glslang::EOpRayQueryProceed:
3476	case glslang::EOpRayQueryGenerateIntersection:
3477	case glslang::EOpRayQueryGetIntersectionType:
3478	case glslang::EOpRayQueryGetIntersectionT:
3479	case glslang::EOpRayQueryGetIntersectionInstanceCustomIndex:
3480	case glslang::EOpRayQueryGetIntersectionInstanceId:
3481	case glslang::EOpRayQueryGetIntersectionInstanceShaderBindingTableRecordOffset:
3482	case glslang::EOpRayQueryGetIntersectionGeometryIndex:
3483	case glslang::EOpRayQueryGetIntersectionPrimitiveIndex:
3484	case glslang::EOpRayQueryGetIntersectionBarycentrics:
3485	case glslang::EOpRayQueryGetIntersectionFrontFace:
3486	case glslang::EOpRayQueryGetIntersectionObjectRayDirection:
3487	case glslang::EOpRayQueryGetIntersectionObjectRayOrigin:
3488	case glslang::EOpRayQueryGetIntersectionObjectToWorld:
3489	case glslang::EOpRayQueryGetIntersectionWorldToObject:
3490	if (arg == 0)
3491	lvalue = true;
3492	break;
3493
3494	case glslang::EOpAtomicAdd:
3495	case glslang::EOpAtomicSubtract:
3496	case glslang::EOpAtomicMin:
3497	case glslang::EOpAtomicMax:
3498	case glslang::EOpAtomicAnd:
3499	case glslang::EOpAtomicOr:
3500	case glslang::EOpAtomicXor:
3501	case glslang::EOpAtomicExchange:
3502	case glslang::EOpAtomicCompSwap:
3503	if (arg == 0)
3504	lvalue = true;
3505	break;
3506
3507	case glslang::EOpFrexp:
3508	if (arg == 1)
3509	lvalue = true;
3510	break;
3511	case glslang::EOpInterpolateAtSample:
3512	case glslang::EOpInterpolateAtOffset:
3513	case glslang::EOpInterpolateAtVertex:
3514	if (arg == 0) {
3515	// If GLSL, use the address of the interpolant argument.
3516	// If HLSL, use an internal version of OpInterolates that takes
3517	// the rvalue of the interpolant. A fixup pass in spirv-opt
3518	// legalization will remove the OpLoad and convert to an lvalue.
3519	// Had to do this because legalization will only propagate a
3520	// builtin into an rvalue.
3521	lvalue = glslangIntermediate->getSource() != glslang::EShSourceHlsl;
3522
3523	// Does it need a swizzle inversion? If so, evaluation is inverted;
3524	// operate first on the swizzle base, then apply the swizzle.
3525	// That is, we transform
3526	//
3527	// interpolate(v.zy) -> interpolate(v).zy
3528	//
3529	if (glslangOperands[0]->getAsOperator() &&
3530	glslangOperands[0]->getAsOperator()->getOp() == glslang::EOpVectorSwizzle)
3531	invertedType = convertGlslangToSpvType(
3532	type: glslangOperands[0]->getAsBinaryNode()->getLeft()->getType());
3533	}
3534	break;
3535	case glslang::EOpAtomicLoad:
3536	case glslang::EOpAtomicStore:
3537	case glslang::EOpAtomicCounterAdd:
3538	case glslang::EOpAtomicCounterSubtract:
3539	case glslang::EOpAtomicCounterMin:
3540	case glslang::EOpAtomicCounterMax:
3541	case glslang::EOpAtomicCounterAnd:
3542	case glslang::EOpAtomicCounterOr:
3543	case glslang::EOpAtomicCounterXor:
3544	case glslang::EOpAtomicCounterExchange:
3545	case glslang::EOpAtomicCounterCompSwap:
3546	if (arg == 0)
3547	lvalue = true;
3548	break;
3549	case glslang::EOpAddCarry:
3550	case glslang::EOpSubBorrow:
3551	if (arg == 2)
3552	lvalue = true;
3553	break;
3554	case glslang::EOpUMulExtended:
3555	case glslang::EOpIMulExtended:
3556	if (arg >= 2)
3557	lvalue = true;
3558	break;
3559	case glslang::EOpCooperativeMatrixLoad:
3560	case glslang::EOpCooperativeMatrixLoadNV:
3561	if (arg == 0 || arg == 1)
3562	lvalue = true;
3563	break;
3564	case glslang::EOpCooperativeMatrixStore:
3565	case glslang::EOpCooperativeMatrixStoreNV:
3566	if (arg == 1)
3567	lvalue = true;
3568	break;
3569	case glslang::EOpSpirvInst:
3570	if (glslangOperands[arg]->getAsTyped()->getQualifier().isSpirvByReference())
3571	lvalue = true;
3572	break;
3573	case glslang::EOpReorderThreadNV:
3574	//Three variants of reorderThreadNV, two of them use hitObjectNV
3575	if (arg == 0 && glslangOperands.size() != 2)
3576	lvalue = true;
3577	break;
3578	case glslang::EOpRayQueryGetIntersectionTriangleVertexPositionsEXT:
3579	if (arg == 0 || arg == 2)
3580	lvalue = true;
3581	break;
3582	default:
3583	break;
3584	}
3585	builder.clearAccessChain();
3586	if (invertedType != spv::NoType && arg == 0)
3587	glslangOperands[0]->getAsBinaryNode()->getLeft()->traverse(this);
3588	else
3589	glslangOperands[arg]->traverse(this);
3590
3591	if (node->getOp() == glslang::EOpCooperativeMatrixLoad ||
3592	node->getOp() == glslang::EOpCooperativeMatrixStore ||
3593	node->getOp() == glslang::EOpCooperativeMatrixLoadNV ||
3594	node->getOp() == glslang::EOpCooperativeMatrixStoreNV) {
3595
3596	if (arg == 1) {
3597	// fold "element" parameter into the access chain
3598	spv::Builder::AccessChain save = builder.getAccessChain();
3599	builder.clearAccessChain();
3600	glslangOperands[2]->traverse(this);
3601
3602	spv::Id elementId = accessChainLoad(type: glslangOperands[2]->getAsTyped()->getType());
3603
3604	builder.setAccessChain(save);
3605
3606	// Point to the first element of the array.
3607	builder.accessChainPush(offset: elementId,
3608	coherentFlags: TranslateCoherent(type: glslangOperands[arg]->getAsTyped()->getType()),
3609	alignment: glslangOperands[arg]->getAsTyped()->getType().getBufferReferenceAlignment());
3610
3611	spv::Builder::AccessChain::CoherentFlags coherentFlags = builder.getAccessChain().coherentFlags;
3612	unsigned int alignment = builder.getAccessChain().alignment;
3613
3614	int memoryAccess = TranslateMemoryAccess(coherentFlags);
3615	if (node->getOp() == glslang::EOpCooperativeMatrixLoad ||
3616	node->getOp() == glslang::EOpCooperativeMatrixLoadNV)
3617	memoryAccess &= ~spv::MemoryAccessMakePointerAvailableKHRMask;
3618	if (node->getOp() == glslang::EOpCooperativeMatrixStore ||
3619	node->getOp() == glslang::EOpCooperativeMatrixStoreNV)
3620	memoryAccess &= ~spv::MemoryAccessMakePointerVisibleKHRMask;
3621	if (builder.getStorageClass(resultId: builder.getAccessChain().base) ==
3622	spv::StorageClassPhysicalStorageBufferEXT) {
3623	memoryAccess = (spv::MemoryAccessMask)(memoryAccess | spv::MemoryAccessAlignedMask);
3624	}
3625
3626	memoryAccessOperands.push_back(x: spv::IdImmediate(false, memoryAccess));
3627
3628	if (memoryAccess & spv::MemoryAccessAlignedMask) {
3629	memoryAccessOperands.push_back(x: spv::IdImmediate(false, alignment));
3630	}
3631
3632	if (memoryAccess &
3633	(spv::MemoryAccessMakePointerAvailableKHRMask | spv::MemoryAccessMakePointerVisibleKHRMask)) {
3634	memoryAccessOperands.push_back(x: spv::IdImmediate(true,
3635	builder.makeUintConstant(u: TranslateMemoryScope(coherentFlags))));
3636	}
3637	} else if (arg == 2) {
3638	continue;
3639	}
3640	}
3641
3642	// for l-values, pass the address, for r-values, pass the value
3643	if (lvalue) {
3644	if (invertedType == spv::NoType && !builder.isSpvLvalue()) {
3645	// SPIR-V cannot represent an l-value containing a swizzle that doesn't
3646	// reduce to a simple access chain. So, we need a temporary vector to
3647	// receive the result, and must later swizzle that into the original
3648	// l-value.
3649	complexLvalues.push_back(x: builder.getAccessChain());
3650	temporaryLvalues.push_back(x: builder.createVariable(
3651	precision: spv::NoPrecision, storageClass: spv::StorageClassFunction,
3652	type: builder.accessChainGetInferredType(), name: "swizzleTemp"));
3653	operands.push_back(x: temporaryLvalues.back());
3654	} else {
3655	operands.push_back(x: builder.accessChainGetLValue());
3656	}
3657	lvalueCoherentFlags = builder.getAccessChain().coherentFlags;
3658	lvalueCoherentFlags |= TranslateCoherent(type: glslangOperands[arg]->getAsTyped()->getType());
3659	} else {
3660	builder.setDebugSourceLocation(line: node->getLoc().line, filename: node->getLoc().getFilename());
3661	glslang::TOperator glslangOp = node->getOp();
3662	if (arg == 1 &&
3663	(glslangOp == glslang::EOpRayQueryGetIntersectionType ||
3664	glslangOp == glslang::EOpRayQueryGetIntersectionT ||
3665	glslangOp == glslang::EOpRayQueryGetIntersectionInstanceCustomIndex ||
3666	glslangOp == glslang::EOpRayQueryGetIntersectionInstanceId ||
3667	glslangOp == glslang::EOpRayQueryGetIntersectionInstanceShaderBindingTableRecordOffset ||
3668	glslangOp == glslang::EOpRayQueryGetIntersectionGeometryIndex ||
3669	glslangOp == glslang::EOpRayQueryGetIntersectionPrimitiveIndex ||
3670	glslangOp == glslang::EOpRayQueryGetIntersectionBarycentrics ||
3671	glslangOp == glslang::EOpRayQueryGetIntersectionFrontFace ||
3672	glslangOp == glslang::EOpRayQueryGetIntersectionObjectRayDirection ||
3673	glslangOp == glslang::EOpRayQueryGetIntersectionObjectRayOrigin ||
3674	glslangOp == glslang::EOpRayQueryGetIntersectionObjectToWorld ||
3675	glslangOp == glslang::EOpRayQueryGetIntersectionWorldToObject ||
3676	glslangOp == glslang::EOpRayQueryGetIntersectionTriangleVertexPositionsEXT
3677	)) {
3678	bool cond = glslangOperands[arg]->getAsConstantUnion()->getConstArray()[0].getBConst();
3679	operands.push_back(x: builder.makeIntConstant(i: cond ? 1 : 0));
3680	} else if ((arg == 10 && glslangOp == glslang::EOpTraceKHR) ||
3681	(arg == 11 && glslangOp == glslang::EOpTraceRayMotionNV) ||
3682	(arg == 1 && glslangOp == glslang::EOpExecuteCallableKHR) ||
3683	(arg == 1 && glslangOp == glslang::EOpHitObjectExecuteShaderNV) ||
3684	(arg == 11 && glslangOp == glslang::EOpHitObjectTraceRayNV) ||
3685	(arg == 12 && glslangOp == glslang::EOpHitObjectTraceRayMotionNV)) {
3686	const int set = glslangOp == glslang::EOpExecuteCallableKHR ? 1 : 0;
3687	const int location = glslangOperands[arg]->getAsConstantUnion()->getConstArray()[0].getUConst();
3688	auto itNode = locationToSymbol[set].find(x: location);
3689	visitSymbol(symbol: itNode->second);
3690	spv::Id symId = getSymbolId(node: itNode->second);
3691	operands.push_back(x: symId);
3692	} else if ((arg == 12 && glslangOp == glslang::EOpHitObjectRecordHitNV) ||
3693	(arg == 13 && glslangOp == glslang::EOpHitObjectRecordHitMotionNV) ||
3694	(arg == 11 && glslangOp == glslang::EOpHitObjectRecordHitWithIndexNV) ||
3695	(arg == 12 && glslangOp == glslang::EOpHitObjectRecordHitWithIndexMotionNV) ||
3696	(arg == 1 && glslangOp == glslang::EOpHitObjectGetAttributesNV)) {
3697	const int location = glslangOperands[arg]->getAsConstantUnion()->getConstArray()[0].getUConst();
3698	const int set = 2;
3699	auto itNode = locationToSymbol[set].find(x: location);
3700	visitSymbol(symbol: itNode->second);
3701	spv::Id symId = getSymbolId(node: itNode->second);
3702	operands.push_back(x: symId);
3703	} else if (glslangOperands[arg]->getAsTyped()->getQualifier().isSpirvLiteral()) {
3704	// Will be translated to a literal value, make a placeholder here
3705	operands.push_back(x: spv::NoResult);
3706	} else {
3707	operands.push_back(x: accessChainLoad(type: glslangOperands[arg]->getAsTyped()->getType()));
3708	}
3709	}
3710	}
3711
3712	builder.setDebugSourceLocation(line: node->getLoc().line, filename: node->getLoc().getFilename());
3713	if (node->getOp() == glslang::EOpCooperativeMatrixLoad ||
3714	node->getOp() == glslang::EOpCooperativeMatrixLoadNV) {
3715	std::vector<spv::IdImmediate> idImmOps;
3716
3717	idImmOps.push_back(x: spv::IdImmediate(true, operands[1])); // buf
3718	if (node->getOp() == glslang::EOpCooperativeMatrixLoad) {
3719	idImmOps.push_back(x: spv::IdImmediate(true, operands[3])); // matrixLayout
3720	idImmOps.push_back(x: spv::IdImmediate(true, operands[2])); // stride
3721	} else {
3722	idImmOps.push_back(x: spv::IdImmediate(true, operands[2])); // stride
3723	idImmOps.push_back(x: spv::IdImmediate(true, operands[3])); // colMajor
3724	}
3725	idImmOps.insert(position: idImmOps.end(), first: memoryAccessOperands.begin(), last: memoryAccessOperands.end());
3726	// get the pointee type
3727	spv::Id typeId = builder.getContainedTypeId(typeId: builder.getTypeId(resultId: operands[0]));
3728	assert(builder.isCooperativeMatrixType(typeId));
3729	// do the op
3730	spv::Id result = node->getOp() == glslang::EOpCooperativeMatrixLoad
3731	? builder.createOp(spv::OpCooperativeMatrixLoadKHR, typeId, operands: idImmOps)
3732	: builder.createOp(spv::OpCooperativeMatrixLoadNV, typeId, operands: idImmOps);
3733	// store the result to the pointer (out param 'm')
3734	builder.createStore(rValue: result, lValue: operands[0]);
3735	result = 0;
3736	} else if (node->getOp() == glslang::EOpCooperativeMatrixStore ||
3737	node->getOp() == glslang::EOpCooperativeMatrixStoreNV) {
3738	std::vector<spv::IdImmediate> idImmOps;
3739
3740	idImmOps.push_back(x: spv::IdImmediate(true, operands[1])); // buf
3741	idImmOps.push_back(x: spv::IdImmediate(true, operands[0])); // object
3742	if (node->getOp() == glslang::EOpCooperativeMatrixStore) {
3743	idImmOps.push_back(x: spv::IdImmediate(true, operands[3])); // matrixLayout
3744	idImmOps.push_back(x: spv::IdImmediate(true, operands[2])); // stride
3745	} else {
3746	idImmOps.push_back(x: spv::IdImmediate(true, operands[2])); // stride
3747	idImmOps.push_back(x: spv::IdImmediate(true, operands[3])); // colMajor
3748	}
3749	idImmOps.insert(position: idImmOps.end(), first: memoryAccessOperands.begin(), last: memoryAccessOperands.end());
3750
3751	if (node->getOp() == glslang::EOpCooperativeMatrixStore)
3752	builder.createNoResultOp(spv::OpCooperativeMatrixStoreKHR, operands: idImmOps);
3753	else
3754	builder.createNoResultOp(spv::OpCooperativeMatrixStoreNV, operands: idImmOps);
3755	result = 0;
3756	} else if (node->getOp() == glslang::EOpRayQueryGetIntersectionTriangleVertexPositionsEXT) {
3757	std::vector<spv::IdImmediate> idImmOps;
3758
3759	idImmOps.push_back(x: spv::IdImmediate(true, operands[0])); // q
3760	idImmOps.push_back(x: spv::IdImmediate(true, operands[1])); // committed
3761
3762	spv::Id typeId = builder.makeArrayType(element: builder.makeVectorType(component: builder.makeFloatType(width: 32), size: 3),
3763	sizeId: builder.makeUintConstant(u: 3), stride: 0);
3764	// do the op
3765
3766	spv::Op spvOp = spv::OpRayQueryGetIntersectionTriangleVertexPositionsKHR;
3767
3768	spv::Id result = builder.createOp(spvOp, typeId, operands: idImmOps);
3769	// store the result to the pointer (out param 'm')
3770	builder.createStore(rValue: result, lValue: operands[2]);
3771	result = 0;
3772	} else if (node->getOp() == glslang::EOpCooperativeMatrixMulAdd) {
3773	uint32_t matrixOperands = 0;
3774
3775	// If the optional operand is present, initialize matrixOperands to that value.
3776	if (glslangOperands.size() == 4 && glslangOperands[3]->getAsConstantUnion()) {
3777	matrixOperands = glslangOperands[3]->getAsConstantUnion()->getConstArray()[0].getIConst();
3778	}
3779
3780	// Determine Cooperative Matrix Operands bits from the signedness of the types.
3781	if (isTypeSignedInt(type: glslangOperands[0]->getAsTyped()->getBasicType()))
3782	matrixOperands |= spv::CooperativeMatrixOperandsMatrixASignedComponentsKHRMask;
3783	if (isTypeSignedInt(type: glslangOperands[1]->getAsTyped()->getBasicType()))
3784	matrixOperands |= spv::CooperativeMatrixOperandsMatrixBSignedComponentsKHRMask;
3785	if (isTypeSignedInt(type: glslangOperands[2]->getAsTyped()->getBasicType()))
3786	matrixOperands |= spv::CooperativeMatrixOperandsMatrixCSignedComponentsKHRMask;
3787	if (isTypeSignedInt(type: node->getBasicType()))
3788	matrixOperands |= spv::CooperativeMatrixOperandsMatrixResultSignedComponentsKHRMask;
3789
3790	std::vector<spv::IdImmediate> idImmOps;
3791	idImmOps.push_back(x: spv::IdImmediate(true, operands[0]));
3792	idImmOps.push_back(x: spv::IdImmediate(true, operands[1]));
3793	idImmOps.push_back(x: spv::IdImmediate(true, operands[2]));
3794	if (matrixOperands != 0)
3795	idImmOps.push_back(x: spv::IdImmediate(false, matrixOperands));
3796
3797	result = builder.createOp(spv::OpCooperativeMatrixMulAddKHR, typeId: resultType(), operands: idImmOps);
3798	} else if (atomic) {
3799	// Handle all atomics
3800	glslang::TBasicType typeProxy = (node->getOp() == glslang::EOpAtomicStore)
3801	? node->getSequence()[0]->getAsTyped()->getBasicType() : node->getBasicType();
3802	result = createAtomicOperation(op: node->getOp(), precision, typeId: resultType(), operands, typeProxy,
3803	lvalueCoherentFlags, opType: node->getType());
3804	} else if (node->getOp() == glslang::EOpSpirvInst) {
3805	const auto& spirvInst = node->getSpirvInstruction();
3806	if (spirvInst.set == "") {
3807	std::vector<spv::IdImmediate> idImmOps;
3808	for (unsigned int i = 0; i < glslangOperands.size(); ++i) {
3809	if (glslangOperands[i]->getAsTyped()->getQualifier().isSpirvLiteral()) {
3810	// Translate the constant to a literal value
3811	std::vector<unsigned> literals;
3812	glslang::TVector<const glslang::TIntermConstantUnion*> constants;
3813	constants.push_back(x: glslangOperands[i]->getAsConstantUnion());
3814	TranslateLiterals(constants, literals);
3815	idImmOps.push_back(x: {false, literals[0]});
3816	} else
3817	idImmOps.push_back(x: {true, operands[i]});
3818	}
3819
3820	if (node->getBasicType() == glslang::EbtVoid)
3821	builder.createNoResultOp(static_cast<spv::Op>(spirvInst.id), operands: idImmOps);
3822	else
3823	result = builder.createOp(static_cast<spv::Op>(spirvInst.id), typeId: resultType(), operands: idImmOps);
3824	} else {
3825	result = builder.createBuiltinCall(
3826	resultType: resultType(), builtins: spirvInst.set == "GLSL.std.450" ? stdBuiltins : getExtBuiltins(name: spirvInst.set.c_str()),
3827	entryPoint: spirvInst.id, args: operands);
3828	}
3829	noReturnValue = node->getBasicType() == glslang::EbtVoid;
3830	} else if (node->getOp() == glslang::EOpDebugPrintf) {
3831	if (!nonSemanticDebugPrintf) {
3832	nonSemanticDebugPrintf = builder.import("NonSemantic.DebugPrintf");
3833	}
3834	result = builder.createBuiltinCall(resultType: builder.makeVoidType(), builtins: nonSemanticDebugPrintf, entryPoint: spv::NonSemanticDebugPrintfDebugPrintf, args: operands);
3835	builder.addExtension(ext: spv::E_SPV_KHR_non_semantic_info);
3836	} else {
3837	// Pass through to generic operations.
3838	switch (glslangOperands.size()) {
3839	case 0:
3840	result = createNoArgOperation(op: node->getOp(), precision, typeId: resultType());
3841	break;
3842	case 1:
3843	{
3844	OpDecorations decorations = { precision,
3845	TranslateNoContractionDecoration(qualifier: node->getType().getQualifier()),
3846	TranslateNonUniformDecoration(qualifier: node->getType().getQualifier()) };
3847	result = createUnaryOperation(
3848	op: node->getOp(), decorations,
3849	typeId: resultType(), operand: operands.front(),
3850	typeProxy: glslangOperands[0]->getAsTyped()->getBasicType(), lvalueCoherentFlags, opType: node->getType());
3851	}
3852	break;
3853	default:
3854	result = createMiscOperation(op: node->getOp(), precision, typeId: resultType(), operands, typeProxy: node->getBasicType());
3855	break;
3856	}
3857
3858	if (invertedType != spv::NoResult)
3859	result = createInvertedSwizzle(precision, *glslangOperands[0]->getAsBinaryNode(), parentResult: result);
3860
3861	for (unsigned int i = 0; i < temporaryLvalues.size(); ++i) {
3862	builder.setAccessChain(complexLvalues[i]);
3863	builder.accessChainStore(rvalue: builder.createLoad(lValue: temporaryLvalues[i], precision: spv::NoPrecision),
3864	nonUniform: TranslateNonUniformDecoration(coherentFlags: complexLvalues[i].coherentFlags));
3865	}
3866	}
3867
3868	if (noReturnValue)
3869	return false;
3870
3871	if (! result) {
3872	logger->missingFunctionality(f: "unknown glslang aggregate");
3873	return true; // pick up a child as a placeholder operand
3874	} else {
3875	builder.clearAccessChain();
3876	builder.setAccessChainRValue(result);
3877	return false;
3878	}
3879	}
3880
3881	// This path handles both if-then-else and ?:
3882	// The if-then-else has a node type of void, while
3883	// ?: has either a void or a non-void node type
3884	//
3885	// Leaving the result, when not void:
3886	// GLSL only has r-values as the result of a :?, but
3887	// if we have an l-value, that can be more efficient if it will
3888	// become the base of a complex r-value expression, because the
3889	// next layer copies r-values into memory to use the access-chain mechanism
3890	bool TGlslangToSpvTraverser::visitSelection(glslang::TVisit /* visit */, glslang::TIntermSelection* node)
3891	{
3892	// see if OpSelect can handle it
3893	const auto isOpSelectable = [&]() {
3894	if (node->getBasicType() == glslang::EbtVoid)
3895	return false;
3896	// OpSelect can do all other types starting with SPV 1.4
3897	if (glslangIntermediate->getSpv().spv < glslang::EShTargetSpv_1_4) {
3898	// pre-1.4, only scalars and vectors can be handled
3899	if ((!node->getType().isScalar() && !node->getType().isVector()))
3900	return false;
3901	}
3902	return true;
3903	};
3904
3905	// See if it simple and safe, or required, to execute both sides.
3906	// Crucially, side effects must be either semantically required or avoided,
3907	// and there are performance trade-offs.
3908	// Return true if required or a good idea (and safe) to execute both sides,
3909	// false otherwise.
3910	const auto bothSidesPolicy = [&]() -> bool {
3911	// do we have both sides?
3912	if (node->getTrueBlock() == nullptr ||
3913	node->getFalseBlock() == nullptr)
3914	return false;
3915
3916	// required? (unless we write additional code to look for side effects
3917	// and make performance trade-offs if none are present)
3918	if (!node->getShortCircuit())
3919	return true;
3920
3921	// if not required to execute both, decide based on performance/practicality...
3922
3923	if (!isOpSelectable())
3924	return false;
3925
3926	assert(node->getType() == node->getTrueBlock() ->getAsTyped()->getType() &&
3927	node->getType() == node->getFalseBlock()->getAsTyped()->getType());
3928
3929	// return true if a single operand to ? : is okay for OpSelect
3930	const auto operandOkay = [](glslang::TIntermTyped* node) {
3931	return node->getAsSymbolNode() || node->getType().getQualifier().isConstant();
3932	};
3933
3934	return operandOkay(node->getTrueBlock() ->getAsTyped()) &&
3935	operandOkay(node->getFalseBlock()->getAsTyped());
3936	};
3937
3938	spv::Id result = spv::NoResult; // upcoming result selecting between trueValue and falseValue
3939	// emit the condition before doing anything with selection
3940	node->getCondition()->traverse(this);
3941	spv::Id condition = accessChainLoad(type: node->getCondition()->getType());
3942
3943	// Find a way of executing both sides and selecting the right result.
3944	const auto executeBothSides = [&]() -> void {
3945	// execute both sides
3946	spv::Id resultType = convertGlslangToSpvType(type: node->getType());
3947	node->getTrueBlock()->traverse(this);
3948	spv::Id trueValue = accessChainLoad(type: node->getTrueBlock()->getAsTyped()->getType());
3949	node->getFalseBlock()->traverse(this);
3950	spv::Id falseValue = accessChainLoad(type: node->getFalseBlock()->getAsTyped()->getType());
3951
3952	builder.setDebugSourceLocation(line: node->getLoc().line, filename: node->getLoc().getFilename());
3953
3954	// done if void
3955	if (node->getBasicType() == glslang::EbtVoid)
3956	return;
3957
3958	// emit code to select between trueValue and falseValue
3959	// see if OpSelect can handle the result type, and that the SPIR-V types
3960	// of the inputs match the result type.
3961	if (isOpSelectable()) {
3962	// Emit OpSelect for this selection.
3963
3964	// smear condition to vector, if necessary (AST is always scalar)
3965	// Before 1.4, smear like for mix(), starting with 1.4, keep it scalar
3966	if (glslangIntermediate->getSpv().spv < glslang::EShTargetSpv_1_4 && builder.isVector(resultId: trueValue)) {
3967	condition = builder.smearScalar(precision: spv::NoPrecision, scalarVal: condition,
3968	vectorType: builder.makeVectorType(component: builder.makeBoolType(),
3969	size: builder.getNumComponents(resultId: trueValue)));
3970	}
3971
3972	// If the types do not match, it is because of mismatched decorations on aggregates.
3973	// Since isOpSelectable only lets us get here for SPIR-V >= 1.4, we can use OpCopyObject
3974	// to get matching types.
3975	if (builder.getTypeId(resultId: trueValue) != resultType) {
3976	trueValue = builder.createUnaryOp(spv::OpCopyLogical, typeId: resultType, operand: trueValue);
3977	}
3978	if (builder.getTypeId(resultId: falseValue) != resultType) {
3979	falseValue = builder.createUnaryOp(spv::OpCopyLogical, typeId: resultType, operand: falseValue);
3980	}
3981
3982	// OpSelect
3983	result = builder.createTriOp(spv::OpSelect, typeId: resultType, operand1: condition, operand2: trueValue, operand3: falseValue);
3984
3985	builder.clearAccessChain();
3986	builder.setAccessChainRValue(result);
3987	} else {
3988	// We need control flow to select the result.
3989	// TODO: Once SPIR-V OpSelect allows arbitrary types, eliminate this path.
3990	result = builder.createVariable(precision: TranslatePrecisionDecoration(type: node->getType()),
3991	storageClass: spv::StorageClassFunction, type: resultType);
3992
3993	// Selection control:
3994	const spv::SelectionControlMask control = TranslateSelectionControl(selectionNode: *node);
3995
3996	// make an "if" based on the value created by the condition
3997	spv::Builder::If ifBuilder(condition, control, builder);
3998
3999	// emit the "then" statement
4000	builder.clearAccessChain();
4001	builder.setAccessChainLValue(result);
4002	multiTypeStore(node->getType(), rValue: trueValue);
4003
4004	ifBuilder.makeBeginElse();
4005	// emit the "else" statement
4006	builder.clearAccessChain();
4007	builder.setAccessChainLValue(result);
4008	multiTypeStore(node->getType(), rValue: falseValue);
4009
4010	// finish off the control flow
4011	ifBuilder.makeEndIf();
4012
4013	builder.clearAccessChain();
4014	builder.setAccessChainLValue(result);
4015	}
4016	};
4017
4018	// Execute the one side needed, as per the condition
4019	const auto executeOneSide = [&]() {
4020	// Always emit control flow.
4021	if (node->getBasicType() != glslang::EbtVoid) {
4022	result = builder.createVariable(precision: TranslatePrecisionDecoration(type: node->getType()), storageClass: spv::StorageClassFunction,
4023	type: convertGlslangToSpvType(type: node->getType()));
4024	}
4025
4026	// Selection control:
4027	const spv::SelectionControlMask control = TranslateSelectionControl(selectionNode: *node);
4028
4029	// make an "if" based on the value created by the condition
4030	spv::Builder::If ifBuilder(condition, control, builder);
4031
4032	// emit the "then" statement
4033	if (node->getTrueBlock() != nullptr) {
4034	node->getTrueBlock()->traverse(this);
4035	if (result != spv::NoResult) {
4036	spv::Id load = accessChainLoad(type: node->getTrueBlock()->getAsTyped()->getType());
4037
4038	builder.clearAccessChain();
4039	builder.setAccessChainLValue(result);
4040	multiTypeStore(node->getType(), rValue: load);
4041	}
4042	}
4043
4044	if (node->getFalseBlock() != nullptr) {
4045	ifBuilder.makeBeginElse();
4046	// emit the "else" statement
4047	node->getFalseBlock()->traverse(this);
4048	if (result != spv::NoResult) {
4049	spv::Id load = accessChainLoad(type: node->getFalseBlock()->getAsTyped()->getType());
4050
4051	builder.clearAccessChain();
4052	builder.setAccessChainLValue(result);
4053	multiTypeStore(node->getType(), rValue: load);
4054	}
4055	}
4056
4057	// finish off the control flow
4058	ifBuilder.makeEndIf();
4059
4060	if (result != spv::NoResult) {
4061	builder.clearAccessChain();
4062	builder.setAccessChainLValue(result);
4063	}
4064	};
4065
4066	// Try for OpSelect (or a requirement to execute both sides)
4067	if (bothSidesPolicy()) {
4068	SpecConstantOpModeGuard spec_constant_op_mode_setter(&builder);
4069	if (node->getType().getQualifier().isSpecConstant())
4070	spec_constant_op_mode_setter.turnOnSpecConstantOpMode();
4071	executeBothSides();
4072	} else
4073	executeOneSide();
4074
4075	return false;
4076	}
4077
4078	bool TGlslangToSpvTraverser::visitSwitch(glslang::TVisit /* visit */, glslang::TIntermSwitch* node)
4079	{
4080	// emit and get the condition before doing anything with switch
4081	node->getCondition()->traverse(this);
4082	spv::Id selector = accessChainLoad(type: node->getCondition()->getAsTyped()->getType());
4083
4084	// Selection control:
4085	const spv::SelectionControlMask control = TranslateSwitchControl(switchNode: *node);
4086
4087	// browse the children to sort out code segments
4088	int defaultSegment = -1;
4089	std::vector<TIntermNode*> codeSegments;
4090	glslang::TIntermSequence& sequence = node->getBody()->getSequence();
4091	std::vector<int> caseValues;
4092	std::vector<int> valueIndexToSegment(sequence.size()); // note: probably not all are used, it is an overestimate
4093	for (glslang::TIntermSequence::iterator c = sequence.begin(); c != sequence.end(); ++c) {
4094	TIntermNode* child = *c;
4095	if (child->getAsBranchNode() && child->getAsBranchNode()->getFlowOp() == glslang::EOpDefault)
4096	defaultSegment = (int)codeSegments.size();
4097	else if (child->getAsBranchNode() && child->getAsBranchNode()->getFlowOp() == glslang::EOpCase) {
4098	valueIndexToSegment[caseValues.size()] = (int)codeSegments.size();
4099	caseValues.push_back(x: child->getAsBranchNode()->getExpression()->getAsConstantUnion()
4100	->getConstArray()[0].getIConst());
4101	} else
4102	codeSegments.push_back(x: child);
4103	}
4104
4105	// handle the case where the last code segment is missing, due to no code
4106	// statements between the last case and the end of the switch statement
4107	if ((caseValues.size() && (int)codeSegments.size() == valueIndexToSegment[caseValues.size() - 1]) ||
4108	(int)codeSegments.size() == defaultSegment)
4109	codeSegments.push_back(x: nullptr);
4110
4111	// make the switch statement
4112	std::vector<spv::Block*> segmentBlocks; // returned, as the blocks allocated in the call
4113	builder.makeSwitch(condition: selector, control, numSegments: (int)codeSegments.size(), caseValues, valueToSegment: valueIndexToSegment, defaultSegment,
4114	segmentBB&: segmentBlocks);
4115
4116	// emit all the code in the segments
4117	breakForLoop.push(x: false);
4118	for (unsigned int s = 0; s < codeSegments.size(); ++s) {
4119	builder.nextSwitchSegment(segmentBB&: segmentBlocks, segment: s);
4120	if (codeSegments[s])
4121	codeSegments[s]->traverse(this);
4122	else
4123	builder.addSwitchBreak();
4124	}
4125	breakForLoop.pop();
4126
4127	builder.endSwitch(segmentBB&: segmentBlocks);
4128
4129	return false;
4130	}
4131
4132	void TGlslangToSpvTraverser::visitConstantUnion(glslang::TIntermConstantUnion* node)
4133	{
4134	if (node->getQualifier().isSpirvLiteral())
4135	return; // Translated to a literal value, skip further processing
4136
4137	int nextConst = 0;
4138	spv::Id constant = createSpvConstantFromConstUnionArray(type: node->getType(), node->getConstArray(), nextConst, specConstant: false);
4139
4140	builder.clearAccessChain();
4141	builder.setAccessChainRValue(constant);
4142	}
4143
4144	bool TGlslangToSpvTraverser::visitLoop(glslang::TVisit /* visit */, glslang::TIntermLoop* node)
4145	{
4146	auto blocks = builder.makeNewLoop();
4147	builder.createBranch(block: &blocks.head);
4148
4149	// Loop control:
4150	std::vector<unsigned int> operands;
4151	const spv::LoopControlMask control = TranslateLoopControl(loopNode: *node, operands);
4152
4153	// Spec requires back edges to target header blocks, and every header block
4154	// must dominate its merge block. Make a header block first to ensure these
4155	// conditions are met. By definition, it will contain OpLoopMerge, followed
4156	// by a block-ending branch. But we don't want to put any other body/test
4157	// instructions in it, since the body/test may have arbitrary instructions,
4158	// including merges of its own.
4159	builder.setBuildPoint(&blocks.head);
4160	builder.setDebugSourceLocation(line: node->getLoc().line, filename: node->getLoc().getFilename());
4161	builder.createLoopMerge(mergeBlock: &blocks.merge, continueBlock: &blocks.continue_target, control, operands);
4162	if (node->testFirst() && node->getTest()) {
4163	spv::Block& test = builder.makeNewBlock();
4164	builder.createBranch(block: &test);
4165
4166	builder.setBuildPoint(&test);
4167	node->getTest()->traverse(this);
4168	spv::Id condition = accessChainLoad(type: node->getTest()->getType());
4169	builder.createConditionalBranch(condition, thenBlock: &blocks.body, elseBlock: &blocks.merge);
4170
4171	builder.setBuildPoint(&blocks.body);
4172	breakForLoop.push(x: true);
4173	if (node->getBody())
4174	node->getBody()->traverse(this);
4175	builder.createBranch(block: &blocks.continue_target);
4176	breakForLoop.pop();
4177
4178	builder.setBuildPoint(&blocks.continue_target);
4179	if (node->getTerminal())
4180	node->getTerminal()->traverse(this);
4181	builder.createBranch(block: &blocks.head);
4182	} else {
4183	builder.setDebugSourceLocation(line: node->getLoc().line, filename: node->getLoc().getFilename());
4184	builder.createBranch(block: &blocks.body);
4185
4186	breakForLoop.push(x: true);
4187	builder.setBuildPoint(&blocks.body);
4188	if (node->getBody())
4189	node->getBody()->traverse(this);
4190	builder.createBranch(block: &blocks.continue_target);
4191	breakForLoop.pop();
4192
4193	builder.setBuildPoint(&blocks.continue_target);
4194	if (node->getTerminal())
4195	node->getTerminal()->traverse(this);
4196	if (node->getTest()) {
4197	node->getTest()->traverse(this);
4198	spv::Id condition =
4199	accessChainLoad(type: node->getTest()->getType());
4200	builder.createConditionalBranch(condition, thenBlock: &blocks.head, elseBlock: &blocks.merge);
4201	} else {
4202	// TODO: unless there was a break/return/discard instruction
4203	// somewhere in the body, this is an infinite loop, so we should
4204	// issue a warning.
4205	builder.createBranch(block: &blocks.head);
4206	}
4207	}
4208	builder.setBuildPoint(&blocks.merge);
4209	builder.closeLoop();
4210	return false;
4211	}
4212
4213	bool TGlslangToSpvTraverser::visitBranch(glslang::TVisit /* visit */, glslang::TIntermBranch* node)
4214	{
4215	if (node->getExpression())
4216	node->getExpression()->traverse(this);
4217
4218	builder.setDebugSourceLocation(line: node->getLoc().line, filename: node->getLoc().getFilename());
4219
4220	switch (node->getFlowOp()) {
4221	case glslang::EOpKill:
4222	if (glslangIntermediate->getSpv().spv >= glslang::EShTargetSpv_1_6) {
4223	if (glslangIntermediate->getSource() == glslang::EShSourceHlsl) {
4224	builder.addCapability(cap: spv::CapabilityDemoteToHelperInvocation);
4225	builder.createNoResultOp(spv::OpDemoteToHelperInvocationEXT);
4226	} else {
4227	builder.makeStatementTerminator(opcode: spv::OpTerminateInvocation, name: "post-terminate-invocation");
4228	}
4229	} else {
4230	builder.makeStatementTerminator(opcode: spv::OpKill, name: "post-discard");
4231	}
4232	break;
4233	case glslang::EOpTerminateInvocation:
4234	builder.addExtension(ext: spv::E_SPV_KHR_terminate_invocation);
4235	builder.makeStatementTerminator(opcode: spv::OpTerminateInvocation, name: "post-terminate-invocation");
4236	break;
4237	case glslang::EOpBreak:
4238	if (breakForLoop.top())
4239	builder.createLoopExit();
4240	else
4241	builder.addSwitchBreak();
4242	break;
4243	case glslang::EOpContinue:
4244	builder.createLoopContinue();
4245	break;
4246	case glslang::EOpReturn:
4247	if (node->getExpression() != nullptr) {
4248	const glslang::TType& glslangReturnType = node->getExpression()->getType();
4249	spv::Id returnId = accessChainLoad(type: glslangReturnType);
4250	if (builder.getTypeId(resultId: returnId) != currentFunction->getReturnType() ||
4251	TranslatePrecisionDecoration(type: glslangReturnType) != currentFunction->getReturnPrecision()) {
4252	builder.clearAccessChain();
4253	spv::Id copyId = builder.createVariable(precision: currentFunction->getReturnPrecision(),
4254	storageClass: spv::StorageClassFunction, type: currentFunction->getReturnType());
4255	builder.setAccessChainLValue(copyId);
4256	multiTypeStore(glslangReturnType, rValue: returnId);
4257	returnId = builder.createLoad(lValue: copyId, precision: currentFunction->getReturnPrecision());
4258	}
4259	builder.makeReturn(implicit: false, retVal: returnId);
4260	} else
4261	builder.makeReturn(implicit: false);
4262
4263	builder.clearAccessChain();
4264	break;
4265
4266	case glslang::EOpDemote:
4267	builder.createNoResultOp(spv::OpDemoteToHelperInvocationEXT);
4268	builder.addExtension(ext: spv::E_SPV_EXT_demote_to_helper_invocation);
4269	builder.addCapability(cap: spv::CapabilityDemoteToHelperInvocationEXT);
4270	break;
4271	case glslang::EOpTerminateRayKHR:
4272	builder.makeStatementTerminator(opcode: spv::OpTerminateRayKHR, name: "post-terminateRayKHR");
4273	break;
4274	case glslang::EOpIgnoreIntersectionKHR:
4275	builder.makeStatementTerminator(opcode: spv::OpIgnoreIntersectionKHR, name: "post-ignoreIntersectionKHR");
4276	break;
4277
4278	default:
4279	assert(0);
4280	break;
4281	}
4282
4283	return false;
4284	}
4285
4286	spv::Id TGlslangToSpvTraverser::createSpvVariable(const glslang::TIntermSymbol* node, spv::Id forcedType)
4287	{
4288	// First, steer off constants, which are not SPIR-V variables, but
4289	// can still have a mapping to a SPIR-V Id.
4290	// This includes specialization constants.
4291	if (node->getQualifier().isConstant()) {
4292	spv::Id result = createSpvConstant(*node);
4293	if (result != spv::NoResult)
4294	return result;
4295	}
4296
4297	// Now, handle actual variables
4298	spv::StorageClass storageClass = TranslateStorageClass(type: node->getType());
4299	spv::Id spvType = forcedType == spv::NoType ? convertGlslangToSpvType(type: node->getType())
4300	: forcedType;
4301
4302	const bool contains16BitType = node->getType().contains16BitFloat() ||
4303	node->getType().contains16BitInt();
4304	if (contains16BitType) {
4305	switch (storageClass) {
4306	case spv::StorageClassInput:
4307	case spv::StorageClassOutput:
4308	builder.addIncorporatedExtension(ext: spv::E_SPV_KHR_16bit_storage, incorporatedVersion: spv::Spv_1_3);
4309	builder.addCapability(cap: spv::CapabilityStorageInputOutput16);
4310	break;
4311	case spv::StorageClassUniform:
4312	builder.addIncorporatedExtension(ext: spv::E_SPV_KHR_16bit_storage, incorporatedVersion: spv::Spv_1_3);
4313	if (node->getType().getQualifier().storage == glslang::EvqBuffer)
4314	builder.addCapability(cap: spv::CapabilityStorageUniformBufferBlock16);
4315	else
4316	builder.addCapability(cap: spv::CapabilityStorageUniform16);
4317	break;
4318	case spv::StorageClassPushConstant:
4319	builder.addIncorporatedExtension(ext: spv::E_SPV_KHR_16bit_storage, incorporatedVersion: spv::Spv_1_3);
4320	builder.addCapability(cap: spv::CapabilityStoragePushConstant16);
4321	break;
4322	case spv::StorageClassStorageBuffer:
4323	case spv::StorageClassPhysicalStorageBufferEXT:
4324	builder.addIncorporatedExtension(ext: spv::E_SPV_KHR_16bit_storage, incorporatedVersion: spv::Spv_1_3);
4325	builder.addCapability(cap: spv::CapabilityStorageUniformBufferBlock16);
4326	break;
4327	default:
4328	if (storageClass == spv::StorageClassWorkgroup &&
4329	node->getType().getBasicType() == glslang::EbtBlock) {
4330	builder.addCapability(cap: spv::CapabilityWorkgroupMemoryExplicitLayout16BitAccessKHR);
4331	break;
4332	}
4333	if (node->getType().contains16BitFloat())
4334	builder.addCapability(cap: spv::CapabilityFloat16);
4335	if (node->getType().contains16BitInt())
4336	builder.addCapability(cap: spv::CapabilityInt16);
4337	break;
4338	}
4339	}
4340
4341	if (node->getType().contains8BitInt()) {
4342	if (storageClass == spv::StorageClassPushConstant) {
4343	builder.addIncorporatedExtension(ext: spv::E_SPV_KHR_8bit_storage, incorporatedVersion: spv::Spv_1_5);
4344	builder.addCapability(cap: spv::CapabilityStoragePushConstant8);
4345	} else if (storageClass == spv::StorageClassUniform) {
4346	builder.addIncorporatedExtension(ext: spv::E_SPV_KHR_8bit_storage, incorporatedVersion: spv::Spv_1_5);
4347	builder.addCapability(cap: spv::CapabilityUniformAndStorageBuffer8BitAccess);
4348	} else if (storageClass == spv::StorageClassStorageBuffer) {
4349	builder.addIncorporatedExtension(ext: spv::E_SPV_KHR_8bit_storage, incorporatedVersion: spv::Spv_1_5);
4350	builder.addCapability(cap: spv::CapabilityStorageBuffer8BitAccess);
4351	} else if (storageClass == spv::StorageClassWorkgroup &&
4352	node->getType().getBasicType() == glslang::EbtBlock) {
4353	builder.addCapability(cap: spv::CapabilityWorkgroupMemoryExplicitLayout8BitAccessKHR);
4354	} else {
4355	builder.addCapability(cap: spv::CapabilityInt8);
4356	}
4357	}
4358
4359	const char* name = node->getName().c_str();
4360	if (glslang::IsAnonymous(name))
4361	name = "";
4362
4363	spv::Id initializer = spv::NoResult;
4364
4365	if (node->getType().getQualifier().storage == glslang::EvqUniform && !node->getConstArray().empty()) {
4366	int nextConst = 0;
4367	initializer = createSpvConstantFromConstUnionArray(type: node->getType(),
4368	node->getConstArray(),
4369	nextConst,
4370	specConstant: false /* specConst */);
4371	} else if (node->getType().getQualifier().isNullInit()) {
4372	initializer = builder.makeNullConstant(typeId: spvType);
4373	}
4374
4375	return builder.createVariable(precision: spv::NoPrecision, storageClass, type: spvType, name, initializer, compilerGenerated: false);
4376	}
4377
4378	// Return type Id of the sampled type.
4379	spv::Id TGlslangToSpvTraverser::getSampledType(const glslang::TSampler& sampler)
4380	{
4381	switch (sampler.type) {
4382	case glslang::EbtInt: return builder.makeIntType(width: 32);
4383	case glslang::EbtUint: return builder.makeUintType(width: 32);
4384	case glslang::EbtFloat: return builder.makeFloatType(width: 32);
4385	case glslang::EbtFloat16:
4386	builder.addExtension(ext: spv::E_SPV_AMD_gpu_shader_half_float_fetch);
4387	builder.addCapability(cap: spv::CapabilityFloat16ImageAMD);
4388	return builder.makeFloatType(width: 16);
4389	case glslang::EbtInt64:
4390	builder.addExtension(ext: spv::E_SPV_EXT_shader_image_int64);
4391	builder.addCapability(cap: spv::CapabilityInt64ImageEXT);
4392	return builder.makeIntType(width: 64);
4393	case glslang::EbtUint64:
4394	builder.addExtension(ext: spv::E_SPV_EXT_shader_image_int64);
4395	builder.addCapability(cap: spv::CapabilityInt64ImageEXT);
4396	return builder.makeUintType(width: 64);
4397	default:
4398	assert(0);
4399	return builder.makeFloatType(width: 32);
4400	}
4401	}
4402
4403	// If node is a swizzle operation, return the type that should be used if
4404	// the swizzle base is first consumed by another operation, before the swizzle
4405	// is applied.
4406	spv::Id TGlslangToSpvTraverser::getInvertedSwizzleType(const glslang::TIntermTyped& node)
4407	{
4408	if (node.getAsOperator() &&
4409	node.getAsOperator()->getOp() == glslang::EOpVectorSwizzle)
4410	return convertGlslangToSpvType(type: node.getAsBinaryNode()->getLeft()->getType());
4411	else
4412	return spv::NoType;
4413	}
4414
4415	// When inverting a swizzle with a parent op, this function
4416	// will apply the swizzle operation to a completed parent operation.
4417	spv::Id TGlslangToSpvTraverser::createInvertedSwizzle(spv::Decoration precision, const glslang::TIntermTyped& node,
4418	spv::Id parentResult)
4419	{
4420	std::vector<unsigned> swizzle;
4421	convertSwizzle(*node.getAsBinaryNode()->getRight()->getAsAggregate(), swizzle);
4422	return builder.createRvalueSwizzle(precision, typeId: convertGlslangToSpvType(type: node.getType()), source: parentResult, channels: swizzle);
4423	}
4424
4425	// Convert a glslang AST swizzle node to a swizzle vector for building SPIR-V.
4426	void TGlslangToSpvTraverser::convertSwizzle(const glslang::TIntermAggregate& node, std::vector<unsigned>& swizzle)
4427	{
4428	const glslang::TIntermSequence& swizzleSequence = node.getSequence();
4429	for (int i = 0; i < (int)swizzleSequence.size(); ++i)
4430	swizzle.push_back(x: swizzleSequence[i]->getAsConstantUnion()->getConstArray()[0].getIConst());
4431	}
4432
4433	// Convert from a glslang type to an SPV type, by calling into a
4434	// recursive version of this function. This establishes the inherited
4435	// layout state rooted from the top-level type.
4436	spv::Id TGlslangToSpvTraverser::convertGlslangToSpvType(const glslang::TType& type, bool forwardReferenceOnly)
4437	{
4438	return convertGlslangToSpvType(type, getExplicitLayout(type), type.getQualifier(), lastBufferBlockMember: false, forwardReferenceOnly);
4439	}
4440
4441	spv::LinkageType TGlslangToSpvTraverser::convertGlslangLinkageToSpv(glslang::TLinkType linkType)
4442	{
4443	switch (linkType) {
4444	case glslang::ELinkExport:
4445	return spv::LinkageTypeExport;
4446	default:
4447	return spv::LinkageTypeMax;
4448	}
4449	}
4450
4451	// Do full recursive conversion of an arbitrary glslang type to a SPIR-V Id.
4452	// explicitLayout can be kept the same throughout the hierarchical recursive walk.
4453	// Mutually recursive with convertGlslangStructToSpvType().
4454	spv::Id TGlslangToSpvTraverser::convertGlslangToSpvType(const glslang::TType& type,
4455	glslang::TLayoutPacking explicitLayout, const glslang::TQualifier& qualifier,
4456	bool lastBufferBlockMember, bool forwardReferenceOnly)
4457	{
4458	spv::Id spvType = spv::NoResult;
4459
4460	switch (type.getBasicType()) {
4461	case glslang::EbtVoid:
4462	spvType = builder.makeVoidType();
4463	assert (! type.isArray());
4464	break;
4465	case glslang::EbtBool:
4466	// "transparent" bool doesn't exist in SPIR-V. The GLSL convention is
4467	// a 32-bit int where non-0 means true.
4468	if (explicitLayout != glslang::ElpNone)
4469	spvType = builder.makeUintType(width: 32);
4470	else
4471	spvType = builder.makeBoolType();
4472	break;
4473	case glslang::EbtInt:
4474	spvType = builder.makeIntType(width: 32);
4475	break;
4476	case glslang::EbtUint:
4477	spvType = builder.makeUintType(width: 32);
4478	break;
4479	case glslang::EbtFloat:
4480	spvType = builder.makeFloatType(width: 32);
4481	break;
4482	case glslang::EbtDouble:
4483	spvType = builder.makeFloatType(width: 64);
4484	break;
4485	case glslang::EbtFloat16:
4486	spvType = builder.makeFloatType(width: 16);
4487	break;
4488	case glslang::EbtInt8:
4489	spvType = builder.makeIntType(width: 8);
4490	break;
4491	case glslang::EbtUint8:
4492	spvType = builder.makeUintType(width: 8);
4493	break;
4494	case glslang::EbtInt16:
4495	spvType = builder.makeIntType(width: 16);
4496	break;
4497	case glslang::EbtUint16:
4498	spvType = builder.makeUintType(width: 16);
4499	break;
4500	case glslang::EbtInt64:
4501	spvType = builder.makeIntType(width: 64);
4502	break;
4503	case glslang::EbtUint64:
4504	spvType = builder.makeUintType(width: 64);
4505	break;
4506	case glslang::EbtAtomicUint:
4507	builder.addCapability(cap: spv::CapabilityAtomicStorage);
4508	spvType = builder.makeUintType(width: 32);
4509	break;
4510	case glslang::EbtAccStruct:
4511	switch (glslangIntermediate->getStage()) {
4512	case EShLangRayGen:
4513	case EShLangIntersect:
4514	case EShLangAnyHit:
4515	case EShLangClosestHit:
4516	case EShLangMiss:
4517	case EShLangCallable:
4518	// these all should have the RayTracingNV/KHR capability already
4519	break;
4520	default:
4521	{
4522	auto& extensions = glslangIntermediate->getRequestedExtensions();
4523	if (extensions.find(x: "GL_EXT_ray_query") != extensions.end()) {
4524	builder.addExtension(ext: spv::E_SPV_KHR_ray_query);
4525	builder.addCapability(cap: spv::CapabilityRayQueryKHR);
4526	}
4527	}
4528	break;
4529	}
4530	spvType = builder.makeAccelerationStructureType();
4531	break;
4532	case glslang::EbtRayQuery:
4533	{
4534	auto& extensions = glslangIntermediate->getRequestedExtensions();
4535	if (extensions.find(x: "GL_EXT_ray_query") != extensions.end()) {
4536	builder.addExtension(ext: spv::E_SPV_KHR_ray_query);
4537	builder.addCapability(cap: spv::CapabilityRayQueryKHR);
4538	}
4539	spvType = builder.makeRayQueryType();
4540	}
4541	break;
4542	case glslang::EbtReference:
4543	{
4544	// Make the forward pointer, then recurse to convert the structure type, then
4545	// patch up the forward pointer with a real pointer type.
4546	if (forwardPointers.find(x: type.getReferentType()) == forwardPointers.end()) {
4547	spv::Id forwardId = builder.makeForwardPointer(spv::StorageClassPhysicalStorageBufferEXT);
4548	forwardPointers[type.getReferentType()] = forwardId;
4549	}
4550	spvType = forwardPointers[type.getReferentType()];
4551	if (!forwardReferenceOnly) {
4552	spv::Id referentType = convertGlslangToSpvType(type: *type.getReferentType());
4553	builder.makePointerFromForwardPointer(spv::StorageClassPhysicalStorageBufferEXT,
4554	forwardPointerType: forwardPointers[type.getReferentType()],
4555	pointee: referentType);
4556	}
4557	}
4558	break;
4559	case glslang::EbtSampler:
4560	{
4561	const glslang::TSampler& sampler = type.getSampler();
4562	if (sampler.isPureSampler()) {
4563	spvType = builder.makeSamplerType();
4564	} else {
4565	// an image is present, make its type
4566	spvType = builder.makeImageType(sampledType: getSampledType(sampler), TranslateDimensionality(sampler),
4567	depth: sampler.isShadow(), arrayed: sampler.isArrayed(), ms: sampler.isMultiSample(),
4568	sampled: sampler.isImageClass() ? 2 : 1, format: TranslateImageFormat(type));
4569	if (sampler.isCombined() &&
4570	(!sampler.isBuffer() || glslangIntermediate->getSpv().spv < glslang::EShTargetSpv_1_6)) {
4571	// Already has both image and sampler, make the combined type. Only combine sampler to
4572	// buffer if before SPIR-V 1.6.
4573	spvType = builder.makeSampledImageType(imageType: spvType);
4574	}
4575	}
4576	}
4577	break;
4578	case glslang::EbtStruct:
4579	case glslang::EbtBlock:
4580	{
4581	// If we've seen this struct type, return it
4582	const glslang::TTypeList* glslangMembers = type.getStruct();
4583
4584	// Try to share structs for different layouts, but not yet for other
4585	// kinds of qualification (primarily not yet including interpolant qualification).
4586	if (! HasNonLayoutQualifiers(type, qualifier))
4587	spvType = structMap[explicitLayout][qualifier.layoutMatrix][glslangMembers];
4588	if (spvType != spv::NoResult)
4589	break;
4590
4591	// else, we haven't seen it...
4592	if (type.getBasicType() == glslang::EbtBlock)
4593	memberRemapper[glslangTypeToIdMap[glslangMembers]].resize(new_size: glslangMembers->size());
4594	spvType = convertGlslangStructToSpvType(type, glslangStruct: glslangMembers, explicitLayout, qualifier);
4595	}
4596	break;
4597	case glslang::EbtString:
4598	// no type used for OpString
4599	return 0;
4600
4601	case glslang::EbtHitObjectNV: {
4602	builder.addExtension(ext: spv::E_SPV_NV_shader_invocation_reorder);
4603	builder.addCapability(cap: spv::CapabilityShaderInvocationReorderNV);
4604	spvType = builder.makeHitObjectNVType();
4605	}
4606	break;
4607	case glslang::EbtSpirvType: {
4608	// GL_EXT_spirv_intrinsics
4609	const auto& spirvType = type.getSpirvType();
4610	const auto& spirvInst = spirvType.spirvInst;
4611
4612	std::vector<spv::IdImmediate> operands;
4613	for (const auto& typeParam : spirvType.typeParams) {
4614	if (typeParam.getAsConstant() != nullptr) {
4615	// Constant expression
4616	auto constant = typeParam.getAsConstant();
4617	if (constant->isLiteral()) {
4618	if (constant->getBasicType() == glslang::EbtFloat) {
4619	float floatValue = static_cast<float>(constant->getConstArray()[0].getDConst());
4620	unsigned literal;
4621	static_assert(sizeof(literal) == sizeof(floatValue), "sizeof(unsigned) != sizeof(float)");
4622	memcpy(dest: &literal, src: &floatValue, n: sizeof(literal));
4623	operands.push_back(x: {false, literal});
4624	} else if (constant->getBasicType() == glslang::EbtInt) {
4625	unsigned literal = constant->getConstArray()[0].getIConst();
4626	operands.push_back(x: {false, literal});
4627	} else if (constant->getBasicType() == glslang::EbtUint) {
4628	unsigned literal = constant->getConstArray()[0].getUConst();
4629	operands.push_back(x: {false, literal});
4630	} else if (constant->getBasicType() == glslang::EbtBool) {
4631	unsigned literal = constant->getConstArray()[0].getBConst();
4632	operands.push_back(x: {false, literal});
4633	} else if (constant->getBasicType() == glslang::EbtString) {
4634	auto str = constant->getConstArray()[0].getSConst()->c_str();
4635	unsigned literal = 0;
4636	char* literalPtr = reinterpret_cast<char*>(&literal);
4637	unsigned charCount = 0;
4638	char ch = 0;
4639	do {
4640	ch = *(str++);
4641	*(literalPtr++) = ch;
4642	++charCount;
4643	if (charCount == 4) {
4644	operands.push_back(x: {false, literal});
4645	literalPtr = reinterpret_cast<char*>(&literal);
4646	charCount = 0;
4647	}
4648	} while (ch != 0);
4649
4650	// Partial literal is padded with 0
4651	if (charCount > 0) {
4652	for (; charCount < 4; ++charCount)
4653	*(literalPtr++) = 0;
4654	operands.push_back(x: {false, literal});
4655	}
4656	} else
4657	assert(0); // Unexpected type
4658	} else
4659	operands.push_back(x: {true, createSpvConstant(*constant)});
4660	} else {
4661	// Type specifier
4662	assert(typeParam.getAsType() != nullptr);
4663	operands.push_back(x: {true, convertGlslangToSpvType(type: *typeParam.getAsType())});
4664	}
4665	}
4666
4667	assert(spirvInst.set == ""); // Currently, couldn't be extended instructions.
4668	spvType = builder.makeGenericType(opcode: static_cast<spv::Op>(spirvInst.id), operands);
4669
4670	break;
4671	}
4672	default:
4673	assert(0);
4674	break;
4675	}
4676
4677	if (type.isMatrix())
4678	spvType = builder.makeMatrixType(component: spvType, cols: type.getMatrixCols(), rows: type.getMatrixRows());
4679	else {
4680	// If this variable has a vector element count greater than 1, create a SPIR-V vector
4681	if (type.getVectorSize() > 1)
4682	spvType = builder.makeVectorType(component: spvType, size: type.getVectorSize());
4683	}
4684
4685	if (type.isCoopMatNV()) {
4686	builder.addCapability(cap: spv::CapabilityCooperativeMatrixNV);
4687	builder.addExtension(ext: spv::E_SPV_NV_cooperative_matrix);
4688
4689	if (type.getBasicType() == glslang::EbtFloat16)
4690	builder.addCapability(cap: spv::CapabilityFloat16);
4691	if (type.getBasicType() == glslang::EbtUint8 ||
4692	type.getBasicType() == glslang::EbtInt8) {
4693	builder.addCapability(cap: spv::CapabilityInt8);
4694	}
4695
4696	spv::Id scope = makeArraySizeId(*type.getTypeParameters()->arraySizes, dim: 1);
4697	spv::Id rows = makeArraySizeId(*type.getTypeParameters()->arraySizes, dim: 2);
4698	spv::Id cols = makeArraySizeId(*type.getTypeParameters()->arraySizes, dim: 3);
4699
4700	spvType = builder.makeCooperativeMatrixTypeNV(component: spvType, scope, rows, cols);
4701	}
4702
4703	if (type.isCoopMatKHR()) {
4704	builder.addCapability(cap: spv::CapabilityCooperativeMatrixKHR);
4705	builder.addExtension(ext: spv::E_SPV_KHR_cooperative_matrix);
4706
4707	if (type.getBasicType() == glslang::EbtFloat16)
4708	builder.addCapability(cap: spv::CapabilityFloat16);
4709	if (type.getBasicType() == glslang::EbtUint8 || type.getBasicType() == glslang::EbtInt8) {
4710	builder.addCapability(cap: spv::CapabilityInt8);
4711	}
4712
4713	spv::Id scope = makeArraySizeId(*type.getTypeParameters()->arraySizes, dim: 0);
4714	spv::Id rows = makeArraySizeId(*type.getTypeParameters()->arraySizes, dim: 1);
4715	spv::Id cols = makeArraySizeId(*type.getTypeParameters()->arraySizes, dim: 2);
4716	spv::Id use = builder.makeUintConstant(u: type.getCoopMatKHRuse());
4717
4718	spvType = builder.makeCooperativeMatrixTypeKHR(component: spvType, scope, rows, cols, use);
4719	}
4720
4721	if (type.isArray()) {
4722	int stride = 0; // keep this 0 unless doing an explicit layout; 0 will mean no decoration, no stride
4723
4724	// Do all but the outer dimension
4725	if (type.getArraySizes()->getNumDims() > 1) {
4726	// We need to decorate array strides for types needing explicit layout, except blocks.
4727	if (explicitLayout != glslang::ElpNone && type.getBasicType() != glslang::EbtBlock) {
4728	// Use a dummy glslang type for querying internal strides of
4729	// arrays of arrays, but using just a one-dimensional array.
4730	glslang::TType simpleArrayType(type, 0); // deference type of the array
4731	while (simpleArrayType.getArraySizes()->getNumDims() > 1)
4732	simpleArrayType.getArraySizes()->dereference();
4733
4734	// Will compute the higher-order strides here, rather than making a whole
4735	// pile of types and doing repetitive recursion on their contents.
4736	stride = getArrayStride(arrayType: simpleArrayType, explicitLayout, qualifier.layoutMatrix);
4737	}
4738
4739	// make the arrays
4740	for (int dim = type.getArraySizes()->getNumDims() - 1; dim > 0; --dim) {
4741	spvType = builder.makeArrayType(element: spvType, sizeId: makeArraySizeId(*type.getArraySizes(), dim), stride);
4742	if (stride > 0)
4743	builder.addDecoration(spvType, spv::DecorationArrayStride, num: stride);
4744	stride *= type.getArraySizes()->getDimSize(dim);
4745	}
4746	} else {
4747	// single-dimensional array, and don't yet have stride
4748
4749	// We need to decorate array strides for types needing explicit layout, except blocks.
4750	if (explicitLayout != glslang::ElpNone && type.getBasicType() != glslang::EbtBlock)
4751	stride = getArrayStride(arrayType: type, explicitLayout, qualifier.layoutMatrix);
4752	}
4753
4754	// Do the outer dimension, which might not be known for a runtime-sized array.
4755	// (Unsized arrays that survive through linking will be runtime-sized arrays)
4756	if (type.isSizedArray())
4757	spvType = builder.makeArrayType(element: spvType, sizeId: makeArraySizeId(*type.getArraySizes(), dim: 0), stride);
4758	else {
4759	if (!lastBufferBlockMember) {
4760	builder.addIncorporatedExtension(ext: "SPV_EXT_descriptor_indexing", incorporatedVersion: spv::Spv_1_5);
4761	builder.addCapability(cap: spv::CapabilityRuntimeDescriptorArrayEXT);
4762	}
4763	spvType = builder.makeRuntimeArray(element: spvType);
4764	}
4765	if (stride > 0)
4766	builder.addDecoration(spvType, spv::DecorationArrayStride, num: stride);
4767	}
4768
4769	return spvType;
4770	}
4771
4772	// Apply SPIR-V decorations to the SPIR-V object (provided by SPIR-V ID). If member index is provided, the
4773	// decorations are applied to this member.
4774	void TGlslangToSpvTraverser::applySpirvDecorate(const glslang::TType& type, spv::Id id, std::optional<int> member)
4775	{
4776	assert(type.getQualifier().hasSpirvDecorate());
4777
4778	const glslang::TSpirvDecorate& spirvDecorate = type.getQualifier().getSpirvDecorate();
4779
4780	// Add spirv_decorate
4781	for (auto& decorate : spirvDecorate.decorates) {
4782	if (!decorate.second.empty()) {
4783	std::vector<unsigned> literals;
4784	TranslateLiterals(constants: decorate.second, literals);
4785	if (member.has_value())
4786	builder.addMemberDecoration(id, member: *member, static_cast<spv::Decoration>(decorate.first), literals);
4787	else
4788	builder.addDecoration(id, static_cast<spv::Decoration>(decorate.first), literals);
4789	} else {
4790	if (member.has_value())
4791	builder.addMemberDecoration(id, member: *member, static_cast<spv::Decoration>(decorate.first));
4792	else
4793	builder.addDecoration(id, static_cast<spv::Decoration>(decorate.first));
4794	}
4795	}
4796
4797	// Add spirv_decorate_id
4798	if (member.has_value()) {
4799	// spirv_decorate_id not applied to members
4800	assert(spirvDecorate.decorateIds.empty());
4801	} else {
4802	for (auto& decorateId : spirvDecorate.decorateIds) {
4803	std::vector<spv::Id> operandIds;
4804	assert(!decorateId.second.empty());
4805	for (auto extraOperand : decorateId.second) {
4806	if (extraOperand->getQualifier().isFrontEndConstant())
4807	operandIds.push_back(x: createSpvConstant(*extraOperand));
4808	else
4809	operandIds.push_back(x: getSymbolId(node: extraOperand->getAsSymbolNode()));
4810	}
4811	builder.addDecorationId(id, static_cast<spv::Decoration>(decorateId.first), operandIds);
4812	}
4813	}
4814
4815	// Add spirv_decorate_string
4816	for (auto& decorateString : spirvDecorate.decorateStrings) {
4817	std::vector<const char*> strings;
4818	assert(!decorateString.second.empty());
4819	for (auto extraOperand : decorateString.second) {
4820	const char* string = extraOperand->getConstArray()[0].getSConst()->c_str();
4821	strings.push_back(x: string);
4822	}
4823	if (member.has_value())
4824	builder.addMemberDecoration(id, member: *member, static_cast<spv::Decoration>(decorateString.first), strings);
4825	else
4826	builder.addDecoration(id, static_cast<spv::Decoration>(decorateString.first), strings);
4827	}
4828	}
4829
4830	// TODO: this functionality should exist at a higher level, in creating the AST
4831	//
4832	// Identify interface members that don't have their required extension turned on.
4833	//
4834	bool TGlslangToSpvTraverser::filterMember(const glslang::TType& member)
4835	{
4836	auto& extensions = glslangIntermediate->getRequestedExtensions();
4837
4838	if (member.getFieldName() == "gl_SecondaryViewportMaskNV" &&
4839	extensions.find(x: "GL_NV_stereo_view_rendering") == extensions.end())
4840	return true;
4841	if (member.getFieldName() == "gl_SecondaryPositionNV" &&
4842	extensions.find(x: "GL_NV_stereo_view_rendering") == extensions.end())
4843	return true;
4844
4845	if (glslangIntermediate->getStage() == EShLangMesh) {
4846	if (member.getFieldName() == "gl_PrimitiveShadingRateEXT" &&
4847	extensions.find(x: "GL_EXT_fragment_shading_rate") == extensions.end())
4848	return true;
4849	}
4850
4851	if (glslangIntermediate->getStage() != EShLangMesh) {
4852	if (member.getFieldName() == "gl_ViewportMask" &&
4853	extensions.find(x: "GL_NV_viewport_array2") == extensions.end())
4854	return true;
4855	if (member.getFieldName() == "gl_PositionPerViewNV" &&
4856	extensions.find(x: "GL_NVX_multiview_per_view_attributes") == extensions.end())
4857	return true;
4858	if (member.getFieldName() == "gl_ViewportMaskPerViewNV" &&
4859	extensions.find(x: "GL_NVX_multiview_per_view_attributes") == extensions.end())
4860	return true;
4861	}
4862
4863	return false;
4864	};
4865
4866	// Do full recursive conversion of a glslang structure (or block) type to a SPIR-V Id.
4867	// explicitLayout can be kept the same throughout the hierarchical recursive walk.
4868	// Mutually recursive with convertGlslangToSpvType().
4869	spv::Id TGlslangToSpvTraverser::convertGlslangStructToSpvType(const glslang::TType& type,
4870	const glslang::TTypeList* glslangMembers,
4871	glslang::TLayoutPacking explicitLayout,
4872	const glslang::TQualifier& qualifier)
4873	{
4874	// Create a vector of struct types for SPIR-V to consume
4875	std::vector<spv::Id> spvMembers;
4876	int memberDelta = 0; // how much the member's index changes from glslang to SPIR-V, normally 0,
4877	// except sometimes for blocks
4878	std::vector<std::pair<glslang::TType*, glslang::TQualifier> > deferredForwardPointers;
4879	for (int i = 0; i < (int)glslangMembers->size(); i++) {
4880	auto& glslangMember = (*glslangMembers)[i];
4881	if (glslangMember.type->hiddenMember()) {
4882	++memberDelta;
4883	if (type.getBasicType() == glslang::EbtBlock)
4884	memberRemapper[glslangTypeToIdMap[glslangMembers]][i] = -1;
4885	} else {
4886	if (type.getBasicType() == glslang::EbtBlock) {
4887	if (filterMember(member: *glslangMember.type)) {
4888	memberDelta++;
4889	memberRemapper[glslangTypeToIdMap[glslangMembers]][i] = -1;
4890	continue;
4891	}
4892	memberRemapper[glslangTypeToIdMap[glslangMembers]][i] = i - memberDelta;
4893	}
4894	// modify just this child's view of the qualifier
4895	glslang::TQualifier memberQualifier = glslangMember.type->getQualifier();
4896	InheritQualifiers(child&: memberQualifier, parent: qualifier);
4897
4898	// manually inherit location
4899	if (! memberQualifier.hasLocation() && qualifier.hasLocation())
4900	memberQualifier.layoutLocation = qualifier.layoutLocation;
4901
4902	// recurse
4903	bool lastBufferBlockMember = qualifier.storage == glslang::EvqBuffer &&
4904	i == (int)glslangMembers->size() - 1;
4905
4906	// Make forward pointers for any pointer members.
4907	if (glslangMember.type->isReference() &&
4908	forwardPointers.find(x: glslangMember.type->getReferentType()) == forwardPointers.end()) {
4909	deferredForwardPointers.push_back(x: std::make_pair(x: glslangMember.type, y&: memberQualifier));
4910	}
4911
4912	// Create the member type.
4913	auto const spvMember = convertGlslangToSpvType(type: *glslangMember.type, explicitLayout, qualifier: memberQualifier, lastBufferBlockMember,
4914	forwardReferenceOnly: glslangMember.type->isReference());
4915	spvMembers.push_back(x: spvMember);
4916
4917	// Update the builder with the type's location so that we can create debug types for the structure members.
4918	// There doesn't exist a "clean" entry point for this information to be passed along to the builder so, for now,
4919	// it is stored in the builder and consumed during the construction of composite debug types.
4920	// TODO: This probably warrants further investigation. This approach was decided to be the least ugly of the
4921	// quick and dirty approaches that were tried.
4922	// Advantages of this approach:
4923	// + Relatively clean. No direct calls into debug type system.
4924	// + Handles nested recursive structures.
4925	// Disadvantages of this approach:
4926	// + Not as clean as desired. Traverser queries/sets persistent state. This is fragile.
4927	// + Table lookup during creation of composite debug types. This really shouldn't be necessary.
4928	if(options.emitNonSemanticShaderDebugInfo) {
4929	builder.debugTypeLocs[spvMember].name = glslangMember.type->getFieldName().c_str();
4930	builder.debugTypeLocs[spvMember].line = glslangMember.loc.line;
4931	builder.debugTypeLocs[spvMember].column = glslangMember.loc.column;
4932	}
4933	}
4934	}
4935
4936	// Make the SPIR-V type
4937	spv::Id spvType = builder.makeStructType(members: spvMembers, name: type.getTypeName().c_str(), compilerGenerated: false);
4938	if (! HasNonLayoutQualifiers(type, qualifier))
4939	structMap[explicitLayout][qualifier.layoutMatrix][glslangMembers] = spvType;
4940
4941	// Decorate it
4942	decorateStructType(type, glslangStruct: glslangMembers, explicitLayout, qualifier, spvType, spvMembers);
4943
4944	for (int i = 0; i < (int)deferredForwardPointers.size(); ++i) {
4945	auto it = deferredForwardPointers[i];
4946	convertGlslangToSpvType(type: *it.first, explicitLayout, qualifier: it.second, lastBufferBlockMember: false);
4947	}
4948
4949	return spvType;
4950	}
4951
4952	void TGlslangToSpvTraverser::decorateStructType(const glslang::TType& type,
4953	const glslang::TTypeList* glslangMembers,
4954	glslang::TLayoutPacking explicitLayout,
4955	const glslang::TQualifier& qualifier,
4956	spv::Id spvType,
4957	const std::vector<spv::Id>& spvMembers)
4958	{
4959	// Name and decorate the non-hidden members
4960	int offset = -1;
4961	bool memberLocationInvalid = type.isArrayOfArrays() ||
4962	(type.isArray() && (type.getQualifier().isArrayedIo(language: glslangIntermediate->getStage()) == false));
4963	for (int i = 0; i < (int)glslangMembers->size(); i++) {
4964	glslang::TType& glslangMember = *(*glslangMembers)[i].type;
4965	int member = i;
4966	if (type.getBasicType() == glslang::EbtBlock) {
4967	member = memberRemapper[glslangTypeToIdMap[glslangMembers]][i];
4968	if (filterMember(member: glslangMember))
4969	continue;
4970	}
4971
4972	// modify just this child's view of the qualifier
4973	glslang::TQualifier memberQualifier = glslangMember.getQualifier();
4974	InheritQualifiers(child&: memberQualifier, parent: qualifier);
4975
4976	// using -1 above to indicate a hidden member
4977	if (member < 0)
4978	continue;
4979
4980	builder.addMemberName(spvType, member, name: glslangMember.getFieldName().c_str());
4981	builder.addMemberDecoration(spvType, member,
4982	TranslateLayoutDecoration(type: glslangMember, matrixLayout: memberQualifier.layoutMatrix));
4983	builder.addMemberDecoration(spvType, member, TranslatePrecisionDecoration(type: glslangMember));
4984	// Add interpolation and auxiliary storage decorations only to
4985	// top-level members of Input and Output storage classes
4986	if (type.getQualifier().storage == glslang::EvqVaryingIn ||
4987	type.getQualifier().storage == glslang::EvqVaryingOut) {
4988	if (type.getBasicType() == glslang::EbtBlock ||
4989	glslangIntermediate->getSource() == glslang::EShSourceHlsl) {
4990	builder.addMemberDecoration(spvType, member, TranslateInterpolationDecoration(qualifier: memberQualifier));
4991	builder.addMemberDecoration(spvType, member, TranslateAuxiliaryStorageDecoration(qualifier: memberQualifier));
4992	addMeshNVDecoration(id: spvType, member, qualifier: memberQualifier);
4993	}
4994	}
4995	builder.addMemberDecoration(spvType, member, TranslateInvariantDecoration(qualifier: memberQualifier));
4996
4997	if (type.getBasicType() == glslang::EbtBlock &&
4998	qualifier.storage == glslang::EvqBuffer) {
4999	// Add memory decorations only to top-level members of shader storage block
5000	std::vector<spv::Decoration> memory;
5001	TranslateMemoryDecoration(qualifier: memberQualifier, memory, useVulkanMemoryModel: glslangIntermediate->usingVulkanMemoryModel());
5002	for (unsigned int i = 0; i < memory.size(); ++i)
5003	builder.addMemberDecoration(spvType, member, memory[i]);
5004	}
5005
5006	// Location assignment was already completed correctly by the front end,
5007	// just track whether a member needs to be decorated.
5008	// Ignore member locations if the container is an array, as that's
5009	// ill-specified and decisions have been made to not allow this.
5010	if (!memberLocationInvalid && memberQualifier.hasLocation())
5011	builder.addMemberDecoration(spvType, member, spv::DecorationLocation, num: memberQualifier.layoutLocation);
5012
5013	// component, XFB, others
5014	if (glslangMember.getQualifier().hasComponent())
5015	builder.addMemberDecoration(spvType, member, spv::DecorationComponent,
5016	num: glslangMember.getQualifier().layoutComponent);
5017	if (glslangMember.getQualifier().hasXfbOffset())
5018	builder.addMemberDecoration(spvType, member, spv::DecorationOffset,
5019	num: glslangMember.getQualifier().layoutXfbOffset);
5020	else if (explicitLayout != glslang::ElpNone) {
5021	// figure out what to do with offset, which is accumulating
5022	int nextOffset;
5023	updateMemberOffset(structType: type, memberType: glslangMember, currentOffset&: offset, nextOffset, explicitLayout, memberQualifier.layoutMatrix);
5024	if (offset >= 0)
5025	builder.addMemberDecoration(spvType, member, spv::DecorationOffset, num: offset);
5026	offset = nextOffset;
5027	}
5028
5029	if (glslangMember.isMatrix() && explicitLayout != glslang::ElpNone)
5030	builder.addMemberDecoration(spvType, member, spv::DecorationMatrixStride,
5031	num: getMatrixStride(matrixType: glslangMember, explicitLayout, memberQualifier.layoutMatrix));
5032
5033	// built-in variable decorations
5034	spv::BuiltIn builtIn = TranslateBuiltInDecoration(builtIn: glslangMember.getQualifier().builtIn, memberDeclaration: true);
5035	if (builtIn != spv::BuiltInMax)
5036	builder.addMemberDecoration(spvType, member, spv::DecorationBuiltIn, num: (int)builtIn);
5037
5038	// nonuniform
5039	builder.addMemberDecoration(spvType, member, TranslateNonUniformDecoration(qualifier: glslangMember.getQualifier()));
5040
5041	if (glslangIntermediate->getHlslFunctionality1() && memberQualifier.semanticName != nullptr) {
5042	builder.addExtension(ext: "SPV_GOOGLE_hlsl_functionality1");
5043	builder.addMemberDecoration(spvType, member, (spv::Decoration)spv::DecorationHlslSemanticGOOGLE,
5044	memberQualifier.semanticName);
5045	}
5046
5047	if (builtIn == spv::BuiltInLayer) {
5048	// SPV_NV_viewport_array2 extension
5049	if (glslangMember.getQualifier().layoutViewportRelative){
5050	builder.addMemberDecoration(spvType, member, (spv::Decoration)spv::DecorationViewportRelativeNV);
5051	builder.addCapability(cap: spv::CapabilityShaderViewportMaskNV);
5052	builder.addExtension(ext: spv::E_SPV_NV_viewport_array2);
5053	}
5054	if (glslangMember.getQualifier().layoutSecondaryViewportRelativeOffset != -2048){
5055	builder.addMemberDecoration(spvType, member,
5056	(spv::Decoration)spv::DecorationSecondaryViewportRelativeNV,
5057	num: glslangMember.getQualifier().layoutSecondaryViewportRelativeOffset);
5058	builder.addCapability(cap: spv::CapabilityShaderStereoViewNV);
5059	builder.addExtension(ext: spv::E_SPV_NV_stereo_view_rendering);
5060	}
5061	}
5062	if (glslangMember.getQualifier().layoutPassthrough) {
5063	builder.addMemberDecoration(spvType, member, (spv::Decoration)spv::DecorationPassthroughNV);
5064	builder.addCapability(cap: spv::CapabilityGeometryShaderPassthroughNV);
5065	builder.addExtension(ext: spv::E_SPV_NV_geometry_shader_passthrough);
5066	}
5067
5068	// Add SPIR-V decorations (GL_EXT_spirv_intrinsics)
5069	if (glslangMember.getQualifier().hasSpirvDecorate())
5070	applySpirvDecorate(type: glslangMember, id: spvType, member);
5071	}
5072
5073	// Decorate the structure
5074	builder.addDecoration(spvType, TranslateLayoutDecoration(type, matrixLayout: qualifier.layoutMatrix));
5075	const auto basicType = type.getBasicType();
5076	const auto typeStorageQualifier = type.getQualifier().storage;
5077	if (basicType == glslang::EbtBlock) {
5078	builder.addDecoration(spvType, TranslateBlockDecoration(storage: typeStorageQualifier, useStorageBuffer: glslangIntermediate->usingStorageBuffer()));
5079	} else if (basicType == glslang::EbtStruct && glslangIntermediate->getSpv().vulkan > 0) {
5080	const auto hasRuntimeArray = !spvMembers.empty() && builder.getOpCode(id: spvMembers.back()) == spv::OpTypeRuntimeArray;
5081	if (hasRuntimeArray) {
5082	builder.addDecoration(spvType, TranslateBlockDecoration(storage: typeStorageQualifier, useStorageBuffer: glslangIntermediate->usingStorageBuffer()));
5083	}
5084	}
5085
5086	if (qualifier.hasHitObjectShaderRecordNV())
5087	builder.addDecoration(spvType, spv::DecorationHitObjectShaderRecordBufferNV);
5088	}
5089
5090	// Turn the expression forming the array size into an id.
5091	// This is not quite trivial, because of specialization constants.
5092	// Sometimes, a raw constant is turned into an Id, and sometimes
5093	// a specialization constant expression is.
5094	spv::Id TGlslangToSpvTraverser::makeArraySizeId(const glslang::TArraySizes& arraySizes, int dim, bool allowZero)
5095	{
5096	// First, see if this is sized with a node, meaning a specialization constant:
5097	glslang::TIntermTyped* specNode = arraySizes.getDimNode(dim);
5098	if (specNode != nullptr) {
5099	builder.clearAccessChain();
5100	SpecConstantOpModeGuard spec_constant_op_mode_setter(&builder);
5101	spec_constant_op_mode_setter.turnOnSpecConstantOpMode();
5102	specNode->traverse(this);
5103	return accessChainLoad(type: specNode->getAsTyped()->getType());
5104	}
5105
5106	// Otherwise, need a compile-time (front end) size, get it:
5107	int size = arraySizes.getDimSize(dim);
5108
5109	if (!allowZero)
5110	assert(size > 0);
5111
5112	return builder.makeUintConstant(u: size);
5113	}
5114
5115	// Wrap the builder's accessChainLoad to:
5116	// - localize handling of RelaxedPrecision
5117	// - use the SPIR-V inferred type instead of another conversion of the glslang type
5118	// (avoids unnecessary work and possible type punning for structures)
5119	// - do conversion of concrete to abstract type
5120	spv::Id TGlslangToSpvTraverser::accessChainLoad(const glslang::TType& type)
5121	{
5122	spv::Id nominalTypeId = builder.accessChainGetInferredType();
5123
5124	spv::Builder::AccessChain::CoherentFlags coherentFlags = builder.getAccessChain().coherentFlags;
5125	coherentFlags |= TranslateCoherent(type);
5126
5127	spv::MemoryAccessMask accessMask = spv::MemoryAccessMask(TranslateMemoryAccess(coherentFlags) & ~spv::MemoryAccessMakePointerAvailableKHRMask);
5128	// If the value being loaded is HelperInvocation, SPIR-V 1.6 is being generated (so that
5129	// SPV_EXT_demote_to_helper_invocation is in core) and the memory model is in use, add
5130	// the Volatile MemoryAccess semantic.
5131	if (type.getQualifier().builtIn == glslang::EbvHelperInvocation &&
5132	glslangIntermediate->usingVulkanMemoryModel() &&
5133	glslangIntermediate->getSpv().spv >= glslang::EShTargetSpv_1_6) {
5134	accessMask = spv::MemoryAccessMask(accessMask | spv::MemoryAccessVolatileMask);
5135	}
5136
5137	unsigned int alignment = builder.getAccessChain().alignment;
5138	alignment |= type.getBufferReferenceAlignment();
5139
5140	spv::Id loadedId = builder.accessChainLoad(precision: TranslatePrecisionDecoration(type),
5141	l_nonUniform: TranslateNonUniformDecoration(coherentFlags: builder.getAccessChain().coherentFlags),
5142	r_nonUniform: TranslateNonUniformDecoration(qualifier: type.getQualifier()),
5143	ResultType: nominalTypeId,
5144	memoryAccess: accessMask,
5145	scope: TranslateMemoryScope(coherentFlags),
5146	alignment);
5147
5148	// Need to convert to abstract types when necessary
5149	if (type.getBasicType() == glslang::EbtBool) {
5150	loadedId = convertLoadedBoolInUniformToUint(type, nominalTypeId, loadedId);
5151	}
5152
5153	return loadedId;
5154	}
5155
5156	// Wrap the builder's accessChainStore to:
5157	// - do conversion of concrete to abstract type
5158	//
5159	// Implicitly uses the existing builder.accessChain as the storage target.
5160	void TGlslangToSpvTraverser::accessChainStore(const glslang::TType& type, spv::Id rvalue)
5161	{
5162	// Need to convert to abstract types when necessary
5163	if (type.getBasicType() == glslang::EbtBool) {
5164	spv::Id nominalTypeId = builder.accessChainGetInferredType();
5165
5166	if (builder.isScalarType(typeId: nominalTypeId)) {
5167	// Conversion for bool
5168	spv::Id boolType = builder.makeBoolType();
5169	if (nominalTypeId != boolType) {
5170	// keep these outside arguments, for determinant order-of-evaluation
5171	spv::Id one = builder.makeUintConstant(u: 1);
5172	spv::Id zero = builder.makeUintConstant(u: 0);
5173	rvalue = builder.createTriOp(spv::OpSelect, typeId: nominalTypeId, operand1: rvalue, operand2: one, operand3: zero);
5174	} else if (builder.getTypeId(resultId: rvalue) != boolType)
5175	rvalue = builder.createBinOp(spv::OpINotEqual, typeId: boolType, operand1: rvalue, operand2: builder.makeUintConstant(u: 0));
5176	} else if (builder.isVectorType(typeId: nominalTypeId)) {
5177	// Conversion for bvec
5178	int vecSize = builder.getNumTypeComponents(typeId: nominalTypeId);
5179	spv::Id bvecType = builder.makeVectorType(component: builder.makeBoolType(), size: vecSize);
5180	if (nominalTypeId != bvecType) {
5181	// keep these outside arguments, for determinant order-of-evaluation
5182	spv::Id one = makeSmearedConstant(constant: builder.makeUintConstant(u: 1), vectorSize: vecSize);
5183	spv::Id zero = makeSmearedConstant(constant: builder.makeUintConstant(u: 0), vectorSize: vecSize);
5184	rvalue = builder.createTriOp(spv::OpSelect, typeId: nominalTypeId, operand1: rvalue, operand2: one, operand3: zero);
5185	} else if (builder.getTypeId(resultId: rvalue) != bvecType)
5186	rvalue = builder.createBinOp(spv::OpINotEqual, typeId: bvecType, operand1: rvalue,
5187	operand2: makeSmearedConstant(constant: builder.makeUintConstant(u: 0), vectorSize: vecSize));
5188	}
5189	}
5190
5191	spv::Builder::AccessChain::CoherentFlags coherentFlags = builder.getAccessChain().coherentFlags;
5192	coherentFlags |= TranslateCoherent(type);
5193
5194	unsigned int alignment = builder.getAccessChain().alignment;
5195	alignment |= type.getBufferReferenceAlignment();
5196
5197	builder.accessChainStore(rvalue, nonUniform: TranslateNonUniformDecoration(coherentFlags: builder.getAccessChain().coherentFlags),
5198	memoryAccess: spv::MemoryAccessMask(TranslateMemoryAccess(coherentFlags) &
5199	~spv::MemoryAccessMakePointerVisibleKHRMask),
5200	scope: TranslateMemoryScope(coherentFlags), alignment);
5201	}
5202
5203	// For storing when types match at the glslang level, but not might match at the
5204	// SPIR-V level.
5205	//
5206	// This especially happens when a single glslang type expands to multiple
5207	// SPIR-V types, like a struct that is used in a member-undecorated way as well
5208	// as in a member-decorated way.
5209	//
5210	// NOTE: This function can handle any store request; if it's not special it
5211	// simplifies to a simple OpStore.
5212	//
5213	// Implicitly uses the existing builder.accessChain as the storage target.
5214	void TGlslangToSpvTraverser::multiTypeStore(const glslang::TType& type, spv::Id rValue)
5215	{
5216	// we only do the complex path here if it's an aggregate
5217	if (! type.isStruct() && ! type.isArray()) {
5218	accessChainStore(type, rvalue: rValue);
5219	return;
5220	}
5221
5222	// and, it has to be a case of type aliasing
5223	spv::Id rType = builder.getTypeId(resultId: rValue);
5224	spv::Id lValue = builder.accessChainGetLValue();
5225	spv::Id lType = builder.getContainedTypeId(typeId: builder.getTypeId(resultId: lValue));
5226	if (lType == rType) {
5227	accessChainStore(type, rvalue: rValue);
5228	return;
5229	}
5230
5231	// Recursively (as needed) copy an aggregate type to a different aggregate type,
5232	// where the two types were the same type in GLSL. This requires member
5233	// by member copy, recursively.
5234
5235	// SPIR-V 1.4 added an instruction to do help do this.
5236	if (glslangIntermediate->getSpv().spv >= glslang::EShTargetSpv_1_4) {
5237	// However, bool in uniform space is changed to int, so
5238	// OpCopyLogical does not work for that.
5239	// TODO: It would be more robust to do a full recursive verification of the types satisfying SPIR-V rules.
5240	bool rBool = builder.containsType(typeId: builder.getTypeId(resultId: rValue), typeOp: spv::OpTypeBool, width: 0);
5241	bool lBool = builder.containsType(typeId: lType, typeOp: spv::OpTypeBool, width: 0);
5242	if (lBool == rBool) {
5243	spv::Id logicalCopy = builder.createUnaryOp(spv::OpCopyLogical, typeId: lType, operand: rValue);
5244	accessChainStore(type, rvalue: logicalCopy);
5245	return;
5246	}
5247	}
5248
5249	// If an array, copy element by element.
5250	if (type.isArray()) {
5251	glslang::TType glslangElementType(type, 0);
5252	spv::Id elementRType = builder.getContainedTypeId(typeId: rType);
5253	for (int index = 0; index < type.getOuterArraySize(); ++index) {
5254	// get the source member
5255	spv::Id elementRValue = builder.createCompositeExtract(composite: rValue, typeId: elementRType, index);
5256
5257	// set up the target storage
5258	builder.clearAccessChain();
5259	builder.setAccessChainLValue(lValue);
5260	builder.accessChainPush(offset: builder.makeIntConstant(i: index), coherentFlags: TranslateCoherent(type),
5261	alignment: type.getBufferReferenceAlignment());
5262
5263	// store the member
5264	multiTypeStore(type: glslangElementType, rValue: elementRValue);
5265	}
5266	} else {
5267	assert(type.isStruct());
5268
5269	// loop over structure members
5270	const glslang::TTypeList& members = *type.getStruct();
5271	for (int m = 0; m < (int)members.size(); ++m) {
5272	const glslang::TType& glslangMemberType = *members[m].type;
5273
5274	// get the source member
5275	spv::Id memberRType = builder.getContainedTypeId(typeId: rType, m);
5276	spv::Id memberRValue = builder.createCompositeExtract(composite: rValue, typeId: memberRType, index: m);
5277
5278	// set up the target storage
5279	builder.clearAccessChain();
5280	builder.setAccessChainLValue(lValue);
5281	builder.accessChainPush(offset: builder.makeIntConstant(i: m), coherentFlags: TranslateCoherent(type),
5282	alignment: type.getBufferReferenceAlignment());
5283
5284	// store the member
5285	multiTypeStore(type: glslangMemberType, rValue: memberRValue);
5286	}
5287	}
5288	}
5289
5290	// Decide whether or not this type should be
5291	// decorated with offsets and strides, and if so
5292	// whether std140 or std430 rules should be applied.
5293	glslang::TLayoutPacking TGlslangToSpvTraverser::getExplicitLayout(const glslang::TType& type) const
5294	{
5295	// has to be a block
5296	if (type.getBasicType() != glslang::EbtBlock)
5297	return glslang::ElpNone;
5298
5299	// has to be a uniform or buffer block or task in/out blocks
5300	if (type.getQualifier().storage != glslang::EvqUniform &&
5301	type.getQualifier().storage != glslang::EvqBuffer &&
5302	type.getQualifier().storage != glslang::EvqShared &&
5303	!type.getQualifier().isTaskMemory())
5304	return glslang::ElpNone;
5305
5306	// return the layout to use
5307	switch (type.getQualifier().layoutPacking) {
5308	case glslang::ElpStd140:
5309	case glslang::ElpStd430:
5310	case glslang::ElpScalar:
5311	return type.getQualifier().layoutPacking;
5312	default:
5313	return glslang::ElpNone;
5314	}
5315	}
5316
5317	// Given an array type, returns the integer stride required for that array
5318	int TGlslangToSpvTraverser::getArrayStride(const glslang::TType& arrayType, glslang::TLayoutPacking explicitLayout,
5319	glslang::TLayoutMatrix matrixLayout)
5320	{
5321	int size;
5322	int stride;
5323	glslangIntermediate->getMemberAlignment(arrayType, size, stride, layoutPacking: explicitLayout,
5324	rowMajor: matrixLayout == glslang::ElmRowMajor);
5325
5326	return stride;
5327	}
5328
5329	// Given a matrix type, or array (of array) of matrixes type, returns the integer stride required for that matrix
5330	// when used as a member of an interface block
5331	int TGlslangToSpvTraverser::getMatrixStride(const glslang::TType& matrixType, glslang::TLayoutPacking explicitLayout,
5332	glslang::TLayoutMatrix matrixLayout)
5333	{
5334	glslang::TType elementType;
5335	elementType.shallowCopy(copyOf: matrixType);
5336	elementType.clearArraySizes();
5337
5338	int size;
5339	int stride;
5340	glslangIntermediate->getMemberAlignment(elementType, size, stride, layoutPacking: explicitLayout,
5341	rowMajor: matrixLayout == glslang::ElmRowMajor);
5342
5343	return stride;
5344	}
5345
5346	// Given a member type of a struct, realign the current offset for it, and compute
5347	// the next (not yet aligned) offset for the next member, which will get aligned
5348	// on the next call.
5349	// 'currentOffset' should be passed in already initialized, ready to modify, and reflecting
5350	// the migration of data from nextOffset -> currentOffset. It should be -1 on the first call.
5351	// -1 means a non-forced member offset (no decoration needed).
5352	void TGlslangToSpvTraverser::updateMemberOffset(const glslang::TType& structType, const glslang::TType& memberType,
5353	int& currentOffset, int& nextOffset, glslang::TLayoutPacking explicitLayout, glslang::TLayoutMatrix matrixLayout)
5354	{
5355	// this will get a positive value when deemed necessary
5356	nextOffset = -1;
5357
5358	// override anything in currentOffset with user-set offset
5359	if (memberType.getQualifier().hasOffset())
5360	currentOffset = memberType.getQualifier().layoutOffset;
5361
5362	// It could be that current linker usage in glslang updated all the layoutOffset,
5363	// in which case the following code does not matter. But, that's not quite right
5364	// once cross-compilation unit GLSL validation is done, as the original user
5365	// settings are needed in layoutOffset, and then the following will come into play.
5366
5367	if (explicitLayout == glslang::ElpNone) {
5368	if (! memberType.getQualifier().hasOffset())
5369	currentOffset = -1;
5370
5371	return;
5372	}
5373
5374	// Getting this far means we need explicit offsets
5375	if (currentOffset < 0)
5376	currentOffset = 0;
5377
5378	// Now, currentOffset is valid (either 0, or from a previous nextOffset),
5379	// but possibly not yet correctly aligned.
5380
5381	int memberSize;
5382	int dummyStride;
5383	int memberAlignment = glslangIntermediate->getMemberAlignment(memberType, size&: memberSize, stride&: dummyStride, layoutPacking: explicitLayout,
5384	rowMajor: matrixLayout == glslang::ElmRowMajor);
5385
5386	bool isVectorLike = memberType.isVector();
5387	if (memberType.isMatrix()) {
5388	if (matrixLayout == glslang::ElmRowMajor)
5389	isVectorLike = memberType.getMatrixRows() == 1;
5390	else
5391	isVectorLike = memberType.getMatrixCols() == 1;
5392	}
5393
5394	// Adjust alignment for HLSL rules
5395	// TODO: make this consistent in early phases of code:
5396	// adjusting this late means inconsistencies with earlier code, which for reflection is an issue
5397	// Until reflection is brought in sync with these adjustments, don't apply to $Global,
5398	// which is the most likely to rely on reflection, and least likely to rely implicit layouts
5399	if (glslangIntermediate->usingHlslOffsets() &&
5400	! memberType.isStruct() && structType.getTypeName().compare(s: "$Global") != 0) {
5401	int componentSize;
5402	int componentAlignment = glslangIntermediate->getBaseAlignmentScalar(memberType, size&: componentSize);
5403	if (! memberType.isArray() && isVectorLike && componentAlignment <= 4)
5404	memberAlignment = componentAlignment;
5405
5406	// Don't add unnecessary padding after this member
5407	if (memberType.isMatrix()) {
5408	if (matrixLayout == glslang::ElmRowMajor)
5409	memberSize -= componentSize * (4 - memberType.getMatrixCols());
5410	else
5411	memberSize -= componentSize * (4 - memberType.getMatrixRows());
5412	} else if (memberType.isArray())
5413	memberSize -= componentSize * (4 - memberType.getVectorSize());
5414	}
5415
5416	// Bump up to member alignment
5417	glslang::RoundToPow2(number&: currentOffset, powerOf2: memberAlignment);
5418
5419	// Bump up to vec4 if there is a bad straddle
5420	if (explicitLayout != glslang::ElpScalar && glslangIntermediate->improperStraddle(type: memberType, size: memberSize,
5421	offset: currentOffset, vectorLike: isVectorLike))
5422	glslang::RoundToPow2(number&: currentOffset, powerOf2: 16);
5423
5424	nextOffset = currentOffset + memberSize;
5425	}
5426
5427	void TGlslangToSpvTraverser::declareUseOfStructMember(const glslang::TTypeList& members, int glslangMember)
5428	{
5429	const glslang::TBuiltInVariable glslangBuiltIn = members[glslangMember].type->getQualifier().builtIn;
5430	switch (glslangBuiltIn)
5431	{
5432	case glslang::EbvPointSize:
5433	case glslang::EbvClipDistance:
5434	case glslang::EbvCullDistance:
5435	case glslang::EbvViewportMaskNV:
5436	case glslang::EbvSecondaryPositionNV:
5437	case glslang::EbvSecondaryViewportMaskNV:
5438	case glslang::EbvPositionPerViewNV:
5439	case glslang::EbvViewportMaskPerViewNV:
5440	case glslang::EbvTaskCountNV:
5441	case glslang::EbvPrimitiveCountNV:
5442	case glslang::EbvPrimitiveIndicesNV:
5443	case glslang::EbvClipDistancePerViewNV:
5444	case glslang::EbvCullDistancePerViewNV:
5445	case glslang::EbvLayerPerViewNV:
5446	case glslang::EbvMeshViewCountNV:
5447	case glslang::EbvMeshViewIndicesNV:
5448	// Generate the associated capability. Delegate to TranslateBuiltInDecoration.
5449	// Alternately, we could just call this for any glslang built-in, since the
5450	// capability already guards against duplicates.
5451	TranslateBuiltInDecoration(builtIn: glslangBuiltIn, memberDeclaration: false);
5452	break;
5453	default:
5454	// Capabilities were already generated when the struct was declared.
5455	break;
5456	}
5457	}
5458
5459	bool TGlslangToSpvTraverser::isShaderEntryPoint(const glslang::TIntermAggregate* node)
5460	{
5461	return node->getName().compare(s: glslangIntermediate->getEntryPointMangledName().c_str()) == 0;
5462	}
5463
5464	// Does parameter need a place to keep writes, separate from the original?
5465	// Assumes called after originalParam(), which filters out block/buffer/opaque-based
5466	// qualifiers such that we should have only in/out/inout/constreadonly here.
5467	bool TGlslangToSpvTraverser::writableParam(glslang::TStorageQualifier qualifier) const
5468	{
5469	assert(qualifier == glslang::EvqIn ||
5470	qualifier == glslang::EvqOut ||
5471	qualifier == glslang::EvqInOut ||
5472	qualifier == glslang::EvqUniform ||
5473	qualifier == glslang::EvqConstReadOnly);
5474	return qualifier != glslang::EvqConstReadOnly &&
5475	qualifier != glslang::EvqUniform;
5476	}
5477
5478	// Is parameter pass-by-original?
5479	bool TGlslangToSpvTraverser::originalParam(glslang::TStorageQualifier qualifier, const glslang::TType& paramType,
5480	bool implicitThisParam)
5481	{
5482	if (implicitThisParam) // implicit this
5483	return true;
5484	if (glslangIntermediate->getSource() == glslang::EShSourceHlsl)
5485	return paramType.getBasicType() == glslang::EbtBlock;
5486	return (paramType.containsOpaque() && !glslangIntermediate->getBindlessMode()) || // sampler, etc.
5487	paramType.getQualifier().isSpirvByReference() || // spirv_by_reference
5488	(paramType.getBasicType() == glslang::EbtBlock && qualifier == glslang::EvqBuffer); // SSBO
5489	}
5490
5491	// Make all the functions, skeletally, without actually visiting their bodies.
5492	void TGlslangToSpvTraverser::makeFunctions(const glslang::TIntermSequence& glslFunctions)
5493	{
5494	const auto getParamDecorations = [&](std::vector<spv::Decoration>& decorations, const glslang::TType& type,
5495	bool useVulkanMemoryModel) {
5496	spv::Decoration paramPrecision = TranslatePrecisionDecoration(type);
5497	if (paramPrecision != spv::NoPrecision)
5498	decorations.push_back(x: paramPrecision);
5499	TranslateMemoryDecoration(qualifier: type.getQualifier(), memory&: decorations, useVulkanMemoryModel);
5500	if (type.isReference()) {
5501	// Original and non-writable params pass the pointer directly and
5502	// use restrict/aliased, others are stored to a pointer in Function
5503	// memory and use RestrictPointer/AliasedPointer.
5504	if (originalParam(qualifier: type.getQualifier().storage, paramType: type, implicitThisParam: false) ||
5505	!writableParam(qualifier: type.getQualifier().storage)) {
5506	// TranslateMemoryDecoration added Restrict decoration already.
5507	if (!type.getQualifier().isRestrict()) {
5508	decorations.push_back(x: spv::DecorationAliased);
5509	}
5510	} else {
5511	decorations.push_back(x: type.getQualifier().isRestrict() ? spv::DecorationRestrictPointerEXT :
5512	spv::DecorationAliasedPointerEXT);
5513	}
5514	}
5515	};
5516
5517	for (int f = 0; f < (int)glslFunctions.size(); ++f) {
5518	glslang::TIntermAggregate* glslFunction = glslFunctions[f]->getAsAggregate();
5519	if (! glslFunction || glslFunction->getOp() != glslang::EOpFunction)
5520	continue;
5521	if (isShaderEntryPoint(node: glslFunction)) {
5522	if (glslangIntermediate->getSource() != glslang::EShSourceHlsl) {
5523	builder.setupDebugFunctionEntry(function: shaderEntry, name: glslangIntermediate->getEntryPointMangledName().c_str(),
5524	line: glslFunction->getLoc().line,
5525	paramTypes: std::vector<spv::Id>(), // main function has no param
5526	paramNames: std::vector<char const*>());
5527	}
5528	continue;
5529	}
5530	// We're on a user function. Set up the basic interface for the function now,
5531	// so that it's available to call. Translating the body will happen later.
5532	//
5533	// Typically (except for a "const in" parameter), an address will be passed to the
5534	// function. What it is an address of varies:
5535	//
5536	// - "in" parameters not marked as "const" can be written to without modifying the calling
5537	// argument so that write needs to be to a copy, hence the address of a copy works.
5538	//
5539	// - "const in" parameters can just be the r-value, as no writes need occur.
5540	//
5541	// - "out" and "inout" arguments can't be done as pointers to the calling argument, because
5542	// GLSL has copy-in/copy-out semantics. They can be handled though with a pointer to a copy.
5543
5544	std::vector<spv::Id> paramTypes;
5545	std::vector<char const*> paramNames;
5546	std::vector<std::vector<spv::Decoration>> paramDecorations; // list of decorations per parameter
5547	glslang::TIntermSequence& parameters = glslFunction->getSequence()[0]->getAsAggregate()->getSequence();
5548
5549	#ifdef ENABLE_HLSL
5550	bool implicitThis = (int)parameters.size() > 0 && parameters[0]->getAsSymbolNode()->getName() ==
5551	glslangIntermediate->implicitThisName;
5552	#else
5553	bool implicitThis = false;
5554	#endif
5555
5556	paramDecorations.resize(new_size: parameters.size());
5557	for (int p = 0; p < (int)parameters.size(); ++p) {
5558	const glslang::TType& paramType = parameters[p]->getAsTyped()->getType();
5559	spv::Id typeId = convertGlslangToSpvType(type: paramType);
5560	if (originalParam(qualifier: paramType.getQualifier().storage, paramType, implicitThisParam: implicitThis && p == 0))
5561	typeId = builder.makePointer(TranslateStorageClass(type: paramType), pointee: typeId);
5562	else if (writableParam(qualifier: paramType.getQualifier().storage))
5563	typeId = builder.makePointer(spv::StorageClassFunction, pointee: typeId);
5564	else
5565	rValueParameters.insert(x: parameters[p]->getAsSymbolNode()->getId());
5566	getParamDecorations(paramDecorations[p], paramType, glslangIntermediate->usingVulkanMemoryModel());
5567	paramTypes.push_back(x: typeId);
5568	}
5569
5570	for (auto const parameter:parameters) {
5571	paramNames.push_back(x: parameter->getAsSymbolNode()->getName().c_str());
5572	}
5573
5574	spv::Block* functionBlock;
5575	spv::Function* function = builder.makeFunctionEntry(
5576	precision: TranslatePrecisionDecoration(type: glslFunction->getType()), returnType: convertGlslangToSpvType(type: glslFunction->getType()),
5577	name: glslFunction->getName().c_str(), linkType: convertGlslangLinkageToSpv(linkType: glslFunction->getLinkType()), paramTypes,
5578	precisions: paramDecorations, entry: &functionBlock);
5579	builder.setupDebugFunctionEntry(function, name: glslFunction->getName().c_str(), line: glslFunction->getLoc().line,
5580	paramTypes, paramNames);
5581	if (implicitThis)
5582	function->setImplicitThis();
5583
5584	// Track function to emit/call later
5585	functionMap[glslFunction->getName().c_str()] = function;
5586
5587	// Set the parameter id's
5588	for (int p = 0; p < (int)parameters.size(); ++p) {
5589	symbolValues[parameters[p]->getAsSymbolNode()->getId()] = function->getParamId(p);
5590	// give a name too
5591	builder.addName(function->getParamId(p), name: parameters[p]->getAsSymbolNode()->getName().c_str());
5592
5593	const glslang::TType& paramType = parameters[p]->getAsTyped()->getType();
5594	if (paramType.contains8BitInt())
5595	builder.addCapability(cap: spv::CapabilityInt8);
5596	if (paramType.contains16BitInt())
5597	builder.addCapability(cap: spv::CapabilityInt16);
5598	if (paramType.contains16BitFloat())
5599	builder.addCapability(cap: spv::CapabilityFloat16);
5600	}
5601	}
5602	}
5603
5604	// Process all the initializers, while skipping the functions and link objects
5605	void TGlslangToSpvTraverser::makeGlobalInitializers(const glslang::TIntermSequence& initializers)
5606	{
5607	builder.setBuildPoint(shaderEntry->getLastBlock());
5608	for (int i = 0; i < (int)initializers.size(); ++i) {
5609	glslang::TIntermAggregate* initializer = initializers[i]->getAsAggregate();
5610	if (initializer && initializer->getOp() != glslang::EOpFunction && initializer->getOp() !=
5611	glslang::EOpLinkerObjects) {
5612
5613	// We're on a top-level node that's not a function. Treat as an initializer, whose
5614	// code goes into the beginning of the entry point.
5615	initializer->traverse(this);
5616	}
5617	}
5618	}
5619	// Walk over all linker objects to create a map for payload and callable data linker objects
5620	// and their location to be used during codegen for OpTraceKHR and OpExecuteCallableKHR
5621	// This is done here since it is possible that these linker objects are not be referenced in the AST
5622	void TGlslangToSpvTraverser::collectRayTracingLinkerObjects()
5623	{
5624	glslang::TIntermAggregate* linkerObjects = glslangIntermediate->findLinkerObjects();
5625	for (auto& objSeq : linkerObjects->getSequence()) {
5626	auto objNode = objSeq->getAsSymbolNode();
5627	if (objNode != nullptr) {
5628	if (objNode->getQualifier().hasLocation()) {
5629	unsigned int location = objNode->getQualifier().layoutLocation;
5630	auto st = objNode->getQualifier().storage;
5631	int set;
5632	switch (st)
5633	{
5634	case glslang::EvqPayload:
5635	case glslang::EvqPayloadIn:
5636	set = 0;
5637	break;
5638	case glslang::EvqCallableData:
5639	case glslang::EvqCallableDataIn:
5640	set = 1;
5641	break;
5642
5643	case glslang::EvqHitObjectAttrNV:
5644	set = 2;
5645	break;
5646
5647	default:
5648	set = -1;
5649	}
5650	if (set != -1)
5651	locationToSymbol[set].insert(x: std::make_pair(x&: location, y&: objNode));
5652	}
5653	}
5654	}
5655	}
5656	// Process all the functions, while skipping initializers.
5657	void TGlslangToSpvTraverser::visitFunctions(const glslang::TIntermSequence& glslFunctions)
5658	{
5659	for (int f = 0; f < (int)glslFunctions.size(); ++f) {
5660	glslang::TIntermAggregate* node = glslFunctions[f]->getAsAggregate();
5661	if (node && (node->getOp() == glslang::EOpFunction || node->getOp() == glslang::EOpLinkerObjects))
5662	node->traverse(this);
5663	}
5664	}
5665
5666	void TGlslangToSpvTraverser::handleFunctionEntry(const glslang::TIntermAggregate* node)
5667	{
5668	// SPIR-V functions should already be in the functionMap from the prepass
5669	// that called makeFunctions().
5670	currentFunction = functionMap[node->getName().c_str()];
5671	spv::Block* functionBlock = currentFunction->getEntryBlock();
5672	builder.setBuildPoint(functionBlock);
5673	builder.enterFunction(function: currentFunction);
5674	}
5675
5676	void TGlslangToSpvTraverser::translateArguments(const glslang::TIntermAggregate& node, std::vector<spv::Id>& arguments,
5677	spv::Builder::AccessChain::CoherentFlags &lvalueCoherentFlags)
5678	{
5679	const glslang::TIntermSequence& glslangArguments = node.getSequence();
5680
5681	glslang::TSampler sampler = {};
5682	bool cubeCompare = false;
5683	bool f16ShadowCompare = false;
5684	if (node.isTexture() || node.isImage()) {
5685	sampler = glslangArguments[0]->getAsTyped()->getType().getSampler();
5686	cubeCompare = sampler.dim == glslang::EsdCube && sampler.arrayed && sampler.shadow;
5687	f16ShadowCompare = sampler.shadow &&
5688	glslangArguments[1]->getAsTyped()->getType().getBasicType() == glslang::EbtFloat16;
5689	}
5690
5691	for (int i = 0; i < (int)glslangArguments.size(); ++i) {
5692	builder.clearAccessChain();
5693	glslangArguments[i]->traverse(this);
5694
5695	// Special case l-value operands
5696	bool lvalue = false;
5697	switch (node.getOp()) {
5698	case glslang::EOpImageAtomicAdd:
5699	case glslang::EOpImageAtomicMin:
5700	case glslang::EOpImageAtomicMax:
5701	case glslang::EOpImageAtomicAnd:
5702	case glslang::EOpImageAtomicOr:
5703	case glslang::EOpImageAtomicXor:
5704	case glslang::EOpImageAtomicExchange:
5705	case glslang::EOpImageAtomicCompSwap:
5706	case glslang::EOpImageAtomicLoad:
5707	case glslang::EOpImageAtomicStore:
5708	if (i == 0)
5709	lvalue = true;
5710	break;
5711	case glslang::EOpSparseImageLoad:
5712	if ((sampler.ms && i == 3) || (! sampler.ms && i == 2))
5713	lvalue = true;
5714	break;
5715	case glslang::EOpSparseTexture:
5716	if (((cubeCompare || f16ShadowCompare) && i == 3) || (! (cubeCompare || f16ShadowCompare) && i == 2))
5717	lvalue = true;
5718	break;
5719	case glslang::EOpSparseTextureClamp:
5720	if (((cubeCompare || f16ShadowCompare) && i == 4) || (! (cubeCompare || f16ShadowCompare) && i == 3))
5721	lvalue = true;
5722	break;
5723	case glslang::EOpSparseTextureLod:
5724	case glslang::EOpSparseTextureOffset:
5725	if ((f16ShadowCompare && i == 4) || (! f16ShadowCompare && i == 3))
5726	lvalue = true;
5727	break;
5728	case glslang::EOpSparseTextureFetch:
5729	if ((sampler.dim != glslang::EsdRect && i == 3) || (sampler.dim == glslang::EsdRect && i == 2))
5730	lvalue = true;
5731	break;
5732	case glslang::EOpSparseTextureFetchOffset:
5733	if ((sampler.dim != glslang::EsdRect && i == 4) || (sampler.dim == glslang::EsdRect && i == 3))
5734	lvalue = true;
5735	break;
5736	case glslang::EOpSparseTextureLodOffset:
5737	case glslang::EOpSparseTextureGrad:
5738	case glslang::EOpSparseTextureOffsetClamp:
5739	if ((f16ShadowCompare && i == 5) || (! f16ShadowCompare && i == 4))
5740	lvalue = true;
5741	break;
5742	case glslang::EOpSparseTextureGradOffset:
5743	case glslang::EOpSparseTextureGradClamp:
5744	if ((f16ShadowCompare && i == 6) || (! f16ShadowCompare && i == 5))
5745	lvalue = true;
5746	break;
5747	case glslang::EOpSparseTextureGradOffsetClamp:
5748	if ((f16ShadowCompare && i == 7) || (! f16ShadowCompare && i == 6))
5749	lvalue = true;
5750	break;
5751	case glslang::EOpSparseTextureGather:
5752	if ((sampler.shadow && i == 3) || (! sampler.shadow && i == 2))
5753	lvalue = true;
5754	break;
5755	case glslang::EOpSparseTextureGatherOffset:
5756	case glslang::EOpSparseTextureGatherOffsets:
5757	if ((sampler.shadow && i == 4) || (! sampler.shadow && i == 3))
5758	lvalue = true;
5759	break;
5760	case glslang::EOpSparseTextureGatherLod:
5761	if (i == 3)
5762	lvalue = true;
5763	break;
5764	case glslang::EOpSparseTextureGatherLodOffset:
5765	case glslang::EOpSparseTextureGatherLodOffsets:
5766	if (i == 4)
5767	lvalue = true;
5768	break;
5769	case glslang::EOpSparseImageLoadLod:
5770	if (i == 3)
5771	lvalue = true;
5772	break;
5773	case glslang::EOpImageSampleFootprintNV:
5774	if (i == 4)
5775	lvalue = true;
5776	break;
5777	case glslang::EOpImageSampleFootprintClampNV:
5778	case glslang::EOpImageSampleFootprintLodNV:
5779	if (i == 5)
5780	lvalue = true;
5781	break;
5782	case glslang::EOpImageSampleFootprintGradNV:
5783	if (i == 6)
5784	lvalue = true;
5785	break;
5786	case glslang::EOpImageSampleFootprintGradClampNV:
5787	if (i == 7)
5788	lvalue = true;
5789	break;
5790	case glslang::EOpRayQueryGetIntersectionTriangleVertexPositionsEXT:
5791	if (i == 2)
5792	lvalue = true;
5793	break;
5794	default:
5795	break;
5796	}
5797
5798	if (lvalue) {
5799	spv::Id lvalue_id = builder.accessChainGetLValue();
5800	arguments.push_back(x: lvalue_id);
5801	lvalueCoherentFlags = builder.getAccessChain().coherentFlags;
5802	builder.addDecoration(lvalue_id, TranslateNonUniformDecoration(coherentFlags: lvalueCoherentFlags));
5803	lvalueCoherentFlags |= TranslateCoherent(type: glslangArguments[i]->getAsTyped()->getType());
5804	} else
5805	arguments.push_back(x: accessChainLoad(type: glslangArguments[i]->getAsTyped()->getType()));
5806	}
5807	}
5808
5809	void TGlslangToSpvTraverser::translateArguments(glslang::TIntermUnary& node, std::vector<spv::Id>& arguments)
5810	{
5811	builder.clearAccessChain();
5812	node.getOperand()->traverse(this);
5813	arguments.push_back(x: accessChainLoad(type: node.getOperand()->getType()));
5814	}
5815
5816	spv::Id TGlslangToSpvTraverser::createImageTextureFunctionCall(glslang::TIntermOperator* node)
5817	{
5818	if (! node->isImage() && ! node->isTexture())
5819	return spv::NoResult;
5820
5821	builder.setDebugSourceLocation(line: node->getLoc().line, filename: node->getLoc().getFilename());
5822
5823	// Process a GLSL texturing op (will be SPV image)
5824
5825	const glslang::TType &imageType = node->getAsAggregate()
5826	? node->getAsAggregate()->getSequence()[0]->getAsTyped()->getType()
5827	: node->getAsUnaryNode()->getOperand()->getAsTyped()->getType();
5828	const glslang::TSampler sampler = imageType.getSampler();
5829	bool f16ShadowCompare = (sampler.shadow && node->getAsAggregate())
5830	? node->getAsAggregate()->getSequence()[1]->getAsTyped()->getType().getBasicType() == glslang::EbtFloat16
5831	: false;
5832
5833	const auto signExtensionMask = [&]() {
5834	if (builder.getSpvVersion() >= spv::Spv_1_4) {
5835	if (sampler.type == glslang::EbtUint)
5836	return spv::ImageOperandsZeroExtendMask;
5837	else if (sampler.type == glslang::EbtInt)
5838	return spv::ImageOperandsSignExtendMask;
5839	}
5840	return spv::ImageOperandsMaskNone;
5841	};
5842
5843	spv::Builder::AccessChain::CoherentFlags lvalueCoherentFlags;
5844
5845	std::vector<spv::Id> arguments;
5846	if (node->getAsAggregate())
5847	translateArguments(node: *node->getAsAggregate(), arguments, lvalueCoherentFlags);
5848	else
5849	translateArguments(node&: *node->getAsUnaryNode(), arguments);
5850	spv::Decoration precision = TranslatePrecisionDecoration(type: node->getType());
5851
5852	spv::Builder::TextureParameters params = { };
5853	params.sampler = arguments[0];
5854
5855	glslang::TCrackedTextureOp cracked;
5856	node->crackTexture(sampler, cracked);
5857
5858	const bool isUnsignedResult = node->getType().getBasicType() == glslang::EbtUint;
5859
5860	if (builder.isSampledImage(resultId: params.sampler) &&
5861	((cracked.query && node->getOp() != glslang::EOpTextureQueryLod) || cracked.fragMask || cracked.fetch)) {
5862	params.sampler = builder.createUnaryOp(spv::OpImage, typeId: builder.getImageType(resultId: params.sampler), operand: params.sampler);
5863	if (imageType.getQualifier().isNonUniform()) {
5864	builder.addDecoration(params.sampler, spv::DecorationNonUniformEXT);
5865	}
5866	}
5867	// Check for queries
5868	if (cracked.query) {
5869	switch (node->getOp()) {
5870	case glslang::EOpImageQuerySize:
5871	case glslang::EOpTextureQuerySize:
5872	if (arguments.size() > 1) {
5873	params.lod = arguments[1];
5874	return builder.createTextureQueryCall(spv::OpImageQuerySizeLod, params, isUnsignedResult);
5875	} else
5876	return builder.createTextureQueryCall(spv::OpImageQuerySize, params, isUnsignedResult);
5877	case glslang::EOpImageQuerySamples:
5878	case glslang::EOpTextureQuerySamples:
5879	return builder.createTextureQueryCall(spv::OpImageQuerySamples, params, isUnsignedResult);
5880	case glslang::EOpTextureQueryLod:
5881	params.coords = arguments[1];
5882	return builder.createTextureQueryCall(spv::OpImageQueryLod, params, isUnsignedResult);
5883	case glslang::EOpTextureQueryLevels:
5884	return builder.createTextureQueryCall(spv::OpImageQueryLevels, params, isUnsignedResult);
5885	case glslang::EOpSparseTexelsResident:
5886	return builder.createUnaryOp(spv::OpImageSparseTexelsResident, typeId: builder.makeBoolType(), operand: arguments[0]);
5887	default:
5888	assert(0);
5889	break;
5890	}
5891	}
5892
5893	int components = node->getType().getVectorSize();
5894
5895	if (node->getOp() == glslang::EOpImageLoad ||
5896	node->getOp() == glslang::EOpImageLoadLod ||
5897	node->getOp() == glslang::EOpTextureFetch ||
5898	node->getOp() == glslang::EOpTextureFetchOffset) {
5899	// These must produce 4 components, per SPIR-V spec. We'll add a conversion constructor if needed.
5900	// This will only happen through the HLSL path for operator[], so we do not have to handle e.g.
5901	// the EOpTexture/Proj/Lod/etc family. It would be harmless to do so, but would need more logic
5902	// here around e.g. which ones return scalars or other types.
5903	components = 4;
5904	}
5905
5906	glslang::TType returnType(node->getType().getBasicType(), glslang::EvqTemporary, components);
5907
5908	auto resultType = [&returnType,this]{ return convertGlslangToSpvType(type: returnType); };
5909
5910	// Check for image functions other than queries
5911	if (node->isImage()) {
5912	std::vector<spv::IdImmediate> operands;
5913	auto opIt = arguments.begin();
5914	spv::IdImmediate image = { true, *(opIt++) };
5915	operands.push_back(x: image);
5916
5917	// Handle subpass operations
5918	// TODO: GLSL should change to have the "MS" only on the type rather than the
5919	// built-in function.
5920	if (cracked.subpass) {
5921	// add on the (0,0) coordinate
5922	spv::Id zero = builder.makeIntConstant(i: 0);
5923	std::vector<spv::Id> comps;
5924	comps.push_back(x: zero);
5925	comps.push_back(x: zero);
5926	spv::IdImmediate coord = { true,
5927	builder.makeCompositeConstant(type: builder.makeVectorType(component: builder.makeIntType(width: 32), size: 2), comps) };
5928	operands.push_back(x: coord);
5929	spv::IdImmediate imageOperands = { false, spv::ImageOperandsMaskNone };
5930	imageOperands.word = imageOperands.word | signExtensionMask();
5931	if (sampler.isMultiSample()) {
5932	imageOperands.word = imageOperands.word | spv::ImageOperandsSampleMask;
5933	}
5934	if (imageOperands.word != spv::ImageOperandsMaskNone) {
5935	operands.push_back(x: imageOperands);
5936	if (sampler.isMultiSample()) {
5937	spv::IdImmediate imageOperand = { true, *(opIt++) };
5938	operands.push_back(x: imageOperand);
5939	}
5940	}
5941	spv::Id result = builder.createOp(spv::OpImageRead, typeId: resultType(), operands);
5942	builder.setPrecision(id: result, precision);
5943	return result;
5944	}
5945
5946	if (cracked.attachmentEXT) {
5947	if (opIt != arguments.end()) {
5948	spv::IdImmediate sample = { true, *opIt };
5949	operands.push_back(x: sample);
5950	}
5951	spv::Id result = builder.createOp(spv::OpColorAttachmentReadEXT, typeId: resultType(), operands);
5952	builder.addExtension(ext: spv::E_SPV_EXT_shader_tile_image);
5953	builder.setPrecision(id: result, precision);
5954	return result;
5955	}
5956
5957	spv::IdImmediate coord = { true, *(opIt++) };
5958	operands.push_back(x: coord);
5959	if (node->getOp() == glslang::EOpImageLoad || node->getOp() == glslang::EOpImageLoadLod) {
5960	spv::ImageOperandsMask mask = spv::ImageOperandsMaskNone;
5961	if (sampler.isMultiSample()) {
5962	mask = mask | spv::ImageOperandsSampleMask;
5963	}
5964	if (cracked.lod) {
5965	builder.addExtension(ext: spv::E_SPV_AMD_shader_image_load_store_lod);
5966	builder.addCapability(cap: spv::CapabilityImageReadWriteLodAMD);
5967	mask = mask | spv::ImageOperandsLodMask;
5968	}
5969	mask = mask | TranslateImageOperands(coherentFlags: TranslateCoherent(type: imageType));
5970	mask = (spv::ImageOperandsMask)(mask & ~spv::ImageOperandsMakeTexelAvailableKHRMask);
5971	mask = mask | signExtensionMask();
5972	if (mask != spv::ImageOperandsMaskNone) {
5973	spv::IdImmediate imageOperands = { false, (unsigned int)mask };
5974	operands.push_back(x: imageOperands);
5975	}
5976	if (mask & spv::ImageOperandsSampleMask) {
5977	spv::IdImmediate imageOperand = { true, *opIt++ };
5978	operands.push_back(x: imageOperand);
5979	}
5980	if (mask & spv::ImageOperandsLodMask) {
5981	spv::IdImmediate imageOperand = { true, *opIt++ };
5982	operands.push_back(x: imageOperand);
5983	}
5984	if (mask & spv::ImageOperandsMakeTexelVisibleKHRMask) {
5985	spv::IdImmediate imageOperand = { true,
5986	builder.makeUintConstant(u: TranslateMemoryScope(coherentFlags: TranslateCoherent(type: imageType))) };
5987	operands.push_back(x: imageOperand);
5988	}
5989
5990	if (builder.getImageTypeFormat(typeId: builder.getImageType(resultId: operands.front().word)) == spv::ImageFormatUnknown)
5991	builder.addCapability(cap: spv::CapabilityStorageImageReadWithoutFormat);
5992
5993	std::vector<spv::Id> result(1, builder.createOp(spv::OpImageRead, typeId: resultType(), operands));
5994	builder.setPrecision(id: result[0], precision);
5995
5996	// If needed, add a conversion constructor to the proper size.
5997	if (components != node->getType().getVectorSize())
5998	result[0] = builder.createConstructor(precision, sources: result, resultTypeId: convertGlslangToSpvType(type: node->getType()));
5999
6000	return result[0];
6001	} else if (node->getOp() == glslang::EOpImageStore || node->getOp() == glslang::EOpImageStoreLod) {
6002
6003	// Push the texel value before the operands
6004	if (sampler.isMultiSample() || cracked.lod) {
6005	spv::IdImmediate texel = { true, *(opIt + 1) };
6006	operands.push_back(x: texel);
6007	} else {
6008	spv::IdImmediate texel = { true, *opIt };
6009	operands.push_back(x: texel);
6010	}
6011
6012	spv::ImageOperandsMask mask = spv::ImageOperandsMaskNone;
6013	if (sampler.isMultiSample()) {
6014	mask = mask | spv::ImageOperandsSampleMask;
6015	}
6016	if (cracked.lod) {
6017	builder.addExtension(ext: spv::E_SPV_AMD_shader_image_load_store_lod);
6018	builder.addCapability(cap: spv::CapabilityImageReadWriteLodAMD);
6019	mask = mask | spv::ImageOperandsLodMask;
6020	}
6021	mask = mask | TranslateImageOperands(coherentFlags: TranslateCoherent(type: imageType));
6022	mask = (spv::ImageOperandsMask)(mask & ~spv::ImageOperandsMakeTexelVisibleKHRMask);
6023	mask = mask | signExtensionMask();
6024	if (mask != spv::ImageOperandsMaskNone) {
6025	spv::IdImmediate imageOperands = { false, (unsigned int)mask };
6026	operands.push_back(x: imageOperands);
6027	}
6028	if (mask & spv::ImageOperandsSampleMask) {
6029	spv::IdImmediate imageOperand = { true, *opIt++ };
6030	operands.push_back(x: imageOperand);
6031	}
6032	if (mask & spv::ImageOperandsLodMask) {
6033	spv::IdImmediate imageOperand = { true, *opIt++ };
6034	operands.push_back(x: imageOperand);
6035	}
6036	if (mask & spv::ImageOperandsMakeTexelAvailableKHRMask) {
6037	spv::IdImmediate imageOperand = { true,
6038	builder.makeUintConstant(u: TranslateMemoryScope(coherentFlags: TranslateCoherent(type: imageType))) };
6039	operands.push_back(x: imageOperand);
6040	}
6041
6042	builder.createNoResultOp(spv::OpImageWrite, operands);
6043	if (builder.getImageTypeFormat(typeId: builder.getImageType(resultId: operands.front().word)) == spv::ImageFormatUnknown)
6044	builder.addCapability(cap: spv::CapabilityStorageImageWriteWithoutFormat);
6045	return spv::NoResult;
6046	} else if (node->getOp() == glslang::EOpSparseImageLoad ||
6047	node->getOp() == glslang::EOpSparseImageLoadLod) {
6048	builder.addCapability(cap: spv::CapabilitySparseResidency);
6049	if (builder.getImageTypeFormat(typeId: builder.getImageType(resultId: operands.front().word)) == spv::ImageFormatUnknown)
6050	builder.addCapability(cap: spv::CapabilityStorageImageReadWithoutFormat);
6051
6052	spv::ImageOperandsMask mask = spv::ImageOperandsMaskNone;
6053	if (sampler.isMultiSample()) {
6054	mask = mask | spv::ImageOperandsSampleMask;
6055	}
6056	if (cracked.lod) {
6057	builder.addExtension(ext: spv::E_SPV_AMD_shader_image_load_store_lod);
6058	builder.addCapability(cap: spv::CapabilityImageReadWriteLodAMD);
6059
6060	mask = mask | spv::ImageOperandsLodMask;
6061	}
6062	mask = mask | TranslateImageOperands(coherentFlags: TranslateCoherent(type: imageType));
6063	mask = (spv::ImageOperandsMask)(mask & ~spv::ImageOperandsMakeTexelAvailableKHRMask);
6064	mask = mask | signExtensionMask();
6065	if (mask != spv::ImageOperandsMaskNone) {
6066	spv::IdImmediate imageOperands = { false, (unsigned int)mask };
6067	operands.push_back(x: imageOperands);
6068	}
6069	if (mask & spv::ImageOperandsSampleMask) {
6070	spv::IdImmediate imageOperand = { true, *opIt++ };
6071	operands.push_back(x: imageOperand);
6072	}
6073	if (mask & spv::ImageOperandsLodMask) {
6074	spv::IdImmediate imageOperand = { true, *opIt++ };
6075	operands.push_back(x: imageOperand);
6076	}
6077	if (mask & spv::ImageOperandsMakeTexelVisibleKHRMask) {
6078	spv::IdImmediate imageOperand = { true, builder.makeUintConstant(u: TranslateMemoryScope(
6079	coherentFlags: TranslateCoherent(type: imageType))) };
6080	operands.push_back(x: imageOperand);
6081	}
6082
6083	// Create the return type that was a special structure
6084	spv::Id texelOut = *opIt;
6085	spv::Id typeId0 = resultType();
6086	spv::Id typeId1 = builder.getDerefTypeId(resultId: texelOut);
6087	spv::Id resultTypeId = builder.makeStructResultType(type0: typeId0, type1: typeId1);
6088
6089	spv::Id resultId = builder.createOp(spv::OpImageSparseRead, typeId: resultTypeId, operands);
6090
6091	// Decode the return type
6092	builder.createStore(rValue: builder.createCompositeExtract(composite: resultId, typeId: typeId1, index: 1), lValue: texelOut);
6093	return builder.createCompositeExtract(composite: resultId, typeId: typeId0, index: 0);
6094	} else {
6095	// Process image atomic operations
6096
6097	// GLSL "IMAGE_PARAMS" will involve in constructing an image texel pointer and this pointer,
6098	// as the first source operand, is required by SPIR-V atomic operations.
6099	// For non-MS, the sample value should be 0
6100	spv::IdImmediate sample = { true, sampler.isMultiSample() ? *(opIt++) : builder.makeUintConstant(u: 0) };
6101	operands.push_back(x: sample);
6102
6103	spv::Id resultTypeId;
6104	glslang::TBasicType typeProxy = node->getBasicType();
6105	// imageAtomicStore has a void return type so base the pointer type on
6106	// the type of the value operand.
6107	if (node->getOp() == glslang::EOpImageAtomicStore) {
6108	resultTypeId = builder.makePointer(spv::StorageClassImage, pointee: builder.getTypeId(resultId: *opIt));
6109	typeProxy = node->getAsAggregate()->getSequence()[0]->getAsTyped()->getType().getSampler().type;
6110	} else {
6111	resultTypeId = builder.makePointer(spv::StorageClassImage, pointee: resultType());
6112	}
6113	spv::Id pointer = builder.createOp(spv::OpImageTexelPointer, typeId: resultTypeId, operands);
6114	if (imageType.getQualifier().nonUniform) {
6115	builder.addDecoration(pointer, spv::DecorationNonUniformEXT);
6116	}
6117
6118	std::vector<spv::Id> operands;
6119	operands.push_back(x: pointer);
6120	for (; opIt != arguments.end(); ++opIt)
6121	operands.push_back(x: *opIt);
6122
6123	return createAtomicOperation(op: node->getOp(), precision, typeId: resultType(), operands, typeProxy,
6124	lvalueCoherentFlags, opType: node->getType());
6125	}
6126	}
6127
6128	// Check for fragment mask functions other than queries
6129	if (cracked.fragMask) {
6130	assert(sampler.ms);
6131
6132	auto opIt = arguments.begin();
6133	std::vector<spv::Id> operands;
6134
6135	operands.push_back(x: params.sampler);
6136	++opIt;
6137
6138	if (sampler.isSubpass()) {
6139	// add on the (0,0) coordinate
6140	spv::Id zero = builder.makeIntConstant(i: 0);
6141	std::vector<spv::Id> comps;
6142	comps.push_back(x: zero);
6143	comps.push_back(x: zero);
6144	operands.push_back(x: builder.makeCompositeConstant(
6145	type: builder.makeVectorType(component: builder.makeIntType(width: 32), size: 2), comps));
6146	}
6147
6148	for (; opIt != arguments.end(); ++opIt)
6149	operands.push_back(x: *opIt);
6150
6151	spv::Op fragMaskOp = spv::OpNop;
6152	if (node->getOp() == glslang::EOpFragmentMaskFetch)
6153	fragMaskOp = spv::OpFragmentMaskFetchAMD;
6154	else if (node->getOp() == glslang::EOpFragmentFetch)
6155	fragMaskOp = spv::OpFragmentFetchAMD;
6156
6157	builder.addExtension(ext: spv::E_SPV_AMD_shader_fragment_mask);
6158	builder.addCapability(cap: spv::CapabilityFragmentMaskAMD);
6159	return builder.createOp(fragMaskOp, typeId: resultType(), operands);
6160	}
6161
6162	// Check for texture functions other than queries
6163	bool sparse = node->isSparseTexture();
6164	bool imageFootprint = node->isImageFootprint();
6165	bool cubeCompare = sampler.dim == glslang::EsdCube && sampler.isArrayed() && sampler.isShadow();
6166
6167	// check for bias argument
6168	bool bias = false;
6169	if (! cracked.lod && ! cracked.grad && ! cracked.fetch && ! cubeCompare) {
6170	int nonBiasArgCount = 2;
6171	if (cracked.gather)
6172	++nonBiasArgCount; // comp argument should be present when bias argument is present
6173
6174	if (f16ShadowCompare)
6175	++nonBiasArgCount;
6176	if (cracked.offset)
6177	++nonBiasArgCount;
6178	else if (cracked.offsets)
6179	++nonBiasArgCount;
6180	if (cracked.grad)
6181	nonBiasArgCount += 2;
6182	if (cracked.lodClamp)
6183	++nonBiasArgCount;
6184	if (sparse)
6185	++nonBiasArgCount;
6186	if (imageFootprint)
6187	//Following three extra arguments
6188	// int granularity, bool coarse, out gl_TextureFootprint2DNV footprint
6189	nonBiasArgCount += 3;
6190	if ((int)arguments.size() > nonBiasArgCount)
6191	bias = true;
6192	}
6193
6194	if (cracked.gather) {
6195	const auto& sourceExtensions = glslangIntermediate->getRequestedExtensions();
6196	if (bias || cracked.lod ||
6197	sourceExtensions.find(x: glslang::E_GL_AMD_texture_gather_bias_lod) != sourceExtensions.end()) {
6198	builder.addExtension(ext: spv::E_SPV_AMD_texture_gather_bias_lod);
6199	builder.addCapability(cap: spv::CapabilityImageGatherBiasLodAMD);
6200	}
6201	}
6202
6203	// set the rest of the arguments
6204
6205	params.coords = arguments[1];
6206	int extraArgs = 0;
6207	bool noImplicitLod = false;
6208
6209	// sort out where Dref is coming from
6210	if (cubeCompare || f16ShadowCompare) {
6211	params.Dref = arguments[2];
6212	++extraArgs;
6213	} else if (sampler.shadow && cracked.gather) {
6214	params.Dref = arguments[2];
6215	++extraArgs;
6216	} else if (sampler.shadow) {
6217	std::vector<spv::Id> indexes;
6218	int dRefComp;
6219	if (cracked.proj)
6220	dRefComp = 2; // "The resulting 3rd component of P in the shadow forms is used as Dref"
6221	else
6222	dRefComp = builder.getNumComponents(resultId: params.coords) - 1;
6223	indexes.push_back(x: dRefComp);
6224	params.Dref = builder.createCompositeExtract(composite: params.coords,
6225	typeId: builder.getScalarTypeId(typeId: builder.getTypeId(resultId: params.coords)), indexes);
6226	}
6227
6228	// lod
6229	if (cracked.lod) {
6230	params.lod = arguments[2 + extraArgs];
6231	++extraArgs;
6232	} else if (glslangIntermediate->getStage() != EShLangFragment &&
6233	!(glslangIntermediate->getStage() == EShLangCompute &&
6234	glslangIntermediate->hasLayoutDerivativeModeNone())) {
6235	// we need to invent the default lod for an explicit lod instruction for a non-fragment stage
6236	noImplicitLod = true;
6237	}
6238
6239	// multisample
6240	if (sampler.isMultiSample()) {
6241	params.sample = arguments[2 + extraArgs]; // For MS, "sample" should be specified
6242	++extraArgs;
6243	}
6244
6245	// gradient
6246	if (cracked.grad) {
6247	params.gradX = arguments[2 + extraArgs];
6248	params.gradY = arguments[3 + extraArgs];
6249	extraArgs += 2;
6250	}
6251
6252	// offset and offsets
6253	if (cracked.offset) {
6254	params.offset = arguments[2 + extraArgs];
6255	++extraArgs;
6256	} else if (cracked.offsets) {
6257	params.offsets = arguments[2 + extraArgs];
6258	++extraArgs;
6259	}
6260
6261	// lod clamp
6262	if (cracked.lodClamp) {
6263	params.lodClamp = arguments[2 + extraArgs];
6264	++extraArgs;
6265	}
6266	// sparse
6267	if (sparse) {
6268	params.texelOut = arguments[2 + extraArgs];
6269	++extraArgs;
6270	}
6271	// gather component
6272	if (cracked.gather && ! sampler.shadow) {
6273	// default component is 0, if missing, otherwise an argument
6274	if (2 + extraArgs < (int)arguments.size()) {
6275	params.component = arguments[2 + extraArgs];
6276	++extraArgs;
6277	} else
6278	params.component = builder.makeIntConstant(i: 0);
6279	}
6280	spv::Id resultStruct = spv::NoResult;
6281	if (imageFootprint) {
6282	//Following three extra arguments
6283	// int granularity, bool coarse, out gl_TextureFootprint2DNV footprint
6284	params.granularity = arguments[2 + extraArgs];
6285	params.coarse = arguments[3 + extraArgs];
6286	resultStruct = arguments[4 + extraArgs];
6287	extraArgs += 3;
6288	}
6289
6290	// bias
6291	if (bias) {
6292	params.bias = arguments[2 + extraArgs];
6293	++extraArgs;
6294	}
6295
6296	if (imageFootprint) {
6297	builder.addExtension(ext: spv::E_SPV_NV_shader_image_footprint);
6298	builder.addCapability(cap: spv::CapabilityImageFootprintNV);
6299
6300
6301	//resultStructType(OpenGL type) contains 5 elements:
6302	//struct gl_TextureFootprint2DNV {
6303	// uvec2 anchor;
6304	// uvec2 offset;
6305	// uvec2 mask;
6306	// uint lod;
6307	// uint granularity;
6308	//};
6309	//or
6310	//struct gl_TextureFootprint3DNV {
6311	// uvec3 anchor;
6312	// uvec3 offset;
6313	// uvec2 mask;
6314	// uint lod;
6315	// uint granularity;
6316	//};
6317	spv::Id resultStructType = builder.getContainedTypeId(typeId: builder.getTypeId(resultId: resultStruct));
6318	assert(builder.isStructType(resultStructType));
6319
6320	//resType (SPIR-V type) contains 6 elements:
6321	//Member 0 must be a Boolean type scalar(LOD),
6322	//Member 1 must be a vector of integer type, whose Signedness operand is 0(anchor),
6323	//Member 2 must be a vector of integer type, whose Signedness operand is 0(offset),
6324	//Member 3 must be a vector of integer type, whose Signedness operand is 0(mask),
6325	//Member 4 must be a scalar of integer type, whose Signedness operand is 0(lod),
6326	//Member 5 must be a scalar of integer type, whose Signedness operand is 0(granularity).
6327	std::vector<spv::Id> members;
6328	members.push_back(x: resultType());
6329	for (int i = 0; i < 5; i++) {
6330	members.push_back(x: builder.getContainedTypeId(typeId: resultStructType, i));
6331	}
6332	spv::Id resType = builder.makeStructType(members, name: "ResType");
6333
6334	//call ImageFootprintNV
6335	spv::Id res = builder.createTextureCall(precision, resultType: resType, sparse, fetch: cracked.fetch, proj: cracked.proj,
6336	gather: cracked.gather, noImplicit: noImplicitLod, params, signExtensionMask());
6337
6338	//copy resType (SPIR-V type) to resultStructType(OpenGL type)
6339	for (int i = 0; i < 5; i++) {
6340	builder.clearAccessChain();
6341	builder.setAccessChainLValue(resultStruct);
6342
6343	//Accessing to a struct we created, no coherent flag is set
6344	spv::Builder::AccessChain::CoherentFlags flags;
6345	flags.clear();
6346
6347	builder.accessChainPush(offset: builder.makeIntConstant(i), coherentFlags: flags, alignment: 0);
6348	builder.accessChainStore(rvalue: builder.createCompositeExtract(composite: res, typeId: builder.getContainedTypeId(typeId: resType, i+1),
6349	index: i+1), nonUniform: TranslateNonUniformDecoration(qualifier: imageType.getQualifier()));
6350	}
6351	return builder.createCompositeExtract(composite: res, typeId: resultType(), index: 0);
6352	}
6353
6354	// projective component (might not to move)
6355	// GLSL: "The texture coordinates consumed from P, not including the last component of P,
6356	// are divided by the last component of P."
6357	// SPIR-V: "... (u [, v] [, w], q)... It may be a vector larger than needed, but all
6358	// unused components will appear after all used components."
6359	if (cracked.proj) {
6360	int projSourceComp = builder.getNumComponents(resultId: params.coords) - 1;
6361	int projTargetComp;
6362	switch (sampler.dim) {
6363	case glslang::Esd1D: projTargetComp = 1; break;
6364	case glslang::Esd2D: projTargetComp = 2; break;
6365	case glslang::EsdRect: projTargetComp = 2; break;
6366	default: projTargetComp = projSourceComp; break;
6367	}
6368	// copy the projective coordinate if we have to
6369	if (projTargetComp != projSourceComp) {
6370	spv::Id projComp = builder.createCompositeExtract(composite: params.coords,
6371	typeId: builder.getScalarTypeId(typeId: builder.getTypeId(resultId: params.coords)), index: projSourceComp);
6372	params.coords = builder.createCompositeInsert(object: projComp, composite: params.coords,
6373	typeId: builder.getTypeId(resultId: params.coords), index: projTargetComp);
6374	}
6375	}
6376
6377	// nonprivate
6378	if (imageType.getQualifier().nonprivate) {
6379	params.nonprivate = true;
6380	}
6381
6382	// volatile
6383	if (imageType.getQualifier().volatil) {
6384	params.volatil = true;
6385	}
6386
6387	std::vector<spv::Id> result( 1,
6388	builder.createTextureCall(precision, resultType: resultType(), sparse, fetch: cracked.fetch, proj: cracked.proj, gather: cracked.gather,
6389	noImplicit: noImplicitLod, params, signExtensionMask())
6390	);
6391
6392	if (components != node->getType().getVectorSize())
6393	result[0] = builder.createConstructor(precision, sources: result, resultTypeId: convertGlslangToSpvType(type: node->getType()));
6394
6395	return result[0];
6396	}
6397
6398	spv::Id TGlslangToSpvTraverser::handleUserFunctionCall(const glslang::TIntermAggregate* node)
6399	{
6400	// Grab the function's pointer from the previously created function
6401	spv::Function* function = functionMap[node->getName().c_str()];
6402	if (! function)
6403	return 0;
6404
6405	const glslang::TIntermSequence& glslangArgs = node->getSequence();
6406	const glslang::TQualifierList& qualifiers = node->getQualifierList();
6407
6408	// See comments in makeFunctions() for details about the semantics for parameter passing.
6409	//
6410	// These imply we need a four step process:
6411	// 1. Evaluate the arguments
6412	// 2. Allocate and make copies of in, out, and inout arguments
6413	// 3. Make the call
6414	// 4. Copy back the results
6415
6416	// 1. Evaluate the arguments and their types
6417	std::vector<spv::Builder::AccessChain> lValues;
6418	std::vector<spv::Id> rValues;
6419	std::vector<const glslang::TType*> argTypes;
6420	for (int a = 0; a < (int)glslangArgs.size(); ++a) {
6421	argTypes.push_back(x: &glslangArgs[a]->getAsTyped()->getType());
6422	// build l-value
6423	builder.clearAccessChain();
6424	glslangArgs[a]->traverse(this);
6425	// keep outputs and pass-by-originals as l-values, evaluate others as r-values
6426	if (originalParam(qualifier: qualifiers[a], paramType: *argTypes[a], implicitThisParam: function->hasImplicitThis() && a == 0) ||
6427	writableParam(qualifier: qualifiers[a])) {
6428	// save l-value
6429	lValues.push_back(x: builder.getAccessChain());
6430	} else {
6431	// process r-value
6432	rValues.push_back(x: accessChainLoad(type: *argTypes.back()));
6433	}
6434	}
6435
6436	// 2. Allocate space for anything needing a copy, and if it's "in" or "inout"
6437	// copy the original into that space.
6438	//
6439	// Also, build up the list of actual arguments to pass in for the call
6440	int lValueCount = 0;
6441	int rValueCount = 0;
6442	std::vector<spv::Id> spvArgs;
6443	for (int a = 0; a < (int)glslangArgs.size(); ++a) {
6444	spv::Id arg;
6445	if (originalParam(qualifier: qualifiers[a], paramType: *argTypes[a], implicitThisParam: function->hasImplicitThis() && a == 0)) {
6446	builder.setAccessChain(lValues[lValueCount]);
6447	arg = builder.accessChainGetLValue();
6448	++lValueCount;
6449	} else if (writableParam(qualifier: qualifiers[a])) {
6450	// need space to hold the copy
6451	arg = builder.createVariable(precision: function->getParamPrecision(param: a), storageClass: spv::StorageClassFunction,
6452	type: builder.getContainedTypeId(typeId: function->getParamType(p: a)), name: "param");
6453	if (qualifiers[a] == glslang::EvqIn || qualifiers[a] == glslang::EvqInOut) {
6454	// need to copy the input into output space
6455	builder.setAccessChain(lValues[lValueCount]);
6456	spv::Id copy = accessChainLoad(type: *argTypes[a]);
6457	builder.clearAccessChain();
6458	builder.setAccessChainLValue(arg);
6459	multiTypeStore(type: *argTypes[a], rValue: copy);
6460	}
6461	++lValueCount;
6462	} else {
6463	// process r-value, which involves a copy for a type mismatch
6464	if (function->getParamType(p: a) != builder.getTypeId(resultId: rValues[rValueCount]) ||
6465	TranslatePrecisionDecoration(type: *argTypes[a]) != function->getParamPrecision(param: a))
6466	{
6467	spv::Id argCopy = builder.createVariable(precision: function->getParamPrecision(param: a), storageClass: spv::StorageClassFunction, type: function->getParamType(p: a), name: "arg");
6468	builder.clearAccessChain();
6469	builder.setAccessChainLValue(argCopy);
6470	multiTypeStore(type: *argTypes[a], rValue: rValues[rValueCount]);
6471	arg = builder.createLoad(lValue: argCopy, precision: function->getParamPrecision(param: a));
6472	} else
6473	arg = rValues[rValueCount];
6474	++rValueCount;
6475	}
6476	spvArgs.push_back(x: arg);
6477	}
6478
6479	// 3. Make the call.
6480	spv::Id result = builder.createFunctionCall(function, spvArgs);
6481	builder.setPrecision(id: result, precision: TranslatePrecisionDecoration(type: node->getType()));
6482	builder.addDecoration(result, TranslateNonUniformDecoration(qualifier: node->getType().getQualifier()));
6483
6484	// 4. Copy back out an "out" arguments.
6485	lValueCount = 0;
6486	for (int a = 0; a < (int)glslangArgs.size(); ++a) {
6487	if (originalParam(qualifier: qualifiers[a], paramType: *argTypes[a], implicitThisParam: function->hasImplicitThis() && a == 0))
6488	++lValueCount;
6489	else if (writableParam(qualifier: qualifiers[a])) {
6490	if (qualifiers[a] == glslang::EvqOut || qualifiers[a] == glslang::EvqInOut) {
6491	spv::Id copy = builder.createLoad(lValue: spvArgs[a], precision: spv::NoPrecision);
6492	builder.addDecoration(copy, TranslateNonUniformDecoration(qualifier: argTypes[a]->getQualifier()));
6493	builder.setAccessChain(lValues[lValueCount]);
6494	multiTypeStore(type: *argTypes[a], rValue: copy);
6495	}
6496	++lValueCount;
6497	}
6498	}
6499
6500	return result;
6501	}
6502
6503	// Translate AST operation to SPV operation, already having SPV-based operands/types.
6504	spv::Id TGlslangToSpvTraverser::createBinaryOperation(glslang::TOperator op, OpDecorations& decorations,
6505	spv::Id typeId, spv::Id left, spv::Id right,
6506	glslang::TBasicType typeProxy, bool reduceComparison)
6507	{
6508	bool isUnsigned = isTypeUnsignedInt(type: typeProxy);
6509	bool isFloat = isTypeFloat(type: typeProxy);
6510	bool isBool = typeProxy == glslang::EbtBool;
6511
6512	spv::Op binOp = spv::OpNop;
6513	bool needMatchingVectors = true; // for non-matrix ops, would a scalar need to smear to match a vector?
6514	bool comparison = false;
6515
6516	switch (op) {
6517	case glslang::EOpAdd:
6518	case glslang::EOpAddAssign:
6519	if (isFloat)
6520	binOp = spv::OpFAdd;
6521	else
6522	binOp = spv::OpIAdd;
6523	break;
6524	case glslang::EOpSub:
6525	case glslang::EOpSubAssign:
6526	if (isFloat)
6527	binOp = spv::OpFSub;
6528	else
6529	binOp = spv::OpISub;
6530	break;
6531	case glslang::EOpMul:
6532	case glslang::EOpMulAssign:
6533	if (isFloat)
6534	binOp = spv::OpFMul;
6535	else
6536	binOp = spv::OpIMul;
6537	break;
6538	case glslang::EOpVectorTimesScalar:
6539	case glslang::EOpVectorTimesScalarAssign:
6540	if (isFloat && (builder.isVector(resultId: left) || builder.isVector(resultId: right))) {
6541	if (builder.isVector(resultId: right))
6542	std::swap(a&: left, b&: right);
6543	assert(builder.isScalar(right));
6544	needMatchingVectors = false;
6545	binOp = spv::OpVectorTimesScalar;
6546	} else if (isFloat)
6547	binOp = spv::OpFMul;
6548	else
6549	binOp = spv::OpIMul;
6550	break;
6551	case glslang::EOpVectorTimesMatrix:
6552	case glslang::EOpVectorTimesMatrixAssign:
6553	binOp = spv::OpVectorTimesMatrix;
6554	break;
6555	case glslang::EOpMatrixTimesVector:
6556	binOp = spv::OpMatrixTimesVector;
6557	break;
6558	case glslang::EOpMatrixTimesScalar:
6559	case glslang::EOpMatrixTimesScalarAssign:
6560	binOp = spv::OpMatrixTimesScalar;
6561	break;
6562	case glslang::EOpMatrixTimesMatrix:
6563	case glslang::EOpMatrixTimesMatrixAssign:
6564	binOp = spv::OpMatrixTimesMatrix;
6565	break;
6566	case glslang::EOpOuterProduct:
6567	binOp = spv::OpOuterProduct;
6568	needMatchingVectors = false;
6569	break;
6570
6571	case glslang::EOpDiv:
6572	case glslang::EOpDivAssign:
6573	if (isFloat)
6574	binOp = spv::OpFDiv;
6575	else if (isUnsigned)
6576	binOp = spv::OpUDiv;
6577	else
6578	binOp = spv::OpSDiv;
6579	break;
6580	case glslang::EOpMod:
6581	case glslang::EOpModAssign:
6582	if (isFloat)
6583	binOp = spv::OpFMod;
6584	else if (isUnsigned)
6585	binOp = spv::OpUMod;
6586	else
6587	binOp = spv::OpSMod;
6588	break;
6589	case glslang::EOpRightShift:
6590	case glslang::EOpRightShiftAssign:
6591	if (isUnsigned)
6592	binOp = spv::OpShiftRightLogical;
6593	else
6594	binOp = spv::OpShiftRightArithmetic;
6595	break;
6596	case glslang::EOpLeftShift:
6597	case glslang::EOpLeftShiftAssign:
6598	binOp = spv::OpShiftLeftLogical;
6599	break;
6600	case glslang::EOpAnd:
6601	case glslang::EOpAndAssign:
6602	binOp = spv::OpBitwiseAnd;
6603	break;
6604	case glslang::EOpLogicalAnd:
6605	needMatchingVectors = false;
6606	binOp = spv::OpLogicalAnd;
6607	break;
6608	case glslang::EOpInclusiveOr:
6609	case glslang::EOpInclusiveOrAssign:
6610	binOp = spv::OpBitwiseOr;
6611	break;
6612	case glslang::EOpLogicalOr:
6613	needMatchingVectors = false;
6614	binOp = spv::OpLogicalOr;
6615	break;
6616	case glslang::EOpExclusiveOr:
6617	case glslang::EOpExclusiveOrAssign:
6618	binOp = spv::OpBitwiseXor;
6619	break;
6620	case glslang::EOpLogicalXor:
6621	needMatchingVectors = false;
6622	binOp = spv::OpLogicalNotEqual;
6623	break;
6624
6625	case glslang::EOpAbsDifference:
6626	binOp = isUnsigned ? spv::OpAbsUSubINTEL : spv::OpAbsISubINTEL;
6627	break;
6628
6629	case glslang::EOpAddSaturate:
6630	binOp = isUnsigned ? spv::OpUAddSatINTEL : spv::OpIAddSatINTEL;
6631	break;
6632
6633	case glslang::EOpSubSaturate:
6634	binOp = isUnsigned ? spv::OpUSubSatINTEL : spv::OpISubSatINTEL;
6635	break;
6636
6637	case glslang::EOpAverage:
6638	binOp = isUnsigned ? spv::OpUAverageINTEL : spv::OpIAverageINTEL;
6639	break;
6640
6641	case glslang::EOpAverageRounded:
6642	binOp = isUnsigned ? spv::OpUAverageRoundedINTEL : spv::OpIAverageRoundedINTEL;
6643	break;
6644
6645	case glslang::EOpMul32x16:
6646	binOp = isUnsigned ? spv::OpUMul32x16INTEL : spv::OpIMul32x16INTEL;
6647	break;
6648
6649	case glslang::EOpExpectEXT:
6650	binOp = spv::OpExpectKHR;
6651	break;
6652
6653	case glslang::EOpLessThan:
6654	case glslang::EOpGreaterThan:
6655	case glslang::EOpLessThanEqual:
6656	case glslang::EOpGreaterThanEqual:
6657	case glslang::EOpEqual:
6658	case glslang::EOpNotEqual:
6659	case glslang::EOpVectorEqual:
6660	case glslang::EOpVectorNotEqual:
6661	comparison = true;
6662	break;
6663	default:
6664	break;
6665	}
6666
6667	// handle mapped binary operations (should be non-comparison)
6668	if (binOp != spv::OpNop) {
6669	assert(comparison == false);
6670	if (builder.isMatrix(resultId: left) || builder.isMatrix(resultId: right) ||
6671	builder.isCooperativeMatrix(resultId: left) || builder.isCooperativeMatrix(resultId: right))
6672	return createBinaryMatrixOperation(binOp, decorations, typeId, left, right);
6673
6674	// No matrix involved; make both operands be the same number of components, if needed
6675	if (needMatchingVectors)
6676	builder.promoteScalar(precision: decorations.precision, left, right);
6677
6678	spv::Id result = builder.createBinOp(binOp, typeId, operand1: left, operand2: right);
6679	decorations.addNoContraction(builder, t: result);
6680	decorations.addNonUniform(builder, t: result);
6681	return builder.setPrecision(id: result, precision: decorations.precision);
6682	}
6683
6684	if (! comparison)
6685	return 0;
6686
6687	// Handle comparison instructions
6688
6689	if (reduceComparison && (op == glslang::EOpEqual || op == glslang::EOpNotEqual)
6690	&& (builder.isVector(resultId: left) || builder.isMatrix(resultId: left) || builder.isAggregate(resultId: left))) {
6691	spv::Id result = builder.createCompositeCompare(precision: decorations.precision, left, right, op == glslang::EOpEqual);
6692	decorations.addNonUniform(builder, t: result);
6693	return result;
6694	}
6695
6696	switch (op) {
6697	case glslang::EOpLessThan:
6698	if (isFloat)
6699	binOp = spv::OpFOrdLessThan;
6700	else if (isUnsigned)
6701	binOp = spv::OpULessThan;
6702	else
6703	binOp = spv::OpSLessThan;
6704	break;
6705	case glslang::EOpGreaterThan:
6706	if (isFloat)
6707	binOp = spv::OpFOrdGreaterThan;
6708	else if (isUnsigned)
6709	binOp = spv::OpUGreaterThan;
6710	else
6711	binOp = spv::OpSGreaterThan;
6712	break;
6713	case glslang::EOpLessThanEqual:
6714	if (isFloat)
6715	binOp = spv::OpFOrdLessThanEqual;
6716	else if (isUnsigned)
6717	binOp = spv::OpULessThanEqual;
6718	else
6719	binOp = spv::OpSLessThanEqual;
6720	break;
6721	case glslang::EOpGreaterThanEqual:
6722	if (isFloat)
6723	binOp = spv::OpFOrdGreaterThanEqual;
6724	else if (isUnsigned)
6725	binOp = spv::OpUGreaterThanEqual;
6726	else
6727	binOp = spv::OpSGreaterThanEqual;
6728	break;
6729	case glslang::EOpEqual:
6730	case glslang::EOpVectorEqual:
6731	if (isFloat)
6732	binOp = spv::OpFOrdEqual;
6733	else if (isBool)
6734	binOp = spv::OpLogicalEqual;
6735	else
6736	binOp = spv::OpIEqual;
6737	break;
6738	case glslang::EOpNotEqual:
6739	case glslang::EOpVectorNotEqual:
6740	if (isFloat)
6741	binOp = spv::OpFUnordNotEqual;
6742	else if (isBool)
6743	binOp = spv::OpLogicalNotEqual;
6744	else
6745	binOp = spv::OpINotEqual;
6746	break;
6747	default:
6748	break;
6749	}
6750
6751	if (binOp != spv::OpNop) {
6752	spv::Id result = builder.createBinOp(binOp, typeId, operand1: left, operand2: right);
6753	decorations.addNoContraction(builder, t: result);
6754	decorations.addNonUniform(builder, t: result);
6755	return builder.setPrecision(id: result, precision: decorations.precision);
6756	}
6757
6758	return 0;
6759	}
6760
6761	//
6762	// Translate AST matrix operation to SPV operation, already having SPV-based operands/types.
6763	// These can be any of:
6764	//
6765	// matrix * scalar
6766	// scalar * matrix
6767	// matrix * matrix linear algebraic
6768	// matrix * vector
6769	// vector * matrix
6770	// matrix * matrix componentwise
6771	// matrix op matrix op in {+, -, /}
6772	// matrix op scalar op in {+, -, /}
6773	// scalar op matrix op in {+, -, /}
6774	//
6775	spv::Id TGlslangToSpvTraverser::createBinaryMatrixOperation(spv::Op op, OpDecorations& decorations, spv::Id typeId,
6776	spv::Id left, spv::Id right)
6777	{
6778	bool firstClass = true;
6779
6780	// First, handle first-class matrix operations (* and matrix/scalar)
6781	switch (op) {
6782	case spv::OpFDiv:
6783	if (builder.isMatrix(resultId: left) && builder.isScalar(resultId: right)) {
6784	// turn matrix / scalar into a multiply...
6785	spv::Id resultType = builder.getTypeId(resultId: right);
6786	right = builder.createBinOp(spv::OpFDiv, typeId: resultType, operand1: builder.makeFpConstant(type: resultType, d: 1.0), operand2: right);
6787	op = spv::OpMatrixTimesScalar;
6788	} else
6789	firstClass = false;
6790	break;
6791	case spv::OpMatrixTimesScalar:
6792	if (builder.isMatrix(resultId: right) || builder.isCooperativeMatrix(resultId: right))
6793	std::swap(a&: left, b&: right);
6794	assert(builder.isScalar(right));
6795	break;
6796	case spv::OpVectorTimesMatrix:
6797	assert(builder.isVector(left));
6798	assert(builder.isMatrix(right));
6799	break;
6800	case spv::OpMatrixTimesVector:
6801	assert(builder.isMatrix(left));
6802	assert(builder.isVector(right));
6803	break;
6804	case spv::OpMatrixTimesMatrix:
6805	assert(builder.isMatrix(left));
6806	assert(builder.isMatrix(right));
6807	break;
6808	default:
6809	firstClass = false;
6810	break;
6811	}
6812
6813	if (builder.isCooperativeMatrix(resultId: left) || builder.isCooperativeMatrix(resultId: right))
6814	firstClass = true;
6815
6816	if (firstClass) {
6817	spv::Id result = builder.createBinOp(op, typeId, operand1: left, operand2: right);
6818	decorations.addNoContraction(builder, t: result);
6819	decorations.addNonUniform(builder, t: result);
6820	return builder.setPrecision(id: result, precision: decorations.precision);
6821	}
6822
6823	// Handle component-wise +, -, *, %, and / for all combinations of type.
6824	// The result type of all of them is the same type as the (a) matrix operand.
6825	// The algorithm is to:
6826	// - break the matrix(es) into vectors
6827	// - smear any scalar to a vector
6828	// - do vector operations
6829	// - make a matrix out the vector results
6830	switch (op) {
6831	case spv::OpFAdd:
6832	case spv::OpFSub:
6833	case spv::OpFDiv:
6834	case spv::OpFMod:
6835	case spv::OpFMul:
6836	{
6837	// one time set up...
6838	bool leftMat = builder.isMatrix(resultId: left);
6839	bool rightMat = builder.isMatrix(resultId: right);
6840	unsigned int numCols = leftMat ? builder.getNumColumns(resultId: left) : builder.getNumColumns(resultId: right);
6841	int numRows = leftMat ? builder.getNumRows(resultId: left) : builder.getNumRows(resultId: right);
6842	spv::Id scalarType = builder.getScalarTypeId(typeId);
6843	spv::Id vecType = builder.makeVectorType(component: scalarType, size: numRows);
6844	std::vector<spv::Id> results;
6845	spv::Id smearVec = spv::NoResult;
6846	if (builder.isScalar(resultId: left))
6847	smearVec = builder.smearScalar(precision: decorations.precision, scalarVal: left, vectorType: vecType);
6848	else if (builder.isScalar(resultId: right))
6849	smearVec = builder.smearScalar(precision: decorations.precision, scalarVal: right, vectorType: vecType);
6850
6851	// do each vector op
6852	for (unsigned int c = 0; c < numCols; ++c) {
6853	std::vector<unsigned int> indexes;
6854	indexes.push_back(x: c);
6855	spv::Id leftVec = leftMat ? builder.createCompositeExtract( composite: left, typeId: vecType, indexes) : smearVec;
6856	spv::Id rightVec = rightMat ? builder.createCompositeExtract(composite: right, typeId: vecType, indexes) : smearVec;
6857	spv::Id result = builder.createBinOp(op, typeId: vecType, operand1: leftVec, operand2: rightVec);
6858	decorations.addNoContraction(builder, t: result);
6859	decorations.addNonUniform(builder, t: result);
6860	results.push_back(x: builder.setPrecision(id: result, precision: decorations.precision));
6861	}
6862
6863	// put the pieces together
6864	spv::Id result = builder.setPrecision(id: builder.createCompositeConstruct(typeId, constituents: results), precision: decorations.precision);
6865	decorations.addNonUniform(builder, t: result);
6866	return result;
6867	}
6868	default:
6869	assert(0);
6870	return spv::NoResult;
6871	}
6872	}
6873
6874	spv::Id TGlslangToSpvTraverser::createUnaryOperation(glslang::TOperator op, OpDecorations& decorations, spv::Id typeId,
6875	spv::Id operand, glslang::TBasicType typeProxy, const spv::Builder::AccessChain::CoherentFlags &lvalueCoherentFlags,
6876	const glslang::TType &opType)
6877	{
6878	spv::Op unaryOp = spv::OpNop;
6879	int extBuiltins = -1;
6880	int libCall = -1;
6881	bool isUnsigned = isTypeUnsignedInt(type: typeProxy);
6882	bool isFloat = isTypeFloat(type: typeProxy);
6883
6884	switch (op) {
6885	case glslang::EOpNegative:
6886	if (isFloat) {
6887	unaryOp = spv::OpFNegate;
6888	if (builder.isMatrixType(typeId))
6889	return createUnaryMatrixOperation(op: unaryOp, decorations, typeId, operand, typeProxy);
6890	} else
6891	unaryOp = spv::OpSNegate;
6892	break;
6893
6894	case glslang::EOpLogicalNot:
6895	case glslang::EOpVectorLogicalNot:
6896	unaryOp = spv::OpLogicalNot;
6897	break;
6898	case glslang::EOpBitwiseNot:
6899	unaryOp = spv::OpNot;
6900	break;
6901
6902	case glslang::EOpDeterminant:
6903	libCall = spv::GLSLstd450Determinant;
6904	break;
6905	case glslang::EOpMatrixInverse:
6906	libCall = spv::GLSLstd450MatrixInverse;
6907	break;
6908	case glslang::EOpTranspose:
6909	unaryOp = spv::OpTranspose;
6910	break;
6911
6912	case glslang::EOpRadians:
6913	libCall = spv::GLSLstd450Radians;
6914	break;
6915	case glslang::EOpDegrees:
6916	libCall = spv::GLSLstd450Degrees;
6917	break;
6918	case glslang::EOpSin:
6919	libCall = spv::GLSLstd450Sin;
6920	break;
6921	case glslang::EOpCos:
6922	libCall = spv::GLSLstd450Cos;
6923	break;
6924	case glslang::EOpTan:
6925	libCall = spv::GLSLstd450Tan;
6926	break;
6927	case glslang::EOpAcos:
6928	libCall = spv::GLSLstd450Acos;
6929	break;
6930	case glslang::EOpAsin:
6931	libCall = spv::GLSLstd450Asin;
6932	break;
6933	case glslang::EOpAtan:
6934	libCall = spv::GLSLstd450Atan;
6935	break;
6936
6937	case glslang::EOpAcosh:
6938	libCall = spv::GLSLstd450Acosh;
6939	break;
6940	case glslang::EOpAsinh:
6941	libCall = spv::GLSLstd450Asinh;
6942	break;
6943	case glslang::EOpAtanh:
6944	libCall = spv::GLSLstd450Atanh;
6945	break;
6946	case glslang::EOpTanh:
6947	libCall = spv::GLSLstd450Tanh;
6948	break;
6949	case glslang::EOpCosh:
6950	libCall = spv::GLSLstd450Cosh;
6951	break;
6952	case glslang::EOpSinh:
6953	libCall = spv::GLSLstd450Sinh;
6954	break;
6955
6956	case glslang::EOpLength:
6957	libCall = spv::GLSLstd450Length;
6958	break;
6959	case glslang::EOpNormalize:
6960	libCall = spv::GLSLstd450Normalize;
6961	break;
6962
6963	case glslang::EOpExp:
6964	libCall = spv::GLSLstd450Exp;
6965	break;
6966	case glslang::EOpLog:
6967	libCall = spv::GLSLstd450Log;
6968	break;
6969	case glslang::EOpExp2:
6970	libCall = spv::GLSLstd450Exp2;
6971	break;
6972	case glslang::EOpLog2:
6973	libCall = spv::GLSLstd450Log2;
6974	break;
6975	case glslang::EOpSqrt:
6976	libCall = spv::GLSLstd450Sqrt;
6977	break;
6978	case glslang::EOpInverseSqrt:
6979	libCall = spv::GLSLstd450InverseSqrt;
6980	break;
6981
6982	case glslang::EOpFloor:
6983	libCall = spv::GLSLstd450Floor;
6984	break;
6985	case glslang::EOpTrunc:
6986	libCall = spv::GLSLstd450Trunc;
6987	break;
6988	case glslang::EOpRound:
6989	libCall = spv::GLSLstd450Round;
6990	break;
6991	case glslang::EOpRoundEven:
6992	libCall = spv::GLSLstd450RoundEven;
6993	break;
6994	case glslang::EOpCeil:
6995	libCall = spv::GLSLstd450Ceil;
6996	break;
6997	case glslang::EOpFract:
6998	libCall = spv::GLSLstd450Fract;
6999	break;
7000
7001	case glslang::EOpIsNan:
7002	unaryOp = spv::OpIsNan;
7003	break;
7004	case glslang::EOpIsInf:
7005	unaryOp = spv::OpIsInf;
7006	break;
7007	case glslang::EOpIsFinite:
7008	unaryOp = spv::OpIsFinite;
7009	break;
7010
7011	case glslang::EOpFloatBitsToInt:
7012	case glslang::EOpFloatBitsToUint:
7013	case glslang::EOpIntBitsToFloat:
7014	case glslang::EOpUintBitsToFloat:
7015	case glslang::EOpDoubleBitsToInt64:
7016	case glslang::EOpDoubleBitsToUint64:
7017	case glslang::EOpInt64BitsToDouble:
7018	case glslang::EOpUint64BitsToDouble:
7019	case glslang::EOpFloat16BitsToInt16:
7020	case glslang::EOpFloat16BitsToUint16:
7021	case glslang::EOpInt16BitsToFloat16:
7022	case glslang::EOpUint16BitsToFloat16:
7023	unaryOp = spv::OpBitcast;
7024	break;
7025
7026	case glslang::EOpPackSnorm2x16:
7027	libCall = spv::GLSLstd450PackSnorm2x16;
7028	break;
7029	case glslang::EOpUnpackSnorm2x16:
7030	libCall = spv::GLSLstd450UnpackSnorm2x16;
7031	break;
7032	case glslang::EOpPackUnorm2x16:
7033	libCall = spv::GLSLstd450PackUnorm2x16;
7034	break;
7035	case glslang::EOpUnpackUnorm2x16:
7036	libCall = spv::GLSLstd450UnpackUnorm2x16;
7037	break;
7038	case glslang::EOpPackHalf2x16:
7039	libCall = spv::GLSLstd450PackHalf2x16;
7040	break;
7041	case glslang::EOpUnpackHalf2x16:
7042	libCall = spv::GLSLstd450UnpackHalf2x16;
7043	break;
7044	case glslang::EOpPackSnorm4x8:
7045	libCall = spv::GLSLstd450PackSnorm4x8;
7046	break;
7047	case glslang::EOpUnpackSnorm4x8:
7048	libCall = spv::GLSLstd450UnpackSnorm4x8;
7049	break;
7050	case glslang::EOpPackUnorm4x8:
7051	libCall = spv::GLSLstd450PackUnorm4x8;
7052	break;
7053	case glslang::EOpUnpackUnorm4x8:
7054	libCall = spv::GLSLstd450UnpackUnorm4x8;
7055	break;
7056	case glslang::EOpPackDouble2x32:
7057	libCall = spv::GLSLstd450PackDouble2x32;
7058	break;
7059	case glslang::EOpUnpackDouble2x32:
7060	libCall = spv::GLSLstd450UnpackDouble2x32;
7061	break;
7062
7063	case glslang::EOpPackInt2x32:
7064	case glslang::EOpUnpackInt2x32:
7065	case glslang::EOpPackUint2x32:
7066	case glslang::EOpUnpackUint2x32:
7067	case glslang::EOpPack16:
7068	case glslang::EOpPack32:
7069	case glslang::EOpPack64:
7070	case glslang::EOpUnpack32:
7071	case glslang::EOpUnpack16:
7072	case glslang::EOpUnpack8:
7073	case glslang::EOpPackInt2x16:
7074	case glslang::EOpUnpackInt2x16:
7075	case glslang::EOpPackUint2x16:
7076	case glslang::EOpUnpackUint2x16:
7077	case glslang::EOpPackInt4x16:
7078	case glslang::EOpUnpackInt4x16:
7079	case glslang::EOpPackUint4x16:
7080	case glslang::EOpUnpackUint4x16:
7081	case glslang::EOpPackFloat2x16:
7082	case glslang::EOpUnpackFloat2x16:
7083	unaryOp = spv::OpBitcast;
7084	break;
7085
7086	case glslang::EOpDPdx:
7087	unaryOp = spv::OpDPdx;
7088	break;
7089	case glslang::EOpDPdy:
7090	unaryOp = spv::OpDPdy;
7091	break;
7092	case glslang::EOpFwidth:
7093	unaryOp = spv::OpFwidth;
7094	break;
7095
7096	case glslang::EOpAny:
7097	unaryOp = spv::OpAny;
7098	break;
7099	case glslang::EOpAll:
7100	unaryOp = spv::OpAll;
7101	break;
7102
7103	case glslang::EOpAbs:
7104	if (isFloat)
7105	libCall = spv::GLSLstd450FAbs;
7106	else
7107	libCall = spv::GLSLstd450SAbs;
7108	break;
7109	case glslang::EOpSign:
7110	if (isFloat)
7111	libCall = spv::GLSLstd450FSign;
7112	else
7113	libCall = spv::GLSLstd450SSign;
7114	break;
7115
7116	case glslang::EOpDPdxFine:
7117	unaryOp = spv::OpDPdxFine;
7118	break;
7119	case glslang::EOpDPdyFine:
7120	unaryOp = spv::OpDPdyFine;
7121	break;
7122	case glslang::EOpFwidthFine:
7123	unaryOp = spv::OpFwidthFine;
7124	break;
7125	case glslang::EOpDPdxCoarse:
7126	unaryOp = spv::OpDPdxCoarse;
7127	break;
7128	case glslang::EOpDPdyCoarse:
7129	unaryOp = spv::OpDPdyCoarse;
7130	break;
7131	case glslang::EOpFwidthCoarse:
7132	unaryOp = spv::OpFwidthCoarse;
7133	break;
7134	case glslang::EOpRayQueryProceed:
7135	unaryOp = spv::OpRayQueryProceedKHR;
7136	break;
7137	case glslang::EOpRayQueryGetRayTMin:
7138	unaryOp = spv::OpRayQueryGetRayTMinKHR;
7139	break;
7140	case glslang::EOpRayQueryGetRayFlags:
7141	unaryOp = spv::OpRayQueryGetRayFlagsKHR;
7142	break;
7143	case glslang::EOpRayQueryGetWorldRayOrigin:
7144	unaryOp = spv::OpRayQueryGetWorldRayOriginKHR;
7145	break;
7146	case glslang::EOpRayQueryGetWorldRayDirection:
7147	unaryOp = spv::OpRayQueryGetWorldRayDirectionKHR;
7148	break;
7149	case glslang::EOpRayQueryGetIntersectionCandidateAABBOpaque:
7150	unaryOp = spv::OpRayQueryGetIntersectionCandidateAABBOpaqueKHR;
7151	break;
7152	case glslang::EOpInterpolateAtCentroid:
7153	if (typeProxy == glslang::EbtFloat16)
7154	builder.addExtension(ext: spv::E_SPV_AMD_gpu_shader_half_float);
7155	libCall = spv::GLSLstd450InterpolateAtCentroid;
7156	break;
7157	case glslang::EOpAtomicCounterIncrement:
7158	case glslang::EOpAtomicCounterDecrement:
7159	case glslang::EOpAtomicCounter:
7160	{
7161	// Handle all of the atomics in one place, in createAtomicOperation()
7162	std::vector<spv::Id> operands;
7163	operands.push_back(x: operand);
7164	return createAtomicOperation(op, precision: decorations.precision, typeId, operands, typeProxy, lvalueCoherentFlags, opType);
7165	}
7166
7167	case glslang::EOpBitFieldReverse:
7168	unaryOp = spv::OpBitReverse;
7169	break;
7170	case glslang::EOpBitCount:
7171	unaryOp = spv::OpBitCount;
7172	break;
7173	case glslang::EOpFindLSB:
7174	libCall = spv::GLSLstd450FindILsb;
7175	break;
7176	case glslang::EOpFindMSB:
7177	if (isUnsigned)
7178	libCall = spv::GLSLstd450FindUMsb;
7179	else
7180	libCall = spv::GLSLstd450FindSMsb;
7181	break;
7182
7183	case glslang::EOpCountLeadingZeros:
7184	builder.addCapability(cap: spv::CapabilityIntegerFunctions2INTEL);
7185	builder.addExtension(ext: "SPV_INTEL_shader_integer_functions2");
7186	unaryOp = spv::OpUCountLeadingZerosINTEL;
7187	break;
7188
7189	case glslang::EOpCountTrailingZeros:
7190	builder.addCapability(cap: spv::CapabilityIntegerFunctions2INTEL);
7191	builder.addExtension(ext: "SPV_INTEL_shader_integer_functions2");
7192	unaryOp = spv::OpUCountTrailingZerosINTEL;
7193	break;
7194
7195	case glslang::EOpBallot:
7196	case glslang::EOpReadFirstInvocation:
7197	case glslang::EOpAnyInvocation:
7198	case glslang::EOpAllInvocations:
7199	case glslang::EOpAllInvocationsEqual:
7200	case glslang::EOpMinInvocations:
7201	case glslang::EOpMaxInvocations:
7202	case glslang::EOpAddInvocations:
7203	case glslang::EOpMinInvocationsNonUniform:
7204	case glslang::EOpMaxInvocationsNonUniform:
7205	case glslang::EOpAddInvocationsNonUniform:
7206	case glslang::EOpMinInvocationsInclusiveScan:
7207	case glslang::EOpMaxInvocationsInclusiveScan:
7208	case glslang::EOpAddInvocationsInclusiveScan:
7209	case glslang::EOpMinInvocationsInclusiveScanNonUniform:
7210	case glslang::EOpMaxInvocationsInclusiveScanNonUniform:
7211	case glslang::EOpAddInvocationsInclusiveScanNonUniform:
7212	case glslang::EOpMinInvocationsExclusiveScan:
7213	case glslang::EOpMaxInvocationsExclusiveScan:
7214	case glslang::EOpAddInvocationsExclusiveScan:
7215	case glslang::EOpMinInvocationsExclusiveScanNonUniform:
7216	case glslang::EOpMaxInvocationsExclusiveScanNonUniform:
7217	case glslang::EOpAddInvocationsExclusiveScanNonUniform:
7218	{
7219	std::vector<spv::Id> operands;
7220	operands.push_back(x: operand);
7221	return createInvocationsOperation(op, typeId, operands, typeProxy);
7222	}
7223	case glslang::EOpSubgroupAll:
7224	case glslang::EOpSubgroupAny:
7225	case glslang::EOpSubgroupAllEqual:
7226	case glslang::EOpSubgroupBroadcastFirst:
7227	case glslang::EOpSubgroupBallot:
7228	case glslang::EOpSubgroupInverseBallot:
7229	case glslang::EOpSubgroupBallotBitCount:
7230	case glslang::EOpSubgroupBallotInclusiveBitCount:
7231	case glslang::EOpSubgroupBallotExclusiveBitCount:
7232	case glslang::EOpSubgroupBallotFindLSB:
7233	case glslang::EOpSubgroupBallotFindMSB:
7234	case glslang::EOpSubgroupAdd:
7235	case glslang::EOpSubgroupMul:
7236	case glslang::EOpSubgroupMin:
7237	case glslang::EOpSubgroupMax:
7238	case glslang::EOpSubgroupAnd:
7239	case glslang::EOpSubgroupOr:
7240	case glslang::EOpSubgroupXor:
7241	case glslang::EOpSubgroupInclusiveAdd:
7242	case glslang::EOpSubgroupInclusiveMul:
7243	case glslang::EOpSubgroupInclusiveMin:
7244	case glslang::EOpSubgroupInclusiveMax:
7245	case glslang::EOpSubgroupInclusiveAnd:
7246	case glslang::EOpSubgroupInclusiveOr:
7247	case glslang::EOpSubgroupInclusiveXor:
7248	case glslang::EOpSubgroupExclusiveAdd:
7249	case glslang::EOpSubgroupExclusiveMul:
7250	case glslang::EOpSubgroupExclusiveMin:
7251	case glslang::EOpSubgroupExclusiveMax:
7252	case glslang::EOpSubgroupExclusiveAnd:
7253	case glslang::EOpSubgroupExclusiveOr:
7254	case glslang::EOpSubgroupExclusiveXor:
7255	case glslang::EOpSubgroupQuadSwapHorizontal:
7256	case glslang::EOpSubgroupQuadSwapVertical:
7257	case glslang::EOpSubgroupQuadSwapDiagonal:
7258	case glslang::EOpSubgroupQuadAll:
7259	case glslang::EOpSubgroupQuadAny: {
7260	std::vector<spv::Id> operands;
7261	operands.push_back(x: operand);
7262	return createSubgroupOperation(op, typeId, operands, typeProxy);
7263	}
7264	case glslang::EOpMbcnt:
7265	extBuiltins = getExtBuiltins(name: spv::E_SPV_AMD_shader_ballot);
7266	libCall = spv::MbcntAMD;
7267	break;
7268
7269	case glslang::EOpCubeFaceIndex:
7270	extBuiltins = getExtBuiltins(name: spv::E_SPV_AMD_gcn_shader);
7271	libCall = spv::CubeFaceIndexAMD;
7272	break;
7273
7274	case glslang::EOpCubeFaceCoord:
7275	extBuiltins = getExtBuiltins(name: spv::E_SPV_AMD_gcn_shader);
7276	libCall = spv::CubeFaceCoordAMD;
7277	break;
7278	case glslang::EOpSubgroupPartition:
7279	unaryOp = spv::OpGroupNonUniformPartitionNV;
7280	break;
7281	case glslang::EOpConstructReference:
7282	unaryOp = spv::OpBitcast;
7283	break;
7284
7285	case glslang::EOpConvUint64ToAccStruct:
7286	case glslang::EOpConvUvec2ToAccStruct:
7287	unaryOp = spv::OpConvertUToAccelerationStructureKHR;
7288	break;
7289
7290	case glslang::EOpHitObjectIsEmptyNV:
7291	unaryOp = spv::OpHitObjectIsEmptyNV;
7292	break;
7293
7294	case glslang::EOpHitObjectIsMissNV:
7295	unaryOp = spv::OpHitObjectIsMissNV;
7296	break;
7297
7298	case glslang::EOpHitObjectIsHitNV:
7299	unaryOp = spv::OpHitObjectIsHitNV;
7300	break;
7301
7302	case glslang::EOpHitObjectGetObjectRayOriginNV:
7303	unaryOp = spv::OpHitObjectGetObjectRayOriginNV;
7304	break;
7305
7306	case glslang::EOpHitObjectGetObjectRayDirectionNV:
7307	unaryOp = spv::OpHitObjectGetObjectRayDirectionNV;
7308	break;
7309
7310	case glslang::EOpHitObjectGetWorldRayOriginNV:
7311	unaryOp = spv::OpHitObjectGetWorldRayOriginNV;
7312	break;
7313
7314	case glslang::EOpHitObjectGetWorldRayDirectionNV:
7315	unaryOp = spv::OpHitObjectGetWorldRayDirectionNV;
7316	break;
7317
7318	case glslang::EOpHitObjectGetObjectToWorldNV:
7319	unaryOp = spv::OpHitObjectGetObjectToWorldNV;
7320	break;
7321
7322	case glslang::EOpHitObjectGetWorldToObjectNV:
7323	unaryOp = spv::OpHitObjectGetWorldToObjectNV;
7324	break;
7325
7326	case glslang::EOpHitObjectGetRayTMinNV:
7327	unaryOp = spv::OpHitObjectGetRayTMinNV;
7328	break;
7329
7330	case glslang::EOpHitObjectGetRayTMaxNV:
7331	unaryOp = spv::OpHitObjectGetRayTMaxNV;
7332	break;
7333
7334	case glslang::EOpHitObjectGetPrimitiveIndexNV:
7335	unaryOp = spv::OpHitObjectGetPrimitiveIndexNV;
7336	break;
7337
7338	case glslang::EOpHitObjectGetInstanceIdNV:
7339	unaryOp = spv::OpHitObjectGetInstanceIdNV;
7340	break;
7341
7342	case glslang::EOpHitObjectGetInstanceCustomIndexNV:
7343	unaryOp = spv::OpHitObjectGetInstanceCustomIndexNV;
7344	break;
7345
7346	case glslang::EOpHitObjectGetGeometryIndexNV:
7347	unaryOp = spv::OpHitObjectGetGeometryIndexNV;
7348	break;
7349
7350	case glslang::EOpHitObjectGetHitKindNV:
7351	unaryOp = spv::OpHitObjectGetHitKindNV;
7352	break;
7353
7354	case glslang::EOpHitObjectGetCurrentTimeNV:
7355	unaryOp = spv::OpHitObjectGetCurrentTimeNV;
7356	break;
7357
7358	case glslang::EOpHitObjectGetShaderBindingTableRecordIndexNV:
7359	unaryOp = spv::OpHitObjectGetShaderBindingTableRecordIndexNV;
7360	break;
7361
7362	case glslang::EOpHitObjectGetShaderRecordBufferHandleNV:
7363	unaryOp = spv::OpHitObjectGetShaderRecordBufferHandleNV;
7364	break;
7365
7366	case glslang::EOpFetchMicroTriangleVertexPositionNV:
7367	unaryOp = spv::OpFetchMicroTriangleVertexPositionNV;
7368	break;
7369
7370	case glslang::EOpFetchMicroTriangleVertexBarycentricNV:
7371	unaryOp = spv::OpFetchMicroTriangleVertexBarycentricNV;
7372	break;
7373
7374	case glslang::EOpCopyObject:
7375	unaryOp = spv::OpCopyObject;
7376	break;
7377
7378	case glslang::EOpDepthAttachmentReadEXT:
7379	builder.addExtension(ext: spv::E_SPV_EXT_shader_tile_image);
7380	builder.addCapability(cap: spv::CapabilityTileImageDepthReadAccessEXT);
7381	unaryOp = spv::OpDepthAttachmentReadEXT;
7382	decorations.precision = spv::NoPrecision;
7383	break;
7384	case glslang::EOpStencilAttachmentReadEXT:
7385	builder.addExtension(ext: spv::E_SPV_EXT_shader_tile_image);
7386	builder.addCapability(cap: spv::CapabilityTileImageStencilReadAccessEXT);
7387	unaryOp = spv::OpStencilAttachmentReadEXT;
7388	decorations.precision = spv::DecorationRelaxedPrecision;
7389	break;
7390
7391	default:
7392	return 0;
7393	}
7394
7395	spv::Id id;
7396	if (libCall >= 0) {
7397	std::vector<spv::Id> args;
7398	args.push_back(x: operand);
7399	id = builder.createBuiltinCall(resultType: typeId, builtins: extBuiltins >= 0 ? extBuiltins : stdBuiltins, entryPoint: libCall, args);
7400	} else {
7401	id = builder.createUnaryOp(unaryOp, typeId, operand);
7402	}
7403
7404	decorations.addNoContraction(builder, t: id);
7405	decorations.addNonUniform(builder, t: id);
7406	return builder.setPrecision(id, precision: decorations.precision);
7407	}
7408
7409	// Create a unary operation on a matrix
7410	spv::Id TGlslangToSpvTraverser::createUnaryMatrixOperation(spv::Op op, OpDecorations& decorations, spv::Id typeId,
7411	spv::Id operand, glslang::TBasicType /* typeProxy */)
7412	{
7413	// Handle unary operations vector by vector.
7414	// The result type is the same type as the original type.
7415	// The algorithm is to:
7416	// - break the matrix into vectors
7417	// - apply the operation to each vector
7418	// - make a matrix out the vector results
7419
7420	// get the types sorted out
7421	int numCols = builder.getNumColumns(resultId: operand);
7422	int numRows = builder.getNumRows(resultId: operand);
7423	spv::Id srcVecType = builder.makeVectorType(component: builder.getScalarTypeId(typeId: builder.getTypeId(resultId: operand)), size: numRows);
7424	spv::Id destVecType = builder.makeVectorType(component: builder.getScalarTypeId(typeId), size: numRows);
7425	std::vector<spv::Id> results;
7426
7427	// do each vector op
7428	for (int c = 0; c < numCols; ++c) {
7429	std::vector<unsigned int> indexes;
7430	indexes.push_back(x: c);
7431	spv::Id srcVec = builder.createCompositeExtract(composite: operand, typeId: srcVecType, indexes);
7432	spv::Id destVec = builder.createUnaryOp(op, typeId: destVecType, operand: srcVec);
7433	decorations.addNoContraction(builder, t: destVec);
7434	decorations.addNonUniform(builder, t: destVec);
7435	results.push_back(x: builder.setPrecision(id: destVec, precision: decorations.precision));
7436	}
7437
7438	// put the pieces together
7439	spv::Id result = builder.setPrecision(id: builder.createCompositeConstruct(typeId, constituents: results), precision: decorations.precision);
7440	decorations.addNonUniform(builder, t: result);
7441	return result;
7442	}
7443
7444	// For converting integers where both the bitwidth and the signedness could
7445	// change, but only do the width change here. The caller is still responsible
7446	// for the signedness conversion.
7447	// destType is the final type that will be converted to, but this function
7448	// may only be doing part of that conversion.
7449	spv::Id TGlslangToSpvTraverser::createIntWidthConversion(glslang::TOperator op, spv::Id operand, int vectorSize, spv::Id destType)
7450	{
7451	// Get the result type width, based on the type to convert to.
7452	int width = 32;
7453	switch(op) {
7454	case glslang::EOpConvInt16ToUint8:
7455	case glslang::EOpConvIntToUint8:
7456	case glslang::EOpConvInt64ToUint8:
7457	case glslang::EOpConvUint16ToInt8:
7458	case glslang::EOpConvUintToInt8:
7459	case glslang::EOpConvUint64ToInt8:
7460	width = 8;
7461	break;
7462	case glslang::EOpConvInt8ToUint16:
7463	case glslang::EOpConvIntToUint16:
7464	case glslang::EOpConvInt64ToUint16:
7465	case glslang::EOpConvUint8ToInt16:
7466	case glslang::EOpConvUintToInt16:
7467	case glslang::EOpConvUint64ToInt16:
7468	width = 16;
7469	break;
7470	case glslang::EOpConvInt8ToUint:
7471	case glslang::EOpConvInt16ToUint:
7472	case glslang::EOpConvInt64ToUint:
7473	case glslang::EOpConvUint8ToInt:
7474	case glslang::EOpConvUint16ToInt:
7475	case glslang::EOpConvUint64ToInt:
7476	width = 32;
7477	break;
7478	case glslang::EOpConvInt8ToUint64:
7479	case glslang::EOpConvInt16ToUint64:
7480	case glslang::EOpConvIntToUint64:
7481	case glslang::EOpConvUint8ToInt64:
7482	case glslang::EOpConvUint16ToInt64:
7483	case glslang::EOpConvUintToInt64:
7484	width = 64;
7485	break;
7486
7487	default:
7488	assert(false && "Default missing");
7489	break;
7490	}
7491
7492	// Get the conversion operation and result type,
7493	// based on the target width, but the source type.
7494	spv::Id type = spv::NoType;
7495	spv::Op convOp = spv::OpNop;
7496	switch(op) {
7497	case glslang::EOpConvInt8ToUint16:
7498	case glslang::EOpConvInt8ToUint:
7499	case glslang::EOpConvInt8ToUint64:
7500	case glslang::EOpConvInt16ToUint8:
7501	case glslang::EOpConvInt16ToUint:
7502	case glslang::EOpConvInt16ToUint64:
7503	case glslang::EOpConvIntToUint8:
7504	case glslang::EOpConvIntToUint16:
7505	case glslang::EOpConvIntToUint64:
7506	case glslang::EOpConvInt64ToUint8:
7507	case glslang::EOpConvInt64ToUint16:
7508	case glslang::EOpConvInt64ToUint:
7509	convOp = spv::OpSConvert;
7510	type = builder.makeIntType(width);
7511	break;
7512	default:
7513	convOp = spv::OpUConvert;
7514	type = builder.makeUintType(width);
7515	break;
7516	}
7517
7518	if (vectorSize > 0)
7519	type = builder.makeVectorType(component: type, size: vectorSize);
7520	else if (builder.getOpCode(id: destType) == spv::OpTypeCooperativeMatrixKHR ||
7521	builder.getOpCode(id: destType) == spv::OpTypeCooperativeMatrixNV) {
7522
7523	type = builder.makeCooperativeMatrixTypeWithSameShape(component: type, otherType: destType);
7524	}
7525
7526	return builder.createUnaryOp(convOp, typeId: type, operand);
7527	}
7528
7529	spv::Id TGlslangToSpvTraverser::createConversion(glslang::TOperator op, OpDecorations& decorations, spv::Id destType,
7530	spv::Id operand, glslang::TBasicType typeProxy)
7531	{
7532	spv::Op convOp = spv::OpNop;
7533	spv::Id zero = 0;
7534	spv::Id one = 0;
7535
7536	int vectorSize = builder.isVectorType(typeId: destType) ? builder.getNumTypeComponents(typeId: destType) : 0;
7537
7538	switch (op) {
7539	case glslang::EOpConvIntToBool:
7540	case glslang::EOpConvUintToBool:
7541	zero = builder.makeUintConstant(u: 0);
7542	zero = makeSmearedConstant(constant: zero, vectorSize);
7543	return builder.createBinOp(spv::OpINotEqual, typeId: destType, operand1: operand, operand2: zero);
7544	case glslang::EOpConvFloatToBool:
7545	zero = builder.makeFloatConstant(f: 0.0F);
7546	zero = makeSmearedConstant(constant: zero, vectorSize);
7547	return builder.createBinOp(spv::OpFUnordNotEqual, typeId: destType, operand1: operand, operand2: zero);
7548	case glslang::EOpConvBoolToFloat:
7549	convOp = spv::OpSelect;
7550	zero = builder.makeFloatConstant(f: 0.0F);
7551	one = builder.makeFloatConstant(f: 1.0F);
7552	break;
7553
7554	case glslang::EOpConvBoolToInt:
7555	case glslang::EOpConvBoolToInt64:
7556	if (op == glslang::EOpConvBoolToInt64) {
7557	zero = builder.makeInt64Constant(i: 0);
7558	one = builder.makeInt64Constant(i: 1);
7559	} else {
7560	zero = builder.makeIntConstant(i: 0);
7561	one = builder.makeIntConstant(i: 1);
7562	}
7563
7564	convOp = spv::OpSelect;
7565	break;
7566
7567	case glslang::EOpConvBoolToUint:
7568	case glslang::EOpConvBoolToUint64:
7569	if (op == glslang::EOpConvBoolToUint64) {
7570	zero = builder.makeUint64Constant(u: 0);
7571	one = builder.makeUint64Constant(u: 1);
7572	} else {
7573	zero = builder.makeUintConstant(u: 0);
7574	one = builder.makeUintConstant(u: 1);
7575	}
7576
7577	convOp = spv::OpSelect;
7578	break;
7579
7580	case glslang::EOpConvInt8ToFloat16:
7581	case glslang::EOpConvInt8ToFloat:
7582	case glslang::EOpConvInt8ToDouble:
7583	case glslang::EOpConvInt16ToFloat16:
7584	case glslang::EOpConvInt16ToFloat:
7585	case glslang::EOpConvInt16ToDouble:
7586	case glslang::EOpConvIntToFloat16:
7587	case glslang::EOpConvIntToFloat:
7588	case glslang::EOpConvIntToDouble:
7589	case glslang::EOpConvInt64ToFloat:
7590	case glslang::EOpConvInt64ToDouble:
7591	case glslang::EOpConvInt64ToFloat16:
7592	convOp = spv::OpConvertSToF;
7593	break;
7594
7595	case glslang::EOpConvUint8ToFloat16:
7596	case glslang::EOpConvUint8ToFloat:
7597	case glslang::EOpConvUint8ToDouble:
7598	case glslang::EOpConvUint16ToFloat16:
7599	case glslang::EOpConvUint16ToFloat:
7600	case glslang::EOpConvUint16ToDouble:
7601	case glslang::EOpConvUintToFloat16:
7602	case glslang::EOpConvUintToFloat:
7603	case glslang::EOpConvUintToDouble:
7604	case glslang::EOpConvUint64ToFloat:
7605	case glslang::EOpConvUint64ToDouble:
7606	case glslang::EOpConvUint64ToFloat16:
7607	convOp = spv::OpConvertUToF;
7608	break;
7609
7610	case glslang::EOpConvFloat16ToInt8:
7611	case glslang::EOpConvFloatToInt8:
7612	case glslang::EOpConvDoubleToInt8:
7613	case glslang::EOpConvFloat16ToInt16:
7614	case glslang::EOpConvFloatToInt16:
7615	case glslang::EOpConvDoubleToInt16:
7616	case glslang::EOpConvFloat16ToInt:
7617	case glslang::EOpConvFloatToInt:
7618	case glslang::EOpConvDoubleToInt:
7619	case glslang::EOpConvFloat16ToInt64:
7620	case glslang::EOpConvFloatToInt64:
7621	case glslang::EOpConvDoubleToInt64:
7622	convOp = spv::OpConvertFToS;
7623	break;
7624
7625	case glslang::EOpConvUint8ToInt8:
7626	case glslang::EOpConvInt8ToUint8:
7627	case glslang::EOpConvUint16ToInt16:
7628	case glslang::EOpConvInt16ToUint16:
7629	case glslang::EOpConvUintToInt:
7630	case glslang::EOpConvIntToUint:
7631	case glslang::EOpConvUint64ToInt64:
7632	case glslang::EOpConvInt64ToUint64:
7633	if (builder.isInSpecConstCodeGenMode()) {
7634	// Build zero scalar or vector for OpIAdd.
7635	if(op == glslang::EOpConvUint8ToInt8 || op == glslang::EOpConvInt8ToUint8) {
7636	zero = builder.makeUint8Constant(u: 0);
7637	} else if (op == glslang::EOpConvUint16ToInt16 || op == glslang::EOpConvInt16ToUint16) {
7638	zero = builder.makeUint16Constant(u: 0);
7639	} else if (op == glslang::EOpConvUint64ToInt64 || op == glslang::EOpConvInt64ToUint64) {
7640	zero = builder.makeUint64Constant(u: 0);
7641	} else {
7642	zero = builder.makeUintConstant(u: 0);
7643	}
7644	zero = makeSmearedConstant(constant: zero, vectorSize);
7645	// Use OpIAdd, instead of OpBitcast to do the conversion when
7646	// generating for OpSpecConstantOp instruction.
7647	return builder.createBinOp(spv::OpIAdd, typeId: destType, operand1: operand, operand2: zero);
7648	}
7649	// For normal run-time conversion instruction, use OpBitcast.
7650	convOp = spv::OpBitcast;
7651	break;
7652
7653	case glslang::EOpConvFloat16ToUint8:
7654	case glslang::EOpConvFloatToUint8:
7655	case glslang::EOpConvDoubleToUint8:
7656	case glslang::EOpConvFloat16ToUint16:
7657	case glslang::EOpConvFloatToUint16:
7658	case glslang::EOpConvDoubleToUint16:
7659	case glslang::EOpConvFloat16ToUint:
7660	case glslang::EOpConvFloatToUint:
7661	case glslang::EOpConvDoubleToUint:
7662	case glslang::EOpConvFloatToUint64:
7663	case glslang::EOpConvDoubleToUint64:
7664	case glslang::EOpConvFloat16ToUint64:
7665	convOp = spv::OpConvertFToU;
7666	break;
7667
7668	case glslang::EOpConvInt8ToBool:
7669	case glslang::EOpConvUint8ToBool:
7670	zero = builder.makeUint8Constant(u: 0);
7671	zero = makeSmearedConstant(constant: zero, vectorSize);
7672	return builder.createBinOp(spv::OpINotEqual, typeId: destType, operand1: operand, operand2: zero);
7673	case glslang::EOpConvInt16ToBool:
7674	case glslang::EOpConvUint16ToBool:
7675	zero = builder.makeUint16Constant(u: 0);
7676	zero = makeSmearedConstant(constant: zero, vectorSize);
7677	return builder.createBinOp(spv::OpINotEqual, typeId: destType, operand1: operand, operand2: zero);
7678	case glslang::EOpConvInt64ToBool:
7679	case glslang::EOpConvUint64ToBool:
7680	zero = builder.makeUint64Constant(u: 0);
7681	zero = makeSmearedConstant(constant: zero, vectorSize);
7682	return builder.createBinOp(spv::OpINotEqual, typeId: destType, operand1: operand, operand2: zero);
7683	case glslang::EOpConvDoubleToBool:
7684	zero = builder.makeDoubleConstant(d: 0.0);
7685	zero = makeSmearedConstant(constant: zero, vectorSize);
7686	return builder.createBinOp(spv::OpFUnordNotEqual, typeId: destType, operand1: operand, operand2: zero);
7687	case glslang::EOpConvFloat16ToBool:
7688	zero = builder.makeFloat16Constant(f16: 0.0F);
7689	zero = makeSmearedConstant(constant: zero, vectorSize);
7690	return builder.createBinOp(spv::OpFUnordNotEqual, typeId: destType, operand1: operand, operand2: zero);
7691	case glslang::EOpConvBoolToDouble:
7692	convOp = spv::OpSelect;
7693	zero = builder.makeDoubleConstant(d: 0.0);
7694	one = builder.makeDoubleConstant(d: 1.0);
7695	break;
7696	case glslang::EOpConvBoolToFloat16:
7697	convOp = spv::OpSelect;
7698	zero = builder.makeFloat16Constant(f16: 0.0F);
7699	one = builder.makeFloat16Constant(f16: 1.0F);
7700	break;
7701	case glslang::EOpConvBoolToInt8:
7702	zero = builder.makeInt8Constant(i: 0);
7703	one = builder.makeInt8Constant(i: 1);
7704	convOp = spv::OpSelect;
7705	break;
7706	case glslang::EOpConvBoolToUint8:
7707	zero = builder.makeUint8Constant(u: 0);
7708	one = builder.makeUint8Constant(u: 1);
7709	convOp = spv::OpSelect;
7710	break;
7711	case glslang::EOpConvBoolToInt16:
7712	zero = builder.makeInt16Constant(i: 0);
7713	one = builder.makeInt16Constant(i: 1);
7714	convOp = spv::OpSelect;
7715	break;
7716	case glslang::EOpConvBoolToUint16:
7717	zero = builder.makeUint16Constant(u: 0);
7718	one = builder.makeUint16Constant(u: 1);
7719	convOp = spv::OpSelect;
7720	break;
7721	case glslang::EOpConvDoubleToFloat:
7722	case glslang::EOpConvFloatToDouble:
7723	case glslang::EOpConvDoubleToFloat16:
7724	case glslang::EOpConvFloat16ToDouble:
7725	case glslang::EOpConvFloatToFloat16:
7726	case glslang::EOpConvFloat16ToFloat:
7727	convOp = spv::OpFConvert;
7728	if (builder.isMatrixType(typeId: destType))
7729	return createUnaryMatrixOperation(op: convOp, decorations, typeId: destType, operand, typeProxy);
7730	break;
7731
7732	case glslang::EOpConvInt8ToInt16:
7733	case glslang::EOpConvInt8ToInt:
7734	case glslang::EOpConvInt8ToInt64:
7735	case glslang::EOpConvInt16ToInt8:
7736	case glslang::EOpConvInt16ToInt:
7737	case glslang::EOpConvInt16ToInt64:
7738	case glslang::EOpConvIntToInt8:
7739	case glslang::EOpConvIntToInt16:
7740	case glslang::EOpConvIntToInt64:
7741	case glslang::EOpConvInt64ToInt8:
7742	case glslang::EOpConvInt64ToInt16:
7743	case glslang::EOpConvInt64ToInt:
7744	convOp = spv::OpSConvert;
7745	break;
7746
7747	case glslang::EOpConvUint8ToUint16:
7748	case glslang::EOpConvUint8ToUint:
7749	case glslang::EOpConvUint8ToUint64:
7750	case glslang::EOpConvUint16ToUint8:
7751	case glslang::EOpConvUint16ToUint:
7752	case glslang::EOpConvUint16ToUint64:
7753	case glslang::EOpConvUintToUint8:
7754	case glslang::EOpConvUintToUint16:
7755	case glslang::EOpConvUintToUint64:
7756	case glslang::EOpConvUint64ToUint8:
7757	case glslang::EOpConvUint64ToUint16:
7758	case glslang::EOpConvUint64ToUint:
7759	convOp = spv::OpUConvert;
7760	break;
7761
7762	case glslang::EOpConvInt8ToUint16:
7763	case glslang::EOpConvInt8ToUint:
7764	case glslang::EOpConvInt8ToUint64:
7765	case glslang::EOpConvInt16ToUint8:
7766	case glslang::EOpConvInt16ToUint:
7767	case glslang::EOpConvInt16ToUint64:
7768	case glslang::EOpConvIntToUint8:
7769	case glslang::EOpConvIntToUint16:
7770	case glslang::EOpConvIntToUint64:
7771	case glslang::EOpConvInt64ToUint8:
7772	case glslang::EOpConvInt64ToUint16:
7773	case glslang::EOpConvInt64ToUint:
7774	case glslang::EOpConvUint8ToInt16:
7775	case glslang::EOpConvUint8ToInt:
7776	case glslang::EOpConvUint8ToInt64:
7777	case glslang::EOpConvUint16ToInt8:
7778	case glslang::EOpConvUint16ToInt:
7779	case glslang::EOpConvUint16ToInt64:
7780	case glslang::EOpConvUintToInt8:
7781	case glslang::EOpConvUintToInt16:
7782	case glslang::EOpConvUintToInt64:
7783	case glslang::EOpConvUint64ToInt8:
7784	case glslang::EOpConvUint64ToInt16:
7785	case glslang::EOpConvUint64ToInt:
7786	// OpSConvert/OpUConvert + OpBitCast
7787	operand = createIntWidthConversion(op, operand, vectorSize, destType);
7788
7789	if (builder.isInSpecConstCodeGenMode()) {
7790	// Build zero scalar or vector for OpIAdd.
7791	switch(op) {
7792	case glslang::EOpConvInt16ToUint8:
7793	case glslang::EOpConvIntToUint8:
7794	case glslang::EOpConvInt64ToUint8:
7795	case glslang::EOpConvUint16ToInt8:
7796	case glslang::EOpConvUintToInt8:
7797	case glslang::EOpConvUint64ToInt8:
7798	zero = builder.makeUint8Constant(u: 0);
7799	break;
7800	case glslang::EOpConvInt8ToUint16:
7801	case glslang::EOpConvIntToUint16:
7802	case glslang::EOpConvInt64ToUint16:
7803	case glslang::EOpConvUint8ToInt16:
7804	case glslang::EOpConvUintToInt16:
7805	case glslang::EOpConvUint64ToInt16:
7806	zero = builder.makeUint16Constant(u: 0);
7807	break;
7808	case glslang::EOpConvInt8ToUint:
7809	case glslang::EOpConvInt16ToUint:
7810	case glslang::EOpConvInt64ToUint:
7811	case glslang::EOpConvUint8ToInt:
7812	case glslang::EOpConvUint16ToInt:
7813	case glslang::EOpConvUint64ToInt:
7814	zero = builder.makeUintConstant(u: 0);
7815	break;
7816	case glslang::EOpConvInt8ToUint64:
7817	case glslang::EOpConvInt16ToUint64:
7818	case glslang::EOpConvIntToUint64:
7819	case glslang::EOpConvUint8ToInt64:
7820	case glslang::EOpConvUint16ToInt64:
7821	case glslang::EOpConvUintToInt64:
7822	zero = builder.makeUint64Constant(u: 0);
7823	break;
7824	default:
7825	assert(false && "Default missing");
7826	break;
7827	}
7828	zero = makeSmearedConstant(constant: zero, vectorSize);
7829	// Use OpIAdd, instead of OpBitcast to do the conversion when
7830	// generating for OpSpecConstantOp instruction.
7831	return builder.createBinOp(spv::OpIAdd, typeId: destType, operand1: operand, operand2: zero);
7832	}
7833	// For normal run-time conversion instruction, use OpBitcast.
7834	convOp = spv::OpBitcast;
7835	break;
7836	case glslang::EOpConvUint64ToPtr:
7837	convOp = spv::OpConvertUToPtr;
7838	break;
7839	case glslang::EOpConvPtrToUint64:
7840	convOp = spv::OpConvertPtrToU;
7841	break;
7842	case glslang::EOpConvPtrToUvec2:
7843	case glslang::EOpConvUvec2ToPtr:
7844	convOp = spv::OpBitcast;
7845	break;
7846
7847	default:
7848	break;
7849	}
7850
7851	spv::Id result = 0;
7852	if (convOp == spv::OpNop)
7853	return result;
7854
7855	if (convOp == spv::OpSelect) {
7856	zero = makeSmearedConstant(constant: zero, vectorSize);
7857	one = makeSmearedConstant(constant: one, vectorSize);
7858	result = builder.createTriOp(convOp, typeId: destType, operand1: operand, operand2: one, operand3: zero);
7859	} else
7860	result = builder.createUnaryOp(convOp, typeId: destType, operand);
7861
7862	result = builder.setPrecision(id: result, precision: decorations.precision);
7863	decorations.addNonUniform(builder, t: result);
7864	return result;
7865	}
7866
7867	spv::Id TGlslangToSpvTraverser::makeSmearedConstant(spv::Id constant, int vectorSize)
7868	{
7869	if (vectorSize == 0)
7870	return constant;
7871
7872	spv::Id vectorTypeId = builder.makeVectorType(component: builder.getTypeId(resultId: constant), size: vectorSize);
7873	std::vector<spv::Id> components;
7874	for (int c = 0; c < vectorSize; ++c)
7875	components.push_back(x: constant);
7876	return builder.makeCompositeConstant(type: vectorTypeId, comps: components);
7877	}
7878
7879	// For glslang ops that map to SPV atomic opCodes
7880	spv::Id TGlslangToSpvTraverser::createAtomicOperation(glslang::TOperator op, spv::Decoration /*precision*/,
7881	spv::Id typeId, std::vector<spv::Id>& operands, glslang::TBasicType typeProxy,
7882	const spv::Builder::AccessChain::CoherentFlags &lvalueCoherentFlags, const glslang::TType &opType)
7883	{
7884	spv::Op opCode = spv::OpNop;
7885
7886	switch (op) {
7887	case glslang::EOpAtomicAdd:
7888	case glslang::EOpImageAtomicAdd:
7889	case glslang::EOpAtomicCounterAdd:
7890	opCode = spv::OpAtomicIAdd;
7891	if (typeProxy == glslang::EbtFloat16 || typeProxy == glslang::EbtFloat || typeProxy == glslang::EbtDouble) {
7892	opCode = spv::OpAtomicFAddEXT;
7893	if (typeProxy == glslang::EbtFloat16 &&
7894	(opType.getVectorSize() == 2 || opType.getVectorSize() == 4)) {
7895	builder.addExtension(ext: spv::E_SPV_NV_shader_atomic_fp16_vector);
7896	builder.addCapability(cap: spv::CapabilityAtomicFloat16VectorNV);
7897	} else {
7898	builder.addExtension(ext: spv::E_SPV_EXT_shader_atomic_float_add);
7899	if (typeProxy == glslang::EbtFloat16) {
7900	builder.addExtension(ext: spv::E_SPV_EXT_shader_atomic_float16_add);
7901	builder.addCapability(cap: spv::CapabilityAtomicFloat16AddEXT);
7902	} else if (typeProxy == glslang::EbtFloat) {
7903	builder.addCapability(cap: spv::CapabilityAtomicFloat32AddEXT);
7904	} else {
7905	builder.addCapability(cap: spv::CapabilityAtomicFloat64AddEXT);
7906	}
7907	}
7908	}
7909	break;
7910	case glslang::EOpAtomicSubtract:
7911	case glslang::EOpAtomicCounterSubtract:
7912	opCode = spv::OpAtomicISub;
7913	break;
7914	case glslang::EOpAtomicMin:
7915	case glslang::EOpImageAtomicMin:
7916	case glslang::EOpAtomicCounterMin:
7917	if (typeProxy == glslang::EbtFloat16 || typeProxy == glslang::EbtFloat || typeProxy == glslang::EbtDouble) {
7918	opCode = spv::OpAtomicFMinEXT;
7919	if (typeProxy == glslang::EbtFloat16 &&
7920	(opType.getVectorSize() == 2 || opType.getVectorSize() == 4)) {
7921	builder.addExtension(ext: spv::E_SPV_NV_shader_atomic_fp16_vector);
7922	builder.addCapability(cap: spv::CapabilityAtomicFloat16VectorNV);
7923	} else {
7924	builder.addExtension(ext: spv::E_SPV_EXT_shader_atomic_float_min_max);
7925	if (typeProxy == glslang::EbtFloat16)
7926	builder.addCapability(cap: spv::CapabilityAtomicFloat16MinMaxEXT);
7927	else if (typeProxy == glslang::EbtFloat)
7928	builder.addCapability(cap: spv::CapabilityAtomicFloat32MinMaxEXT);
7929	else
7930	builder.addCapability(cap: spv::CapabilityAtomicFloat64MinMaxEXT);
7931	}
7932	} else if (typeProxy == glslang::EbtUint || typeProxy == glslang::EbtUint64) {
7933	opCode = spv::OpAtomicUMin;
7934	} else {
7935	opCode = spv::OpAtomicSMin;
7936	}
7937	break;
7938	case glslang::EOpAtomicMax:
7939	case glslang::EOpImageAtomicMax:
7940	case glslang::EOpAtomicCounterMax:
7941	if (typeProxy == glslang::EbtFloat16 || typeProxy == glslang::EbtFloat || typeProxy == glslang::EbtDouble) {
7942	opCode = spv::OpAtomicFMaxEXT;
7943	if (typeProxy == glslang::EbtFloat16 &&
7944	(opType.getVectorSize() == 2 || opType.getVectorSize() == 4)) {
7945	builder.addExtension(ext: spv::E_SPV_NV_shader_atomic_fp16_vector);
7946	builder.addCapability(cap: spv::CapabilityAtomicFloat16VectorNV);
7947	} else {
7948	builder.addExtension(ext: spv::E_SPV_EXT_shader_atomic_float_min_max);
7949	if (typeProxy == glslang::EbtFloat16)
7950	builder.addCapability(cap: spv::CapabilityAtomicFloat16MinMaxEXT);
7951	else if (typeProxy == glslang::EbtFloat)
7952	builder.addCapability(cap: spv::CapabilityAtomicFloat32MinMaxEXT);
7953	else
7954	builder.addCapability(cap: spv::CapabilityAtomicFloat64MinMaxEXT);
7955	}
7956	} else if (typeProxy == glslang::EbtUint || typeProxy == glslang::EbtUint64) {
7957	opCode = spv::OpAtomicUMax;
7958	} else {
7959	opCode = spv::OpAtomicSMax;
7960	}
7961	break;
7962	case glslang::EOpAtomicAnd:
7963	case glslang::EOpImageAtomicAnd:
7964	case glslang::EOpAtomicCounterAnd:
7965	opCode = spv::OpAtomicAnd;
7966	break;
7967	case glslang::EOpAtomicOr:
7968	case glslang::EOpImageAtomicOr:
7969	case glslang::EOpAtomicCounterOr:
7970	opCode = spv::OpAtomicOr;
7971	break;
7972	case glslang::EOpAtomicXor:
7973	case glslang::EOpImageAtomicXor:
7974	case glslang::EOpAtomicCounterXor:
7975	opCode = spv::OpAtomicXor;
7976	break;
7977	case glslang::EOpAtomicExchange:
7978	case glslang::EOpImageAtomicExchange:
7979	case glslang::EOpAtomicCounterExchange:
7980	if ((typeProxy == glslang::EbtFloat16) &&
7981	(opType.getVectorSize() == 2 || opType.getVectorSize() == 4)) {
7982	builder.addExtension(ext: spv::E_SPV_NV_shader_atomic_fp16_vector);
7983	builder.addCapability(cap: spv::CapabilityAtomicFloat16VectorNV);
7984	}
7985
7986	opCode = spv::OpAtomicExchange;
7987	break;
7988	case glslang::EOpAtomicCompSwap:
7989	case glslang::EOpImageAtomicCompSwap:
7990	case glslang::EOpAtomicCounterCompSwap:
7991	opCode = spv::OpAtomicCompareExchange;
7992	break;
7993	case glslang::EOpAtomicCounterIncrement:
7994	opCode = spv::OpAtomicIIncrement;
7995	break;
7996	case glslang::EOpAtomicCounterDecrement:
7997	opCode = spv::OpAtomicIDecrement;
7998	break;
7999	case glslang::EOpAtomicCounter:
8000	case glslang::EOpImageAtomicLoad:
8001	case glslang::EOpAtomicLoad:
8002	opCode = spv::OpAtomicLoad;
8003	break;
8004	case glslang::EOpAtomicStore:
8005	case glslang::EOpImageAtomicStore:
8006	opCode = spv::OpAtomicStore;
8007	break;
8008	default:
8009	assert(0);
8010	break;
8011	}
8012
8013	if (typeProxy == glslang::EbtInt64 || typeProxy == glslang::EbtUint64)
8014	builder.addCapability(cap: spv::CapabilityInt64Atomics);
8015
8016	// Sort out the operands
8017	// - mapping from glslang -> SPV
8018	// - there are extra SPV operands that are optional in glslang
8019	// - compare-exchange swaps the value and comparator
8020	// - compare-exchange has an extra memory semantics
8021	// - EOpAtomicCounterDecrement needs a post decrement
8022	spv::Id pointerId = 0, compareId = 0, valueId = 0;
8023	// scope defaults to Device in the old model, QueueFamilyKHR in the new model
8024	spv::Id scopeId;
8025	if (glslangIntermediate->usingVulkanMemoryModel()) {
8026	scopeId = builder.makeUintConstant(u: spv::ScopeQueueFamilyKHR);
8027	} else {
8028	scopeId = builder.makeUintConstant(u: spv::ScopeDevice);
8029	}
8030	// semantics default to relaxed
8031	spv::Id semanticsId = builder.makeUintConstant(u: lvalueCoherentFlags.isVolatile() &&
8032	glslangIntermediate->usingVulkanMemoryModel() ?
8033	spv::MemorySemanticsVolatileMask :
8034	spv::MemorySemanticsMaskNone);
8035	spv::Id semanticsId2 = semanticsId;
8036
8037	pointerId = operands[0];
8038	if (opCode == spv::OpAtomicIIncrement || opCode == spv::OpAtomicIDecrement) {
8039	// no additional operands
8040	} else if (opCode == spv::OpAtomicCompareExchange) {
8041	compareId = operands[1];
8042	valueId = operands[2];
8043	if (operands.size() > 3) {
8044	scopeId = operands[3];
8045	semanticsId = builder.makeUintConstant(
8046	u: builder.getConstantScalar(resultId: operands[4]) | builder.getConstantScalar(resultId: operands[5]));
8047	semanticsId2 = builder.makeUintConstant(
8048	u: builder.getConstantScalar(resultId: operands[6]) | builder.getConstantScalar(resultId: operands[7]));
8049	}
8050	} else if (opCode == spv::OpAtomicLoad) {
8051	if (operands.size() > 1) {
8052	scopeId = operands[1];
8053	semanticsId = builder.makeUintConstant(
8054	u: builder.getConstantScalar(resultId: operands[2]) | builder.getConstantScalar(resultId: operands[3]));
8055	}
8056	} else {
8057	// atomic store or RMW
8058	valueId = operands[1];
8059	if (operands.size() > 2) {
8060	scopeId = operands[2];
8061	semanticsId = builder.makeUintConstant
8062	(u: builder.getConstantScalar(resultId: operands[3]) | builder.getConstantScalar(resultId: operands[4]));
8063	}
8064	}
8065
8066	// Check for capabilities
8067	unsigned semanticsImmediate = builder.getConstantScalar(resultId: semanticsId) | builder.getConstantScalar(resultId: semanticsId2);
8068	if (semanticsImmediate & (spv::MemorySemanticsMakeAvailableKHRMask |
8069	spv::MemorySemanticsMakeVisibleKHRMask |
8070	spv::MemorySemanticsOutputMemoryKHRMask |
8071	spv::MemorySemanticsVolatileMask)) {
8072	builder.addCapability(cap: spv::CapabilityVulkanMemoryModelKHR);
8073	}
8074
8075	if (builder.getConstantScalar(resultId: scopeId) == spv::ScopeQueueFamily) {
8076	builder.addCapability(cap: spv::CapabilityVulkanMemoryModelKHR);
8077	}
8078
8079	if (glslangIntermediate->usingVulkanMemoryModel() && builder.getConstantScalar(resultId: scopeId) == spv::ScopeDevice) {
8080	builder.addCapability(cap: spv::CapabilityVulkanMemoryModelDeviceScopeKHR);
8081	}
8082
8083	std::vector<spv::Id> spvAtomicOperands; // hold the spv operands
8084	spvAtomicOperands.reserve(n: 6);
8085	spvAtomicOperands.push_back(x: pointerId);
8086	spvAtomicOperands.push_back(x: scopeId);
8087	spvAtomicOperands.push_back(x: semanticsId);
8088	if (opCode == spv::OpAtomicCompareExchange) {
8089	spvAtomicOperands.push_back(x: semanticsId2);
8090	spvAtomicOperands.push_back(x: valueId);
8091	spvAtomicOperands.push_back(x: compareId);
8092	} else if (opCode != spv::OpAtomicLoad && opCode != spv::OpAtomicIIncrement && opCode != spv::OpAtomicIDecrement) {
8093	spvAtomicOperands.push_back(x: valueId);
8094	}
8095
8096	if (opCode == spv::OpAtomicStore) {
8097	builder.createNoResultOp(opCode, operands: spvAtomicOperands);
8098	return 0;
8099	} else {
8100	spv::Id resultId = builder.createOp(opCode, typeId, operands: spvAtomicOperands);
8101
8102	// GLSL and HLSL atomic-counter decrement return post-decrement value,
8103	// while SPIR-V returns pre-decrement value. Translate between these semantics.
8104	if (op == glslang::EOpAtomicCounterDecrement)
8105	resultId = builder.createBinOp(spv::OpISub, typeId, operand1: resultId, operand2: builder.makeIntConstant(i: 1));
8106
8107	return resultId;
8108	}
8109	}
8110
8111	// Create group invocation operations.
8112	spv::Id TGlslangToSpvTraverser::createInvocationsOperation(glslang::TOperator op, spv::Id typeId,
8113	std::vector<spv::Id>& operands, glslang::TBasicType typeProxy)
8114	{
8115	bool isUnsigned = isTypeUnsignedInt(type: typeProxy);
8116	bool isFloat = isTypeFloat(type: typeProxy);
8117
8118	spv::Op opCode = spv::OpNop;
8119	std::vector<spv::IdImmediate> spvGroupOperands;
8120	spv::GroupOperation groupOperation = spv::GroupOperationMax;
8121
8122	if (op == glslang::EOpBallot || op == glslang::EOpReadFirstInvocation ||
8123	op == glslang::EOpReadInvocation) {
8124	builder.addExtension(ext: spv::E_SPV_KHR_shader_ballot);
8125	builder.addCapability(cap: spv::CapabilitySubgroupBallotKHR);
8126	} else if (op == glslang::EOpAnyInvocation ||
8127	op == glslang::EOpAllInvocations ||
8128	op == glslang::EOpAllInvocationsEqual) {
8129	builder.addExtension(ext: spv::E_SPV_KHR_subgroup_vote);
8130	builder.addCapability(cap: spv::CapabilitySubgroupVoteKHR);
8131	} else {
8132	builder.addCapability(cap: spv::CapabilityGroups);
8133	if (op == glslang::EOpMinInvocationsNonUniform ||
8134	op == glslang::EOpMaxInvocationsNonUniform ||
8135	op == glslang::EOpAddInvocationsNonUniform ||
8136	op == glslang::EOpMinInvocationsInclusiveScanNonUniform ||
8137	op == glslang::EOpMaxInvocationsInclusiveScanNonUniform ||
8138	op == glslang::EOpAddInvocationsInclusiveScanNonUniform ||
8139	op == glslang::EOpMinInvocationsExclusiveScanNonUniform ||
8140	op == glslang::EOpMaxInvocationsExclusiveScanNonUniform ||
8141	op == glslang::EOpAddInvocationsExclusiveScanNonUniform)
8142	builder.addExtension(ext: spv::E_SPV_AMD_shader_ballot);
8143
8144	switch (op) {
8145	case glslang::EOpMinInvocations:
8146	case glslang::EOpMaxInvocations:
8147	case glslang::EOpAddInvocations:
8148	case glslang::EOpMinInvocationsNonUniform:
8149	case glslang::EOpMaxInvocationsNonUniform:
8150	case glslang::EOpAddInvocationsNonUniform:
8151	groupOperation = spv::GroupOperationReduce;
8152	break;
8153	case glslang::EOpMinInvocationsInclusiveScan:
8154	case glslang::EOpMaxInvocationsInclusiveScan:
8155	case glslang::EOpAddInvocationsInclusiveScan:
8156	case glslang::EOpMinInvocationsInclusiveScanNonUniform:
8157	case glslang::EOpMaxInvocationsInclusiveScanNonUniform:
8158	case glslang::EOpAddInvocationsInclusiveScanNonUniform:
8159	groupOperation = spv::GroupOperationInclusiveScan;
8160	break;
8161	case glslang::EOpMinInvocationsExclusiveScan:
8162	case glslang::EOpMaxInvocationsExclusiveScan:
8163	case glslang::EOpAddInvocationsExclusiveScan:
8164	case glslang::EOpMinInvocationsExclusiveScanNonUniform:
8165	case glslang::EOpMaxInvocationsExclusiveScanNonUniform:
8166	case glslang::EOpAddInvocationsExclusiveScanNonUniform:
8167	groupOperation = spv::GroupOperationExclusiveScan;
8168	break;
8169	default:
8170	break;
8171	}
8172	spv::IdImmediate scope = { true, builder.makeUintConstant(u: spv::ScopeSubgroup) };
8173	spvGroupOperands.push_back(x: scope);
8174	if (groupOperation != spv::GroupOperationMax) {
8175	spv::IdImmediate groupOp = { false, (unsigned)groupOperation };
8176	spvGroupOperands.push_back(x: groupOp);
8177	}
8178	}
8179
8180	for (auto opIt = operands.begin(); opIt != operands.end(); ++opIt) {
8181	spv::IdImmediate op = { true, *opIt };
8182	spvGroupOperands.push_back(x: op);
8183	}
8184
8185	switch (op) {
8186	case glslang::EOpAnyInvocation:
8187	opCode = spv::OpSubgroupAnyKHR;
8188	break;
8189	case glslang::EOpAllInvocations:
8190	opCode = spv::OpSubgroupAllKHR;
8191	break;
8192	case glslang::EOpAllInvocationsEqual:
8193	opCode = spv::OpSubgroupAllEqualKHR;
8194	break;
8195	case glslang::EOpReadInvocation:
8196	opCode = spv::OpSubgroupReadInvocationKHR;
8197	if (builder.isVectorType(typeId))
8198	return CreateInvocationsVectorOperation(op: opCode, groupOperation, typeId, operands);
8199	break;
8200	case glslang::EOpReadFirstInvocation:
8201	opCode = spv::OpSubgroupFirstInvocationKHR;
8202	if (builder.isVectorType(typeId))
8203	return CreateInvocationsVectorOperation(op: opCode, groupOperation, typeId, operands);
8204	break;
8205	case glslang::EOpBallot:
8206	{
8207	// NOTE: According to the spec, the result type of "OpSubgroupBallotKHR" must be a 4 component vector of 32
8208	// bit integer types. The GLSL built-in function "ballotARB()" assumes the maximum number of invocations in
8209	// a subgroup is 64. Thus, we have to convert uvec4.xy to uint64_t as follow:
8210	//
8211	// result = Bitcast(SubgroupBallotKHR(Predicate).xy)
8212	//
8213	spv::Id uintType = builder.makeUintType(width: 32);
8214	spv::Id uvec4Type = builder.makeVectorType(component: uintType, size: 4);
8215	spv::Id result = builder.createOp(spv::OpSubgroupBallotKHR, typeId: uvec4Type, operands: spvGroupOperands);
8216
8217	std::vector<spv::Id> components;
8218	components.push_back(x: builder.createCompositeExtract(composite: result, typeId: uintType, index: 0));
8219	components.push_back(x: builder.createCompositeExtract(composite: result, typeId: uintType, index: 1));
8220
8221	spv::Id uvec2Type = builder.makeVectorType(component: uintType, size: 2);
8222	return builder.createUnaryOp(spv::OpBitcast, typeId,
8223	operand: builder.createCompositeConstruct(typeId: uvec2Type, constituents: components));
8224	}
8225
8226	case glslang::EOpMinInvocations:
8227	case glslang::EOpMaxInvocations:
8228	case glslang::EOpAddInvocations:
8229	case glslang::EOpMinInvocationsInclusiveScan:
8230	case glslang::EOpMaxInvocationsInclusiveScan:
8231	case glslang::EOpAddInvocationsInclusiveScan:
8232	case glslang::EOpMinInvocationsExclusiveScan:
8233	case glslang::EOpMaxInvocationsExclusiveScan:
8234	case glslang::EOpAddInvocationsExclusiveScan:
8235	if (op == glslang::EOpMinInvocations ||
8236	op == glslang::EOpMinInvocationsInclusiveScan ||
8237	op == glslang::EOpMinInvocationsExclusiveScan) {
8238	if (isFloat)
8239	opCode = spv::OpGroupFMin;
8240	else {
8241	if (isUnsigned)
8242	opCode = spv::OpGroupUMin;
8243	else
8244	opCode = spv::OpGroupSMin;
8245	}
8246	} else if (op == glslang::EOpMaxInvocations ||
8247	op == glslang::EOpMaxInvocationsInclusiveScan ||
8248	op == glslang::EOpMaxInvocationsExclusiveScan) {
8249	if (isFloat)
8250	opCode = spv::OpGroupFMax;
8251	else {
8252	if (isUnsigned)
8253	opCode = spv::OpGroupUMax;
8254	else
8255	opCode = spv::OpGroupSMax;
8256	}
8257	} else {
8258	if (isFloat)
8259	opCode = spv::OpGroupFAdd;
8260	else
8261	opCode = spv::OpGroupIAdd;
8262	}
8263
8264	if (builder.isVectorType(typeId))
8265	return CreateInvocationsVectorOperation(op: opCode, groupOperation, typeId, operands);
8266
8267	break;
8268	case glslang::EOpMinInvocationsNonUniform:
8269	case glslang::EOpMaxInvocationsNonUniform:
8270	case glslang::EOpAddInvocationsNonUniform:
8271	case glslang::EOpMinInvocationsInclusiveScanNonUniform:
8272	case glslang::EOpMaxInvocationsInclusiveScanNonUniform:
8273	case glslang::EOpAddInvocationsInclusiveScanNonUniform:
8274	case glslang::EOpMinInvocationsExclusiveScanNonUniform:
8275	case glslang::EOpMaxInvocationsExclusiveScanNonUniform:
8276	case glslang::EOpAddInvocationsExclusiveScanNonUniform:
8277	if (op == glslang::EOpMinInvocationsNonUniform ||
8278	op == glslang::EOpMinInvocationsInclusiveScanNonUniform ||
8279	op == glslang::EOpMinInvocationsExclusiveScanNonUniform) {
8280	if (isFloat)
8281	opCode = spv::OpGroupFMinNonUniformAMD;
8282	else {
8283	if (isUnsigned)
8284	opCode = spv::OpGroupUMinNonUniformAMD;
8285	else
8286	opCode = spv::OpGroupSMinNonUniformAMD;
8287	}
8288	}
8289	else if (op == glslang::EOpMaxInvocationsNonUniform ||
8290	op == glslang::EOpMaxInvocationsInclusiveScanNonUniform ||
8291	op == glslang::EOpMaxInvocationsExclusiveScanNonUniform) {
8292	if (isFloat)
8293	opCode = spv::OpGroupFMaxNonUniformAMD;
8294	else {
8295	if (isUnsigned)
8296	opCode = spv::OpGroupUMaxNonUniformAMD;
8297	else
8298	opCode = spv::OpGroupSMaxNonUniformAMD;
8299	}
8300	}
8301	else {
8302	if (isFloat)
8303	opCode = spv::OpGroupFAddNonUniformAMD;
8304	else
8305	opCode = spv::OpGroupIAddNonUniformAMD;
8306	}
8307
8308	if (builder.isVectorType(typeId))
8309	return CreateInvocationsVectorOperation(op: opCode, groupOperation, typeId, operands);
8310
8311	break;
8312	default:
8313	logger->missingFunctionality(f: "invocation operation");
8314	return spv::NoResult;
8315	}
8316
8317	assert(opCode != spv::OpNop);
8318	return builder.createOp(opCode, typeId, operands: spvGroupOperands);
8319	}
8320
8321	// Create group invocation operations on a vector
8322	spv::Id TGlslangToSpvTraverser::CreateInvocationsVectorOperation(spv::Op op, spv::GroupOperation groupOperation,
8323	spv::Id typeId, std::vector<spv::Id>& operands)
8324	{
8325	assert(op == spv::OpGroupFMin || op == spv::OpGroupUMin || op == spv::OpGroupSMin ||
8326	op == spv::OpGroupFMax || op == spv::OpGroupUMax || op == spv::OpGroupSMax ||
8327	op == spv::OpGroupFAdd || op == spv::OpGroupIAdd || op == spv::OpGroupBroadcast ||
8328	op == spv::OpSubgroupReadInvocationKHR || op == spv::OpSubgroupFirstInvocationKHR ||
8329	op == spv::OpGroupFMinNonUniformAMD || op == spv::OpGroupUMinNonUniformAMD ||
8330	op == spv::OpGroupSMinNonUniformAMD ||
8331	op == spv::OpGroupFMaxNonUniformAMD || op == spv::OpGroupUMaxNonUniformAMD ||
8332	op == spv::OpGroupSMaxNonUniformAMD ||
8333	op == spv::OpGroupFAddNonUniformAMD || op == spv::OpGroupIAddNonUniformAMD);
8334
8335	// Handle group invocation operations scalar by scalar.
8336	// The result type is the same type as the original type.
8337	// The algorithm is to:
8338	// - break the vector into scalars
8339	// - apply the operation to each scalar
8340	// - make a vector out the scalar results
8341
8342	// get the types sorted out
8343	int numComponents = builder.getNumComponents(resultId: operands[0]);
8344	spv::Id scalarType = builder.getScalarTypeId(typeId: builder.getTypeId(resultId: operands[0]));
8345	std::vector<spv::Id> results;
8346
8347	// do each scalar op
8348	for (int comp = 0; comp < numComponents; ++comp) {
8349	std::vector<unsigned int> indexes;
8350	indexes.push_back(x: comp);
8351	spv::IdImmediate scalar = { true, builder.createCompositeExtract(composite: operands[0], typeId: scalarType, indexes) };
8352	std::vector<spv::IdImmediate> spvGroupOperands;
8353	if (op == spv::OpSubgroupReadInvocationKHR) {
8354	spvGroupOperands.push_back(x: scalar);
8355	spv::IdImmediate operand = { true, operands[1] };
8356	spvGroupOperands.push_back(x: operand);
8357	} else if (op == spv::OpSubgroupFirstInvocationKHR) {
8358	spvGroupOperands.push_back(x: scalar);
8359	} else if (op == spv::OpGroupBroadcast) {
8360	spv::IdImmediate scope = { true, builder.makeUintConstant(u: spv::ScopeSubgroup) };
8361	spvGroupOperands.push_back(x: scope);
8362	spvGroupOperands.push_back(x: scalar);
8363	spv::IdImmediate operand = { true, operands[1] };
8364	spvGroupOperands.push_back(x: operand);
8365	} else {
8366	spv::IdImmediate scope = { true, builder.makeUintConstant(u: spv::ScopeSubgroup) };
8367	spvGroupOperands.push_back(x: scope);
8368	spv::IdImmediate groupOp = { false, (unsigned)groupOperation };
8369	spvGroupOperands.push_back(x: groupOp);
8370	spvGroupOperands.push_back(x: scalar);
8371	}
8372
8373	results.push_back(x: builder.createOp(op, typeId: scalarType, operands: spvGroupOperands));
8374	}
8375
8376	// put the pieces together
8377	return builder.createCompositeConstruct(typeId, constituents: results);
8378	}
8379
8380	// Create subgroup invocation operations.
8381	spv::Id TGlslangToSpvTraverser::createSubgroupOperation(glslang::TOperator op, spv::Id typeId,
8382	std::vector<spv::Id>& operands, glslang::TBasicType typeProxy)
8383	{
8384	// Add the required capabilities.
8385	switch (op) {
8386	case glslang::EOpSubgroupElect:
8387	builder.addCapability(cap: spv::CapabilityGroupNonUniform);
8388	break;
8389	case glslang::EOpSubgroupQuadAll:
8390	case glslang::EOpSubgroupQuadAny:
8391	builder.addExtension(ext: spv::E_SPV_KHR_quad_control);
8392	builder.addCapability(cap: spv::CapabilityQuadControlKHR);
8393	[[fallthrough]];
8394	case glslang::EOpSubgroupAll:
8395	case glslang::EOpSubgroupAny:
8396	case glslang::EOpSubgroupAllEqual:
8397	builder.addCapability(cap: spv::CapabilityGroupNonUniform);
8398	builder.addCapability(cap: spv::CapabilityGroupNonUniformVote);
8399	break;
8400	case glslang::EOpSubgroupBroadcast:
8401	case glslang::EOpSubgroupBroadcastFirst:
8402	case glslang::EOpSubgroupBallot:
8403	case glslang::EOpSubgroupInverseBallot:
8404	case glslang::EOpSubgroupBallotBitExtract:
8405	case glslang::EOpSubgroupBallotBitCount:
8406	case glslang::EOpSubgroupBallotInclusiveBitCount:
8407	case glslang::EOpSubgroupBallotExclusiveBitCount:
8408	case glslang::EOpSubgroupBallotFindLSB:
8409	case glslang::EOpSubgroupBallotFindMSB:
8410	builder.addCapability(cap: spv::CapabilityGroupNonUniform);
8411	builder.addCapability(cap: spv::CapabilityGroupNonUniformBallot);
8412	break;
8413	case glslang::EOpSubgroupRotate:
8414	case glslang::EOpSubgroupClusteredRotate:
8415	builder.addExtension(ext: spv::E_SPV_KHR_subgroup_rotate);
8416	builder.addCapability(cap: spv::CapabilityGroupNonUniformRotateKHR);
8417	break;
8418	case glslang::EOpSubgroupShuffle:
8419	case glslang::EOpSubgroupShuffleXor:
8420	builder.addCapability(cap: spv::CapabilityGroupNonUniform);
8421	builder.addCapability(cap: spv::CapabilityGroupNonUniformShuffle);
8422	break;
8423	case glslang::EOpSubgroupShuffleUp:
8424	case glslang::EOpSubgroupShuffleDown:
8425	builder.addCapability(cap: spv::CapabilityGroupNonUniform);
8426	builder.addCapability(cap: spv::CapabilityGroupNonUniformShuffleRelative);
8427	break;
8428	case glslang::EOpSubgroupAdd:
8429	case glslang::EOpSubgroupMul:
8430	case glslang::EOpSubgroupMin:
8431	case glslang::EOpSubgroupMax:
8432	case glslang::EOpSubgroupAnd:
8433	case glslang::EOpSubgroupOr:
8434	case glslang::EOpSubgroupXor:
8435	case glslang::EOpSubgroupInclusiveAdd:
8436	case glslang::EOpSubgroupInclusiveMul:
8437	case glslang::EOpSubgroupInclusiveMin:
8438	case glslang::EOpSubgroupInclusiveMax:
8439	case glslang::EOpSubgroupInclusiveAnd:
8440	case glslang::EOpSubgroupInclusiveOr:
8441	case glslang::EOpSubgroupInclusiveXor:
8442	case glslang::EOpSubgroupExclusiveAdd:
8443	case glslang::EOpSubgroupExclusiveMul:
8444	case glslang::EOpSubgroupExclusiveMin:
8445	case glslang::EOpSubgroupExclusiveMax:
8446	case glslang::EOpSubgroupExclusiveAnd:
8447	case glslang::EOpSubgroupExclusiveOr:
8448	case glslang::EOpSubgroupExclusiveXor:
8449	builder.addCapability(cap: spv::CapabilityGroupNonUniform);
8450	builder.addCapability(cap: spv::CapabilityGroupNonUniformArithmetic);
8451	break;
8452	case glslang::EOpSubgroupClusteredAdd:
8453	case glslang::EOpSubgroupClusteredMul:
8454	case glslang::EOpSubgroupClusteredMin:
8455	case glslang::EOpSubgroupClusteredMax:
8456	case glslang::EOpSubgroupClusteredAnd:
8457	case glslang::EOpSubgroupClusteredOr:
8458	case glslang::EOpSubgroupClusteredXor:
8459	builder.addCapability(cap: spv::CapabilityGroupNonUniform);
8460	builder.addCapability(cap: spv::CapabilityGroupNonUniformClustered);
8461	break;
8462	case glslang::EOpSubgroupQuadBroadcast:
8463	case glslang::EOpSubgroupQuadSwapHorizontal:
8464	case glslang::EOpSubgroupQuadSwapVertical:
8465	case glslang::EOpSubgroupQuadSwapDiagonal:
8466	builder.addCapability(cap: spv::CapabilityGroupNonUniform);
8467	builder.addCapability(cap: spv::CapabilityGroupNonUniformQuad);
8468	break;
8469	case glslang::EOpSubgroupPartitionedAdd:
8470	case glslang::EOpSubgroupPartitionedMul:
8471	case glslang::EOpSubgroupPartitionedMin:
8472	case glslang::EOpSubgroupPartitionedMax:
8473	case glslang::EOpSubgroupPartitionedAnd:
8474	case glslang::EOpSubgroupPartitionedOr:
8475	case glslang::EOpSubgroupPartitionedXor:
8476	case glslang::EOpSubgroupPartitionedInclusiveAdd:
8477	case glslang::EOpSubgroupPartitionedInclusiveMul:
8478	case glslang::EOpSubgroupPartitionedInclusiveMin:
8479	case glslang::EOpSubgroupPartitionedInclusiveMax:
8480	case glslang::EOpSubgroupPartitionedInclusiveAnd:
8481	case glslang::EOpSubgroupPartitionedInclusiveOr:
8482	case glslang::EOpSubgroupPartitionedInclusiveXor:
8483	case glslang::EOpSubgroupPartitionedExclusiveAdd:
8484	case glslang::EOpSubgroupPartitionedExclusiveMul:
8485	case glslang::EOpSubgroupPartitionedExclusiveMin:
8486	case glslang::EOpSubgroupPartitionedExclusiveMax:
8487	case glslang::EOpSubgroupPartitionedExclusiveAnd:
8488	case glslang::EOpSubgroupPartitionedExclusiveOr:
8489	case glslang::EOpSubgroupPartitionedExclusiveXor:
8490	builder.addExtension(ext: spv::E_SPV_NV_shader_subgroup_partitioned);
8491	builder.addCapability(cap: spv::CapabilityGroupNonUniformPartitionedNV);
8492	break;
8493	default: assert(0 && "Unhandled subgroup operation!");
8494	}
8495
8496
8497	const bool isUnsigned = isTypeUnsignedInt(type: typeProxy);
8498	const bool isFloat = isTypeFloat(type: typeProxy);
8499	const bool isBool = typeProxy == glslang::EbtBool;
8500
8501	spv::Op opCode = spv::OpNop;
8502
8503	// Figure out which opcode to use.
8504	switch (op) {
8505	case glslang::EOpSubgroupElect: opCode = spv::OpGroupNonUniformElect; break;
8506	case glslang::EOpSubgroupQuadAll: opCode = spv::OpGroupNonUniformQuadAllKHR; break;
8507	case glslang::EOpSubgroupAll: opCode = spv::OpGroupNonUniformAll; break;
8508	case glslang::EOpSubgroupQuadAny: opCode = spv::OpGroupNonUniformQuadAnyKHR; break;
8509	case glslang::EOpSubgroupAny: opCode = spv::OpGroupNonUniformAny; break;
8510	case glslang::EOpSubgroupAllEqual: opCode = spv::OpGroupNonUniformAllEqual; break;
8511	case glslang::EOpSubgroupBroadcast: opCode = spv::OpGroupNonUniformBroadcast; break;
8512	case glslang::EOpSubgroupBroadcastFirst: opCode = spv::OpGroupNonUniformBroadcastFirst; break;
8513	case glslang::EOpSubgroupBallot: opCode = spv::OpGroupNonUniformBallot; break;
8514	case glslang::EOpSubgroupInverseBallot: opCode = spv::OpGroupNonUniformInverseBallot; break;
8515	case glslang::EOpSubgroupBallotBitExtract: opCode = spv::OpGroupNonUniformBallotBitExtract; break;
8516	case glslang::EOpSubgroupBallotBitCount:
8517	case glslang::EOpSubgroupBallotInclusiveBitCount:
8518	case glslang::EOpSubgroupBallotExclusiveBitCount: opCode = spv::OpGroupNonUniformBallotBitCount; break;
8519	case glslang::EOpSubgroupBallotFindLSB: opCode = spv::OpGroupNonUniformBallotFindLSB; break;
8520	case glslang::EOpSubgroupBallotFindMSB: opCode = spv::OpGroupNonUniformBallotFindMSB; break;
8521	case glslang::EOpSubgroupShuffle: opCode = spv::OpGroupNonUniformShuffle; break;
8522	case glslang::EOpSubgroupShuffleXor: opCode = spv::OpGroupNonUniformShuffleXor; break;
8523	case glslang::EOpSubgroupShuffleUp: opCode = spv::OpGroupNonUniformShuffleUp; break;
8524	case glslang::EOpSubgroupShuffleDown: opCode = spv::OpGroupNonUniformShuffleDown; break;
8525	case glslang::EOpSubgroupRotate:
8526	case glslang::EOpSubgroupClusteredRotate: opCode = spv::OpGroupNonUniformRotateKHR; break;
8527	case glslang::EOpSubgroupAdd:
8528	case glslang::EOpSubgroupInclusiveAdd:
8529	case glslang::EOpSubgroupExclusiveAdd:
8530	case glslang::EOpSubgroupClusteredAdd:
8531	case glslang::EOpSubgroupPartitionedAdd:
8532	case glslang::EOpSubgroupPartitionedInclusiveAdd:
8533	case glslang::EOpSubgroupPartitionedExclusiveAdd:
8534	if (isFloat) {
8535	opCode = spv::OpGroupNonUniformFAdd;
8536	} else {
8537	opCode = spv::OpGroupNonUniformIAdd;
8538	}
8539	break;
8540	case glslang::EOpSubgroupMul:
8541	case glslang::EOpSubgroupInclusiveMul:
8542	case glslang::EOpSubgroupExclusiveMul:
8543	case glslang::EOpSubgroupClusteredMul:
8544	case glslang::EOpSubgroupPartitionedMul:
8545	case glslang::EOpSubgroupPartitionedInclusiveMul:
8546	case glslang::EOpSubgroupPartitionedExclusiveMul:
8547	if (isFloat) {
8548	opCode = spv::OpGroupNonUniformFMul;
8549	} else {
8550	opCode = spv::OpGroupNonUniformIMul;
8551	}
8552	break;
8553	case glslang::EOpSubgroupMin:
8554	case glslang::EOpSubgroupInclusiveMin:
8555	case glslang::EOpSubgroupExclusiveMin:
8556	case glslang::EOpSubgroupClusteredMin:
8557	case glslang::EOpSubgroupPartitionedMin:
8558	case glslang::EOpSubgroupPartitionedInclusiveMin:
8559	case glslang::EOpSubgroupPartitionedExclusiveMin:
8560	if (isFloat) {
8561	opCode = spv::OpGroupNonUniformFMin;
8562	} else if (isUnsigned) {
8563	opCode = spv::OpGroupNonUniformUMin;
8564	} else {
8565	opCode = spv::OpGroupNonUniformSMin;
8566	}
8567	break;
8568	case glslang::EOpSubgroupMax:
8569	case glslang::EOpSubgroupInclusiveMax:
8570	case glslang::EOpSubgroupExclusiveMax:
8571	case glslang::EOpSubgroupClusteredMax:
8572	case glslang::EOpSubgroupPartitionedMax:
8573	case glslang::EOpSubgroupPartitionedInclusiveMax:
8574	case glslang::EOpSubgroupPartitionedExclusiveMax:
8575	if (isFloat) {
8576	opCode = spv::OpGroupNonUniformFMax;
8577	} else if (isUnsigned) {
8578	opCode = spv::OpGroupNonUniformUMax;
8579	} else {
8580	opCode = spv::OpGroupNonUniformSMax;
8581	}
8582	break;
8583	case glslang::EOpSubgroupAnd:
8584	case glslang::EOpSubgroupInclusiveAnd:
8585	case glslang::EOpSubgroupExclusiveAnd:
8586	case glslang::EOpSubgroupClusteredAnd:
8587	case glslang::EOpSubgroupPartitionedAnd:
8588	case glslang::EOpSubgroupPartitionedInclusiveAnd:
8589	case glslang::EOpSubgroupPartitionedExclusiveAnd:
8590	if (isBool) {
8591	opCode = spv::OpGroupNonUniformLogicalAnd;
8592	} else {
8593	opCode = spv::OpGroupNonUniformBitwiseAnd;
8594	}
8595	break;
8596	case glslang::EOpSubgroupOr:
8597	case glslang::EOpSubgroupInclusiveOr:
8598	case glslang::EOpSubgroupExclusiveOr:
8599	case glslang::EOpSubgroupClusteredOr:
8600	case glslang::EOpSubgroupPartitionedOr:
8601	case glslang::EOpSubgroupPartitionedInclusiveOr:
8602	case glslang::EOpSubgroupPartitionedExclusiveOr:
8603	if (isBool) {
8604	opCode = spv::OpGroupNonUniformLogicalOr;
8605	} else {
8606	opCode = spv::OpGroupNonUniformBitwiseOr;
8607	}
8608	break;
8609	case glslang::EOpSubgroupXor:
8610	case glslang::EOpSubgroupInclusiveXor:
8611	case glslang::EOpSubgroupExclusiveXor:
8612	case glslang::EOpSubgroupClusteredXor:
8613	case glslang::EOpSubgroupPartitionedXor:
8614	case glslang::EOpSubgroupPartitionedInclusiveXor:
8615	case glslang::EOpSubgroupPartitionedExclusiveXor:
8616	if (isBool) {
8617	opCode = spv::OpGroupNonUniformLogicalXor;
8618	} else {
8619	opCode = spv::OpGroupNonUniformBitwiseXor;
8620	}
8621	break;
8622	case glslang::EOpSubgroupQuadBroadcast: opCode = spv::OpGroupNonUniformQuadBroadcast; break;
8623	case glslang::EOpSubgroupQuadSwapHorizontal:
8624	case glslang::EOpSubgroupQuadSwapVertical:
8625	case glslang::EOpSubgroupQuadSwapDiagonal: opCode = spv::OpGroupNonUniformQuadSwap; break;
8626	default: assert(0 && "Unhandled subgroup operation!");
8627	}
8628
8629	// get the right Group Operation
8630	spv::GroupOperation groupOperation = spv::GroupOperationMax;
8631	switch (op) {
8632	default:
8633	break;
8634	case glslang::EOpSubgroupBallotBitCount:
8635	case glslang::EOpSubgroupAdd:
8636	case glslang::EOpSubgroupMul:
8637	case glslang::EOpSubgroupMin:
8638	case glslang::EOpSubgroupMax:
8639	case glslang::EOpSubgroupAnd:
8640	case glslang::EOpSubgroupOr:
8641	case glslang::EOpSubgroupXor:
8642	groupOperation = spv::GroupOperationReduce;
8643	break;
8644	case glslang::EOpSubgroupBallotInclusiveBitCount:
8645	case glslang::EOpSubgroupInclusiveAdd:
8646	case glslang::EOpSubgroupInclusiveMul:
8647	case glslang::EOpSubgroupInclusiveMin:
8648	case glslang::EOpSubgroupInclusiveMax:
8649	case glslang::EOpSubgroupInclusiveAnd:
8650	case glslang::EOpSubgroupInclusiveOr:
8651	case glslang::EOpSubgroupInclusiveXor:
8652	groupOperation = spv::GroupOperationInclusiveScan;
8653	break;
8654	case glslang::EOpSubgroupBallotExclusiveBitCount:
8655	case glslang::EOpSubgroupExclusiveAdd:
8656	case glslang::EOpSubgroupExclusiveMul:
8657	case glslang::EOpSubgroupExclusiveMin:
8658	case glslang::EOpSubgroupExclusiveMax:
8659	case glslang::EOpSubgroupExclusiveAnd:
8660	case glslang::EOpSubgroupExclusiveOr:
8661	case glslang::EOpSubgroupExclusiveXor:
8662	groupOperation = spv::GroupOperationExclusiveScan;
8663	break;
8664	case glslang::EOpSubgroupClusteredAdd:
8665	case glslang::EOpSubgroupClusteredMul:
8666	case glslang::EOpSubgroupClusteredMin:
8667	case glslang::EOpSubgroupClusteredMax:
8668	case glslang::EOpSubgroupClusteredAnd:
8669	case glslang::EOpSubgroupClusteredOr:
8670	case glslang::EOpSubgroupClusteredXor:
8671	groupOperation = spv::GroupOperationClusteredReduce;
8672	break;
8673	case glslang::EOpSubgroupPartitionedAdd:
8674	case glslang::EOpSubgroupPartitionedMul:
8675	case glslang::EOpSubgroupPartitionedMin:
8676	case glslang::EOpSubgroupPartitionedMax:
8677	case glslang::EOpSubgroupPartitionedAnd:
8678	case glslang::EOpSubgroupPartitionedOr:
8679	case glslang::EOpSubgroupPartitionedXor:
8680	groupOperation = spv::GroupOperationPartitionedReduceNV;
8681	break;
8682	case glslang::EOpSubgroupPartitionedInclusiveAdd:
8683	case glslang::EOpSubgroupPartitionedInclusiveMul:
8684	case glslang::EOpSubgroupPartitionedInclusiveMin:
8685	case glslang::EOpSubgroupPartitionedInclusiveMax:
8686	case glslang::EOpSubgroupPartitionedInclusiveAnd:
8687	case glslang::EOpSubgroupPartitionedInclusiveOr:
8688	case glslang::EOpSubgroupPartitionedInclusiveXor:
8689	groupOperation = spv::GroupOperationPartitionedInclusiveScanNV;
8690	break;
8691	case glslang::EOpSubgroupPartitionedExclusiveAdd:
8692	case glslang::EOpSubgroupPartitionedExclusiveMul:
8693	case glslang::EOpSubgroupPartitionedExclusiveMin:
8694	case glslang::EOpSubgroupPartitionedExclusiveMax:
8695	case glslang::EOpSubgroupPartitionedExclusiveAnd:
8696	case glslang::EOpSubgroupPartitionedExclusiveOr:
8697	case glslang::EOpSubgroupPartitionedExclusiveXor:
8698	groupOperation = spv::GroupOperationPartitionedExclusiveScanNV;
8699	break;
8700	}
8701
8702	// build the instruction
8703	std::vector<spv::IdImmediate> spvGroupOperands;
8704
8705	// Every operation begins with the Execution Scope operand.
8706	spv::IdImmediate executionScope = { true, builder.makeUintConstant(u: spv::ScopeSubgroup) };
8707	// All other ops need the execution scope. Quad Control Ops don't need scope, it's always Quad.
8708	if (opCode != spv::OpGroupNonUniformQuadAllKHR && opCode != spv::OpGroupNonUniformQuadAnyKHR) {
8709	spvGroupOperands.push_back(x: executionScope);
8710	}
8711
8712	// Next, for all operations that use a Group Operation, push that as an operand.
8713	if (groupOperation != spv::GroupOperationMax) {
8714	spv::IdImmediate groupOperand = { false, (unsigned)groupOperation };
8715	spvGroupOperands.push_back(x: groupOperand);
8716	}
8717
8718	// Push back the operands next.
8719	for (auto opIt = operands.cbegin(); opIt != operands.cend(); ++opIt) {
8720	spv::IdImmediate operand = { true, *opIt };
8721	spvGroupOperands.push_back(x: operand);
8722	}
8723
8724	// Some opcodes have additional operands.
8725	spv::Id directionId = spv::NoResult;
8726	switch (op) {
8727	default: break;
8728	case glslang::EOpSubgroupQuadSwapHorizontal: directionId = builder.makeUintConstant(u: 0); break;
8729	case glslang::EOpSubgroupQuadSwapVertical: directionId = builder.makeUintConstant(u: 1); break;
8730	case glslang::EOpSubgroupQuadSwapDiagonal: directionId = builder.makeUintConstant(u: 2); break;
8731	}
8732	if (directionId != spv::NoResult) {
8733	spv::IdImmediate direction = { true, directionId };
8734	spvGroupOperands.push_back(x: direction);
8735	}
8736
8737	return builder.createOp(opCode, typeId, operands: spvGroupOperands);
8738	}
8739
8740	spv::Id TGlslangToSpvTraverser::createMiscOperation(glslang::TOperator op, spv::Decoration precision,
8741	spv::Id typeId, std::vector<spv::Id>& operands, glslang::TBasicType typeProxy)
8742	{
8743	bool isUnsigned = isTypeUnsignedInt(type: typeProxy);
8744	bool isFloat = isTypeFloat(type: typeProxy);
8745
8746	spv::Op opCode = spv::OpNop;
8747	int extBuiltins = -1;
8748	int libCall = -1;
8749	size_t consumedOperands = operands.size();
8750	spv::Id typeId0 = 0;
8751	if (consumedOperands > 0)
8752	typeId0 = builder.getTypeId(resultId: operands[0]);
8753	spv::Id typeId1 = 0;
8754	if (consumedOperands > 1)
8755	typeId1 = builder.getTypeId(resultId: operands[1]);
8756	spv::Id frexpIntType = 0;
8757
8758	switch (op) {
8759	case glslang::EOpMin:
8760	if (isFloat)
8761	libCall = nanMinMaxClamp ? spv::GLSLstd450NMin : spv::GLSLstd450FMin;
8762	else if (isUnsigned)
8763	libCall = spv::GLSLstd450UMin;
8764	else
8765	libCall = spv::GLSLstd450SMin;
8766	builder.promoteScalar(precision, left&: operands.front(), right&: operands.back());
8767	break;
8768	case glslang::EOpModf:
8769	libCall = spv::GLSLstd450Modf;
8770	break;
8771	case glslang::EOpMax:
8772	if (isFloat)
8773	libCall = nanMinMaxClamp ? spv::GLSLstd450NMax : spv::GLSLstd450FMax;
8774	else if (isUnsigned)
8775	libCall = spv::GLSLstd450UMax;
8776	else
8777	libCall = spv::GLSLstd450SMax;
8778	builder.promoteScalar(precision, left&: operands.front(), right&: operands.back());
8779	break;
8780	case glslang::EOpPow:
8781	libCall = spv::GLSLstd450Pow;
8782	break;
8783	case glslang::EOpDot:
8784	opCode = spv::OpDot;
8785	break;
8786	case glslang::EOpAtan:
8787	libCall = spv::GLSLstd450Atan2;
8788	break;
8789
8790	case glslang::EOpClamp:
8791	if (isFloat)
8792	libCall = nanMinMaxClamp ? spv::GLSLstd450NClamp : spv::GLSLstd450FClamp;
8793	else if (isUnsigned)
8794	libCall = spv::GLSLstd450UClamp;
8795	else
8796	libCall = spv::GLSLstd450SClamp;
8797	builder.promoteScalar(precision, left&: operands.front(), right&: operands[1]);
8798	builder.promoteScalar(precision, left&: operands.front(), right&: operands[2]);
8799	break;
8800	case glslang::EOpMix:
8801	if (! builder.isBoolType(typeId: builder.getScalarTypeId(typeId: builder.getTypeId(resultId: operands.back())))) {
8802	assert(isFloat);
8803	libCall = spv::GLSLstd450FMix;
8804	} else {
8805	opCode = spv::OpSelect;
8806	std::swap(a&: operands.front(), b&: operands.back());
8807	}
8808	builder.promoteScalar(precision, left&: operands.front(), right&: operands.back());
8809	break;
8810	case glslang::EOpStep:
8811	libCall = spv::GLSLstd450Step;
8812	builder.promoteScalar(precision, left&: operands.front(), right&: operands.back());
8813	break;
8814	case glslang::EOpSmoothStep:
8815	libCall = spv::GLSLstd450SmoothStep;
8816	builder.promoteScalar(precision, left&: operands[0], right&: operands[2]);
8817	builder.promoteScalar(precision, left&: operands[1], right&: operands[2]);
8818	break;
8819
8820	case glslang::EOpDistance:
8821	libCall = spv::GLSLstd450Distance;
8822	break;
8823	case glslang::EOpCross:
8824	libCall = spv::GLSLstd450Cross;
8825	break;
8826	case glslang::EOpFaceForward:
8827	libCall = spv::GLSLstd450FaceForward;
8828	break;
8829	case glslang::EOpReflect:
8830	libCall = spv::GLSLstd450Reflect;
8831	break;
8832	case glslang::EOpRefract:
8833	libCall = spv::GLSLstd450Refract;
8834	break;
8835	case glslang::EOpBarrier:
8836	{
8837	// This is for the extended controlBarrier function, with four operands.
8838	// The unextended barrier() goes through createNoArgOperation.
8839	assert(operands.size() == 4);
8840	unsigned int executionScope = builder.getConstantScalar(resultId: operands[0]);
8841	unsigned int memoryScope = builder.getConstantScalar(resultId: operands[1]);
8842	unsigned int semantics = builder.getConstantScalar(resultId: operands[2]) | builder.getConstantScalar(resultId: operands[3]);
8843	builder.createControlBarrier(execution: (spv::Scope)executionScope, memory: (spv::Scope)memoryScope,
8844	(spv::MemorySemanticsMask)semantics);
8845	if (semantics & (spv::MemorySemanticsMakeAvailableKHRMask |
8846	spv::MemorySemanticsMakeVisibleKHRMask |
8847	spv::MemorySemanticsOutputMemoryKHRMask |
8848	spv::MemorySemanticsVolatileMask)) {
8849	builder.addCapability(cap: spv::CapabilityVulkanMemoryModelKHR);
8850	}
8851	if (glslangIntermediate->usingVulkanMemoryModel() && (executionScope == spv::ScopeDevice ||
8852	memoryScope == spv::ScopeDevice)) {
8853	builder.addCapability(cap: spv::CapabilityVulkanMemoryModelDeviceScopeKHR);
8854	}
8855	return 0;
8856	}
8857	break;
8858	case glslang::EOpMemoryBarrier:
8859	{
8860	// This is for the extended memoryBarrier function, with three operands.
8861	// The unextended memoryBarrier() goes through createNoArgOperation.
8862	assert(operands.size() == 3);
8863	unsigned int memoryScope = builder.getConstantScalar(resultId: operands[0]);
8864	unsigned int semantics = builder.getConstantScalar(resultId: operands[1]) | builder.getConstantScalar(resultId: operands[2]);
8865	builder.createMemoryBarrier(executionScope: (spv::Scope)memoryScope, memorySemantics: (spv::MemorySemanticsMask)semantics);
8866	if (semantics & (spv::MemorySemanticsMakeAvailableKHRMask |
8867	spv::MemorySemanticsMakeVisibleKHRMask |
8868	spv::MemorySemanticsOutputMemoryKHRMask |
8869	spv::MemorySemanticsVolatileMask)) {
8870	builder.addCapability(cap: spv::CapabilityVulkanMemoryModelKHR);
8871	}
8872	if (glslangIntermediate->usingVulkanMemoryModel() && memoryScope == spv::ScopeDevice) {
8873	builder.addCapability(cap: spv::CapabilityVulkanMemoryModelDeviceScopeKHR);
8874	}
8875	return 0;
8876	}
8877	break;
8878
8879	case glslang::EOpInterpolateAtSample:
8880	if (typeProxy == glslang::EbtFloat16)
8881	builder.addExtension(ext: spv::E_SPV_AMD_gpu_shader_half_float);
8882	libCall = spv::GLSLstd450InterpolateAtSample;
8883	break;
8884	case glslang::EOpInterpolateAtOffset:
8885	if (typeProxy == glslang::EbtFloat16)
8886	builder.addExtension(ext: spv::E_SPV_AMD_gpu_shader_half_float);
8887	libCall = spv::GLSLstd450InterpolateAtOffset;
8888	break;
8889	case glslang::EOpAddCarry:
8890	opCode = spv::OpIAddCarry;
8891	typeId = builder.makeStructResultType(type0: typeId0, type1: typeId0);
8892	consumedOperands = 2;
8893	break;
8894	case glslang::EOpSubBorrow:
8895	opCode = spv::OpISubBorrow;
8896	typeId = builder.makeStructResultType(type0: typeId0, type1: typeId0);
8897	consumedOperands = 2;
8898	break;
8899	case glslang::EOpUMulExtended:
8900	opCode = spv::OpUMulExtended;
8901	typeId = builder.makeStructResultType(type0: typeId0, type1: typeId0);
8902	consumedOperands = 2;
8903	break;
8904	case glslang::EOpIMulExtended:
8905	opCode = spv::OpSMulExtended;
8906	typeId = builder.makeStructResultType(type0: typeId0, type1: typeId0);
8907	consumedOperands = 2;
8908	break;
8909	case glslang::EOpBitfieldExtract:
8910	if (isUnsigned)
8911	opCode = spv::OpBitFieldUExtract;
8912	else
8913	opCode = spv::OpBitFieldSExtract;
8914	break;
8915	case glslang::EOpBitfieldInsert:
8916	opCode = spv::OpBitFieldInsert;
8917	break;
8918
8919	case glslang::EOpFma:
8920	libCall = spv::GLSLstd450Fma;
8921	break;
8922	case glslang::EOpFrexp:
8923	{
8924	libCall = spv::GLSLstd450FrexpStruct;
8925	assert(builder.isPointerType(typeId1));
8926	typeId1 = builder.getContainedTypeId(typeId: typeId1);
8927	int width = builder.getScalarTypeWidth(typeId: typeId1);
8928	if (width == 16)
8929	// Using 16-bit exp operand, enable extension SPV_AMD_gpu_shader_int16
8930	builder.addExtension(ext: spv::E_SPV_AMD_gpu_shader_int16);
8931	if (builder.getNumComponents(resultId: operands[0]) == 1)
8932	frexpIntType = builder.makeIntegerType(width, hasSign: true);
8933	else
8934	frexpIntType = builder.makeVectorType(component: builder.makeIntegerType(width, hasSign: true),
8935	size: builder.getNumComponents(resultId: operands[0]));
8936	typeId = builder.makeStructResultType(type0: typeId0, type1: frexpIntType);
8937	consumedOperands = 1;
8938	}
8939	break;
8940	case glslang::EOpLdexp:
8941	libCall = spv::GLSLstd450Ldexp;
8942	break;
8943
8944	case glslang::EOpReadInvocation:
8945	return createInvocationsOperation(op, typeId, operands, typeProxy);
8946
8947	case glslang::EOpSubgroupBroadcast:
8948	case glslang::EOpSubgroupBallotBitExtract:
8949	case glslang::EOpSubgroupShuffle:
8950	case glslang::EOpSubgroupShuffleXor:
8951	case glslang::EOpSubgroupShuffleUp:
8952	case glslang::EOpSubgroupShuffleDown:
8953	case glslang::EOpSubgroupRotate:
8954	case glslang::EOpSubgroupClusteredRotate:
8955	case glslang::EOpSubgroupClusteredAdd:
8956	case glslang::EOpSubgroupClusteredMul:
8957	case glslang::EOpSubgroupClusteredMin:
8958	case glslang::EOpSubgroupClusteredMax:
8959	case glslang::EOpSubgroupClusteredAnd:
8960	case glslang::EOpSubgroupClusteredOr:
8961	case glslang::EOpSubgroupClusteredXor:
8962	case glslang::EOpSubgroupQuadBroadcast:
8963	case glslang::EOpSubgroupPartitionedAdd:
8964	case glslang::EOpSubgroupPartitionedMul:
8965	case glslang::EOpSubgroupPartitionedMin:
8966	case glslang::EOpSubgroupPartitionedMax:
8967	case glslang::EOpSubgroupPartitionedAnd:
8968	case glslang::EOpSubgroupPartitionedOr:
8969	case glslang::EOpSubgroupPartitionedXor:
8970	case glslang::EOpSubgroupPartitionedInclusiveAdd:
8971	case glslang::EOpSubgroupPartitionedInclusiveMul:
8972	case glslang::EOpSubgroupPartitionedInclusiveMin:
8973	case glslang::EOpSubgroupPartitionedInclusiveMax:
8974	case glslang::EOpSubgroupPartitionedInclusiveAnd:
8975	case glslang::EOpSubgroupPartitionedInclusiveOr:
8976	case glslang::EOpSubgroupPartitionedInclusiveXor:
8977	case glslang::EOpSubgroupPartitionedExclusiveAdd:
8978	case glslang::EOpSubgroupPartitionedExclusiveMul:
8979	case glslang::EOpSubgroupPartitionedExclusiveMin:
8980	case glslang::EOpSubgroupPartitionedExclusiveMax:
8981	case glslang::EOpSubgroupPartitionedExclusiveAnd:
8982	case glslang::EOpSubgroupPartitionedExclusiveOr:
8983	case glslang::EOpSubgroupPartitionedExclusiveXor:
8984	return createSubgroupOperation(op, typeId, operands, typeProxy);
8985
8986	case glslang::EOpSwizzleInvocations:
8987	extBuiltins = getExtBuiltins(name: spv::E_SPV_AMD_shader_ballot);
8988	libCall = spv::SwizzleInvocationsAMD;
8989	break;
8990	case glslang::EOpSwizzleInvocationsMasked:
8991	extBuiltins = getExtBuiltins(name: spv::E_SPV_AMD_shader_ballot);
8992	libCall = spv::SwizzleInvocationsMaskedAMD;
8993	break;
8994	case glslang::EOpWriteInvocation:
8995	extBuiltins = getExtBuiltins(name: spv::E_SPV_AMD_shader_ballot);
8996	libCall = spv::WriteInvocationAMD;
8997	break;
8998
8999	case glslang::EOpMin3:
9000	extBuiltins = getExtBuiltins(name: spv::E_SPV_AMD_shader_trinary_minmax);
9001	if (isFloat)
9002	libCall = spv::FMin3AMD;
9003	else {
9004	if (isUnsigned)
9005	libCall = spv::UMin3AMD;
9006	else
9007	libCall = spv::SMin3AMD;
9008	}
9009	break;
9010	case glslang::EOpMax3:
9011	extBuiltins = getExtBuiltins(name: spv::E_SPV_AMD_shader_trinary_minmax);
9012	if (isFloat)
9013	libCall = spv::FMax3AMD;
9014	else {
9015	if (isUnsigned)
9016	libCall = spv::UMax3AMD;
9017	else
9018	libCall = spv::SMax3AMD;
9019	}
9020	break;
9021	case glslang::EOpMid3:
9022	extBuiltins = getExtBuiltins(name: spv::E_SPV_AMD_shader_trinary_minmax);
9023	if (isFloat)
9024	libCall = spv::FMid3AMD;
9025	else {
9026	if (isUnsigned)
9027	libCall = spv::UMid3AMD;
9028	else
9029	libCall = spv::SMid3AMD;
9030	}
9031	break;
9032
9033	case glslang::EOpInterpolateAtVertex:
9034	if (typeProxy == glslang::EbtFloat16)
9035	builder.addExtension(ext: spv::E_SPV_AMD_gpu_shader_half_float);
9036	extBuiltins = getExtBuiltins(name: spv::E_SPV_AMD_shader_explicit_vertex_parameter);
9037	libCall = spv::InterpolateAtVertexAMD;
9038	break;
9039
9040	case glslang::EOpReportIntersection:
9041	typeId = builder.makeBoolType();
9042	opCode = spv::OpReportIntersectionKHR;
9043	break;
9044	case glslang::EOpTraceNV:
9045	builder.createNoResultOp(spv::OpTraceNV, operands);
9046	return 0;
9047	case glslang::EOpTraceRayMotionNV:
9048	builder.addExtension(ext: spv::E_SPV_NV_ray_tracing_motion_blur);
9049	builder.addCapability(cap: spv::CapabilityRayTracingMotionBlurNV);
9050	builder.createNoResultOp(spv::OpTraceRayMotionNV, operands);
9051	return 0;
9052	case glslang::EOpTraceKHR:
9053	builder.createNoResultOp(spv::OpTraceRayKHR, operands);
9054	return 0;
9055	case glslang::EOpExecuteCallableNV:
9056	builder.createNoResultOp(spv::OpExecuteCallableNV, operands);
9057	return 0;
9058	case glslang::EOpExecuteCallableKHR:
9059	builder.createNoResultOp(spv::OpExecuteCallableKHR, operands);
9060	return 0;
9061
9062	case glslang::EOpRayQueryInitialize:
9063	builder.createNoResultOp(spv::OpRayQueryInitializeKHR, operands);
9064	return 0;
9065	case glslang::EOpRayQueryTerminate:
9066	builder.createNoResultOp(spv::OpRayQueryTerminateKHR, operands);
9067	return 0;
9068	case glslang::EOpRayQueryGenerateIntersection:
9069	builder.createNoResultOp(spv::OpRayQueryGenerateIntersectionKHR, operands);
9070	return 0;
9071	case glslang::EOpRayQueryConfirmIntersection:
9072	builder.createNoResultOp(spv::OpRayQueryConfirmIntersectionKHR, operands);
9073	return 0;
9074	case glslang::EOpRayQueryProceed:
9075	typeId = builder.makeBoolType();
9076	opCode = spv::OpRayQueryProceedKHR;
9077	break;
9078	case glslang::EOpRayQueryGetIntersectionType:
9079	typeId = builder.makeUintType(width: 32);
9080	opCode = spv::OpRayQueryGetIntersectionTypeKHR;
9081	break;
9082	case glslang::EOpRayQueryGetRayTMin:
9083	typeId = builder.makeFloatType(width: 32);
9084	opCode = spv::OpRayQueryGetRayTMinKHR;
9085	break;
9086	case glslang::EOpRayQueryGetRayFlags:
9087	typeId = builder.makeIntType(width: 32);
9088	opCode = spv::OpRayQueryGetRayFlagsKHR;
9089	break;
9090	case glslang::EOpRayQueryGetIntersectionT:
9091	typeId = builder.makeFloatType(width: 32);
9092	opCode = spv::OpRayQueryGetIntersectionTKHR;
9093	break;
9094	case glslang::EOpRayQueryGetIntersectionInstanceCustomIndex:
9095	typeId = builder.makeIntType(width: 32);
9096	opCode = spv::OpRayQueryGetIntersectionInstanceCustomIndexKHR;
9097	break;
9098	case glslang::EOpRayQueryGetIntersectionInstanceId:
9099	typeId = builder.makeIntType(width: 32);
9100	opCode = spv::OpRayQueryGetIntersectionInstanceIdKHR;
9101	break;
9102	case glslang::EOpRayQueryGetIntersectionInstanceShaderBindingTableRecordOffset:
9103	typeId = builder.makeUintType(width: 32);
9104	opCode = spv::OpRayQueryGetIntersectionInstanceShaderBindingTableRecordOffsetKHR;
9105	break;
9106	case glslang::EOpRayQueryGetIntersectionGeometryIndex:
9107	typeId = builder.makeIntType(width: 32);
9108	opCode = spv::OpRayQueryGetIntersectionGeometryIndexKHR;
9109	break;
9110	case glslang::EOpRayQueryGetIntersectionPrimitiveIndex:
9111	typeId = builder.makeIntType(width: 32);
9112	opCode = spv::OpRayQueryGetIntersectionPrimitiveIndexKHR;
9113	break;
9114	case glslang::EOpRayQueryGetIntersectionBarycentrics:
9115	typeId = builder.makeVectorType(component: builder.makeFloatType(width: 32), size: 2);
9116	opCode = spv::OpRayQueryGetIntersectionBarycentricsKHR;
9117	break;
9118	case glslang::EOpRayQueryGetIntersectionFrontFace:
9119	typeId = builder.makeBoolType();
9120	opCode = spv::OpRayQueryGetIntersectionFrontFaceKHR;
9121	break;
9122	case glslang::EOpRayQueryGetIntersectionCandidateAABBOpaque:
9123	typeId = builder.makeBoolType();
9124	opCode = spv::OpRayQueryGetIntersectionCandidateAABBOpaqueKHR;
9125	break;
9126	case glslang::EOpRayQueryGetIntersectionObjectRayDirection:
9127	typeId = builder.makeVectorType(component: builder.makeFloatType(width: 32), size: 3);
9128	opCode = spv::OpRayQueryGetIntersectionObjectRayDirectionKHR;
9129	break;
9130	case glslang::EOpRayQueryGetIntersectionObjectRayOrigin:
9131	typeId = builder.makeVectorType(component: builder.makeFloatType(width: 32), size: 3);
9132	opCode = spv::OpRayQueryGetIntersectionObjectRayOriginKHR;
9133	break;
9134	case glslang::EOpRayQueryGetWorldRayDirection:
9135	typeId = builder.makeVectorType(component: builder.makeFloatType(width: 32), size: 3);
9136	opCode = spv::OpRayQueryGetWorldRayDirectionKHR;
9137	break;
9138	case glslang::EOpRayQueryGetWorldRayOrigin:
9139	typeId = builder.makeVectorType(component: builder.makeFloatType(width: 32), size: 3);
9140	opCode = spv::OpRayQueryGetWorldRayOriginKHR;
9141	break;
9142	case glslang::EOpRayQueryGetIntersectionObjectToWorld:
9143	typeId = builder.makeMatrixType(component: builder.makeFloatType(width: 32), cols: 4, rows: 3);
9144	opCode = spv::OpRayQueryGetIntersectionObjectToWorldKHR;
9145	break;
9146	case glslang::EOpRayQueryGetIntersectionWorldToObject:
9147	typeId = builder.makeMatrixType(component: builder.makeFloatType(width: 32), cols: 4, rows: 3);
9148	opCode = spv::OpRayQueryGetIntersectionWorldToObjectKHR;
9149	break;
9150	case glslang::EOpWritePackedPrimitiveIndices4x8NV:
9151	builder.createNoResultOp(spv::OpWritePackedPrimitiveIndices4x8NV, operands);
9152	return 0;
9153	case glslang::EOpEmitMeshTasksEXT:
9154	if (taskPayloadID)
9155	operands.push_back(x: taskPayloadID);
9156	// As per SPV_EXT_mesh_shader make it a terminating instruction in the current block
9157	builder.makeStatementTerminator(opcode: spv::OpEmitMeshTasksEXT, operands, name: "post-OpEmitMeshTasksEXT");
9158	return 0;
9159	case glslang::EOpSetMeshOutputsEXT:
9160	builder.createNoResultOp(spv::OpSetMeshOutputsEXT, operands);
9161	return 0;
9162	case glslang::EOpCooperativeMatrixMulAddNV:
9163	opCode = spv::OpCooperativeMatrixMulAddNV;
9164	break;
9165	case glslang::EOpHitObjectTraceRayNV:
9166	builder.createNoResultOp(spv::OpHitObjectTraceRayNV, operands);
9167	return 0;
9168	case glslang::EOpHitObjectTraceRayMotionNV:
9169	builder.createNoResultOp(spv::OpHitObjectTraceRayMotionNV, operands);
9170	return 0;
9171	case glslang::EOpHitObjectRecordHitNV:
9172	builder.createNoResultOp(spv::OpHitObjectRecordHitNV, operands);
9173	return 0;
9174	case glslang::EOpHitObjectRecordHitMotionNV:
9175	builder.createNoResultOp(spv::OpHitObjectRecordHitMotionNV, operands);
9176	return 0;
9177	case glslang::EOpHitObjectRecordHitWithIndexNV:
9178	builder.createNoResultOp(spv::OpHitObjectRecordHitWithIndexNV, operands);
9179	return 0;
9180	case glslang::EOpHitObjectRecordHitWithIndexMotionNV:
9181	builder.createNoResultOp(spv::OpHitObjectRecordHitWithIndexMotionNV, operands);
9182	return 0;
9183	case glslang::EOpHitObjectRecordMissNV:
9184	builder.createNoResultOp(spv::OpHitObjectRecordMissNV, operands);
9185	return 0;
9186	case glslang::EOpHitObjectRecordMissMotionNV:
9187	builder.createNoResultOp(spv::OpHitObjectRecordMissMotionNV, operands);
9188	return 0;
9189	case glslang::EOpHitObjectExecuteShaderNV:
9190	builder.createNoResultOp(spv::OpHitObjectExecuteShaderNV, operands);
9191	return 0;
9192	case glslang::EOpHitObjectIsEmptyNV:
9193	typeId = builder.makeBoolType();
9194	opCode = spv::OpHitObjectIsEmptyNV;
9195	break;
9196	case glslang::EOpHitObjectIsMissNV:
9197	typeId = builder.makeBoolType();
9198	opCode = spv::OpHitObjectIsMissNV;
9199	break;
9200	case glslang::EOpHitObjectIsHitNV:
9201	typeId = builder.makeBoolType();
9202	opCode = spv::OpHitObjectIsHitNV;
9203	break;
9204	case glslang::EOpHitObjectGetRayTMinNV:
9205	typeId = builder.makeFloatType(width: 32);
9206	opCode = spv::OpHitObjectGetRayTMinNV;
9207	break;
9208	case glslang::EOpHitObjectGetRayTMaxNV:
9209	typeId = builder.makeFloatType(width: 32);
9210	opCode = spv::OpHitObjectGetRayTMaxNV;
9211	break;
9212	case glslang::EOpHitObjectGetObjectRayOriginNV:
9213	typeId = builder.makeVectorType(component: builder.makeFloatType(width: 32), size: 3);
9214	opCode = spv::OpHitObjectGetObjectRayOriginNV;
9215	break;
9216	case glslang::EOpHitObjectGetObjectRayDirectionNV:
9217	typeId = builder.makeVectorType(component: builder.makeFloatType(width: 32), size: 3);
9218	opCode = spv::OpHitObjectGetObjectRayDirectionNV;
9219	break;
9220	case glslang::EOpHitObjectGetWorldRayOriginNV:
9221	typeId = builder.makeVectorType(component: builder.makeFloatType(width: 32), size: 3);
9222	opCode = spv::OpHitObjectGetWorldRayOriginNV;
9223	break;
9224	case glslang::EOpHitObjectGetWorldRayDirectionNV:
9225	typeId = builder.makeVectorType(component: builder.makeFloatType(width: 32), size: 3);
9226	opCode = spv::OpHitObjectGetWorldRayDirectionNV;
9227	break;
9228	case glslang::EOpHitObjectGetWorldToObjectNV:
9229	typeId = builder.makeMatrixType(component: builder.makeFloatType(width: 32), cols: 4, rows: 3);
9230	opCode = spv::OpHitObjectGetWorldToObjectNV;
9231	break;
9232	case glslang::EOpHitObjectGetObjectToWorldNV:
9233	typeId = builder.makeMatrixType(component: builder.makeFloatType(width: 32), cols: 4, rows: 3);
9234	opCode = spv::OpHitObjectGetObjectToWorldNV;
9235	break;
9236	case glslang::EOpHitObjectGetInstanceCustomIndexNV:
9237	typeId = builder.makeIntegerType(width: 32, hasSign: 1);
9238	opCode = spv::OpHitObjectGetInstanceCustomIndexNV;
9239	break;
9240	case glslang::EOpHitObjectGetInstanceIdNV:
9241	typeId = builder.makeIntegerType(width: 32, hasSign: 1);
9242	opCode = spv::OpHitObjectGetInstanceIdNV;
9243	break;
9244	case glslang::EOpHitObjectGetGeometryIndexNV:
9245	typeId = builder.makeIntegerType(width: 32, hasSign: 1);
9246	opCode = spv::OpHitObjectGetGeometryIndexNV;
9247	break;
9248	case glslang::EOpHitObjectGetPrimitiveIndexNV:
9249	typeId = builder.makeIntegerType(width: 32, hasSign: 1);
9250	opCode = spv::OpHitObjectGetPrimitiveIndexNV;
9251	break;
9252	case glslang::EOpHitObjectGetHitKindNV:
9253	typeId = builder.makeIntegerType(width: 32, hasSign: 0);
9254	opCode = spv::OpHitObjectGetHitKindNV;
9255	break;
9256	case glslang::EOpHitObjectGetCurrentTimeNV:
9257	typeId = builder.makeFloatType(width: 32);
9258	opCode = spv::OpHitObjectGetCurrentTimeNV;
9259	break;
9260	case glslang::EOpHitObjectGetShaderBindingTableRecordIndexNV:
9261	typeId = builder.makeIntegerType(width: 32, hasSign: 0);
9262	opCode = spv::OpHitObjectGetShaderBindingTableRecordIndexNV;
9263	return 0;
9264	case glslang::EOpHitObjectGetAttributesNV:
9265	builder.createNoResultOp(spv::OpHitObjectGetAttributesNV, operands);
9266	return 0;
9267	case glslang::EOpHitObjectGetShaderRecordBufferHandleNV:
9268	typeId = builder.makeVectorType(component: builder.makeUintType(width: 32), size: 2);
9269	opCode = spv::OpHitObjectGetShaderRecordBufferHandleNV;
9270	break;
9271	case glslang::EOpReorderThreadNV: {
9272	if (operands.size() == 2) {
9273	builder.createNoResultOp(spv::OpReorderThreadWithHintNV, operands);
9274	} else {
9275	builder.createNoResultOp(spv::OpReorderThreadWithHitObjectNV, operands);
9276	}
9277	return 0;
9278
9279	}
9280
9281	case glslang::EOpImageSampleWeightedQCOM:
9282	typeId = builder.makeVectorType(component: builder.makeFloatType(width: 32), size: 4);
9283	opCode = spv::OpImageSampleWeightedQCOM;
9284	addImageProcessingQCOMDecoration(id: operands[2], decor: spv::DecorationWeightTextureQCOM);
9285	break;
9286	case glslang::EOpImageBoxFilterQCOM:
9287	typeId = builder.makeVectorType(component: builder.makeFloatType(width: 32), size: 4);
9288	opCode = spv::OpImageBoxFilterQCOM;
9289	break;
9290	case glslang::EOpImageBlockMatchSADQCOM:
9291	typeId = builder.makeVectorType(component: builder.makeFloatType(width: 32), size: 4);
9292	opCode = spv::OpImageBlockMatchSADQCOM;
9293	addImageProcessingQCOMDecoration(id: operands[0], decor: spv::DecorationBlockMatchTextureQCOM);
9294	addImageProcessingQCOMDecoration(id: operands[2], decor: spv::DecorationBlockMatchTextureQCOM);
9295	break;
9296	case glslang::EOpImageBlockMatchSSDQCOM:
9297	typeId = builder.makeVectorType(component: builder.makeFloatType(width: 32), size: 4);
9298	opCode = spv::OpImageBlockMatchSSDQCOM;
9299	addImageProcessingQCOMDecoration(id: operands[0], decor: spv::DecorationBlockMatchTextureQCOM);
9300	addImageProcessingQCOMDecoration(id: operands[2], decor: spv::DecorationBlockMatchTextureQCOM);
9301	break;
9302
9303	case glslang::EOpFetchMicroTriangleVertexBarycentricNV:
9304	typeId = builder.makeVectorType(component: builder.makeFloatType(width: 32), size: 2);
9305	opCode = spv::OpFetchMicroTriangleVertexBarycentricNV;
9306	break;
9307
9308	case glslang::EOpFetchMicroTriangleVertexPositionNV:
9309	typeId = builder.makeVectorType(component: builder.makeFloatType(width: 32), size: 3);
9310	opCode = spv::OpFetchMicroTriangleVertexPositionNV;
9311	break;
9312
9313	case glslang::EOpImageBlockMatchWindowSSDQCOM:
9314	typeId = builder.makeVectorType(component: builder.makeFloatType(width: 32), size: 4);
9315	opCode = spv::OpImageBlockMatchWindowSSDQCOM;
9316	addImageProcessing2QCOMDecoration(id: operands[0], isForGather: false);
9317	addImageProcessing2QCOMDecoration(id: operands[2], isForGather: false);
9318	break;
9319	case glslang::EOpImageBlockMatchWindowSADQCOM:
9320	typeId = builder.makeVectorType(component: builder.makeFloatType(width: 32), size: 4);
9321	opCode = spv::OpImageBlockMatchWindowSADQCOM;
9322	addImageProcessing2QCOMDecoration(id: operands[0], isForGather: false);
9323	addImageProcessing2QCOMDecoration(id: operands[2], isForGather: false);
9324	break;
9325	case glslang::EOpImageBlockMatchGatherSSDQCOM:
9326	typeId = builder.makeVectorType(component: builder.makeFloatType(width: 32), size: 4);
9327	opCode = spv::OpImageBlockMatchGatherSSDQCOM;
9328	addImageProcessing2QCOMDecoration(id: operands[0], isForGather: true);
9329	addImageProcessing2QCOMDecoration(id: operands[2], isForGather: true);
9330	break;
9331	case glslang::EOpImageBlockMatchGatherSADQCOM:
9332	typeId = builder.makeVectorType(component: builder.makeFloatType(width: 32), size: 4);
9333	opCode = spv::OpImageBlockMatchGatherSADQCOM;
9334	addImageProcessing2QCOMDecoration(id: operands[0], isForGather: true);
9335	addImageProcessing2QCOMDecoration(id: operands[2], isForGather: true);
9336	break;
9337	default:
9338	return 0;
9339	}
9340
9341	spv::Id id = 0;
9342	if (libCall >= 0) {
9343	// Use an extended instruction from the standard library.
9344	// Construct the call arguments, without modifying the original operands vector.
9345	// We might need the remaining arguments, e.g. in the EOpFrexp case.
9346	std::vector<spv::Id> callArguments(consumedOperands);
9347	for (size_t i = 0; i < consumedOperands; ++i)
9348	callArguments[i] = operands[i];
9349	id = builder.createBuiltinCall(resultType: typeId, builtins: extBuiltins >= 0 ? extBuiltins : stdBuiltins, entryPoint: libCall, args: callArguments);
9350	} else if (opCode == spv::OpDot && !isFloat) {
9351	// int dot(int, int)
9352	// NOTE: never called for scalar/vector1, this is turned into simple mul before this can be reached
9353	const int componentCount = builder.getNumComponents(resultId: operands[0]);
9354	spv::Id mulOp = builder.createBinOp(spv::OpIMul, typeId: builder.getTypeId(resultId: operands[0]), operand1: operands[0], operand2: operands[1]);
9355	builder.setPrecision(id: mulOp, precision);
9356	id = builder.createCompositeExtract(composite: mulOp, typeId, index: 0);
9357	for (int i = 1; i < componentCount; ++i) {
9358	builder.setPrecision(id, precision);
9359	id = builder.createBinOp(spv::OpIAdd, typeId, operand1: id, operand2: builder.createCompositeExtract(composite: mulOp, typeId, index: i));
9360	}
9361	} else {
9362	switch (consumedOperands) {
9363	case 0:
9364	// should all be handled by visitAggregate and createNoArgOperation
9365	assert(0);
9366	return 0;
9367	case 1:
9368	// should all be handled by createUnaryOperation
9369	assert(0);
9370	return 0;
9371	case 2:
9372	id = builder.createBinOp(opCode, typeId, operand1: operands[0], operand2: operands[1]);
9373	break;
9374	default:
9375	// anything 3 or over doesn't have l-value operands, so all should be consumed
9376	assert(consumedOperands == operands.size());
9377	id = builder.createOp(opCode, typeId, operands);
9378	break;
9379	}
9380	}
9381
9382	// Decode the return types that were structures
9383	switch (op) {
9384	case glslang::EOpAddCarry:
9385	case glslang::EOpSubBorrow:
9386	builder.createStore(rValue: builder.createCompositeExtract(composite: id, typeId: typeId0, index: 1), lValue: operands[2]);
9387	id = builder.createCompositeExtract(composite: id, typeId: typeId0, index: 0);
9388	break;
9389	case glslang::EOpUMulExtended:
9390	case glslang::EOpIMulExtended:
9391	builder.createStore(rValue: builder.createCompositeExtract(composite: id, typeId: typeId0, index: 0), lValue: operands[3]);
9392	builder.createStore(rValue: builder.createCompositeExtract(composite: id, typeId: typeId0, index: 1), lValue: operands[2]);
9393	break;
9394	case glslang::EOpFrexp:
9395	{
9396	assert(operands.size() == 2);
9397	if (builder.isFloatType(typeId: builder.getScalarTypeId(typeId: typeId1))) {
9398	// "exp" is floating-point type (from HLSL intrinsic)
9399	spv::Id member1 = builder.createCompositeExtract(composite: id, typeId: frexpIntType, index: 1);
9400	member1 = builder.createUnaryOp(spv::OpConvertSToF, typeId: typeId1, operand: member1);
9401	builder.createStore(rValue: member1, lValue: operands[1]);
9402	} else
9403	// "exp" is integer type (from GLSL built-in function)
9404	builder.createStore(rValue: builder.createCompositeExtract(composite: id, typeId: frexpIntType, index: 1), lValue: operands[1]);
9405	id = builder.createCompositeExtract(composite: id, typeId: typeId0, index: 0);
9406	}
9407	break;
9408	default:
9409	break;
9410	}
9411
9412	return builder.setPrecision(id, precision);
9413	}
9414
9415	// Intrinsics with no arguments (or no return value, and no precision).
9416	spv::Id TGlslangToSpvTraverser::createNoArgOperation(glslang::TOperator op, spv::Decoration precision, spv::Id typeId)
9417	{
9418	// GLSL memory barriers use queuefamily scope in new model, device scope in old model
9419	spv::Scope memoryBarrierScope = glslangIntermediate->usingVulkanMemoryModel() ?
9420	spv::ScopeQueueFamilyKHR : spv::ScopeDevice;
9421
9422	switch (op) {
9423	case glslang::EOpBarrier:
9424	if (glslangIntermediate->getStage() == EShLangTessControl) {
9425	if (glslangIntermediate->usingVulkanMemoryModel()) {
9426	builder.createControlBarrier(execution: spv::ScopeWorkgroup, memory: spv::ScopeWorkgroup,
9427	spv::MemorySemanticsOutputMemoryKHRMask |
9428	spv::MemorySemanticsAcquireReleaseMask);
9429	builder.addCapability(cap: spv::CapabilityVulkanMemoryModelKHR);
9430	} else {
9431	builder.createControlBarrier(execution: spv::ScopeWorkgroup, memory: spv::ScopeInvocation, spv::MemorySemanticsMaskNone);
9432	}
9433	} else {
9434	builder.createControlBarrier(execution: spv::ScopeWorkgroup, memory: spv::ScopeWorkgroup,
9435	spv::MemorySemanticsWorkgroupMemoryMask |
9436	spv::MemorySemanticsAcquireReleaseMask);
9437	}
9438	return 0;
9439	case glslang::EOpMemoryBarrier:
9440	builder.createMemoryBarrier(executionScope: memoryBarrierScope, memorySemantics: spv::MemorySemanticsAllMemory |
9441	spv::MemorySemanticsAcquireReleaseMask);
9442	return 0;
9443	case glslang::EOpMemoryBarrierBuffer:
9444	builder.createMemoryBarrier(executionScope: memoryBarrierScope, memorySemantics: spv::MemorySemanticsUniformMemoryMask |
9445	spv::MemorySemanticsAcquireReleaseMask);
9446	return 0;
9447	case glslang::EOpMemoryBarrierShared:
9448	builder.createMemoryBarrier(executionScope: memoryBarrierScope, memorySemantics: spv::MemorySemanticsWorkgroupMemoryMask |
9449	spv::MemorySemanticsAcquireReleaseMask);
9450	return 0;
9451	case glslang::EOpGroupMemoryBarrier:
9452	builder.createMemoryBarrier(executionScope: spv::ScopeWorkgroup, memorySemantics: spv::MemorySemanticsAllMemory |
9453	spv::MemorySemanticsAcquireReleaseMask);
9454	return 0;
9455	case glslang::EOpMemoryBarrierAtomicCounter:
9456	builder.createMemoryBarrier(executionScope: memoryBarrierScope, memorySemantics: spv::MemorySemanticsAtomicCounterMemoryMask |
9457	spv::MemorySemanticsAcquireReleaseMask);
9458	return 0;
9459	case glslang::EOpMemoryBarrierImage:
9460	builder.createMemoryBarrier(executionScope: memoryBarrierScope, memorySemantics: spv::MemorySemanticsImageMemoryMask |
9461	spv::MemorySemanticsAcquireReleaseMask);
9462	return 0;
9463	case glslang::EOpAllMemoryBarrierWithGroupSync:
9464	builder.createControlBarrier(execution: spv::ScopeWorkgroup, memory: spv::ScopeDevice,
9465	spv::MemorySemanticsAllMemory |
9466	spv::MemorySemanticsAcquireReleaseMask);
9467	return 0;
9468	case glslang::EOpDeviceMemoryBarrier:
9469	builder.createMemoryBarrier(executionScope: spv::ScopeDevice, memorySemantics: spv::MemorySemanticsUniformMemoryMask |
9470	spv::MemorySemanticsImageMemoryMask |
9471	spv::MemorySemanticsAcquireReleaseMask);
9472	return 0;
9473	case glslang::EOpDeviceMemoryBarrierWithGroupSync:
9474	builder.createControlBarrier(execution: spv::ScopeWorkgroup, memory: spv::ScopeDevice, spv::MemorySemanticsUniformMemoryMask |
9475	spv::MemorySemanticsImageMemoryMask |
9476	spv::MemorySemanticsAcquireReleaseMask);
9477	return 0;
9478	case glslang::EOpWorkgroupMemoryBarrier:
9479	builder.createMemoryBarrier(executionScope: spv::ScopeWorkgroup, memorySemantics: spv::MemorySemanticsWorkgroupMemoryMask |
9480	spv::MemorySemanticsAcquireReleaseMask);
9481	return 0;
9482	case glslang::EOpWorkgroupMemoryBarrierWithGroupSync:
9483	builder.createControlBarrier(execution: spv::ScopeWorkgroup, memory: spv::ScopeWorkgroup,
9484	spv::MemorySemanticsWorkgroupMemoryMask |
9485	spv::MemorySemanticsAcquireReleaseMask);
9486	return 0;
9487	case glslang::EOpSubgroupBarrier:
9488	builder.createControlBarrier(execution: spv::ScopeSubgroup, memory: spv::ScopeSubgroup, spv::MemorySemanticsAllMemory |
9489	spv::MemorySemanticsAcquireReleaseMask);
9490	return spv::NoResult;
9491	case glslang::EOpSubgroupMemoryBarrier:
9492	builder.createMemoryBarrier(executionScope: spv::ScopeSubgroup, memorySemantics: spv::MemorySemanticsAllMemory |
9493	spv::MemorySemanticsAcquireReleaseMask);
9494	return spv::NoResult;
9495	case glslang::EOpSubgroupMemoryBarrierBuffer:
9496	builder.createMemoryBarrier(executionScope: spv::ScopeSubgroup, memorySemantics: spv::MemorySemanticsUniformMemoryMask |
9497	spv::MemorySemanticsAcquireReleaseMask);
9498	return spv::NoResult;
9499	case glslang::EOpSubgroupMemoryBarrierImage:
9500	builder.createMemoryBarrier(executionScope: spv::ScopeSubgroup, memorySemantics: spv::MemorySemanticsImageMemoryMask |
9501	spv::MemorySemanticsAcquireReleaseMask);
9502	return spv::NoResult;
9503	case glslang::EOpSubgroupMemoryBarrierShared:
9504	builder.createMemoryBarrier(executionScope: spv::ScopeSubgroup, memorySemantics: spv::MemorySemanticsWorkgroupMemoryMask |
9505	spv::MemorySemanticsAcquireReleaseMask);
9506	return spv::NoResult;
9507
9508	case glslang::EOpEmitVertex:
9509	builder.createNoResultOp(spv::OpEmitVertex);
9510	return 0;
9511	case glslang::EOpEndPrimitive:
9512	builder.createNoResultOp(spv::OpEndPrimitive);
9513	return 0;
9514
9515	case glslang::EOpSubgroupElect: {
9516	std::vector<spv::Id> operands;
9517	return createSubgroupOperation(op, typeId, operands, typeProxy: glslang::EbtVoid);
9518	}
9519	case glslang::EOpTime:
9520	{
9521	std::vector<spv::Id> args; // Dummy arguments
9522	spv::Id id = builder.createBuiltinCall(resultType: typeId, builtins: getExtBuiltins(name: spv::E_SPV_AMD_gcn_shader), entryPoint: spv::TimeAMD, args);
9523	return builder.setPrecision(id, precision);
9524	}
9525	case glslang::EOpIgnoreIntersectionNV:
9526	builder.createNoResultOp(spv::OpIgnoreIntersectionNV);
9527	return 0;
9528	case glslang::EOpTerminateRayNV:
9529	builder.createNoResultOp(spv::OpTerminateRayNV);
9530	return 0;
9531	case glslang::EOpRayQueryInitialize:
9532	builder.createNoResultOp(spv::OpRayQueryInitializeKHR);
9533	return 0;
9534	case glslang::EOpRayQueryTerminate:
9535	builder.createNoResultOp(spv::OpRayQueryTerminateKHR);
9536	return 0;
9537	case glslang::EOpRayQueryGenerateIntersection:
9538	builder.createNoResultOp(spv::OpRayQueryGenerateIntersectionKHR);
9539	return 0;
9540	case glslang::EOpRayQueryConfirmIntersection:
9541	builder.createNoResultOp(spv::OpRayQueryConfirmIntersectionKHR);
9542	return 0;
9543	case glslang::EOpBeginInvocationInterlock:
9544	builder.createNoResultOp(spv::OpBeginInvocationInterlockEXT);
9545	return 0;
9546	case glslang::EOpEndInvocationInterlock:
9547	builder.createNoResultOp(spv::OpEndInvocationInterlockEXT);
9548	return 0;
9549
9550	case glslang::EOpIsHelperInvocation:
9551	{
9552	std::vector<spv::Id> args; // Dummy arguments
9553	builder.addExtension(ext: spv::E_SPV_EXT_demote_to_helper_invocation);
9554	builder.addCapability(cap: spv::CapabilityDemoteToHelperInvocationEXT);
9555	return builder.createOp(spv::OpIsHelperInvocationEXT, typeId, operands: args);
9556	}
9557
9558	case glslang::EOpReadClockSubgroupKHR: {
9559	std::vector<spv::Id> args;
9560	args.push_back(x: builder.makeUintConstant(u: spv::ScopeSubgroup));
9561	builder.addExtension(ext: spv::E_SPV_KHR_shader_clock);
9562	builder.addCapability(cap: spv::CapabilityShaderClockKHR);
9563	return builder.createOp(spv::OpReadClockKHR, typeId, operands: args);
9564	}
9565
9566	case glslang::EOpReadClockDeviceKHR: {
9567	std::vector<spv::Id> args;
9568	args.push_back(x: builder.makeUintConstant(u: spv::ScopeDevice));
9569	builder.addExtension(ext: spv::E_SPV_KHR_shader_clock);
9570	builder.addCapability(cap: spv::CapabilityShaderClockKHR);
9571	return builder.createOp(spv::OpReadClockKHR, typeId, operands: args);
9572	}
9573	case glslang::EOpStencilAttachmentReadEXT:
9574	case glslang::EOpDepthAttachmentReadEXT:
9575	{
9576	builder.addExtension(ext: spv::E_SPV_EXT_shader_tile_image);
9577
9578	spv::Decoration precision;
9579	spv::Op spv_op;
9580	if (op == glslang::EOpStencilAttachmentReadEXT)
9581	{
9582	precision = spv::DecorationRelaxedPrecision;
9583	spv_op = spv::OpStencilAttachmentReadEXT;
9584	builder.addCapability(cap: spv::CapabilityTileImageStencilReadAccessEXT);
9585	}
9586	else
9587	{
9588	precision = spv::NoPrecision;
9589	spv_op = spv::OpDepthAttachmentReadEXT;
9590	builder.addCapability(cap: spv::CapabilityTileImageDepthReadAccessEXT);
9591	}
9592
9593	std::vector<spv::Id> args; // Dummy args
9594	spv::Id result = builder.createOp(spv_op, typeId, operands: args);
9595	return builder.setPrecision(id: result, precision);
9596	}
9597	default:
9598	break;
9599	}
9600
9601	logger->missingFunctionality(f: "unknown operation with no arguments");
9602
9603	return 0;
9604	}
9605
9606	spv::Id TGlslangToSpvTraverser::getSymbolId(const glslang::TIntermSymbol* symbol)
9607	{
9608	auto iter = symbolValues.find(x: symbol->getId());
9609	spv::Id id;
9610	if (symbolValues.end() != iter) {
9611	id = iter->second;
9612	return id;
9613	}
9614
9615	// it was not found, create it
9616	spv::BuiltIn builtIn = TranslateBuiltInDecoration(builtIn: symbol->getQualifier().builtIn, memberDeclaration: false);
9617	auto forcedType = getForcedType(glslangBuiltIn: symbol->getQualifier().builtIn, glslangType: symbol->getType());
9618
9619	// There are pairs of symbols that map to the same SPIR-V built-in:
9620	// gl_ObjectToWorldEXT and gl_ObjectToWorld3x4EXT, and gl_WorldToObjectEXT
9621	// and gl_WorldToObject3x4EXT. SPIR-V forbids having two OpVariables
9622	// with the same BuiltIn in the same storage class, so we must re-use one.
9623	const bool mayNeedToReuseBuiltIn =
9624	builtIn == spv::BuiltInObjectToWorldKHR ||
9625	builtIn == spv::BuiltInWorldToObjectKHR;
9626
9627	if (mayNeedToReuseBuiltIn) {
9628	auto iter = builtInVariableIds.find(x: uint32_t(builtIn));
9629	if (builtInVariableIds.end() != iter) {
9630	id = iter->second;
9631	symbolValues[symbol->getId()] = id;
9632	if (forcedType.second != spv::NoType)
9633	forceType[id] = forcedType.second;
9634	return id;
9635	}
9636	}
9637
9638	id = createSpvVariable(node: symbol, forcedType: forcedType.first);
9639
9640	if (mayNeedToReuseBuiltIn) {
9641	builtInVariableIds.insert(x: {uint32_t(builtIn), id});
9642	}
9643
9644	symbolValues[symbol->getId()] = id;
9645	if (forcedType.second != spv::NoType)
9646	forceType[id] = forcedType.second;
9647
9648	if (symbol->getBasicType() != glslang::EbtBlock) {
9649	builder.addDecoration(id, TranslatePrecisionDecoration(type: symbol->getType()));
9650	builder.addDecoration(id, TranslateInterpolationDecoration(qualifier: symbol->getType().getQualifier()));
9651	builder.addDecoration(id, TranslateAuxiliaryStorageDecoration(qualifier: symbol->getType().getQualifier()));
9652	addMeshNVDecoration(id, /*member*/ -1, qualifier: symbol->getType().getQualifier());
9653	if (symbol->getQualifier().hasComponent())
9654	builder.addDecoration(id, spv::DecorationComponent, num: symbol->getQualifier().layoutComponent);
9655	if (symbol->getQualifier().hasIndex())
9656	builder.addDecoration(id, spv::DecorationIndex, num: symbol->getQualifier().layoutIndex);
9657	if (symbol->getType().getQualifier().hasSpecConstantId())
9658	builder.addDecoration(id, spv::DecorationSpecId, num: symbol->getType().getQualifier().layoutSpecConstantId);
9659	// atomic counters use this:
9660	if (symbol->getQualifier().hasOffset())
9661	builder.addDecoration(id, spv::DecorationOffset, num: symbol->getQualifier().layoutOffset);
9662	}
9663
9664	if (symbol->getQualifier().hasLocation()) {
9665	if (!(glslangIntermediate->isRayTracingStage() &&
9666	(glslangIntermediate->IsRequestedExtension(extension: glslang::E_GL_EXT_ray_tracing) ||
9667	glslangIntermediate->IsRequestedExtension(extension: glslang::E_GL_NV_shader_invocation_reorder))
9668	&& (builder.getStorageClass(resultId: id) == spv::StorageClassRayPayloadKHR ||
9669	builder.getStorageClass(resultId: id) == spv::StorageClassIncomingRayPayloadKHR ||
9670	builder.getStorageClass(resultId: id) == spv::StorageClassCallableDataKHR ||
9671	builder.getStorageClass(resultId: id) == spv::StorageClassIncomingCallableDataKHR ||
9672	builder.getStorageClass(resultId: id) == spv::StorageClassHitObjectAttributeNV))) {
9673	// Location values are used to link TraceRayKHR/ExecuteCallableKHR/HitObjectGetAttributesNV
9674	// to corresponding variables but are not valid in SPIRV since they are supported only
9675	// for Input/Output Storage classes.
9676	builder.addDecoration(id, spv::DecorationLocation, num: symbol->getQualifier().layoutLocation);
9677	}
9678	}
9679
9680	builder.addDecoration(id, TranslateInvariantDecoration(qualifier: symbol->getType().getQualifier()));
9681	if (symbol->getQualifier().hasStream() && glslangIntermediate->isMultiStream()) {
9682	builder.addCapability(cap: spv::CapabilityGeometryStreams);
9683	builder.addDecoration(id, spv::DecorationStream, num: symbol->getQualifier().layoutStream);
9684	}
9685	if (symbol->getQualifier().hasSet())
9686	builder.addDecoration(id, spv::DecorationDescriptorSet, num: symbol->getQualifier().layoutSet);
9687	else if (IsDescriptorResource(type: symbol->getType())) {
9688	// default to 0
9689	builder.addDecoration(id, spv::DecorationDescriptorSet, num: 0);
9690	}
9691	if (symbol->getQualifier().hasBinding())
9692	builder.addDecoration(id, spv::DecorationBinding, num: symbol->getQualifier().layoutBinding);
9693	else if (IsDescriptorResource(type: symbol->getType())) {
9694	// default to 0
9695	builder.addDecoration(id, spv::DecorationBinding, num: 0);
9696	}
9697	if (symbol->getQualifier().hasAttachment())
9698	builder.addDecoration(id, spv::DecorationInputAttachmentIndex, num: symbol->getQualifier().layoutAttachment);
9699	if (glslangIntermediate->getXfbMode()) {
9700	builder.addCapability(cap: spv::CapabilityTransformFeedback);
9701	if (symbol->getQualifier().hasXfbBuffer()) {
9702	builder.addDecoration(id, spv::DecorationXfbBuffer, num: symbol->getQualifier().layoutXfbBuffer);
9703	unsigned stride = glslangIntermediate->getXfbStride(buffer: symbol->getQualifier().layoutXfbBuffer);
9704	if (stride != glslang::TQualifier::layoutXfbStrideEnd)
9705	builder.addDecoration(id, spv::DecorationXfbStride, num: stride);
9706	}
9707	if (symbol->getQualifier().hasXfbOffset())
9708	builder.addDecoration(id, spv::DecorationOffset, num: symbol->getQualifier().layoutXfbOffset);
9709	}
9710
9711	// add built-in variable decoration
9712	if (builtIn != spv::BuiltInMax) {
9713	// WorkgroupSize deprecated in spirv1.6
9714	if (glslangIntermediate->getSpv().spv < glslang::EShTargetSpv_1_6 ||
9715	builtIn != spv::BuiltInWorkgroupSize)
9716	builder.addDecoration(id, spv::DecorationBuiltIn, num: (int)builtIn);
9717	}
9718
9719	// Add volatile decoration to HelperInvocation for spirv1.6 and beyond
9720	if (builtIn == spv::BuiltInHelperInvocation &&
9721	!glslangIntermediate->usingVulkanMemoryModel() &&
9722	glslangIntermediate->getSpv().spv >= glslang::EShTargetSpv_1_6) {
9723	builder.addDecoration(id, spv::DecorationVolatile);
9724	}
9725
9726	// Subgroup builtins which have input storage class are volatile for ray tracing stages.
9727	if (symbol->getType().isImage() || symbol->getQualifier().isPipeInput()) {
9728	std::vector<spv::Decoration> memory;
9729	TranslateMemoryDecoration(qualifier: symbol->getType().getQualifier(), memory,
9730	useVulkanMemoryModel: glslangIntermediate->usingVulkanMemoryModel());
9731	for (unsigned int i = 0; i < memory.size(); ++i)
9732	builder.addDecoration(id, memory[i]);
9733	}
9734
9735	if (builtIn == spv::BuiltInSampleMask) {
9736	spv::Decoration decoration;
9737	// GL_NV_sample_mask_override_coverage extension
9738	if (glslangIntermediate->getLayoutOverrideCoverage())
9739	decoration = (spv::Decoration)spv::DecorationOverrideCoverageNV;
9740	else
9741	decoration = (spv::Decoration)spv::DecorationMax;
9742	builder.addDecoration(id, decoration);
9743	if (decoration != spv::DecorationMax) {
9744	builder.addCapability(cap: spv::CapabilitySampleMaskOverrideCoverageNV);
9745	builder.addExtension(ext: spv::E_SPV_NV_sample_mask_override_coverage);
9746	}
9747	}
9748	else if (builtIn == spv::BuiltInLayer) {
9749	// SPV_NV_viewport_array2 extension
9750	if (symbol->getQualifier().layoutViewportRelative) {
9751	builder.addDecoration(id, (spv::Decoration)spv::DecorationViewportRelativeNV);
9752	builder.addCapability(cap: spv::CapabilityShaderViewportMaskNV);
9753	builder.addExtension(ext: spv::E_SPV_NV_viewport_array2);
9754	}
9755	if (symbol->getQualifier().layoutSecondaryViewportRelativeOffset != -2048) {
9756	builder.addDecoration(id, (spv::Decoration)spv::DecorationSecondaryViewportRelativeNV,
9757	num: symbol->getQualifier().layoutSecondaryViewportRelativeOffset);
9758	builder.addCapability(cap: spv::CapabilityShaderStereoViewNV);
9759	builder.addExtension(ext: spv::E_SPV_NV_stereo_view_rendering);
9760	}
9761	}
9762
9763	if (symbol->getQualifier().layoutPassthrough) {
9764	builder.addDecoration(id, spv::DecorationPassthroughNV);
9765	builder.addCapability(cap: spv::CapabilityGeometryShaderPassthroughNV);
9766	builder.addExtension(ext: spv::E_SPV_NV_geometry_shader_passthrough);
9767	}
9768	if (symbol->getQualifier().pervertexNV) {
9769	builder.addDecoration(id, spv::DecorationPerVertexNV);
9770	builder.addCapability(cap: spv::CapabilityFragmentBarycentricNV);
9771	builder.addExtension(ext: spv::E_SPV_NV_fragment_shader_barycentric);
9772	}
9773
9774	if (symbol->getQualifier().pervertexEXT) {
9775	builder.addDecoration(id, spv::DecorationPerVertexKHR);
9776	builder.addCapability(cap: spv::CapabilityFragmentBarycentricKHR);
9777	builder.addExtension(ext: spv::E_SPV_KHR_fragment_shader_barycentric);
9778	}
9779
9780	if (glslangIntermediate->getHlslFunctionality1() && symbol->getType().getQualifier().semanticName != nullptr) {
9781	builder.addExtension(ext: "SPV_GOOGLE_hlsl_functionality1");
9782	builder.addDecoration(id, (spv::Decoration)spv::DecorationHlslSemanticGOOGLE,
9783	symbol->getType().getQualifier().semanticName);
9784	}
9785
9786	if (symbol->isReference()) {
9787	builder.addDecoration(id, symbol->getType().getQualifier().restrict ?
9788	spv::DecorationRestrictPointerEXT : spv::DecorationAliasedPointerEXT);
9789	}
9790
9791	// Add SPIR-V decorations (GL_EXT_spirv_intrinsics)
9792	if (symbol->getType().getQualifier().hasSpirvDecorate())
9793	applySpirvDecorate(type: symbol->getType(), id, member: {});
9794
9795	return id;
9796	}
9797
9798	// add per-primitive, per-view. per-task decorations to a struct member (member >= 0) or an object
9799	void TGlslangToSpvTraverser::addMeshNVDecoration(spv::Id id, int member, const glslang::TQualifier& qualifier)
9800	{
9801	bool isMeshShaderExt = (glslangIntermediate->getRequestedExtensions().find(x: glslang::E_GL_EXT_mesh_shader) !=
9802	glslangIntermediate->getRequestedExtensions().end());
9803
9804	if (member >= 0) {
9805	if (qualifier.perPrimitiveNV) {
9806	// Need to add capability/extension for fragment shader.
9807	// Mesh shader already adds this by default.
9808	if (glslangIntermediate->getStage() == EShLangFragment) {
9809	if(isMeshShaderExt) {
9810	builder.addCapability(cap: spv::CapabilityMeshShadingEXT);
9811	builder.addExtension(ext: spv::E_SPV_EXT_mesh_shader);
9812	} else {
9813	builder.addCapability(cap: spv::CapabilityMeshShadingNV);
9814	builder.addExtension(ext: spv::E_SPV_NV_mesh_shader);
9815	}
9816	}
9817	builder.addMemberDecoration(id, member: (unsigned)member, spv::DecorationPerPrimitiveNV);
9818	}
9819	if (qualifier.perViewNV)
9820	builder.addMemberDecoration(id, member: (unsigned)member, spv::DecorationPerViewNV);
9821	if (qualifier.perTaskNV)
9822	builder.addMemberDecoration(id, member: (unsigned)member, spv::DecorationPerTaskNV);
9823	} else {
9824	if (qualifier.perPrimitiveNV) {
9825	// Need to add capability/extension for fragment shader.
9826	// Mesh shader already adds this by default.
9827	if (glslangIntermediate->getStage() == EShLangFragment) {
9828	if(isMeshShaderExt) {
9829	builder.addCapability(cap: spv::CapabilityMeshShadingEXT);
9830	builder.addExtension(ext: spv::E_SPV_EXT_mesh_shader);
9831	} else {
9832	builder.addCapability(cap: spv::CapabilityMeshShadingNV);
9833	builder.addExtension(ext: spv::E_SPV_NV_mesh_shader);
9834	}
9835	}
9836	builder.addDecoration(id, spv::DecorationPerPrimitiveNV);
9837	}
9838	if (qualifier.perViewNV)
9839	builder.addDecoration(id, spv::DecorationPerViewNV);
9840	if (qualifier.perTaskNV)
9841	builder.addDecoration(id, spv::DecorationPerTaskNV);
9842	}
9843	}
9844
9845	bool TGlslangToSpvTraverser::hasQCOMImageProceessingDecoration(spv::Id id, spv::Decoration decor)
9846	{
9847	std::vector<spv::Decoration> &decoVec = idToQCOMDecorations[id];
9848	for ( auto d : decoVec ) {
9849	if ( d == decor )
9850	return true;
9851	}
9852	return false;
9853	}
9854
9855	void TGlslangToSpvTraverser::addImageProcessingQCOMDecoration(spv::Id id, spv::Decoration decor)
9856	{
9857	spv::Op opc = builder.getOpCode(id);
9858	if (opc == spv::OpSampledImage) {
9859	id = builder.getIdOperand(resultId: id, idx: 0);
9860	opc = builder.getOpCode(id);
9861	}
9862
9863	if (opc == spv::OpLoad) {
9864	spv::Id texid = builder.getIdOperand(resultId: id, idx: 0);
9865	if (!hasQCOMImageProceessingDecoration(id: texid, decor)) {//
9866	builder.addDecoration(texid, decor);
9867	idToQCOMDecorations[texid].push_back(x: decor);
9868	}
9869	}
9870	}
9871
9872	void TGlslangToSpvTraverser::addImageProcessing2QCOMDecoration(spv::Id id, bool isForGather)
9873	{
9874	if (isForGather) {
9875	return addImageProcessingQCOMDecoration(id, decor: spv::DecorationBlockMatchTextureQCOM);
9876	}
9877
9878	auto addDecor =
9879	[this](spv::Id id, spv::Decoration decor) {
9880	spv::Id tsopc = this->builder.getOpCode(id);
9881	if (tsopc == spv::OpLoad) {
9882	spv::Id tsid = this->builder.getIdOperand(resultId: id, idx: 0);
9883	if (this->glslangIntermediate->getSpv().spv >= glslang::EShTargetSpv_1_4) {
9884	assert(iOSet.count(tsid) > 0);
9885	}
9886	if (!hasQCOMImageProceessingDecoration(id: tsid, decor)) {
9887	this->builder.addDecoration(tsid, decor);
9888	idToQCOMDecorations[tsid].push_back(x: decor);
9889	}
9890	}
9891	};
9892
9893	spv::Id opc = builder.getOpCode(id);
9894	bool isInterfaceObject = (opc != spv::OpSampledImage);
9895
9896	if (!isInterfaceObject) {
9897	addDecor(builder.getIdOperand(resultId: id, idx: 0), spv::DecorationBlockMatchTextureQCOM);
9898	addDecor(builder.getIdOperand(resultId: id, idx: 1), spv::DecorationBlockMatchSamplerQCOM);
9899	} else {
9900	addDecor(id, spv::DecorationBlockMatchTextureQCOM);
9901	addDecor(id, spv::DecorationBlockMatchSamplerQCOM);
9902	}
9903	}
9904
9905	// Make a full tree of instructions to build a SPIR-V specialization constant,
9906	// or regular constant if possible.
9907	//
9908	// TBD: this is not yet done, nor verified to be the best design, it does do the leaf symbols though
9909	//
9910	// Recursively walk the nodes. The nodes form a tree whose leaves are
9911	// regular constants, which themselves are trees that createSpvConstant()
9912	// recursively walks. So, this function walks the "top" of the tree:
9913	// - emit specialization constant-building instructions for specConstant
9914	// - when running into a non-spec-constant, switch to createSpvConstant()
9915	spv::Id TGlslangToSpvTraverser::createSpvConstant(const glslang::TIntermTyped& node)
9916	{
9917	assert(node.getQualifier().isConstant());
9918
9919	// Handle front-end constants first (non-specialization constants).
9920	if (! node.getQualifier().specConstant) {
9921	// hand off to the non-spec-constant path
9922	assert(node.getAsConstantUnion() != nullptr || node.getAsSymbolNode() != nullptr);
9923	int nextConst = 0;
9924	return createSpvConstantFromConstUnionArray(type: node.getType(), node.getAsConstantUnion() ?
9925	node.getAsConstantUnion()->getConstArray() : node.getAsSymbolNode()->getConstArray(),
9926	nextConst, specConstant: false);
9927	}
9928
9929	// We now know we have a specialization constant to build
9930
9931	// Extra capabilities may be needed.
9932	if (node.getType().contains8BitInt())
9933	builder.addCapability(cap: spv::CapabilityInt8);
9934	if (node.getType().contains16BitFloat())
9935	builder.addCapability(cap: spv::CapabilityFloat16);
9936	if (node.getType().contains16BitInt())
9937	builder.addCapability(cap: spv::CapabilityInt16);
9938	if (node.getType().contains64BitInt())
9939	builder.addCapability(cap: spv::CapabilityInt64);
9940	if (node.getType().containsDouble())
9941	builder.addCapability(cap: spv::CapabilityFloat64);
9942
9943	// gl_WorkGroupSize is a special case until the front-end handles hierarchical specialization constants,
9944	// even then, it's specialization ids are handled by special case syntax in GLSL: layout(local_size_x = ...
9945	if (node.getType().getQualifier().builtIn == glslang::EbvWorkGroupSize) {
9946	std::vector<spv::Id> dimConstId;
9947	for (int dim = 0; dim < 3; ++dim) {
9948	bool specConst = (glslangIntermediate->getLocalSizeSpecId(dim) != glslang::TQualifier::layoutNotSet);
9949	dimConstId.push_back(x: builder.makeUintConstant(u: glslangIntermediate->getLocalSize(dim), specConstant: specConst));
9950	if (specConst) {
9951	builder.addDecoration(dimConstId.back(), spv::DecorationSpecId,
9952	num: glslangIntermediate->getLocalSizeSpecId(dim));
9953	}
9954	}
9955	return builder.makeCompositeConstant(type: builder.makeVectorType(component: builder.makeUintType(width: 32), size: 3), comps: dimConstId, specConst: true);
9956	}
9957
9958	// An AST node labelled as specialization constant should be a symbol node.
9959	// Its initializer should either be a sub tree with constant nodes, or a constant union array.
9960	if (auto* sn = node.getAsSymbolNode()) {
9961	spv::Id result;
9962	if (auto* sub_tree = sn->getConstSubtree()) {
9963	// Traverse the constant constructor sub tree like generating normal run-time instructions.
9964	// During the AST traversal, if the node is marked as 'specConstant', SpecConstantOpModeGuard
9965	// will set the builder into spec constant op instruction generating mode.
9966	sub_tree->traverse(this);
9967	result = accessChainLoad(type: sub_tree->getType());
9968	} else if (auto* const_union_array = &sn->getConstArray()) {
9969	int nextConst = 0;
9970	result = createSpvConstantFromConstUnionArray(type: sn->getType(), *const_union_array, nextConst, specConstant: true);
9971	} else {
9972	logger->missingFunctionality(f: "Invalid initializer for spec onstant.");
9973	return spv::NoResult;
9974	}
9975	builder.addName(result, name: sn->getName().c_str());
9976	return result;
9977	}
9978
9979	// Neither a front-end constant node, nor a specialization constant node with constant union array or
9980	// constant sub tree as initializer.
9981	logger->missingFunctionality(f: "Neither a front-end constant nor a spec constant.");
9982	return spv::NoResult;
9983	}
9984
9985	// Use 'consts' as the flattened glslang source of scalar constants to recursively
9986	// build the aggregate SPIR-V constant.
9987	//
9988	// If there are not enough elements present in 'consts', 0 will be substituted;
9989	// an empty 'consts' can be used to create a fully zeroed SPIR-V constant.
9990	//
9991	spv::Id TGlslangToSpvTraverser::createSpvConstantFromConstUnionArray(const glslang::TType& glslangType,
9992	const glslang::TConstUnionArray& consts, int& nextConst, bool specConstant)
9993	{
9994	// vector of constants for SPIR-V
9995	std::vector<spv::Id> spvConsts;
9996
9997	// Type is used for struct and array constants
9998	spv::Id typeId = convertGlslangToSpvType(type: glslangType);
9999
10000	if (glslangType.isArray()) {
10001	glslang::TType elementType(glslangType, 0);
10002	for (int i = 0; i < glslangType.getOuterArraySize(); ++i)
10003	spvConsts.push_back(x: createSpvConstantFromConstUnionArray(glslangType: elementType, consts, nextConst, specConstant: false));
10004	} else if (glslangType.isMatrix()) {
10005	glslang::TType vectorType(glslangType, 0);
10006	for (int col = 0; col < glslangType.getMatrixCols(); ++col)
10007	spvConsts.push_back(x: createSpvConstantFromConstUnionArray(glslangType: vectorType, consts, nextConst, specConstant: false));
10008	} else if (glslangType.isCoopMat()) {
10009	glslang::TType componentType(glslangType.getBasicType());
10010	spvConsts.push_back(x: createSpvConstantFromConstUnionArray(glslangType: componentType, consts, nextConst, specConstant: false));
10011	} else if (glslangType.isStruct()) {
10012	glslang::TVector<glslang::TTypeLoc>::const_iterator iter;
10013	for (iter = glslangType.getStruct()->begin(); iter != glslangType.getStruct()->end(); ++iter)
10014	spvConsts.push_back(x: createSpvConstantFromConstUnionArray(glslangType: *iter->type, consts, nextConst, specConstant: false));
10015	} else if (glslangType.getVectorSize() > 1) {
10016	for (unsigned int i = 0; i < (unsigned int)glslangType.getVectorSize(); ++i) {
10017	bool zero = nextConst >= consts.size();
10018	switch (glslangType.getBasicType()) {
10019	case glslang::EbtInt:
10020	spvConsts.push_back(x: builder.makeIntConstant(i: zero ? 0 : consts[nextConst].getIConst()));
10021	break;
10022	case glslang::EbtUint:
10023	spvConsts.push_back(x: builder.makeUintConstant(u: zero ? 0 : consts[nextConst].getUConst()));
10024	break;
10025	case glslang::EbtFloat:
10026	spvConsts.push_back(x: builder.makeFloatConstant(f: zero ? 0.0F : (float)consts[nextConst].getDConst()));
10027	break;
10028	case glslang::EbtBool:
10029	spvConsts.push_back(x: builder.makeBoolConstant(b: zero ? false : consts[nextConst].getBConst()));
10030	break;
10031	case glslang::EbtInt8:
10032	builder.addCapability(cap: spv::CapabilityInt8);
10033	spvConsts.push_back(x: builder.makeInt8Constant(i: zero ? 0 : consts[nextConst].getI8Const()));
10034	break;
10035	case glslang::EbtUint8:
10036	builder.addCapability(cap: spv::CapabilityInt8);
10037	spvConsts.push_back(x: builder.makeUint8Constant(u: zero ? 0 : consts[nextConst].getU8Const()));
10038	break;
10039	case glslang::EbtInt16:
10040	builder.addCapability(cap: spv::CapabilityInt16);
10041	spvConsts.push_back(x: builder.makeInt16Constant(i: zero ? 0 : consts[nextConst].getI16Const()));
10042	break;
10043	case glslang::EbtUint16:
10044	builder.addCapability(cap: spv::CapabilityInt16);
10045	spvConsts.push_back(x: builder.makeUint16Constant(u: zero ? 0 : consts[nextConst].getU16Const()));
10046	break;
10047	case glslang::EbtInt64:
10048	spvConsts.push_back(x: builder.makeInt64Constant(i: zero ? 0 : consts[nextConst].getI64Const()));
10049	break;
10050	case glslang::EbtUint64:
10051	spvConsts.push_back(x: builder.makeUint64Constant(u: zero ? 0 : consts[nextConst].getU64Const()));
10052	break;
10053	case glslang::EbtDouble:
10054	spvConsts.push_back(x: builder.makeDoubleConstant(d: zero ? 0.0 : consts[nextConst].getDConst()));
10055	break;
10056	case glslang::EbtFloat16:
10057	builder.addCapability(cap: spv::CapabilityFloat16);
10058	spvConsts.push_back(x: builder.makeFloat16Constant(f16: zero ? 0.0F : (float)consts[nextConst].getDConst()));
10059	break;
10060	default:
10061	assert(0);
10062	break;
10063	}
10064	++nextConst;
10065	}
10066	} else {
10067	// we have a non-aggregate (scalar) constant
10068	bool zero = nextConst >= consts.size();
10069	spv::Id scalar = 0;
10070	switch (glslangType.getBasicType()) {
10071	case glslang::EbtInt:
10072	scalar = builder.makeIntConstant(i: zero ? 0 : consts[nextConst].getIConst(), specConstant);
10073	break;
10074	case glslang::EbtUint:
10075	scalar = builder.makeUintConstant(u: zero ? 0 : consts[nextConst].getUConst(), specConstant);
10076	break;
10077	case glslang::EbtFloat:
10078	scalar = builder.makeFloatConstant(f: zero ? 0.0F : (float)consts[nextConst].getDConst(), specConstant);
10079	break;
10080	case glslang::EbtBool:
10081	scalar = builder.makeBoolConstant(b: zero ? false : consts[nextConst].getBConst(), specConstant);
10082	break;
10083	case glslang::EbtInt8:
10084	builder.addCapability(cap: spv::CapabilityInt8);
10085	scalar = builder.makeInt8Constant(i: zero ? 0 : consts[nextConst].getI8Const(), specConstant);
10086	break;
10087	case glslang::EbtUint8:
10088	builder.addCapability(cap: spv::CapabilityInt8);
10089	scalar = builder.makeUint8Constant(u: zero ? 0 : consts[nextConst].getU8Const(), specConstant);
10090	break;
10091	case glslang::EbtInt16:
10092	builder.addCapability(cap: spv::CapabilityInt16);
10093	scalar = builder.makeInt16Constant(i: zero ? 0 : consts[nextConst].getI16Const(), specConstant);
10094	break;
10095	case glslang::EbtUint16:
10096	builder.addCapability(cap: spv::CapabilityInt16);
10097	scalar = builder.makeUint16Constant(u: zero ? 0 : consts[nextConst].getU16Const(), specConstant);
10098	break;
10099	case glslang::EbtInt64:
10100	scalar = builder.makeInt64Constant(i: zero ? 0 : consts[nextConst].getI64Const(), specConstant);
10101	break;
10102	case glslang::EbtUint64:
10103	scalar = builder.makeUint64Constant(u: zero ? 0 : consts[nextConst].getU64Const(), specConstant);
10104	break;
10105	case glslang::EbtDouble:
10106	scalar = builder.makeDoubleConstant(d: zero ? 0.0 : consts[nextConst].getDConst(), specConstant);
10107	break;
10108	case glslang::EbtFloat16:
10109	builder.addCapability(cap: spv::CapabilityFloat16);
10110	scalar = builder.makeFloat16Constant(f16: zero ? 0.0F : (float)consts[nextConst].getDConst(), specConstant);
10111	break;
10112	case glslang::EbtReference:
10113	scalar = builder.makeUint64Constant(u: zero ? 0 : consts[nextConst].getU64Const(), specConstant);
10114	scalar = builder.createUnaryOp(spv::OpBitcast, typeId, operand: scalar);
10115	break;
10116	case glslang::EbtString:
10117	scalar = builder.getStringId(str: consts[nextConst].getSConst()->c_str());
10118	break;
10119	default:
10120	assert(0);
10121	break;
10122	}
10123	++nextConst;
10124	return scalar;
10125	}
10126
10127	return builder.makeCompositeConstant(type: typeId, comps: spvConsts);
10128	}
10129
10130	// Return true if the node is a constant or symbol whose reading has no
10131	// non-trivial observable cost or effect.
10132	bool TGlslangToSpvTraverser::isTrivialLeaf(const glslang::TIntermTyped* node)
10133	{
10134	// don't know what this is
10135	if (node == nullptr)
10136	return false;
10137
10138	// a constant is safe
10139	if (node->getAsConstantUnion() != nullptr)
10140	return true;
10141
10142	// not a symbol means non-trivial
10143	if (node->getAsSymbolNode() == nullptr)
10144	return false;
10145
10146	// a symbol, depends on what's being read
10147	switch (node->getType().getQualifier().storage) {
10148	case glslang::EvqTemporary:
10149	case glslang::EvqGlobal:
10150	case glslang::EvqIn:
10151	case glslang::EvqInOut:
10152	case glslang::EvqConst:
10153	case glslang::EvqConstReadOnly:
10154	case glslang::EvqUniform:
10155	return true;
10156	default:
10157	return false;
10158	}
10159	}
10160
10161	// A node is trivial if it is a single operation with no side effects.
10162	// HLSL (and/or vectors) are always trivial, as it does not short circuit.
10163	// Otherwise, error on the side of saying non-trivial.
10164	// Return true if trivial.
10165	bool TGlslangToSpvTraverser::isTrivial(const glslang::TIntermTyped* node)
10166	{
10167	if (node == nullptr)
10168	return false;
10169
10170	// count non scalars as trivial, as well as anything coming from HLSL
10171	if (! node->getType().isScalarOrVec1() || glslangIntermediate->getSource() == glslang::EShSourceHlsl)
10172	return true;
10173
10174	// symbols and constants are trivial
10175	if (isTrivialLeaf(node))
10176	return true;
10177
10178	// otherwise, it needs to be a simple operation or one or two leaf nodes
10179
10180	// not a simple operation
10181	const glslang::TIntermBinary* binaryNode = node->getAsBinaryNode();
10182	const glslang::TIntermUnary* unaryNode = node->getAsUnaryNode();
10183	if (binaryNode == nullptr && unaryNode == nullptr)
10184	return false;
10185
10186	// not on leaf nodes
10187	if (binaryNode && (! isTrivialLeaf(node: binaryNode->getLeft()) || ! isTrivialLeaf(node: binaryNode->getRight())))
10188	return false;
10189
10190	if (unaryNode && ! isTrivialLeaf(node: unaryNode->getOperand())) {
10191	return false;
10192	}
10193
10194	switch (node->getAsOperator()->getOp()) {
10195	case glslang::EOpLogicalNot:
10196	case glslang::EOpConvIntToBool:
10197	case glslang::EOpConvUintToBool:
10198	case glslang::EOpConvFloatToBool:
10199	case glslang::EOpConvDoubleToBool:
10200	case glslang::EOpEqual:
10201	case glslang::EOpNotEqual:
10202	case glslang::EOpLessThan:
10203	case glslang::EOpGreaterThan:
10204	case glslang::EOpLessThanEqual:
10205	case glslang::EOpGreaterThanEqual:
10206	case glslang::EOpIndexDirect:
10207	case glslang::EOpIndexDirectStruct:
10208	case glslang::EOpLogicalXor:
10209	case glslang::EOpAny:
10210	case glslang::EOpAll:
10211	return true;
10212	default:
10213	return false;
10214	}
10215	}
10216
10217	// Emit short-circuiting code, where 'right' is never evaluated unless
10218	// the left side is true (for &&) or false (for ||).
10219	spv::Id TGlslangToSpvTraverser::createShortCircuit(glslang::TOperator op, glslang::TIntermTyped& left,
10220	glslang::TIntermTyped& right)
10221	{
10222	spv::Id boolTypeId = builder.makeBoolType();
10223
10224	// emit left operand
10225	builder.clearAccessChain();
10226	left.traverse(this);
10227	spv::Id leftId = accessChainLoad(type: left.getType());
10228
10229	// Operands to accumulate OpPhi operands
10230	std::vector<spv::Id> phiOperands;
10231	phiOperands.reserve(n: 4);
10232	// accumulate left operand's phi information
10233	phiOperands.push_back(x: leftId);
10234	phiOperands.push_back(x: builder.getBuildPoint()->getId());
10235
10236	// Make the two kinds of operation symmetric with a "!"
10237	// || => emit "if (! left) result = right"
10238	// && => emit "if ( left) result = right"
10239	//
10240	// TODO: this runtime "not" for || could be avoided by adding functionality
10241	// to 'builder' to have an "else" without an "then"
10242	if (op == glslang::EOpLogicalOr)
10243	leftId = builder.createUnaryOp(spv::OpLogicalNot, typeId: boolTypeId, operand: leftId);
10244
10245	// make an "if" based on the left value
10246	spv::Builder::If ifBuilder(leftId, spv::SelectionControlMaskNone, builder);
10247
10248	// emit right operand as the "then" part of the "if"
10249	builder.clearAccessChain();
10250	right.traverse(this);
10251	spv::Id rightId = accessChainLoad(type: right.getType());
10252
10253	// accumulate left operand's phi information
10254	phiOperands.push_back(x: rightId);
10255	phiOperands.push_back(x: builder.getBuildPoint()->getId());
10256
10257	// finish the "if"
10258	ifBuilder.makeEndIf();
10259
10260	// phi together the two results
10261	return builder.createOp(spv::OpPhi, typeId: boolTypeId, operands: phiOperands);
10262	}
10263
10264	// Return type Id of the imported set of extended instructions corresponds to the name.
10265	// Import this set if it has not been imported yet.
10266	spv::Id TGlslangToSpvTraverser::getExtBuiltins(const char* name)
10267	{
10268	if (extBuiltinMap.find(x: name) != extBuiltinMap.end())
10269	return extBuiltinMap[name];
10270	else {
10271	spv::Id extBuiltins = builder.import(name);
10272	extBuiltinMap[name] = extBuiltins;
10273	return extBuiltins;
10274	}
10275	}
10276
10277	}; // end anonymous namespace
10278
10279	namespace QtShaderTools {
10280	namespace glslang {
10281
10282	void GetSpirvVersion(std::string& version)
10283	{
10284	const int bufSize = 100;
10285	char buf[bufSize];
10286	snprintf(s: buf, maxlen: bufSize, format: "0x%08x, Revision %d", spv::Version, spv::Revision);
10287	version = buf;
10288	}
10289
10290	// For low-order part of the generator's magic number. Bump up
10291	// when there is a change in the style (e.g., if SSA form changes,
10292	// or a different instruction sequence to do something gets used).
10293	int GetSpirvGeneratorVersion()
10294	{
10295	// return 1; // start
10296	// return 2; // EOpAtomicCounterDecrement gets a post decrement, to map between GLSL -> SPIR-V
10297	// return 3; // change/correct barrier-instruction operands, to match memory model group decisions
10298	// return 4; // some deeper access chains: for dynamic vector component, and local Boolean component
10299	// return 5; // make OpArrayLength result type be an int with signedness of 0
10300	// return 6; // revert version 5 change, which makes a different (new) kind of incorrect code,
10301	// versions 4 and 6 each generate OpArrayLength as it has long been done
10302	// return 7; // GLSL volatile keyword maps to both SPIR-V decorations Volatile and Coherent
10303	// return 8; // switch to new dead block eliminator; use OpUnreachable
10304	// return 9; // don't include opaque function parameters in OpEntryPoint global's operand list
10305	// return 10; // Generate OpFUnordNotEqual for != comparisons
10306	return 11; // Make OpEmitMeshTasksEXT a terminal instruction
10307	}
10308
10309	// Write SPIR-V out to a binary file
10310	bool OutputSpvBin(const std::vector<unsigned int>& spirv, const char* baseName)
10311	{
10312	std::ofstream out;
10313	out.open(s: baseName, mode: std::ios::binary | std::ios::out);
10314	if (out.fail()) {
10315	printf(format: "ERROR: Failed to open file: %s\n", baseName);
10316	return false;
10317	}
10318	for (int i = 0; i < (int)spirv.size(); ++i) {
10319	unsigned int word = spirv[i];
10320	out.write(s: (const char*)&word, n: 4);
10321	}
10322	out.close();
10323	return true;
10324	}
10325
10326	// Write SPIR-V out to a text file with 32-bit hexadecimal words
10327	bool OutputSpvHex(const std::vector<unsigned int>& spirv, const char* baseName, const char* varName)
10328	{
10329	std::ofstream out;
10330	out.open(s: baseName, mode: std::ios::binary | std::ios::out);
10331	if (out.fail()) {
10332	printf(format: "ERROR: Failed to open file: %s\n", baseName);
10333	return false;
10334	}
10335	out << "\t// " <<
10336	GetSpirvGeneratorVersion() <<
10337	GLSLANG_VERSION_MAJOR << "." << GLSLANG_VERSION_MINOR << "." << GLSLANG_VERSION_PATCH <<
10338	GLSLANG_VERSION_FLAVOR << std::endl;
10339	if (varName != nullptr) {
10340	out << "\t #pragma once" << std::endl;
10341	out << "const uint32_t " << varName << "[] = {" << std::endl;
10342	}
10343	const int WORDS_PER_LINE = 8;
10344	for (int i = 0; i < (int)spirv.size(); i += WORDS_PER_LINE) {
10345	out << "\t";
10346	for (int j = 0; j < WORDS_PER_LINE && i + j < (int)spirv.size(); ++j) {
10347	const unsigned int word = spirv[i + j];
10348	out << "0x" << std::hex << std::setw(8) << std::setfill('0') << word;
10349	if (i + j + 1 < (int)spirv.size()) {
10350	out << ",";
10351	}
10352	}
10353	out << std::endl;
10354	}
10355	if (varName != nullptr) {
10356	out << "};";
10357	out << std::endl;
10358	}
10359	out.close();
10360	return true;
10361	}
10362
10363	//
10364	// Set up the glslang traversal
10365	//
10366	void GlslangToSpv(const TIntermediate& intermediate, std::vector<unsigned int>& spirv, SpvOptions* options)
10367	{
10368	spv::SpvBuildLogger logger;
10369	GlslangToSpv(intermediate, spirv, logger: &logger, options);
10370	}
10371
10372	void GlslangToSpv(const TIntermediate& intermediate, std::vector<unsigned int>& spirv,
10373	spv::SpvBuildLogger* logger, SpvOptions* options)
10374	{
10375	TIntermNode* root = intermediate.getTreeRoot();
10376
10377	if (root == nullptr)
10378	return;
10379
10380	SpvOptions defaultOptions;
10381	if (options == nullptr)
10382	options = &defaultOptions;
10383
10384	GetThreadPoolAllocator().push();
10385
10386	TGlslangToSpvTraverser it(intermediate.getSpv().spv, &intermediate, logger, *options);
10387	root->traverse(&it);
10388	it.finishSpv(compileOnly: options->compileOnly);
10389	it.dumpSpv(out&: spirv);
10390
10391	#if ENABLE_OPT
10392	// If from HLSL, run spirv-opt to "legalize" the SPIR-V for Vulkan
10393	// eg. forward and remove memory writes of opaque types.
10394	bool prelegalization = intermediate.getSource() == EShSourceHlsl;
10395	if ((prelegalization || options->optimizeSize) && !options->disableOptimizer) {
10396	SpirvToolsTransform(intermediate, spirv, logger, options);
10397	prelegalization = false;
10398	}
10399	else if (options->stripDebugInfo) {
10400	// Strip debug info even if optimization is disabled.
10401	SpirvToolsStripDebugInfo(intermediate, spirv, logger);
10402	}
10403
10404	if (options->validate)
10405	SpirvToolsValidate(intermediate, spirv, logger, prelegalization);
10406
10407	if (options->disassemble)
10408	SpirvToolsDisassemble(std::cout, spirv);
10409
10410	#endif
10411
10412	GetThreadPoolAllocator().pop();
10413	}
10414
10415	} // end namespace glslang
10416	} // namespace QtShaderTools
10417