1	//
2	// Copyright (C) 2014-2016 LunarG, Inc.
3	// Copyright (C) 2015-2020 Google, Inc.
4	// Copyright (C) 2017, 2022-2024 Arm Limited.
5	// Modifications Copyright (C) 2020 Advanced Micro Devices, Inc. All rights reserved.
6	//
7	// All rights reserved.
8	//
9	// Redistribution and use in source and binary forms, with or without
10	// modification, are permitted provided that the following conditions
11	// are met:
12	//
13	// Redistributions of source code must retain the above copyright
14	// notice, this list of conditions and the following disclaimer.
15	//
16	// Redistributions in binary form must reproduce the above
17	// copyright notice, this list of conditions and the following
18	// disclaimer in the documentation and/or other materials provided
19	// with the distribution.
20	//
21	// Neither the name of 3Dlabs Inc. Ltd. nor the names of its
22	// contributors may be used to endorse or promote products derived
23	// from this software without specific prior written permission.
24	//
25	// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
26	// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
27	// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
28	// FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
29	// COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
30	// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
31	// BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
32	// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
33	// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
34	// LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
35	// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
36	// POSSIBILITY OF SUCH DAMAGE.
37
38	//
39	// Visit the nodes in the glslang intermediate tree representation to
40	// translate them to SPIR-V.
41	//
42
43	#include "spirv.hpp"
44	#include "GlslangToSpv.h"
45	#include "SpvBuilder.h"
46	#include "SpvTools.h"
47	namespace spv {
48	#include "GLSL.std.450.h"
49	#include "GLSL.ext.KHR.h"
50	#include "GLSL.ext.EXT.h"
51	#include "GLSL.ext.AMD.h"
52	#include "GLSL.ext.NV.h"
53	#include "GLSL.ext.ARM.h"
54	#include "GLSL.ext.QCOM.h"
55	#include "NonSemanticDebugPrintf.h"
56	}
57
58	// Glslang includes
59	#include "../glslang/MachineIndependent/localintermediate.h"
60	#include "../glslang/MachineIndependent/SymbolTable.h"
61	#include "../glslang/Include/Common.h"
62
63	// Build-time generated includes
64	#include "glslang/build_info.h"
65
66	#include <fstream>
67	#include <iomanip>
68	#include <list>
69	#include <map>
70	#include <optional>
71	#include <stack>
72	#include <string>
73	#include <vector>
74
75	namespace {
76
77	namespace {
78	class SpecConstantOpModeGuard {
79	public:
80	SpecConstantOpModeGuard(spv::Builder* builder)
81	: builder_(builder) {
82	previous_flag_ = builder->isInSpecConstCodeGenMode();
83	}
84	~SpecConstantOpModeGuard() {
85	previous_flag_ ? builder_->setToSpecConstCodeGenMode()
86	: builder_->setToNormalCodeGenMode();
87	}
88	void turnOnSpecConstantOpMode() {
89	builder_->setToSpecConstCodeGenMode();
90	}
91
92	private:
93	spv::Builder* builder_;
94	bool previous_flag_;
95	};
96
97	struct OpDecorations {
98	public:
99	OpDecorations(spv::Decoration precision, spv::Decoration noContraction, spv::Decoration nonUniform) :
100	precision(precision)
101	,
102	noContraction(noContraction),
103	nonUniform(nonUniform)
104	{ }
105
106	spv::Decoration precision;
107
108	void addNoContraction(spv::Builder& builder, spv::Id t) { builder.addDecoration(t, noContraction); }
109	void addNonUniform(spv::Builder& builder, spv::Id t) { builder.addDecoration(t, nonUniform); }
110	protected:
111	spv::Decoration noContraction;
112	spv::Decoration nonUniform;
113	};
114
115	} // namespace
116
117	using namespace QtShaderTools;
118
119	//
120	// The main holder of information for translating glslang to SPIR-V.
121	//
122	// Derives from the AST walking base class.
123	//
124	class TGlslangToSpvTraverser : public glslang::TIntermTraverser {
125	public:
126	TGlslangToSpvTraverser(unsigned int spvVersion, const glslang::TIntermediate*, spv::SpvBuildLogger* logger,
127	glslang::SpvOptions& options);
128	virtual ~TGlslangToSpvTraverser() { }
129
130	bool visitAggregate(glslang::TVisit, glslang::TIntermAggregate*);
131	bool visitBinary(glslang::TVisit, glslang::TIntermBinary*);
132	void visitConstantUnion(glslang::TIntermConstantUnion*);
133	bool visitSelection(glslang::TVisit, glslang::TIntermSelection*);
134	bool visitSwitch(glslang::TVisit, glslang::TIntermSwitch*);
135	void visitSymbol(glslang::TIntermSymbol* symbol);
136	bool visitUnary(glslang::TVisit, glslang::TIntermUnary*);
137	bool visitLoop(glslang::TVisit, glslang::TIntermLoop*);
138	bool visitBranch(glslang::TVisit visit, glslang::TIntermBranch*);
139
140	void finishSpv(bool compileOnly);
141	void dumpSpv(std::vector<unsigned int>& out);
142
143	protected:
144	TGlslangToSpvTraverser(TGlslangToSpvTraverser&);
145	TGlslangToSpvTraverser& operator=(TGlslangToSpvTraverser&);
146
147	spv::Decoration TranslateInterpolationDecoration(const glslang::TQualifier& qualifier);
148	spv::Decoration TranslateAuxiliaryStorageDecoration(const glslang::TQualifier& qualifier);
149	spv::Decoration TranslateNonUniformDecoration(const glslang::TQualifier& qualifier);
150	spv::Decoration TranslateNonUniformDecoration(const spv::Builder::AccessChain::CoherentFlags& coherentFlags);
151	spv::Builder::AccessChain::CoherentFlags TranslateCoherent(const glslang::TType& type);
152	spv::MemoryAccessMask TranslateMemoryAccess(const spv::Builder::AccessChain::CoherentFlags &coherentFlags);
153	spv::ImageOperandsMask TranslateImageOperands(const spv::Builder::AccessChain::CoherentFlags &coherentFlags);
154	spv::Scope TranslateMemoryScope(const spv::Builder::AccessChain::CoherentFlags &coherentFlags);
155	spv::BuiltIn TranslateBuiltInDecoration(glslang::TBuiltInVariable, bool memberDeclaration);
156	spv::ImageFormat TranslateImageFormat(const glslang::TType& type);
157	spv::SelectionControlMask TranslateSelectionControl(const glslang::TIntermSelection&) const;
158	spv::SelectionControlMask TranslateSwitchControl(const glslang::TIntermSwitch&) const;
159	spv::LoopControlMask TranslateLoopControl(const glslang::TIntermLoop&, std::vector<unsigned int>& operands) const;
160	spv::StorageClass TranslateStorageClass(const glslang::TType&);
161	void TranslateLiterals(const glslang::TVector<const glslang::TIntermConstantUnion*>&, std::vector<unsigned>&) const;
162	void addIndirectionIndexCapabilities(const glslang::TType& baseType, const glslang::TType& indexType);
163	spv::Id createSpvVariable(const glslang::TIntermSymbol*, spv::Id forcedType);
164	spv::Id getSampledType(const glslang::TSampler&);
165	spv::Id getInvertedSwizzleType(const glslang::TIntermTyped&);
166	spv::Id createInvertedSwizzle(spv::Decoration precision, const glslang::TIntermTyped&, spv::Id parentResult);
167	void convertSwizzle(const glslang::TIntermAggregate&, std::vector<unsigned>& swizzle);
168	spv::Id convertGlslangToSpvType(const glslang::TType& type, bool forwardReferenceOnly = false);
169	spv::Id convertGlslangToSpvType(const glslang::TType& type, glslang::TLayoutPacking, const glslang::TQualifier&,
170	bool lastBufferBlockMember, bool forwardReferenceOnly = false);
171	void applySpirvDecorate(const glslang::TType& type, spv::Id id, std::optional<int> member);
172	bool filterMember(const glslang::TType& member);
173	spv::Id convertGlslangStructToSpvType(const glslang::TType&, const glslang::TTypeList* glslangStruct,
174	glslang::TLayoutPacking, const glslang::TQualifier&);
175	spv::LinkageType convertGlslangLinkageToSpv(glslang::TLinkType glslangLinkType);
176	void decorateStructType(const glslang::TType&, const glslang::TTypeList* glslangStruct, glslang::TLayoutPacking,
177	const glslang::TQualifier&, spv::Id, const std::vector<spv::Id>& spvMembers);
178	spv::Id makeArraySizeId(const glslang::TArraySizes&, int dim, bool allowZero = false, bool boolType = false);
179	spv::Id accessChainLoad(const glslang::TType& type);
180	void accessChainStore(const glslang::TType& type, spv::Id rvalue);
181	void multiTypeStore(const glslang::TType&, spv::Id rValue);
182	spv::Id convertLoadedBoolInUniformToUint(const glslang::TType& type, spv::Id nominalTypeId, spv::Id loadedId);
183	glslang::TLayoutPacking getExplicitLayout(const glslang::TType& type) const;
184	int getArrayStride(const glslang::TType& arrayType, glslang::TLayoutPacking, glslang::TLayoutMatrix);
185	int getMatrixStride(const glslang::TType& matrixType, glslang::TLayoutPacking, glslang::TLayoutMatrix);
186	void updateMemberOffset(const glslang::TType& structType, const glslang::TType& memberType, int& currentOffset,
187	int& nextOffset, glslang::TLayoutPacking, glslang::TLayoutMatrix);
188	void declareUseOfStructMember(const glslang::TTypeList& members, int glslangMember);
189
190	bool isShaderEntryPoint(const glslang::TIntermAggregate* node);
191	bool writableParam(glslang::TStorageQualifier) const;
192	bool originalParam(glslang::TStorageQualifier, const glslang::TType&, bool implicitThisParam);
193	void makeFunctions(const glslang::TIntermSequence&);
194	void makeGlobalInitializers(const glslang::TIntermSequence&);
195	void collectRayTracingLinkerObjects();
196	void visitFunctions(const glslang::TIntermSequence&);
197	void handleFunctionEntry(const glslang::TIntermAggregate* node);
198	void translateArguments(const glslang::TIntermAggregate& node, std::vector<spv::Id>& arguments,
199	spv::Builder::AccessChain::CoherentFlags &lvalueCoherentFlags);
200	void translateArguments(glslang::TIntermUnary& node, std::vector<spv::Id>& arguments);
201	spv::Id createImageTextureFunctionCall(glslang::TIntermOperator* node);
202	spv::Id handleUserFunctionCall(const glslang::TIntermAggregate*);
203
204	spv::Id createBinaryOperation(glslang::TOperator op, OpDecorations&, spv::Id typeId, spv::Id left, spv::Id right,
205	glslang::TBasicType typeProxy, bool reduceComparison = true);
206	spv::Id createBinaryMatrixOperation(spv::Op, OpDecorations&, spv::Id typeId, spv::Id left, spv::Id right);
207	spv::Id createUnaryOperation(glslang::TOperator op, OpDecorations&, spv::Id typeId, spv::Id operand,
208	glslang::TBasicType typeProxy,
209	const spv::Builder::AccessChain::CoherentFlags &lvalueCoherentFlags,
210	const glslang::TType &opType);
211	spv::Id createUnaryMatrixOperation(spv::Op op, OpDecorations&, spv::Id typeId, spv::Id operand,
212	glslang::TBasicType typeProxy);
213	spv::Id createConversion(glslang::TOperator op, OpDecorations&, spv::Id destTypeId, spv::Id operand,
214	glslang::TBasicType resultBasicType, glslang::TBasicType operandBasicType);
215	spv::Id createIntWidthConversion(spv::Id operand, int vectorSize, spv::Id destType,
216	glslang::TBasicType resultBasicType, glslang::TBasicType operandBasicType);
217	spv::Id makeSmearedConstant(spv::Id constant, int vectorSize);
218	spv::Id createAtomicOperation(glslang::TOperator op, spv::Decoration precision, spv::Id typeId,
219	std::vector<spv::Id>& operands, glslang::TBasicType typeProxy,
220	const spv::Builder::AccessChain::CoherentFlags &lvalueCoherentFlags,
221	const glslang::TType &opType);
222	spv::Id createInvocationsOperation(glslang::TOperator op, spv::Id typeId, std::vector<spv::Id>& operands,
223	glslang::TBasicType typeProxy);
224	spv::Id CreateInvocationsVectorOperation(spv::Op op, spv::GroupOperation groupOperation,
225	spv::Id typeId, std::vector<spv::Id>& operands);
226	spv::Id createSubgroupOperation(glslang::TOperator op, spv::Id typeId, std::vector<spv::Id>& operands,
227	glslang::TBasicType typeProxy);
228	spv::Id createMiscOperation(glslang::TOperator op, spv::Decoration precision, spv::Id typeId,
229	std::vector<spv::Id>& operands, glslang::TBasicType typeProxy);
230	spv::Id createNoArgOperation(glslang::TOperator op, spv::Decoration precision, spv::Id typeId);
231	spv::Id getSymbolId(const glslang::TIntermSymbol* node);
232	void addMeshNVDecoration(spv::Id id, int member, const glslang::TQualifier & qualifier);
233	bool hasQCOMImageProceessingDecoration(spv::Id id, spv::Decoration decor);
234	void addImageProcessingQCOMDecoration(spv::Id id, spv::Decoration decor);
235	void addImageProcessing2QCOMDecoration(spv::Id id, bool isForGather);
236	spv::Id createSpvConstant(const glslang::TIntermTyped&);
237	spv::Id createSpvConstantFromConstUnionArray(const glslang::TType& type, const glslang::TConstUnionArray&,
238	int& nextConst, bool specConstant);
239	bool isTrivialLeaf(const glslang::TIntermTyped* node);
240	bool isTrivial(const glslang::TIntermTyped* node);
241	spv::Id createShortCircuit(glslang::TOperator, glslang::TIntermTyped& left, glslang::TIntermTyped& right);
242	spv::Id getExtBuiltins(const char* name);
243	std::pair<spv::Id, spv::Id> getForcedType(glslang::TBuiltInVariable builtIn, const glslang::TType&);
244	spv::Id translateForcedType(spv::Id object);
245	spv::Id createCompositeConstruct(spv::Id typeId, std::vector<spv::Id> constituents);
246
247	glslang::SpvOptions& options;
248	spv::Function* shaderEntry;
249	spv::Function* currentFunction;
250	spv::Instruction* entryPoint;
251	int sequenceDepth;
252
253	spv::SpvBuildLogger* logger;
254
255	// There is a 1:1 mapping between a spv builder and a module; this is thread safe
256	spv::Builder builder;
257	bool inEntryPoint;
258	bool entryPointTerminated;
259	bool linkageOnly; // true when visiting the set of objects in the AST present only for
260	// establishing interface, whether or not they were statically used
261	std::set<spv::Id> iOSet; // all input/output variables from either static use or declaration of interface
262	const glslang::TIntermediate* glslangIntermediate;
263	bool nanMinMaxClamp; // true if use NMin/NMax/NClamp instead of FMin/FMax/FClamp
264	spv::Id stdBuiltins;
265	spv::Id nonSemanticDebugPrintf;
266	std::unordered_map<std::string, spv::Id> extBuiltinMap;
267
268	std::unordered_map<long long, spv::Id> symbolValues;
269	std::unordered_map<uint32_t, spv::Id> builtInVariableIds;
270	std::unordered_set<long long> rValueParameters; // set of formal function parameters passed as rValues,
271	// rather than a pointer
272	std::unordered_map<std::string, spv::Function*> functionMap;
273	std::unordered_map<const glslang::TTypeList*, spv::Id> structMap[glslang::ElpCount][glslang::ElmCount];
274	// for mapping glslang block indices to spv indices (e.g., due to hidden members):
275	std::unordered_map<long long, std::vector<int>> memberRemapper;
276	// for mapping glslang symbol struct to symbol Id
277	std::unordered_map<const glslang::TTypeList*, long long> glslangTypeToIdMap;
278	std::stack<bool> breakForLoop; // false means break for switch
279	std::unordered_map<std::string, const glslang::TIntermSymbol*> counterOriginator;
280	// Map pointee types for EbtReference to their forward pointers
281	std::map<const glslang::TType *, spv::Id> forwardPointers;
282	// Type forcing, for when SPIR-V wants a different type than the AST,
283	// requiring local translation to and from SPIR-V type on every access.
284	// Maps <builtin-variable-id -> AST-required-type-id>
285	std::unordered_map<spv::Id, spv::Id> forceType;
286	// Used by Task shader while generating opearnds for OpEmitMeshTasksEXT
287	spv::Id taskPayloadID;
288	// Used later for generating OpTraceKHR/OpExecuteCallableKHR/OpHitObjectRecordHit*/OpHitObjectGetShaderBindingTableData
289	std::unordered_map<unsigned int, glslang::TIntermSymbol *> locationToSymbol[4];
290	std::unordered_map<spv::Id, std::vector<spv::Decoration> > idToQCOMDecorations;
291	};
292
293	//
294	// Helper functions for translating glslang representations to SPIR-V enumerants.
295	//
296
297	// Translate glslang profile to SPIR-V source language.
298	spv::SourceLanguage TranslateSourceLanguage(glslang::EShSource source, EProfile profile)
299	{
300	switch (source) {
301	case glslang::EShSourceGlsl:
302	switch (profile) {
303	case ENoProfile:
304	case ECoreProfile:
305	case ECompatibilityProfile:
306	return spv::SourceLanguageGLSL;
307	case EEsProfile:
308	return spv::SourceLanguageESSL;
309	default:
310	return spv::SourceLanguageUnknown;
311	}
312	case glslang::EShSourceHlsl:
313	return spv::SourceLanguageHLSL;
314	default:
315	return spv::SourceLanguageUnknown;
316	}
317	}
318
319	// Translate glslang language (stage) to SPIR-V execution model.
320	spv::ExecutionModel TranslateExecutionModel(EShLanguage stage, bool isMeshShaderEXT = false)
321	{
322	switch (stage) {
323	case EShLangVertex: return spv::ExecutionModelVertex;
324	case EShLangFragment: return spv::ExecutionModelFragment;
325	case EShLangCompute: return spv::ExecutionModelGLCompute;
326	case EShLangTessControl: return spv::ExecutionModelTessellationControl;
327	case EShLangTessEvaluation: return spv::ExecutionModelTessellationEvaluation;
328	case EShLangGeometry: return spv::ExecutionModelGeometry;
329	case EShLangRayGen: return spv::ExecutionModelRayGenerationKHR;
330	case EShLangIntersect: return spv::ExecutionModelIntersectionKHR;
331	case EShLangAnyHit: return spv::ExecutionModelAnyHitKHR;
332	case EShLangClosestHit: return spv::ExecutionModelClosestHitKHR;
333	case EShLangMiss: return spv::ExecutionModelMissKHR;
334	case EShLangCallable: return spv::ExecutionModelCallableKHR;
335	case EShLangTask: return (isMeshShaderEXT)? spv::ExecutionModelTaskEXT : spv::ExecutionModelTaskNV;
336	case EShLangMesh: return (isMeshShaderEXT)? spv::ExecutionModelMeshEXT: spv::ExecutionModelMeshNV;
337	default:
338	assert(0);
339	return spv::ExecutionModelFragment;
340	}
341	}
342
343	// Translate glslang sampler type to SPIR-V dimensionality.
344	spv::Dim TranslateDimensionality(const glslang::TSampler& sampler)
345	{
346	switch (sampler.dim) {
347	case glslang::Esd1D: return spv::Dim1D;
348	case glslang::Esd2D: return spv::Dim2D;
349	case glslang::Esd3D: return spv::Dim3D;
350	case glslang::EsdCube: return spv::DimCube;
351	case glslang::EsdRect: return spv::DimRect;
352	case glslang::EsdBuffer: return spv::DimBuffer;
353	case glslang::EsdSubpass: return spv::DimSubpassData;
354	case glslang::EsdAttachmentEXT: return spv::DimTileImageDataEXT;
355	default:
356	assert(0);
357	return spv::Dim2D;
358	}
359	}
360
361	// Translate glslang precision to SPIR-V precision decorations.
362	spv::Decoration TranslatePrecisionDecoration(glslang::TPrecisionQualifier glslangPrecision)
363	{
364	switch (glslangPrecision) {
365	case glslang::EpqLow: return spv::DecorationRelaxedPrecision;
366	case glslang::EpqMedium: return spv::DecorationRelaxedPrecision;
367	default:
368	return spv::NoPrecision;
369	}
370	}
371
372	// Translate glslang type to SPIR-V precision decorations.
373	spv::Decoration TranslatePrecisionDecoration(const glslang::TType& type)
374	{
375	return TranslatePrecisionDecoration(glslangPrecision: type.getQualifier().precision);
376	}
377
378	// Translate glslang type to SPIR-V block decorations.
379	spv::Decoration TranslateBlockDecoration(const glslang::TStorageQualifier storage, bool useStorageBuffer)
380	{
381	switch (storage) {
382	case glslang::EvqUniform: return spv::DecorationBlock;
383	case glslang::EvqBuffer: return useStorageBuffer ? spv::DecorationBlock : spv::DecorationBufferBlock;
384	case glslang::EvqVaryingIn: return spv::DecorationBlock;
385	case glslang::EvqVaryingOut: return spv::DecorationBlock;
386	case glslang::EvqShared: return spv::DecorationBlock;
387	case glslang::EvqPayload: return spv::DecorationBlock;
388	case glslang::EvqPayloadIn: return spv::DecorationBlock;
389	case glslang::EvqHitAttr: return spv::DecorationBlock;
390	case glslang::EvqCallableData: return spv::DecorationBlock;
391	case glslang::EvqCallableDataIn: return spv::DecorationBlock;
392	case glslang::EvqHitObjectAttrNV: return spv::DecorationBlock;
393	default:
394	assert(0);
395	break;
396	}
397
398	return spv::DecorationMax;
399	}
400
401	// Translate glslang type to SPIR-V memory decorations.
402	void TranslateMemoryDecoration(const glslang::TQualifier& qualifier, std::vector<spv::Decoration>& memory,
403	bool useVulkanMemoryModel)
404	{
405	if (!useVulkanMemoryModel) {
406	if (qualifier.isVolatile()) {
407	memory.push_back(x: spv::DecorationVolatile);
408	memory.push_back(x: spv::DecorationCoherent);
409	} else if (qualifier.isCoherent()) {
410	memory.push_back(x: spv::DecorationCoherent);
411	}
412	}
413	if (qualifier.isRestrict())
414	memory.push_back(x: spv::DecorationRestrict);
415	if (qualifier.isReadOnly())
416	memory.push_back(x: spv::DecorationNonWritable);
417	if (qualifier.isWriteOnly())
418	memory.push_back(x: spv::DecorationNonReadable);
419	}
420
421	// Translate glslang type to SPIR-V layout decorations.
422	spv::Decoration TranslateLayoutDecoration(const glslang::TType& type, glslang::TLayoutMatrix matrixLayout)
423	{
424	if (type.isMatrix()) {
425	switch (matrixLayout) {
426	case glslang::ElmRowMajor:
427	return spv::DecorationRowMajor;
428	case glslang::ElmColumnMajor:
429	return spv::DecorationColMajor;
430	default:
431	// opaque layouts don't need a majorness
432	return spv::DecorationMax;
433	}
434	} else {
435	switch (type.getBasicType()) {
436	default:
437	return spv::DecorationMax;
438	break;
439	case glslang::EbtBlock:
440	switch (type.getQualifier().storage) {
441	case glslang::EvqShared:
442	case glslang::EvqUniform:
443	case glslang::EvqBuffer:
444	switch (type.getQualifier().layoutPacking) {
445	case glslang::ElpShared: return spv::DecorationGLSLShared;
446	case glslang::ElpPacked: return spv::DecorationGLSLPacked;
447	default:
448	return spv::DecorationMax;
449	}
450	case glslang::EvqVaryingIn:
451	case glslang::EvqVaryingOut:
452	if (type.getQualifier().isTaskMemory()) {
453	switch (type.getQualifier().layoutPacking) {
454	case glslang::ElpShared: return spv::DecorationGLSLShared;
455	case glslang::ElpPacked: return spv::DecorationGLSLPacked;
456	default: break;
457	}
458	} else {
459	assert(type.getQualifier().layoutPacking == glslang::ElpNone);
460	}
461	return spv::DecorationMax;
462	case glslang::EvqPayload:
463	case glslang::EvqPayloadIn:
464	case glslang::EvqHitAttr:
465	case glslang::EvqCallableData:
466	case glslang::EvqCallableDataIn:
467	case glslang::EvqHitObjectAttrNV:
468	return spv::DecorationMax;
469	default:
470	assert(0);
471	return spv::DecorationMax;
472	}
473	}
474	}
475	}
476
477	// Translate glslang type to SPIR-V interpolation decorations.
478	// Returns spv::DecorationMax when no decoration
479	// should be applied.
480	spv::Decoration TGlslangToSpvTraverser::TranslateInterpolationDecoration(const glslang::TQualifier& qualifier)
481	{
482	if (qualifier.smooth)
483	// Smooth decoration doesn't exist in SPIR-V 1.0
484	return spv::DecorationMax;
485	else if (qualifier.isNonPerspective())
486	return spv::DecorationNoPerspective;
487	else if (qualifier.flat)
488	return spv::DecorationFlat;
489	else if (qualifier.isExplicitInterpolation()) {
490	builder.addExtension(ext: spv::E_SPV_AMD_shader_explicit_vertex_parameter);
491	return spv::DecorationExplicitInterpAMD;
492	}
493	else
494	return spv::DecorationMax;
495	}
496
497	// Translate glslang type to SPIR-V auxiliary storage decorations.
498	// Returns spv::DecorationMax when no decoration
499	// should be applied.
500	spv::Decoration TGlslangToSpvTraverser::TranslateAuxiliaryStorageDecoration(const glslang::TQualifier& qualifier)
501	{
502	if (qualifier.centroid)
503	return spv::DecorationCentroid;
504	else if (qualifier.patch)
505	return spv::DecorationPatch;
506	else if (qualifier.sample) {
507	builder.addCapability(cap: spv::CapabilitySampleRateShading);
508	return spv::DecorationSample;
509	}
510
511	return spv::DecorationMax;
512	}
513
514	// If glslang type is invariant, return SPIR-V invariant decoration.
515	spv::Decoration TranslateInvariantDecoration(const glslang::TQualifier& qualifier)
516	{
517	if (qualifier.invariant)
518	return spv::DecorationInvariant;
519	else
520	return spv::DecorationMax;
521	}
522
523	// If glslang type is noContraction, return SPIR-V NoContraction decoration.
524	spv::Decoration TranslateNoContractionDecoration(const glslang::TQualifier& qualifier)
525	{
526	if (qualifier.isNoContraction())
527	return spv::DecorationNoContraction;
528	else
529	return spv::DecorationMax;
530	}
531
532	// If glslang type is nonUniform, return SPIR-V NonUniform decoration.
533	spv::Decoration TGlslangToSpvTraverser::TranslateNonUniformDecoration(const glslang::TQualifier& qualifier)
534	{
535	if (qualifier.isNonUniform()) {
536	builder.addIncorporatedExtension(ext: "SPV_EXT_descriptor_indexing", incorporatedVersion: spv::Spv_1_5);
537	builder.addCapability(cap: spv::CapabilityShaderNonUniformEXT);
538	return spv::DecorationNonUniformEXT;
539	} else
540	return spv::DecorationMax;
541	}
542
543	// If lvalue flags contains nonUniform, return SPIR-V NonUniform decoration.
544	spv::Decoration TGlslangToSpvTraverser::TranslateNonUniformDecoration(
545	const spv::Builder::AccessChain::CoherentFlags& coherentFlags)
546	{
547	if (coherentFlags.isNonUniform()) {
548	builder.addIncorporatedExtension(ext: "SPV_EXT_descriptor_indexing", incorporatedVersion: spv::Spv_1_5);
549	builder.addCapability(cap: spv::CapabilityShaderNonUniformEXT);
550	return spv::DecorationNonUniformEXT;
551	} else
552	return spv::DecorationMax;
553	}
554
555	spv::MemoryAccessMask TGlslangToSpvTraverser::TranslateMemoryAccess(
556	const spv::Builder::AccessChain::CoherentFlags &coherentFlags)
557	{
558	spv::MemoryAccessMask mask = spv::MemoryAccessMaskNone;
559
560	if (!glslangIntermediate->usingVulkanMemoryModel() || coherentFlags.isImage)
561	return mask;
562
563	if (coherentFlags.isVolatile() || coherentFlags.anyCoherent()) {
564	mask = mask | spv::MemoryAccessMakePointerAvailableKHRMask |
565	spv::MemoryAccessMakePointerVisibleKHRMask;
566	}
567
568	if (coherentFlags.nonprivate) {
569	mask = mask | spv::MemoryAccessNonPrivatePointerKHRMask;
570	}
571	if (coherentFlags.volatil) {
572	mask = mask | spv::MemoryAccessVolatileMask;
573	}
574	if (mask != spv::MemoryAccessMaskNone) {
575	builder.addCapability(cap: spv::CapabilityVulkanMemoryModelKHR);
576	}
577
578	return mask;
579	}
580
581	spv::ImageOperandsMask TGlslangToSpvTraverser::TranslateImageOperands(
582	const spv::Builder::AccessChain::CoherentFlags &coherentFlags)
583	{
584	spv::ImageOperandsMask mask = spv::ImageOperandsMaskNone;
585
586	if (!glslangIntermediate->usingVulkanMemoryModel())
587	return mask;
588
589	if (coherentFlags.volatil ||
590	coherentFlags.anyCoherent()) {
591	mask = mask | spv::ImageOperandsMakeTexelAvailableKHRMask |
592	spv::ImageOperandsMakeTexelVisibleKHRMask;
593	}
594	if (coherentFlags.nonprivate) {
595	mask = mask | spv::ImageOperandsNonPrivateTexelKHRMask;
596	}
597	if (coherentFlags.volatil) {
598	mask = mask | spv::ImageOperandsVolatileTexelKHRMask;
599	}
600	if (mask != spv::ImageOperandsMaskNone) {
601	builder.addCapability(cap: spv::CapabilityVulkanMemoryModelKHR);
602	}
603
604	return mask;
605	}
606
607	spv::Builder::AccessChain::CoherentFlags TGlslangToSpvTraverser::TranslateCoherent(const glslang::TType& type)
608	{
609	spv::Builder::AccessChain::CoherentFlags flags = {};
610	flags.coherent = type.getQualifier().coherent;
611	flags.devicecoherent = type.getQualifier().devicecoherent;
612	flags.queuefamilycoherent = type.getQualifier().queuefamilycoherent;
613	// shared variables are implicitly workgroupcoherent in GLSL.
614	flags.workgroupcoherent = type.getQualifier().workgroupcoherent ||
615	type.getQualifier().storage == glslang::EvqShared;
616	flags.subgroupcoherent = type.getQualifier().subgroupcoherent;
617	flags.shadercallcoherent = type.getQualifier().shadercallcoherent;
618	flags.volatil = type.getQualifier().volatil;
619	// *coherent variables are implicitly nonprivate in GLSL
620	flags.nonprivate = type.getQualifier().nonprivate ||
621	flags.anyCoherent() ||
622	flags.volatil;
623	flags.isImage = type.getBasicType() == glslang::EbtSampler;
624	flags.nonUniform = type.getQualifier().nonUniform;
625	return flags;
626	}
627
628	spv::Scope TGlslangToSpvTraverser::TranslateMemoryScope(
629	const spv::Builder::AccessChain::CoherentFlags &coherentFlags)
630	{
631	spv::Scope scope = spv::ScopeMax;
632
633	if (coherentFlags.volatil || coherentFlags.coherent) {
634	// coherent defaults to Device scope in the old model, QueueFamilyKHR scope in the new model
635	scope = glslangIntermediate->usingVulkanMemoryModel() ? spv::ScopeQueueFamilyKHR : spv::ScopeDevice;
636	} else if (coherentFlags.devicecoherent) {
637	scope = spv::ScopeDevice;
638	} else if (coherentFlags.queuefamilycoherent) {
639	scope = spv::ScopeQueueFamilyKHR;
640	} else if (coherentFlags.workgroupcoherent) {
641	scope = spv::ScopeWorkgroup;
642	} else if (coherentFlags.subgroupcoherent) {
643	scope = spv::ScopeSubgroup;
644	} else if (coherentFlags.shadercallcoherent) {
645	scope = spv::ScopeShaderCallKHR;
646	}
647	if (glslangIntermediate->usingVulkanMemoryModel() && scope == spv::ScopeDevice) {
648	builder.addCapability(cap: spv::CapabilityVulkanMemoryModelDeviceScopeKHR);
649	}
650
651	return scope;
652	}
653
654	// Translate a glslang built-in variable to a SPIR-V built in decoration. Also generate
655	// associated capabilities when required. For some built-in variables, a capability
656	// is generated only when using the variable in an executable instruction, but not when
657	// just declaring a struct member variable with it. This is true for PointSize,
658	// ClipDistance, and CullDistance.
659	spv::BuiltIn TGlslangToSpvTraverser::TranslateBuiltInDecoration(glslang::TBuiltInVariable builtIn,
660	bool memberDeclaration)
661	{
662	switch (builtIn) {
663	case glslang::EbvPointSize:
664	// Defer adding the capability until the built-in is actually used.
665	if (! memberDeclaration) {
666	switch (glslangIntermediate->getStage()) {
667	case EShLangGeometry:
668	builder.addCapability(cap: spv::CapabilityGeometryPointSize);
669	break;
670	case EShLangTessControl:
671	case EShLangTessEvaluation:
672	builder.addCapability(cap: spv::CapabilityTessellationPointSize);
673	break;
674	default:
675	break;
676	}
677	}
678	return spv::BuiltInPointSize;
679
680	case glslang::EbvPosition: return spv::BuiltInPosition;
681	case glslang::EbvVertexId: return spv::BuiltInVertexId;
682	case glslang::EbvInstanceId: return spv::BuiltInInstanceId;
683	case glslang::EbvVertexIndex: return spv::BuiltInVertexIndex;
684	case glslang::EbvInstanceIndex: return spv::BuiltInInstanceIndex;
685
686	case glslang::EbvFragCoord: return spv::BuiltInFragCoord;
687	case glslang::EbvPointCoord: return spv::BuiltInPointCoord;
688	case glslang::EbvFace: return spv::BuiltInFrontFacing;
689	case glslang::EbvFragDepth: return spv::BuiltInFragDepth;
690
691	case glslang::EbvNumWorkGroups: return spv::BuiltInNumWorkgroups;
692	case glslang::EbvWorkGroupSize: return spv::BuiltInWorkgroupSize;
693	case glslang::EbvWorkGroupId: return spv::BuiltInWorkgroupId;
694	case glslang::EbvLocalInvocationId: return spv::BuiltInLocalInvocationId;
695	case glslang::EbvLocalInvocationIndex: return spv::BuiltInLocalInvocationIndex;
696	case glslang::EbvGlobalInvocationId: return spv::BuiltInGlobalInvocationId;
697
698	// These *Distance capabilities logically belong here, but if the member is declared and
699	// then never used, consumers of SPIR-V prefer the capability not be declared.
700	// They are now generated when used, rather than here when declared.
701	// Potentially, the specification should be more clear what the minimum
702	// use needed is to trigger the capability.
703	//
704	case glslang::EbvClipDistance:
705	if (!memberDeclaration)
706	builder.addCapability(cap: spv::CapabilityClipDistance);
707	return spv::BuiltInClipDistance;
708
709	case glslang::EbvCullDistance:
710	if (!memberDeclaration)
711	builder.addCapability(cap: spv::CapabilityCullDistance);
712	return spv::BuiltInCullDistance;
713
714	case glslang::EbvViewportIndex:
715	if (glslangIntermediate->getStage() == EShLangGeometry ||
716	glslangIntermediate->getStage() == EShLangFragment) {
717	builder.addCapability(cap: spv::CapabilityMultiViewport);
718	}
719	if (glslangIntermediate->getStage() == EShLangVertex ||
720	glslangIntermediate->getStage() == EShLangTessControl ||
721	glslangIntermediate->getStage() == EShLangTessEvaluation) {
722
723	if (builder.getSpvVersion() < spv::Spv_1_5) {
724	builder.addIncorporatedExtension(ext: spv::E_SPV_EXT_shader_viewport_index_layer, incorporatedVersion: spv::Spv_1_5);
725	builder.addCapability(cap: spv::CapabilityShaderViewportIndexLayerEXT);
726	}
727	else
728	builder.addCapability(cap: spv::CapabilityShaderViewportIndex);
729	}
730	return spv::BuiltInViewportIndex;
731
732	case glslang::EbvSampleId:
733	builder.addCapability(cap: spv::CapabilitySampleRateShading);
734	return spv::BuiltInSampleId;
735
736	case glslang::EbvSamplePosition:
737	builder.addCapability(cap: spv::CapabilitySampleRateShading);
738	return spv::BuiltInSamplePosition;
739
740	case glslang::EbvSampleMask:
741	return spv::BuiltInSampleMask;
742
743	case glslang::EbvLayer:
744	if (glslangIntermediate->getStage() == EShLangMesh) {
745	return spv::BuiltInLayer;
746	}
747	if (glslangIntermediate->getStage() == EShLangGeometry ||
748	glslangIntermediate->getStage() == EShLangFragment) {
749	builder.addCapability(cap: spv::CapabilityGeometry);
750	}
751	if (glslangIntermediate->getStage() == EShLangVertex ||
752	glslangIntermediate->getStage() == EShLangTessControl ||
753	glslangIntermediate->getStage() == EShLangTessEvaluation) {
754
755	if (builder.getSpvVersion() < spv::Spv_1_5) {
756	builder.addIncorporatedExtension(ext: spv::E_SPV_EXT_shader_viewport_index_layer, incorporatedVersion: spv::Spv_1_5);
757	builder.addCapability(cap: spv::CapabilityShaderViewportIndexLayerEXT);
758	} else
759	builder.addCapability(cap: spv::CapabilityShaderLayer);
760	}
761	return spv::BuiltInLayer;
762
763	case glslang::EbvBaseVertex:
764	builder.addIncorporatedExtension(ext: spv::E_SPV_KHR_shader_draw_parameters, incorporatedVersion: spv::Spv_1_3);
765	builder.addCapability(cap: spv::CapabilityDrawParameters);
766	return spv::BuiltInBaseVertex;
767
768	case glslang::EbvBaseInstance:
769	builder.addIncorporatedExtension(ext: spv::E_SPV_KHR_shader_draw_parameters, incorporatedVersion: spv::Spv_1_3);
770	builder.addCapability(cap: spv::CapabilityDrawParameters);
771	return spv::BuiltInBaseInstance;
772
773	case glslang::EbvDrawId:
774	builder.addIncorporatedExtension(ext: spv::E_SPV_KHR_shader_draw_parameters, incorporatedVersion: spv::Spv_1_3);
775	builder.addCapability(cap: spv::CapabilityDrawParameters);
776	return spv::BuiltInDrawIndex;
777
778	case glslang::EbvPrimitiveId:
779	if (glslangIntermediate->getStage() == EShLangFragment)
780	builder.addCapability(cap: spv::CapabilityGeometry);
781	return spv::BuiltInPrimitiveId;
782
783	case glslang::EbvFragStencilRef:
784	builder.addExtension(ext: spv::E_SPV_EXT_shader_stencil_export);
785	builder.addCapability(cap: spv::CapabilityStencilExportEXT);
786	return spv::BuiltInFragStencilRefEXT;
787
788	case glslang::EbvShadingRateKHR:
789	builder.addExtension(ext: spv::E_SPV_KHR_fragment_shading_rate);
790	builder.addCapability(cap: spv::CapabilityFragmentShadingRateKHR);
791	return spv::BuiltInShadingRateKHR;
792
793	case glslang::EbvPrimitiveShadingRateKHR:
794	builder.addExtension(ext: spv::E_SPV_KHR_fragment_shading_rate);
795	builder.addCapability(cap: spv::CapabilityFragmentShadingRateKHR);
796	return spv::BuiltInPrimitiveShadingRateKHR;
797
798	case glslang::EbvInvocationId: return spv::BuiltInInvocationId;
799	case glslang::EbvTessLevelInner: return spv::BuiltInTessLevelInner;
800	case glslang::EbvTessLevelOuter: return spv::BuiltInTessLevelOuter;
801	case glslang::EbvTessCoord: return spv::BuiltInTessCoord;
802	case glslang::EbvPatchVertices: return spv::BuiltInPatchVertices;
803	case glslang::EbvHelperInvocation: return spv::BuiltInHelperInvocation;
804
805	case glslang::EbvSubGroupSize:
806	builder.addExtension(ext: spv::E_SPV_KHR_shader_ballot);
807	builder.addCapability(cap: spv::CapabilitySubgroupBallotKHR);
808	return spv::BuiltInSubgroupSize;
809
810	case glslang::EbvSubGroupInvocation:
811	builder.addExtension(ext: spv::E_SPV_KHR_shader_ballot);
812	builder.addCapability(cap: spv::CapabilitySubgroupBallotKHR);
813	return spv::BuiltInSubgroupLocalInvocationId;
814
815	case glslang::EbvSubGroupEqMask:
816	builder.addExtension(ext: spv::E_SPV_KHR_shader_ballot);
817	builder.addCapability(cap: spv::CapabilitySubgroupBallotKHR);
818	return spv::BuiltInSubgroupEqMask;
819
820	case glslang::EbvSubGroupGeMask:
821	builder.addExtension(ext: spv::E_SPV_KHR_shader_ballot);
822	builder.addCapability(cap: spv::CapabilitySubgroupBallotKHR);
823	return spv::BuiltInSubgroupGeMask;
824
825	case glslang::EbvSubGroupGtMask:
826	builder.addExtension(ext: spv::E_SPV_KHR_shader_ballot);
827	builder.addCapability(cap: spv::CapabilitySubgroupBallotKHR);
828	return spv::BuiltInSubgroupGtMask;
829
830	case glslang::EbvSubGroupLeMask:
831	builder.addExtension(ext: spv::E_SPV_KHR_shader_ballot);
832	builder.addCapability(cap: spv::CapabilitySubgroupBallotKHR);
833	return spv::BuiltInSubgroupLeMask;
834
835	case glslang::EbvSubGroupLtMask:
836	builder.addExtension(ext: spv::E_SPV_KHR_shader_ballot);
837	builder.addCapability(cap: spv::CapabilitySubgroupBallotKHR);
838	return spv::BuiltInSubgroupLtMask;
839
840	case glslang::EbvNumSubgroups:
841	builder.addCapability(cap: spv::CapabilityGroupNonUniform);
842	return spv::BuiltInNumSubgroups;
843
844	case glslang::EbvSubgroupID:
845	builder.addCapability(cap: spv::CapabilityGroupNonUniform);
846	return spv::BuiltInSubgroupId;
847
848	case glslang::EbvSubgroupSize2:
849	builder.addCapability(cap: spv::CapabilityGroupNonUniform);
850	return spv::BuiltInSubgroupSize;
851
852	case glslang::EbvSubgroupInvocation2:
853	builder.addCapability(cap: spv::CapabilityGroupNonUniform);
854	return spv::BuiltInSubgroupLocalInvocationId;
855
856	case glslang::EbvSubgroupEqMask2:
857	builder.addCapability(cap: spv::CapabilityGroupNonUniform);
858	builder.addCapability(cap: spv::CapabilityGroupNonUniformBallot);
859	return spv::BuiltInSubgroupEqMask;
860
861	case glslang::EbvSubgroupGeMask2:
862	builder.addCapability(cap: spv::CapabilityGroupNonUniform);
863	builder.addCapability(cap: spv::CapabilityGroupNonUniformBallot);
864	return spv::BuiltInSubgroupGeMask;
865
866	case glslang::EbvSubgroupGtMask2:
867	builder.addCapability(cap: spv::CapabilityGroupNonUniform);
868	builder.addCapability(cap: spv::CapabilityGroupNonUniformBallot);
869	return spv::BuiltInSubgroupGtMask;
870
871	case glslang::EbvSubgroupLeMask2:
872	builder.addCapability(cap: spv::CapabilityGroupNonUniform);
873	builder.addCapability(cap: spv::CapabilityGroupNonUniformBallot);
874	return spv::BuiltInSubgroupLeMask;
875
876	case glslang::EbvSubgroupLtMask2:
877	builder.addCapability(cap: spv::CapabilityGroupNonUniform);
878	builder.addCapability(cap: spv::CapabilityGroupNonUniformBallot);
879	return spv::BuiltInSubgroupLtMask;
880
881	case glslang::EbvBaryCoordNoPersp:
882	builder.addExtension(ext: spv::E_SPV_AMD_shader_explicit_vertex_parameter);
883	return spv::BuiltInBaryCoordNoPerspAMD;
884
885	case glslang::EbvBaryCoordNoPerspCentroid:
886	builder.addExtension(ext: spv::E_SPV_AMD_shader_explicit_vertex_parameter);
887	return spv::BuiltInBaryCoordNoPerspCentroidAMD;
888
889	case glslang::EbvBaryCoordNoPerspSample:
890	builder.addExtension(ext: spv::E_SPV_AMD_shader_explicit_vertex_parameter);
891	return spv::BuiltInBaryCoordNoPerspSampleAMD;
892
893	case glslang::EbvBaryCoordSmooth:
894	builder.addExtension(ext: spv::E_SPV_AMD_shader_explicit_vertex_parameter);
895	return spv::BuiltInBaryCoordSmoothAMD;
896
897	case glslang::EbvBaryCoordSmoothCentroid:
898	builder.addExtension(ext: spv::E_SPV_AMD_shader_explicit_vertex_parameter);
899	return spv::BuiltInBaryCoordSmoothCentroidAMD;
900
901	case glslang::EbvBaryCoordSmoothSample:
902	builder.addExtension(ext: spv::E_SPV_AMD_shader_explicit_vertex_parameter);
903	return spv::BuiltInBaryCoordSmoothSampleAMD;
904
905	case glslang::EbvBaryCoordPullModel:
906	builder.addExtension(ext: spv::E_SPV_AMD_shader_explicit_vertex_parameter);
907	return spv::BuiltInBaryCoordPullModelAMD;
908
909	case glslang::EbvDeviceIndex:
910	builder.addIncorporatedExtension(ext: spv::E_SPV_KHR_device_group, incorporatedVersion: spv::Spv_1_3);
911	builder.addCapability(cap: spv::CapabilityDeviceGroup);
912	return spv::BuiltInDeviceIndex;
913
914	case glslang::EbvViewIndex:
915	builder.addIncorporatedExtension(ext: spv::E_SPV_KHR_multiview, incorporatedVersion: spv::Spv_1_3);
916	builder.addCapability(cap: spv::CapabilityMultiView);
917	return spv::BuiltInViewIndex;
918
919	case glslang::EbvFragSizeEXT:
920	builder.addExtension(ext: spv::E_SPV_EXT_fragment_invocation_density);
921	builder.addCapability(cap: spv::CapabilityFragmentDensityEXT);
922	return spv::BuiltInFragSizeEXT;
923
924	case glslang::EbvFragInvocationCountEXT:
925	builder.addExtension(ext: spv::E_SPV_EXT_fragment_invocation_density);
926	builder.addCapability(cap: spv::CapabilityFragmentDensityEXT);
927	return spv::BuiltInFragInvocationCountEXT;
928
929	case glslang::EbvViewportMaskNV:
930	if (!memberDeclaration) {
931	builder.addExtension(ext: spv::E_SPV_NV_viewport_array2);
932	builder.addCapability(cap: spv::CapabilityShaderViewportMaskNV);
933	}
934	return spv::BuiltInViewportMaskNV;
935	case glslang::EbvSecondaryPositionNV:
936	if (!memberDeclaration) {
937	builder.addExtension(ext: spv::E_SPV_NV_stereo_view_rendering);
938	builder.addCapability(cap: spv::CapabilityShaderStereoViewNV);
939	}
940	return spv::BuiltInSecondaryPositionNV;
941	case glslang::EbvSecondaryViewportMaskNV:
942	if (!memberDeclaration) {
943	builder.addExtension(ext: spv::E_SPV_NV_stereo_view_rendering);
944	builder.addCapability(cap: spv::CapabilityShaderStereoViewNV);
945	}
946	return spv::BuiltInSecondaryViewportMaskNV;
947	case glslang::EbvPositionPerViewNV:
948	if (!memberDeclaration) {
949	builder.addExtension(ext: spv::E_SPV_NVX_multiview_per_view_attributes);
950	builder.addCapability(cap: spv::CapabilityPerViewAttributesNV);
951	}
952	return spv::BuiltInPositionPerViewNV;
953	case glslang::EbvViewportMaskPerViewNV:
954	if (!memberDeclaration) {
955	builder.addExtension(ext: spv::E_SPV_NVX_multiview_per_view_attributes);
956	builder.addCapability(cap: spv::CapabilityPerViewAttributesNV);
957	}
958	return spv::BuiltInViewportMaskPerViewNV;
959	case glslang::EbvFragFullyCoveredNV:
960	builder.addExtension(ext: spv::E_SPV_EXT_fragment_fully_covered);
961	builder.addCapability(cap: spv::CapabilityFragmentFullyCoveredEXT);
962	return spv::BuiltInFullyCoveredEXT;
963	case glslang::EbvFragmentSizeNV:
964	builder.addExtension(ext: spv::E_SPV_NV_shading_rate);
965	builder.addCapability(cap: spv::CapabilityShadingRateNV);
966	return spv::BuiltInFragmentSizeNV;
967	case glslang::EbvInvocationsPerPixelNV:
968	builder.addExtension(ext: spv::E_SPV_NV_shading_rate);
969	builder.addCapability(cap: spv::CapabilityShadingRateNV);
970	return spv::BuiltInInvocationsPerPixelNV;
971
972	// ray tracing
973	case glslang::EbvLaunchId:
974	return spv::BuiltInLaunchIdKHR;
975	case glslang::EbvLaunchSize:
976	return spv::BuiltInLaunchSizeKHR;
977	case glslang::EbvWorldRayOrigin:
978	return spv::BuiltInWorldRayOriginKHR;
979	case glslang::EbvWorldRayDirection:
980	return spv::BuiltInWorldRayDirectionKHR;
981	case glslang::EbvObjectRayOrigin:
982	return spv::BuiltInObjectRayOriginKHR;
983	case glslang::EbvObjectRayDirection:
984	return spv::BuiltInObjectRayDirectionKHR;
985	case glslang::EbvRayTmin:
986	return spv::BuiltInRayTminKHR;
987	case glslang::EbvRayTmax:
988	return spv::BuiltInRayTmaxKHR;
989	case glslang::EbvCullMask:
990	return spv::BuiltInCullMaskKHR;
991	case glslang::EbvPositionFetch:
992	return spv::BuiltInHitTriangleVertexPositionsKHR;
993	case glslang::EbvInstanceCustomIndex:
994	return spv::BuiltInInstanceCustomIndexKHR;
995	case glslang::EbvHitKind:
996	return spv::BuiltInHitKindKHR;
997	case glslang::EbvObjectToWorld:
998	case glslang::EbvObjectToWorld3x4:
999	return spv::BuiltInObjectToWorldKHR;
1000	case glslang::EbvWorldToObject:
1001	case glslang::EbvWorldToObject3x4:
1002	return spv::BuiltInWorldToObjectKHR;
1003	case glslang::EbvIncomingRayFlags:
1004	return spv::BuiltInIncomingRayFlagsKHR;
1005	case glslang::EbvGeometryIndex:
1006	return spv::BuiltInRayGeometryIndexKHR;
1007	case glslang::EbvCurrentRayTimeNV:
1008	builder.addExtension(ext: spv::E_SPV_NV_ray_tracing_motion_blur);
1009	builder.addCapability(cap: spv::CapabilityRayTracingMotionBlurNV);
1010	return spv::BuiltInCurrentRayTimeNV;
1011	case glslang::EbvMicroTrianglePositionNV:
1012	builder.addCapability(cap: spv::CapabilityRayTracingDisplacementMicromapNV);
1013	builder.addExtension(ext: "SPV_NV_displacement_micromap");
1014	return spv::BuiltInHitMicroTriangleVertexPositionsNV;
1015	case glslang::EbvMicroTriangleBaryNV:
1016	builder.addCapability(cap: spv::CapabilityRayTracingDisplacementMicromapNV);
1017	builder.addExtension(ext: "SPV_NV_displacement_micromap");
1018	return spv::BuiltInHitMicroTriangleVertexBarycentricsNV;
1019	case glslang::EbvHitKindFrontFacingMicroTriangleNV:
1020	builder.addCapability(cap: spv::CapabilityRayTracingDisplacementMicromapNV);
1021	builder.addExtension(ext: "SPV_NV_displacement_micromap");
1022	return spv::BuiltInHitKindFrontFacingMicroTriangleNV;
1023	case glslang::EbvHitKindBackFacingMicroTriangleNV:
1024	builder.addCapability(cap: spv::CapabilityRayTracingDisplacementMicromapNV);
1025	builder.addExtension(ext: "SPV_NV_displacement_micromap");
1026	return spv::BuiltInHitKindBackFacingMicroTriangleNV;
1027
1028	// barycentrics
1029	case glslang::EbvBaryCoordNV:
1030	builder.addExtension(ext: spv::E_SPV_NV_fragment_shader_barycentric);
1031	builder.addCapability(cap: spv::CapabilityFragmentBarycentricNV);
1032	return spv::BuiltInBaryCoordNV;
1033	case glslang::EbvBaryCoordNoPerspNV:
1034	builder.addExtension(ext: spv::E_SPV_NV_fragment_shader_barycentric);
1035	builder.addCapability(cap: spv::CapabilityFragmentBarycentricNV);
1036	return spv::BuiltInBaryCoordNoPerspNV;
1037
1038	case glslang::EbvBaryCoordEXT:
1039	builder.addExtension(ext: spv::E_SPV_KHR_fragment_shader_barycentric);
1040	builder.addCapability(cap: spv::CapabilityFragmentBarycentricKHR);
1041	return spv::BuiltInBaryCoordKHR;
1042	case glslang::EbvBaryCoordNoPerspEXT:
1043	builder.addExtension(ext: spv::E_SPV_KHR_fragment_shader_barycentric);
1044	builder.addCapability(cap: spv::CapabilityFragmentBarycentricKHR);
1045	return spv::BuiltInBaryCoordNoPerspKHR;
1046
1047	// mesh shaders
1048	case glslang::EbvTaskCountNV:
1049	return spv::BuiltInTaskCountNV;
1050	case glslang::EbvPrimitiveCountNV:
1051	return spv::BuiltInPrimitiveCountNV;
1052	case glslang::EbvPrimitiveIndicesNV:
1053	return spv::BuiltInPrimitiveIndicesNV;
1054	case glslang::EbvClipDistancePerViewNV:
1055	return spv::BuiltInClipDistancePerViewNV;
1056	case glslang::EbvCullDistancePerViewNV:
1057	return spv::BuiltInCullDistancePerViewNV;
1058	case glslang::EbvLayerPerViewNV:
1059	return spv::BuiltInLayerPerViewNV;
1060	case glslang::EbvMeshViewCountNV:
1061	return spv::BuiltInMeshViewCountNV;
1062	case glslang::EbvMeshViewIndicesNV:
1063	return spv::BuiltInMeshViewIndicesNV;
1064
1065	// SPV_EXT_mesh_shader
1066	case glslang::EbvPrimitivePointIndicesEXT:
1067	return spv::BuiltInPrimitivePointIndicesEXT;
1068	case glslang::EbvPrimitiveLineIndicesEXT:
1069	return spv::BuiltInPrimitiveLineIndicesEXT;
1070	case glslang::EbvPrimitiveTriangleIndicesEXT:
1071	return spv::BuiltInPrimitiveTriangleIndicesEXT;
1072	case glslang::EbvCullPrimitiveEXT:
1073	return spv::BuiltInCullPrimitiveEXT;
1074
1075	// sm builtins
1076	case glslang::EbvWarpsPerSM:
1077	builder.addExtension(ext: spv::E_SPV_NV_shader_sm_builtins);
1078	builder.addCapability(cap: spv::CapabilityShaderSMBuiltinsNV);
1079	return spv::BuiltInWarpsPerSMNV;
1080	case glslang::EbvSMCount:
1081	builder.addExtension(ext: spv::E_SPV_NV_shader_sm_builtins);
1082	builder.addCapability(cap: spv::CapabilityShaderSMBuiltinsNV);
1083	return spv::BuiltInSMCountNV;
1084	case glslang::EbvWarpID:
1085	builder.addExtension(ext: spv::E_SPV_NV_shader_sm_builtins);
1086	builder.addCapability(cap: spv::CapabilityShaderSMBuiltinsNV);
1087	return spv::BuiltInWarpIDNV;
1088	case glslang::EbvSMID:
1089	builder.addExtension(ext: spv::E_SPV_NV_shader_sm_builtins);
1090	builder.addCapability(cap: spv::CapabilityShaderSMBuiltinsNV);
1091	return spv::BuiltInSMIDNV;
1092
1093	// ARM builtins
1094	case glslang::EbvCoreCountARM:
1095	builder.addExtension(ext: spv::E_SPV_ARM_core_builtins);
1096	builder.addCapability(cap: spv::CapabilityCoreBuiltinsARM);
1097	return spv::BuiltInCoreCountARM;
1098	case glslang::EbvCoreIDARM:
1099	builder.addExtension(ext: spv::E_SPV_ARM_core_builtins);
1100	builder.addCapability(cap: spv::CapabilityCoreBuiltinsARM);
1101	return spv::BuiltInCoreIDARM;
1102	case glslang::EbvCoreMaxIDARM:
1103	builder.addExtension(ext: spv::E_SPV_ARM_core_builtins);
1104	builder.addCapability(cap: spv::CapabilityCoreBuiltinsARM);
1105	return spv::BuiltInCoreMaxIDARM;
1106	case glslang::EbvWarpIDARM:
1107	builder.addExtension(ext: spv::E_SPV_ARM_core_builtins);
1108	builder.addCapability(cap: spv::CapabilityCoreBuiltinsARM);
1109	return spv::BuiltInWarpIDARM;
1110	case glslang::EbvWarpMaxIDARM:
1111	builder.addExtension(ext: spv::E_SPV_ARM_core_builtins);
1112	builder.addCapability(cap: spv::CapabilityCoreBuiltinsARM);
1113	return spv::BuiltInWarpMaxIDARM;
1114
1115	default:
1116	return spv::BuiltInMax;
1117	}
1118	}
1119
1120	// Translate glslang image layout format to SPIR-V image format.
1121	spv::ImageFormat TGlslangToSpvTraverser::TranslateImageFormat(const glslang::TType& type)
1122	{
1123	assert(type.getBasicType() == glslang::EbtSampler);
1124
1125	// Check for capabilities
1126	switch (type.getQualifier().getFormat()) {
1127	case glslang::ElfRg32f:
1128	case glslang::ElfRg16f:
1129	case glslang::ElfR11fG11fB10f:
1130	case glslang::ElfR16f:
1131	case glslang::ElfRgba16:
1132	case glslang::ElfRgb10A2:
1133	case glslang::ElfRg16:
1134	case glslang::ElfRg8:
1135	case glslang::ElfR16:
1136	case glslang::ElfR8:
1137	case glslang::ElfRgba16Snorm:
1138	case glslang::ElfRg16Snorm:
1139	case glslang::ElfRg8Snorm:
1140	case glslang::ElfR16Snorm:
1141	case glslang::ElfR8Snorm:
1142
1143	case glslang::ElfRg32i:
1144	case glslang::ElfRg16i:
1145	case glslang::ElfRg8i:
1146	case glslang::ElfR16i:
1147	case glslang::ElfR8i:
1148
1149	case glslang::ElfRgb10a2ui:
1150	case glslang::ElfRg32ui:
1151	case glslang::ElfRg16ui:
1152	case glslang::ElfRg8ui:
1153	case glslang::ElfR16ui:
1154	case glslang::ElfR8ui:
1155	builder.addCapability(cap: spv::CapabilityStorageImageExtendedFormats);
1156	break;
1157
1158	case glslang::ElfR64ui:
1159	case glslang::ElfR64i:
1160	builder.addExtension(ext: spv::E_SPV_EXT_shader_image_int64);
1161	builder.addCapability(cap: spv::CapabilityInt64ImageEXT);
1162	break;
1163	default:
1164	break;
1165	}
1166
1167	// do the translation
1168	switch (type.getQualifier().getFormat()) {
1169	case glslang::ElfNone: return spv::ImageFormatUnknown;
1170	case glslang::ElfRgba32f: return spv::ImageFormatRgba32f;
1171	case glslang::ElfRgba16f: return spv::ImageFormatRgba16f;
1172	case glslang::ElfR32f: return spv::ImageFormatR32f;
1173	case glslang::ElfRgba8: return spv::ImageFormatRgba8;
1174	case glslang::ElfRgba8Snorm: return spv::ImageFormatRgba8Snorm;
1175	case glslang::ElfRg32f: return spv::ImageFormatRg32f;
1176	case glslang::ElfRg16f: return spv::ImageFormatRg16f;
1177	case glslang::ElfR11fG11fB10f: return spv::ImageFormatR11fG11fB10f;
1178	case glslang::ElfR16f: return spv::ImageFormatR16f;
1179	case glslang::ElfRgba16: return spv::ImageFormatRgba16;
1180	case glslang::ElfRgb10A2: return spv::ImageFormatRgb10A2;
1181	case glslang::ElfRg16: return spv::ImageFormatRg16;
1182	case glslang::ElfRg8: return spv::ImageFormatRg8;
1183	case glslang::ElfR16: return spv::ImageFormatR16;
1184	case glslang::ElfR8: return spv::ImageFormatR8;
1185	case glslang::ElfRgba16Snorm: return spv::ImageFormatRgba16Snorm;
1186	case glslang::ElfRg16Snorm: return spv::ImageFormatRg16Snorm;
1187	case glslang::ElfRg8Snorm: return spv::ImageFormatRg8Snorm;
1188	case glslang::ElfR16Snorm: return spv::ImageFormatR16Snorm;
1189	case glslang::ElfR8Snorm: return spv::ImageFormatR8Snorm;
1190	case glslang::ElfRgba32i: return spv::ImageFormatRgba32i;
1191	case glslang::ElfRgba16i: return spv::ImageFormatRgba16i;
1192	case glslang::ElfRgba8i: return spv::ImageFormatRgba8i;
1193	case glslang::ElfR32i: return spv::ImageFormatR32i;
1194	case glslang::ElfRg32i: return spv::ImageFormatRg32i;
1195	case glslang::ElfRg16i: return spv::ImageFormatRg16i;
1196	case glslang::ElfRg8i: return spv::ImageFormatRg8i;
1197	case glslang::ElfR16i: return spv::ImageFormatR16i;
1198	case glslang::ElfR8i: return spv::ImageFormatR8i;
1199	case glslang::ElfRgba32ui: return spv::ImageFormatRgba32ui;
1200	case glslang::ElfRgba16ui: return spv::ImageFormatRgba16ui;
1201	case glslang::ElfRgba8ui: return spv::ImageFormatRgba8ui;
1202	case glslang::ElfR32ui: return spv::ImageFormatR32ui;
1203	case glslang::ElfRg32ui: return spv::ImageFormatRg32ui;
1204	case glslang::ElfRg16ui: return spv::ImageFormatRg16ui;
1205	case glslang::ElfRgb10a2ui: return spv::ImageFormatRgb10a2ui;
1206	case glslang::ElfRg8ui: return spv::ImageFormatRg8ui;
1207	case glslang::ElfR16ui: return spv::ImageFormatR16ui;
1208	case glslang::ElfR8ui: return spv::ImageFormatR8ui;
1209	case glslang::ElfR64ui: return spv::ImageFormatR64ui;
1210	case glslang::ElfR64i: return spv::ImageFormatR64i;
1211	default: return spv::ImageFormatMax;
1212	}
1213	}
1214
1215	spv::SelectionControlMask TGlslangToSpvTraverser::TranslateSelectionControl(
1216	const glslang::TIntermSelection& selectionNode) const
1217	{
1218	if (selectionNode.getFlatten())
1219	return spv::SelectionControlFlattenMask;
1220	if (selectionNode.getDontFlatten())
1221	return spv::SelectionControlDontFlattenMask;
1222	return spv::SelectionControlMaskNone;
1223	}
1224
1225	spv::SelectionControlMask TGlslangToSpvTraverser::TranslateSwitchControl(const glslang::TIntermSwitch& switchNode)
1226	const
1227	{
1228	if (switchNode.getFlatten())
1229	return spv::SelectionControlFlattenMask;
1230	if (switchNode.getDontFlatten())
1231	return spv::SelectionControlDontFlattenMask;
1232	return spv::SelectionControlMaskNone;
1233	}
1234
1235	// return a non-0 dependency if the dependency argument must be set
1236	spv::LoopControlMask TGlslangToSpvTraverser::TranslateLoopControl(const glslang::TIntermLoop& loopNode,
1237	std::vector<unsigned int>& operands) const
1238	{
1239	spv::LoopControlMask control = spv::LoopControlMaskNone;
1240
1241	if (loopNode.getDontUnroll())
1242	control = control | spv::LoopControlDontUnrollMask;
1243	if (loopNode.getUnroll())
1244	control = control | spv::LoopControlUnrollMask;
1245	if (unsigned(loopNode.getLoopDependency()) == glslang::TIntermLoop::dependencyInfinite)
1246	control = control | spv::LoopControlDependencyInfiniteMask;
1247	else if (loopNode.getLoopDependency() > 0) {
1248	control = control | spv::LoopControlDependencyLengthMask;
1249	operands.push_back(x: (unsigned int)loopNode.getLoopDependency());
1250	}
1251	if (glslangIntermediate->getSpv().spv >= glslang::EShTargetSpv_1_4) {
1252	if (loopNode.getMinIterations() > 0) {
1253	control = control | spv::LoopControlMinIterationsMask;
1254	operands.push_back(x: loopNode.getMinIterations());
1255	}
1256	if (loopNode.getMaxIterations() < glslang::TIntermLoop::iterationsInfinite) {
1257	control = control | spv::LoopControlMaxIterationsMask;
1258	operands.push_back(x: loopNode.getMaxIterations());
1259	}
1260	if (loopNode.getIterationMultiple() > 1) {
1261	control = control | spv::LoopControlIterationMultipleMask;
1262	operands.push_back(x: loopNode.getIterationMultiple());
1263	}
1264	if (loopNode.getPeelCount() > 0) {
1265	control = control | spv::LoopControlPeelCountMask;
1266	operands.push_back(x: loopNode.getPeelCount());
1267	}
1268	if (loopNode.getPartialCount() > 0) {
1269	control = control | spv::LoopControlPartialCountMask;
1270	operands.push_back(x: loopNode.getPartialCount());
1271	}
1272	}
1273
1274	return control;
1275	}
1276
1277	// Translate glslang type to SPIR-V storage class.
1278	spv::StorageClass TGlslangToSpvTraverser::TranslateStorageClass(const glslang::TType& type)
1279	{
1280	if (type.getBasicType() == glslang::EbtRayQuery || type.getBasicType() == glslang::EbtHitObjectNV)
1281	return spv::StorageClassPrivate;
1282	if (type.getQualifier().isSpirvByReference()) {
1283	if (type.getQualifier().isParamInput() || type.getQualifier().isParamOutput())
1284	return spv::StorageClassFunction;
1285	}
1286	if (type.getQualifier().isPipeInput())
1287	return spv::StorageClassInput;
1288	if (type.getQualifier().isPipeOutput())
1289	return spv::StorageClassOutput;
1290	if (type.getQualifier().storage == glslang::EvqTileImageEXT || type.isAttachmentEXT()) {
1291	builder.addExtension(ext: spv::E_SPV_EXT_shader_tile_image);
1292	builder.addCapability(cap: spv::CapabilityTileImageColorReadAccessEXT);
1293	return spv::StorageClassTileImageEXT;
1294	}
1295
1296	if (glslangIntermediate->getSource() != glslang::EShSourceHlsl ||
1297	type.getQualifier().storage == glslang::EvqUniform) {
1298	if (type.isAtomic())
1299	return spv::StorageClassAtomicCounter;
1300	if (type.containsOpaque() && !glslangIntermediate->getBindlessMode())
1301	return spv::StorageClassUniformConstant;
1302	}
1303
1304	if (type.getQualifier().isUniformOrBuffer() &&
1305	type.getQualifier().isShaderRecord()) {
1306	return spv::StorageClassShaderRecordBufferKHR;
1307	}
1308
1309	if (glslangIntermediate->usingStorageBuffer() && type.getQualifier().storage == glslang::EvqBuffer) {
1310	builder.addIncorporatedExtension(ext: spv::E_SPV_KHR_storage_buffer_storage_class, incorporatedVersion: spv::Spv_1_3);
1311	return spv::StorageClassStorageBuffer;
1312	}
1313
1314	if (type.getQualifier().isUniformOrBuffer()) {
1315	if (type.getQualifier().isPushConstant())
1316	return spv::StorageClassPushConstant;
1317	if (type.getBasicType() == glslang::EbtBlock)
1318	return spv::StorageClassUniform;
1319	return spv::StorageClassUniformConstant;
1320	}
1321
1322	if (type.getQualifier().storage == glslang::EvqShared && type.getBasicType() == glslang::EbtBlock) {
1323	builder.addExtension(ext: spv::E_SPV_KHR_workgroup_memory_explicit_layout);
1324	builder.addCapability(cap: spv::CapabilityWorkgroupMemoryExplicitLayoutKHR);
1325	return spv::StorageClassWorkgroup;
1326	}
1327
1328	switch (type.getQualifier().storage) {
1329	case glslang::EvqGlobal: return spv::StorageClassPrivate;
1330	case glslang::EvqConstReadOnly: return spv::StorageClassFunction;
1331	case glslang::EvqTemporary: return spv::StorageClassFunction;
1332	case glslang::EvqShared: return spv::StorageClassWorkgroup;
1333	case glslang::EvqPayload: return spv::StorageClassRayPayloadKHR;
1334	case glslang::EvqPayloadIn: return spv::StorageClassIncomingRayPayloadKHR;
1335	case glslang::EvqHitAttr: return spv::StorageClassHitAttributeKHR;
1336	case glslang::EvqCallableData: return spv::StorageClassCallableDataKHR;
1337	case glslang::EvqCallableDataIn: return spv::StorageClassIncomingCallableDataKHR;
1338	case glslang::EvqtaskPayloadSharedEXT : return spv::StorageClassTaskPayloadWorkgroupEXT;
1339	case glslang::EvqHitObjectAttrNV: return spv::StorageClassHitObjectAttributeNV;
1340	case glslang::EvqSpirvStorageClass: return static_cast<spv::StorageClass>(type.getQualifier().spirvStorageClass);
1341	default:
1342	assert(0);
1343	break;
1344	}
1345
1346	return spv::StorageClassFunction;
1347	}
1348
1349	// Translate glslang constants to SPIR-V literals
1350	void TGlslangToSpvTraverser::TranslateLiterals(const glslang::TVector<const glslang::TIntermConstantUnion*>& constants,
1351	std::vector<unsigned>& literals) const
1352	{
1353	for (auto constant : constants) {
1354	if (constant->getBasicType() == glslang::EbtFloat) {
1355	float floatValue = static_cast<float>(constant->getConstArray()[0].getDConst());
1356	unsigned literal;
1357	static_assert(sizeof(literal) == sizeof(floatValue), "sizeof(unsigned) != sizeof(float)");
1358	memcpy(dest: &literal, src: &floatValue, n: sizeof(literal));
1359	literals.push_back(x: literal);
1360	} else if (constant->getBasicType() == glslang::EbtInt) {
1361	unsigned literal = constant->getConstArray()[0].getIConst();
1362	literals.push_back(x: literal);
1363	} else if (constant->getBasicType() == glslang::EbtUint) {
1364	unsigned literal = constant->getConstArray()[0].getUConst();
1365	literals.push_back(x: literal);
1366	} else if (constant->getBasicType() == glslang::EbtBool) {
1367	unsigned literal = constant->getConstArray()[0].getBConst();
1368	literals.push_back(x: literal);
1369	} else if (constant->getBasicType() == glslang::EbtString) {
1370	auto str = constant->getConstArray()[0].getSConst()->c_str();
1371	unsigned literal = 0;
1372	char* literalPtr = reinterpret_cast<char*>(&literal);
1373	unsigned charCount = 0;
1374	char ch = 0;
1375	do {
1376	ch = *(str++);
1377	*(literalPtr++) = ch;
1378	++charCount;
1379	if (charCount == 4) {
1380	literals.push_back(x: literal);
1381	literalPtr = reinterpret_cast<char*>(&literal);
1382	charCount = 0;
1383	}
1384	} while (ch != 0);
1385
1386	// Partial literal is padded with 0
1387	if (charCount > 0) {
1388	for (; charCount < 4; ++charCount)
1389	*(literalPtr++) = 0;
1390	literals.push_back(x: literal);
1391	}
1392	} else
1393	assert(0); // Unexpected type
1394	}
1395	}
1396
1397	// Add capabilities pertaining to how an array is indexed.
1398	void TGlslangToSpvTraverser::addIndirectionIndexCapabilities(const glslang::TType& baseType,
1399	const glslang::TType& indexType)
1400	{
1401	if (indexType.getQualifier().isNonUniform()) {
1402	// deal with an asserted non-uniform index
1403	// SPV_EXT_descriptor_indexing already added in TranslateNonUniformDecoration
1404	if (baseType.getBasicType() == glslang::EbtSampler) {
1405	if (baseType.getQualifier().hasAttachment())
1406	builder.addCapability(cap: spv::CapabilityInputAttachmentArrayNonUniformIndexingEXT);
1407	else if (baseType.isImage() && baseType.getSampler().isBuffer())
1408	builder.addCapability(cap: spv::CapabilityStorageTexelBufferArrayNonUniformIndexingEXT);
1409	else if (baseType.isTexture() && baseType.getSampler().isBuffer())
1410	builder.addCapability(cap: spv::CapabilityUniformTexelBufferArrayNonUniformIndexingEXT);
1411	else if (baseType.isImage())
1412	builder.addCapability(cap: spv::CapabilityStorageImageArrayNonUniformIndexingEXT);
1413	else if (baseType.isTexture())
1414	builder.addCapability(cap: spv::CapabilitySampledImageArrayNonUniformIndexingEXT);
1415	} else if (baseType.getBasicType() == glslang::EbtBlock) {
1416	if (baseType.getQualifier().storage == glslang::EvqBuffer)
1417	builder.addCapability(cap: spv::CapabilityStorageBufferArrayNonUniformIndexingEXT);
1418	else if (baseType.getQualifier().storage == glslang::EvqUniform)
1419	builder.addCapability(cap: spv::CapabilityUniformBufferArrayNonUniformIndexingEXT);
1420	}
1421	} else {
1422	// assume a dynamically uniform index
1423	if (baseType.getBasicType() == glslang::EbtSampler) {
1424	if (baseType.getQualifier().hasAttachment()) {
1425	builder.addIncorporatedExtension(ext: "SPV_EXT_descriptor_indexing", incorporatedVersion: spv::Spv_1_5);
1426	builder.addCapability(cap: spv::CapabilityInputAttachmentArrayDynamicIndexingEXT);
1427	} else if (baseType.isImage() && baseType.getSampler().isBuffer()) {
1428	builder.addIncorporatedExtension(ext: "SPV_EXT_descriptor_indexing", incorporatedVersion: spv::Spv_1_5);
1429	builder.addCapability(cap: spv::CapabilityStorageTexelBufferArrayDynamicIndexingEXT);
1430	} else if (baseType.isTexture() && baseType.getSampler().isBuffer()) {
1431	builder.addIncorporatedExtension(ext: "SPV_EXT_descriptor_indexing", incorporatedVersion: spv::Spv_1_5);
1432	builder.addCapability(cap: spv::CapabilityUniformTexelBufferArrayDynamicIndexingEXT);
1433	}
1434	}
1435	}
1436	}
1437
1438	// Return whether or not the given type is something that should be tied to a
1439	// descriptor set.
1440	bool IsDescriptorResource(const glslang::TType& type)
1441	{
1442	// uniform and buffer blocks are included, unless it is a push_constant
1443	if (type.getBasicType() == glslang::EbtBlock)
1444	return type.getQualifier().isUniformOrBuffer() &&
1445	! type.getQualifier().isShaderRecord() &&
1446	! type.getQualifier().isPushConstant();
1447
1448	// non block...
1449	// basically samplerXXX/subpass/sampler/texture are all included
1450	// if they are the global-scope-class, not the function parameter
1451	// (or local, if they ever exist) class.
1452	if (type.getBasicType() == glslang::EbtSampler ||
1453	type.getBasicType() == glslang::EbtAccStruct)
1454	return type.getQualifier().isUniformOrBuffer();
1455
1456	// None of the above.
1457	return false;
1458	}
1459
1460	void InheritQualifiers(glslang::TQualifier& child, const glslang::TQualifier& parent)
1461	{
1462	if (child.layoutMatrix == glslang::ElmNone)
1463	child.layoutMatrix = parent.layoutMatrix;
1464
1465	if (parent.invariant)
1466	child.invariant = true;
1467	if (parent.flat)
1468	child.flat = true;
1469	if (parent.centroid)
1470	child.centroid = true;
1471	if (parent.nopersp)
1472	child.nopersp = true;
1473	if (parent.explicitInterp)
1474	child.explicitInterp = true;
1475	if (parent.perPrimitiveNV)
1476	child.perPrimitiveNV = true;
1477	if (parent.perViewNV)
1478	child.perViewNV = true;
1479	if (parent.perTaskNV)
1480	child.perTaskNV = true;
1481	if (parent.storage == glslang::EvqtaskPayloadSharedEXT)
1482	child.storage = glslang::EvqtaskPayloadSharedEXT;
1483	if (parent.patch)
1484	child.patch = true;
1485	if (parent.sample)
1486	child.sample = true;
1487	if (parent.coherent)
1488	child.coherent = true;
1489	if (parent.devicecoherent)
1490	child.devicecoherent = true;
1491	if (parent.queuefamilycoherent)
1492	child.queuefamilycoherent = true;
1493	if (parent.workgroupcoherent)
1494	child.workgroupcoherent = true;
1495	if (parent.subgroupcoherent)
1496	child.subgroupcoherent = true;
1497	if (parent.shadercallcoherent)
1498	child.shadercallcoherent = true;
1499	if (parent.nonprivate)
1500	child.nonprivate = true;
1501	if (parent.volatil)
1502	child.volatil = true;
1503	if (parent.restrict)
1504	child.restrict = true;
1505	if (parent.readonly)
1506	child.readonly = true;
1507	if (parent.writeonly)
1508	child.writeonly = true;
1509	if (parent.nonUniform)
1510	child.nonUniform = true;
1511	}
1512
1513	bool HasNonLayoutQualifiers(const glslang::TType& type, const glslang::TQualifier& qualifier)
1514	{
1515	// This should list qualifiers that simultaneous satisfy:
1516	// - struct members might inherit from a struct declaration
1517	// (note that non-block structs don't explicitly inherit,
1518	// only implicitly, meaning no decoration involved)
1519	// - affect decorations on the struct members
1520	// (note smooth does not, and expecting something like volatile
1521	// to effect the whole object)
1522	// - are not part of the offset/st430/etc or row/column-major layout
1523	return qualifier.invariant || (qualifier.hasLocation() && type.getBasicType() == glslang::EbtBlock);
1524	}
1525
1526	//
1527	// Implement the TGlslangToSpvTraverser class.
1528	//
1529
1530	TGlslangToSpvTraverser::TGlslangToSpvTraverser(unsigned int spvVersion,
1531	const glslang::TIntermediate* glslangIntermediate,
1532	spv::SpvBuildLogger* buildLogger, glslang::SpvOptions& options) :
1533	TIntermTraverser(true, false, true),
1534	options(options),
1535	shaderEntry(nullptr), currentFunction(nullptr),
1536	sequenceDepth(0), logger(buildLogger),
1537	builder(spvVersion, (glslang::GetKhronosToolId() << 16) | glslang::GetSpirvGeneratorVersion(), logger),
1538	inEntryPoint(false), entryPointTerminated(false), linkageOnly(false),
1539	glslangIntermediate(glslangIntermediate),
1540	nanMinMaxClamp(glslangIntermediate->getNanMinMaxClamp()),
1541	nonSemanticDebugPrintf(0),
1542	taskPayloadID(0)
1543	{
1544	bool isMeshShaderExt = (glslangIntermediate->getRequestedExtensions().find(x: glslang::E_GL_EXT_mesh_shader) !=
1545	glslangIntermediate->getRequestedExtensions().end());
1546	spv::ExecutionModel executionModel = TranslateExecutionModel(stage: glslangIntermediate->getStage(), isMeshShaderEXT: isMeshShaderExt);
1547
1548	builder.clearAccessChain();
1549	builder.setSource(lang: TranslateSourceLanguage(source: glslangIntermediate->getSource(), profile: glslangIntermediate->getProfile()),
1550	version: glslangIntermediate->getVersion());
1551
1552	if (options.emitNonSemanticShaderDebugSource)
1553	this->options.emitNonSemanticShaderDebugInfo = true;
1554	if (options.emitNonSemanticShaderDebugInfo)
1555	this->options.generateDebugInfo = true;
1556
1557	if (this->options.generateDebugInfo) {
1558	if (this->options.emitNonSemanticShaderDebugInfo) {
1559	builder.setEmitNonSemanticShaderDebugInfo(this->options.emitNonSemanticShaderDebugSource);
1560	}
1561	else {
1562	builder.setEmitSpirvDebugInfo();
1563	}
1564	builder.setDebugMainSourceFile(glslangIntermediate->getSourceFile());
1565
1566	// Set the source shader's text. If for SPV version 1.0, include
1567	// a preamble in comments stating the OpModuleProcessed instructions.
1568	// Otherwise, emit those as actual instructions.
1569	std::string text;
1570	const std::vector<std::string>& processes = glslangIntermediate->getProcesses();
1571	for (int p = 0; p < (int)processes.size(); ++p) {
1572	if (glslangIntermediate->getSpv().spv < glslang::EShTargetSpv_1_1) {
1573	text.append(s: "// OpModuleProcessed ");
1574	text.append(str: processes[p]);
1575	text.append(s: "\n");
1576	} else
1577	builder.addModuleProcessed(p: processes[p]);
1578	}
1579	if (glslangIntermediate->getSpv().spv < glslang::EShTargetSpv_1_1 && (int)processes.size() > 0)
1580	text.append(s: "#line 1\n");
1581	text.append(str: glslangIntermediate->getSourceText());
1582	builder.setSourceText(text);
1583	// Pass name and text for all included files
1584	const std::map<std::string, std::string>& include_txt = glslangIntermediate->getIncludeText();
1585	for (auto iItr = include_txt.begin(); iItr != include_txt.end(); ++iItr)
1586	builder.addInclude(name: iItr->first, text: iItr->second);
1587	}
1588
1589	builder.setUseReplicatedComposites(glslangIntermediate->usingReplicatedComposites());
1590
1591	stdBuiltins = builder.import("GLSL.std.450");
1592
1593	spv::AddressingModel addressingModel = spv::AddressingModelLogical;
1594	spv::MemoryModel memoryModel = spv::MemoryModelGLSL450;
1595
1596	if (glslangIntermediate->usingPhysicalStorageBuffer()) {
1597	addressingModel = spv::AddressingModelPhysicalStorageBuffer64EXT;
1598	builder.addIncorporatedExtension(ext: spv::E_SPV_KHR_physical_storage_buffer, incorporatedVersion: spv::Spv_1_5);
1599	builder.addCapability(cap: spv::CapabilityPhysicalStorageBufferAddressesEXT);
1600	}
1601	if (glslangIntermediate->usingVulkanMemoryModel()) {
1602	memoryModel = spv::MemoryModelVulkanKHR;
1603	builder.addCapability(cap: spv::CapabilityVulkanMemoryModelKHR);
1604	builder.addIncorporatedExtension(ext: spv::E_SPV_KHR_vulkan_memory_model, incorporatedVersion: spv::Spv_1_5);
1605	}
1606	builder.setMemoryModel(addr: addressingModel, mem: memoryModel);
1607
1608	if (glslangIntermediate->usingVariablePointers()) {
1609	builder.addCapability(cap: spv::CapabilityVariablePointers);
1610	}
1611
1612	// If not linking, there is no entry point
1613	if (!options.compileOnly) {
1614	shaderEntry = builder.makeEntryPoint(glslangIntermediate->getEntryPointName().c_str());
1615	entryPoint =
1616	builder.addEntryPoint(executionModel, shaderEntry, name: glslangIntermediate->getEntryPointName().c_str());
1617	}
1618
1619	// Add the source extensions
1620	const auto& sourceExtensions = glslangIntermediate->getRequestedExtensions();
1621	for (auto it = sourceExtensions.begin(); it != sourceExtensions.end(); ++it)
1622	builder.addSourceExtension(ext: it->c_str());
1623
1624	// Add the top-level modes for this shader.
1625
1626	if (glslangIntermediate->getXfbMode()) {
1627	builder.addCapability(cap: spv::CapabilityTransformFeedback);
1628	builder.addExecutionMode(shaderEntry, mode: spv::ExecutionModeXfb);
1629	}
1630
1631	if (glslangIntermediate->getLayoutPrimitiveCulling()) {
1632	builder.addCapability(cap: spv::CapabilityRayTraversalPrimitiveCullingKHR);
1633	}
1634
1635	if (glslangIntermediate->getSubgroupUniformControlFlow()) {
1636	builder.addExtension(ext: spv::E_SPV_KHR_subgroup_uniform_control_flow);
1637	builder.addExecutionMode(shaderEntry, mode: spv::ExecutionModeSubgroupUniformControlFlowKHR);
1638	}
1639	if (glslangIntermediate->getMaximallyReconverges()) {
1640	builder.addExtension(ext: spv::E_SPV_KHR_maximal_reconvergence);
1641	builder.addExecutionMode(shaderEntry, mode: spv::ExecutionModeMaximallyReconvergesKHR);
1642	}
1643
1644	if (glslangIntermediate->getQuadDerivMode())
1645	{
1646	builder.addCapability(cap: spv::CapabilityQuadControlKHR);
1647	builder.addExtension(ext: spv::E_SPV_KHR_quad_control);
1648	builder.addExecutionMode(shaderEntry, mode: spv::ExecutionModeQuadDerivativesKHR);
1649	}
1650
1651	if (glslangIntermediate->getReqFullQuadsMode())
1652	{
1653	builder.addCapability(cap: spv::CapabilityQuadControlKHR);
1654	builder.addExtension(ext: spv::E_SPV_KHR_quad_control);
1655	builder.addExecutionMode(shaderEntry, mode: spv::ExecutionModeRequireFullQuadsKHR);
1656	}
1657
1658	unsigned int mode;
1659	switch (glslangIntermediate->getStage()) {
1660	case EShLangVertex:
1661	builder.addCapability(cap: spv::CapabilityShader);
1662	break;
1663
1664	case EShLangFragment:
1665	builder.addCapability(cap: spv::CapabilityShader);
1666	if (glslangIntermediate->getPixelCenterInteger())
1667	builder.addExecutionMode(shaderEntry, mode: spv::ExecutionModePixelCenterInteger);
1668
1669	if (glslangIntermediate->getOriginUpperLeft())
1670	builder.addExecutionMode(shaderEntry, mode: spv::ExecutionModeOriginUpperLeft);
1671	else
1672	builder.addExecutionMode(shaderEntry, mode: spv::ExecutionModeOriginLowerLeft);
1673
1674	if (glslangIntermediate->getEarlyFragmentTests())
1675	builder.addExecutionMode(shaderEntry, mode: spv::ExecutionModeEarlyFragmentTests);
1676
1677	if (glslangIntermediate->getEarlyAndLateFragmentTestsAMD())
1678	{
1679	builder.addExecutionMode(shaderEntry, mode: spv::ExecutionModeEarlyAndLateFragmentTestsAMD);
1680	builder.addExtension(ext: spv::E_SPV_AMD_shader_early_and_late_fragment_tests);
1681	}
1682
1683	if (glslangIntermediate->getPostDepthCoverage()) {
1684	builder.addCapability(cap: spv::CapabilitySampleMaskPostDepthCoverage);
1685	builder.addExecutionMode(shaderEntry, mode: spv::ExecutionModePostDepthCoverage);
1686	builder.addExtension(ext: spv::E_SPV_KHR_post_depth_coverage);
1687	}
1688
1689	if (glslangIntermediate->getNonCoherentColorAttachmentReadEXT()) {
1690	builder.addCapability(cap: spv::CapabilityTileImageColorReadAccessEXT);
1691	builder.addExecutionMode(shaderEntry, mode: spv::ExecutionModeNonCoherentColorAttachmentReadEXT);
1692	builder.addExtension(ext: spv::E_SPV_EXT_shader_tile_image);
1693	}
1694
1695	if (glslangIntermediate->getNonCoherentDepthAttachmentReadEXT()) {
1696	builder.addCapability(cap: spv::CapabilityTileImageDepthReadAccessEXT);
1697	builder.addExecutionMode(shaderEntry, mode: spv::ExecutionModeNonCoherentDepthAttachmentReadEXT);
1698	builder.addExtension(ext: spv::E_SPV_EXT_shader_tile_image);
1699	}
1700
1701	if (glslangIntermediate->getNonCoherentStencilAttachmentReadEXT()) {
1702	builder.addCapability(cap: spv::CapabilityTileImageStencilReadAccessEXT);
1703	builder.addExecutionMode(shaderEntry, mode: spv::ExecutionModeNonCoherentStencilAttachmentReadEXT);
1704	builder.addExtension(ext: spv::E_SPV_EXT_shader_tile_image);
1705	}
1706
1707	if (glslangIntermediate->isDepthReplacing())
1708	builder.addExecutionMode(shaderEntry, mode: spv::ExecutionModeDepthReplacing);
1709
1710	if (glslangIntermediate->isStencilReplacing())
1711	builder.addExecutionMode(shaderEntry, mode: spv::ExecutionModeStencilRefReplacingEXT);
1712
1713	switch(glslangIntermediate->getDepth()) {
1714	case glslang::EldGreater: mode = spv::ExecutionModeDepthGreater; break;
1715	case glslang::EldLess: mode = spv::ExecutionModeDepthLess; break;
1716	case glslang::EldUnchanged: mode = spv::ExecutionModeDepthUnchanged; break;
1717	default: mode = spv::ExecutionModeMax; break;
1718	}
1719
1720	if (mode != spv::ExecutionModeMax)
1721	builder.addExecutionMode(shaderEntry, mode: (spv::ExecutionMode)mode);
1722
1723	switch (glslangIntermediate->getStencil()) {
1724	case glslang::ElsRefUnchangedFrontAMD: mode = spv::ExecutionModeStencilRefUnchangedFrontAMD; break;
1725	case glslang::ElsRefGreaterFrontAMD: mode = spv::ExecutionModeStencilRefGreaterFrontAMD; break;
1726	case glslang::ElsRefLessFrontAMD: mode = spv::ExecutionModeStencilRefLessFrontAMD; break;
1727	case glslang::ElsRefUnchangedBackAMD: mode = spv::ExecutionModeStencilRefUnchangedBackAMD; break;
1728	case glslang::ElsRefGreaterBackAMD: mode = spv::ExecutionModeStencilRefGreaterBackAMD; break;
1729	case glslang::ElsRefLessBackAMD: mode = spv::ExecutionModeStencilRefLessBackAMD; break;
1730	default: mode = spv::ExecutionModeMax; break;
1731	}
1732
1733	if (mode != spv::ExecutionModeMax)
1734	builder.addExecutionMode(shaderEntry, mode: (spv::ExecutionMode)mode);
1735	switch (glslangIntermediate->getInterlockOrdering()) {
1736	case glslang::EioPixelInterlockOrdered: mode = spv::ExecutionModePixelInterlockOrderedEXT;
1737	break;
1738	case glslang::EioPixelInterlockUnordered: mode = spv::ExecutionModePixelInterlockUnorderedEXT;
1739	break;
1740	case glslang::EioSampleInterlockOrdered: mode = spv::ExecutionModeSampleInterlockOrderedEXT;
1741	break;
1742	case glslang::EioSampleInterlockUnordered: mode = spv::ExecutionModeSampleInterlockUnorderedEXT;
1743	break;
1744	case glslang::EioShadingRateInterlockOrdered: mode = spv::ExecutionModeShadingRateInterlockOrderedEXT;
1745	break;
1746	case glslang::EioShadingRateInterlockUnordered: mode = spv::ExecutionModeShadingRateInterlockUnorderedEXT;
1747	break;
1748	default: mode = spv::ExecutionModeMax;
1749	break;
1750	}
1751	if (mode != spv::ExecutionModeMax) {
1752	builder.addExecutionMode(shaderEntry, mode: (spv::ExecutionMode)mode);
1753	if (mode == spv::ExecutionModeShadingRateInterlockOrderedEXT ||
1754	mode == spv::ExecutionModeShadingRateInterlockUnorderedEXT) {
1755	builder.addCapability(cap: spv::CapabilityFragmentShaderShadingRateInterlockEXT);
1756	} else if (mode == spv::ExecutionModePixelInterlockOrderedEXT ||
1757	mode == spv::ExecutionModePixelInterlockUnorderedEXT) {
1758	builder.addCapability(cap: spv::CapabilityFragmentShaderPixelInterlockEXT);
1759	} else {
1760	builder.addCapability(cap: spv::CapabilityFragmentShaderSampleInterlockEXT);
1761	}
1762	builder.addExtension(ext: spv::E_SPV_EXT_fragment_shader_interlock);
1763	}
1764	break;
1765
1766	case EShLangCompute: {
1767	builder.addCapability(cap: spv::CapabilityShader);
1768	bool needSizeId = false;
1769	for (int dim = 0; dim < 3; ++dim) {
1770	if ((glslangIntermediate->getLocalSizeSpecId(dim) != glslang::TQualifier::layoutNotSet)) {
1771	needSizeId = true;
1772	break;
1773	}
1774	}
1775	if (glslangIntermediate->getSpv().spv >= glslang::EShTargetSpv_1_6 && needSizeId) {
1776	std::vector<spv::Id> dimConstId;
1777	for (int dim = 0; dim < 3; ++dim) {
1778	bool specConst = (glslangIntermediate->getLocalSizeSpecId(dim) != glslang::TQualifier::layoutNotSet);
1779	dimConstId.push_back(x: builder.makeUintConstant(u: glslangIntermediate->getLocalSize(dim), specConstant: specConst));
1780	if (specConst) {
1781	builder.addDecoration(dimConstId.back(), spv::DecorationSpecId,
1782	num: glslangIntermediate->getLocalSizeSpecId(dim));
1783	needSizeId = true;
1784	}
1785	}
1786	builder.addExecutionModeId(shaderEntry, mode: spv::ExecutionModeLocalSizeId, operandIds: dimConstId);
1787	} else {
1788	builder.addExecutionMode(shaderEntry, mode: spv::ExecutionModeLocalSize, value1: glslangIntermediate->getLocalSize(dim: 0),
1789	value2: glslangIntermediate->getLocalSize(dim: 1),
1790	value3: glslangIntermediate->getLocalSize(dim: 2));
1791	}
1792	if (glslangIntermediate->getLayoutDerivativeModeNone() == glslang::LayoutDerivativeGroupQuads) {
1793	builder.addCapability(cap: spv::CapabilityComputeDerivativeGroupQuadsNV);
1794	builder.addExecutionMode(shaderEntry, mode: spv::ExecutionModeDerivativeGroupQuadsNV);
1795	builder.addExtension(ext: spv::E_SPV_NV_compute_shader_derivatives);
1796	} else if (glslangIntermediate->getLayoutDerivativeModeNone() == glslang::LayoutDerivativeGroupLinear) {
1797	builder.addCapability(cap: spv::CapabilityComputeDerivativeGroupLinearNV);
1798	builder.addExecutionMode(shaderEntry, mode: spv::ExecutionModeDerivativeGroupLinearNV);
1799	builder.addExtension(ext: spv::E_SPV_NV_compute_shader_derivatives);
1800	}
1801	break;
1802	}
1803	case EShLangTessEvaluation:
1804	case EShLangTessControl:
1805	builder.addCapability(cap: spv::CapabilityTessellation);
1806
1807	glslang::TLayoutGeometry primitive;
1808
1809	if (glslangIntermediate->getStage() == EShLangTessControl) {
1810	builder.addExecutionMode(shaderEntry, mode: spv::ExecutionModeOutputVertices,
1811	value1: glslangIntermediate->getVertices());
1812	primitive = glslangIntermediate->getOutputPrimitive();
1813	} else {
1814	primitive = glslangIntermediate->getInputPrimitive();
1815	}
1816
1817	switch (primitive) {
1818	case glslang::ElgTriangles: mode = spv::ExecutionModeTriangles; break;
1819	case glslang::ElgQuads: mode = spv::ExecutionModeQuads; break;
1820	case glslang::ElgIsolines: mode = spv::ExecutionModeIsolines; break;
1821	default: mode = spv::ExecutionModeMax; break;
1822	}
1823	if (mode != spv::ExecutionModeMax)
1824	builder.addExecutionMode(shaderEntry, mode: (spv::ExecutionMode)mode);
1825
1826	switch (glslangIntermediate->getVertexSpacing()) {
1827	case glslang::EvsEqual: mode = spv::ExecutionModeSpacingEqual; break;
1828	case glslang::EvsFractionalEven: mode = spv::ExecutionModeSpacingFractionalEven; break;
1829	case glslang::EvsFractionalOdd: mode = spv::ExecutionModeSpacingFractionalOdd; break;
1830	default: mode = spv::ExecutionModeMax; break;
1831	}
1832	if (mode != spv::ExecutionModeMax)
1833	builder.addExecutionMode(shaderEntry, mode: (spv::ExecutionMode)mode);
1834
1835	switch (glslangIntermediate->getVertexOrder()) {
1836	case glslang::EvoCw: mode = spv::ExecutionModeVertexOrderCw; break;
1837	case glslang::EvoCcw: mode = spv::ExecutionModeVertexOrderCcw; break;
1838	default: mode = spv::ExecutionModeMax; break;
1839	}
1840	if (mode != spv::ExecutionModeMax)
1841	builder.addExecutionMode(shaderEntry, mode: (spv::ExecutionMode)mode);
1842
1843	if (glslangIntermediate->getPointMode())
1844	builder.addExecutionMode(shaderEntry, mode: spv::ExecutionModePointMode);
1845	break;
1846
1847	case EShLangGeometry:
1848	builder.addCapability(cap: spv::CapabilityGeometry);
1849	switch (glslangIntermediate->getInputPrimitive()) {
1850	case glslang::ElgPoints: mode = spv::ExecutionModeInputPoints; break;
1851	case glslang::ElgLines: mode = spv::ExecutionModeInputLines; break;
1852	case glslang::ElgLinesAdjacency: mode = spv::ExecutionModeInputLinesAdjacency; break;
1853	case glslang::ElgTriangles: mode = spv::ExecutionModeTriangles; break;
1854	case glslang::ElgTrianglesAdjacency: mode = spv::ExecutionModeInputTrianglesAdjacency; break;
1855	default: mode = spv::ExecutionModeMax; break;
1856	}
1857	if (mode != spv::ExecutionModeMax)
1858	builder.addExecutionMode(shaderEntry, mode: (spv::ExecutionMode)mode);
1859
1860	builder.addExecutionMode(shaderEntry, mode: spv::ExecutionModeInvocations, value1: glslangIntermediate->getInvocations());
1861
1862	switch (glslangIntermediate->getOutputPrimitive()) {
1863	case glslang::ElgPoints: mode = spv::ExecutionModeOutputPoints; break;
1864	case glslang::ElgLineStrip: mode = spv::ExecutionModeOutputLineStrip; break;
1865	case glslang::ElgTriangleStrip: mode = spv::ExecutionModeOutputTriangleStrip; break;
1866	default: mode = spv::ExecutionModeMax; break;
1867	}
1868	if (mode != spv::ExecutionModeMax)
1869	builder.addExecutionMode(shaderEntry, mode: (spv::ExecutionMode)mode);
1870	builder.addExecutionMode(shaderEntry, mode: spv::ExecutionModeOutputVertices, value1: glslangIntermediate->getVertices());
1871	break;
1872
1873	case EShLangRayGen:
1874	case EShLangIntersect:
1875	case EShLangAnyHit:
1876	case EShLangClosestHit:
1877	case EShLangMiss:
1878	case EShLangCallable:
1879	{
1880	auto& extensions = glslangIntermediate->getRequestedExtensions();
1881	if (extensions.find(x: "GL_NV_ray_tracing") == extensions.end()) {
1882	builder.addCapability(cap: spv::CapabilityRayTracingKHR);
1883	builder.addExtension(ext: "SPV_KHR_ray_tracing");
1884	}
1885	else {
1886	builder.addCapability(cap: spv::CapabilityRayTracingNV);
1887	builder.addExtension(ext: "SPV_NV_ray_tracing");
1888	}
1889	if (glslangIntermediate->getStage() != EShLangRayGen && glslangIntermediate->getStage() != EShLangCallable) {
1890	if (extensions.find(x: "GL_EXT_ray_cull_mask") != extensions.end()) {
1891	builder.addCapability(cap: spv::CapabilityRayCullMaskKHR);
1892	builder.addExtension(ext: "SPV_KHR_ray_cull_mask");
1893	}
1894	if (extensions.find(x: "GL_EXT_ray_tracing_position_fetch") != extensions.end()) {
1895	builder.addCapability(cap: spv::CapabilityRayTracingPositionFetchKHR);
1896	builder.addExtension(ext: "SPV_KHR_ray_tracing_position_fetch");
1897	}
1898	}
1899	break;
1900	}
1901	case EShLangTask:
1902	case EShLangMesh:
1903	if(isMeshShaderExt) {
1904	builder.addCapability(cap: spv::CapabilityMeshShadingEXT);
1905	builder.addExtension(ext: spv::E_SPV_EXT_mesh_shader);
1906	} else {
1907	builder.addCapability(cap: spv::CapabilityMeshShadingNV);
1908	builder.addExtension(ext: spv::E_SPV_NV_mesh_shader);
1909	}
1910	if (glslangIntermediate->getSpv().spv >= glslang::EShTargetSpv_1_6) {
1911	std::vector<spv::Id> dimConstId;
1912	for (int dim = 0; dim < 3; ++dim) {
1913	bool specConst = (glslangIntermediate->getLocalSizeSpecId(dim) != glslang::TQualifier::layoutNotSet);
1914	dimConstId.push_back(x: builder.makeUintConstant(u: glslangIntermediate->getLocalSize(dim), specConstant: specConst));
1915	if (specConst) {
1916	builder.addDecoration(dimConstId.back(), spv::DecorationSpecId,
1917	num: glslangIntermediate->getLocalSizeSpecId(dim));
1918	}
1919	}
1920	builder.addExecutionModeId(shaderEntry, mode: spv::ExecutionModeLocalSizeId, operandIds: dimConstId);
1921	} else {
1922	builder.addExecutionMode(shaderEntry, mode: spv::ExecutionModeLocalSize, value1: glslangIntermediate->getLocalSize(dim: 0),
1923	value2: glslangIntermediate->getLocalSize(dim: 1),
1924	value3: glslangIntermediate->getLocalSize(dim: 2));
1925	}
1926	if (glslangIntermediate->getStage() == EShLangMesh) {
1927	builder.addExecutionMode(shaderEntry, mode: spv::ExecutionModeOutputVertices,
1928	value1: glslangIntermediate->getVertices());
1929	builder.addExecutionMode(shaderEntry, mode: spv::ExecutionModeOutputPrimitivesNV,
1930	value1: glslangIntermediate->getPrimitives());
1931
1932	switch (glslangIntermediate->getOutputPrimitive()) {
1933	case glslang::ElgPoints: mode = spv::ExecutionModeOutputPoints; break;
1934	case glslang::ElgLines: mode = spv::ExecutionModeOutputLinesNV; break;
1935	case glslang::ElgTriangles: mode = spv::ExecutionModeOutputTrianglesNV; break;
1936	default: mode = spv::ExecutionModeMax; break;
1937	}
1938	if (mode != spv::ExecutionModeMax)
1939	builder.addExecutionMode(shaderEntry, mode: (spv::ExecutionMode)mode);
1940	}
1941	break;
1942
1943	default:
1944	break;
1945	}
1946
1947	//
1948	// Add SPIR-V requirements (GL_EXT_spirv_intrinsics)
1949	//
1950	if (glslangIntermediate->hasSpirvRequirement()) {
1951	const glslang::TSpirvRequirement& spirvRequirement = glslangIntermediate->getSpirvRequirement();
1952
1953	// Add SPIR-V extension requirement
1954	for (auto& extension : spirvRequirement.extensions)
1955	builder.addExtension(ext: extension.c_str());
1956
1957	// Add SPIR-V capability requirement
1958	for (auto capability : spirvRequirement.capabilities)
1959	builder.addCapability(cap: static_cast<spv::Capability>(capability));
1960	}
1961
1962	//
1963	// Add SPIR-V execution mode qualifiers (GL_EXT_spirv_intrinsics)
1964	//
1965	if (glslangIntermediate->hasSpirvExecutionMode()) {
1966	const glslang::TSpirvExecutionMode spirvExecutionMode = glslangIntermediate->getSpirvExecutionMode();
1967
1968	// Add spirv_execution_mode
1969	for (auto& mode : spirvExecutionMode.modes) {
1970	if (!mode.second.empty()) {
1971	std::vector<unsigned> literals;
1972	TranslateLiterals(constants: mode.second, literals);
1973	builder.addExecutionMode(shaderEntry, mode: static_cast<spv::ExecutionMode>(mode.first), literals);
1974	} else
1975	builder.addExecutionMode(shaderEntry, mode: static_cast<spv::ExecutionMode>(mode.first));
1976	}
1977
1978	// Add spirv_execution_mode_id
1979	for (auto& modeId : spirvExecutionMode.modeIds) {
1980	std::vector<spv::Id> operandIds;
1981	assert(!modeId.second.empty());
1982	for (auto extraOperand : modeId.second) {
1983	if (extraOperand->getType().getQualifier().isSpecConstant())
1984	operandIds.push_back(x: getSymbolId(node: extraOperand->getAsSymbolNode()));
1985	else
1986	operandIds.push_back(x: createSpvConstant(*extraOperand));
1987	}
1988	builder.addExecutionModeId(shaderEntry, mode: static_cast<spv::ExecutionMode>(modeId.first), operandIds);
1989	}
1990	}
1991	}
1992
1993	// Finish creating SPV, after the traversal is complete.
1994	void TGlslangToSpvTraverser::finishSpv(bool compileOnly)
1995	{
1996	// If not linking, an entry point is not expected
1997	if (!compileOnly) {
1998	// Finish the entry point function
1999	if (!entryPointTerminated) {
2000	builder.setBuildPoint(shaderEntry->getLastBlock());
2001	builder.leaveFunction();
2002	}
2003
2004	// finish off the entry-point SPV instruction by adding the Input/Output <id>
2005	entryPoint->reserveOperands(count: iOSet.size());
2006	for (auto id : iOSet)
2007	entryPoint->addIdOperand(id);
2008	}
2009
2010	// Add capabilities, extensions, remove unneeded decorations, etc.,
2011	// based on the resulting SPIR-V.
2012	// Note: WebGPU code generation must have the opportunity to aggressively
2013	// prune unreachable merge blocks and continue targets.
2014	builder.postProcess(compileOnly);
2015	}
2016
2017	// Write the SPV into 'out'.
2018	void TGlslangToSpvTraverser::dumpSpv(std::vector<unsigned int>& out)
2019	{
2020	builder.dump(out);
2021	}
2022
2023	//
2024	// Implement the traversal functions.
2025	//
2026	// Return true from interior nodes to have the external traversal
2027	// continue on to children. Return false if children were
2028	// already processed.
2029	//
2030
2031	//
2032	// Symbols can turn into
2033	// - uniform/input reads
2034	// - output writes
2035	// - complex lvalue base setups: foo.bar[3].... , where we see foo and start up an access chain
2036	// - something simple that degenerates into the last bullet
2037	//
2038	void TGlslangToSpvTraverser::visitSymbol(glslang::TIntermSymbol* symbol)
2039	{
2040	// We update the line information even though no code might be generated here
2041	// This is helpful to yield correct lines for control flow instructions
2042	if (!linkageOnly) {
2043	builder.setDebugSourceLocation(line: symbol->getLoc().line, filename: symbol->getLoc().getFilename());
2044	}
2045
2046	if (symbol->getBasicType() == glslang::EbtFunction) {
2047	return;
2048	}
2049
2050	SpecConstantOpModeGuard spec_constant_op_mode_setter(&builder);
2051	if (symbol->getType().isStruct())
2052	glslangTypeToIdMap[symbol->getType().getStruct()] = symbol->getId();
2053
2054	if (symbol->getType().getQualifier().isSpecConstant())
2055	spec_constant_op_mode_setter.turnOnSpecConstantOpMode();
2056	#ifdef ENABLE_HLSL
2057	// Skip symbol handling if it is string-typed
2058	if (symbol->getBasicType() == glslang::EbtString)
2059	return;
2060	#endif
2061
2062	// getSymbolId() will set up all the IO decorations on the first call.
2063	// Formal function parameters were mapped during makeFunctions().
2064	spv::Id id = getSymbolId(node: symbol);
2065
2066	if (symbol->getType().getQualifier().isTaskPayload())
2067	taskPayloadID = id; // cache the taskPayloadID to be used it as operand for OpEmitMeshTasksEXT
2068
2069	if (builder.isPointer(resultId: id)) {
2070	if (!symbol->getType().getQualifier().isParamInput() &&
2071	!symbol->getType().getQualifier().isParamOutput()) {
2072	// Include all "static use" and "linkage only" interface variables on the OpEntryPoint instruction
2073	// Consider adding to the OpEntryPoint interface list.
2074	// Only looking at structures if they have at least one member.
2075	if (!symbol->getType().isStruct() || symbol->getType().getStruct()->size() > 0) {
2076	spv::StorageClass sc = builder.getStorageClass(resultId: id);
2077	// Before SPIR-V 1.4, we only want to include Input and Output.
2078	// Starting with SPIR-V 1.4, we want all globals.
2079	if ((glslangIntermediate->getSpv().spv >= glslang::EShTargetSpv_1_4 && builder.isGlobalVariable(resultId: id)) ||
2080	(sc == spv::StorageClassInput || sc == spv::StorageClassOutput)) {
2081	iOSet.insert(x: id);
2082	}
2083	}
2084	}
2085
2086	// If the SPIR-V type is required to be different than the AST type
2087	// (for ex SubgroupMasks or 3x4 ObjectToWorld/WorldToObject matrices),
2088	// translate now from the SPIR-V type to the AST type, for the consuming
2089	// operation.
2090	// Note this turns it from an l-value to an r-value.
2091	// Currently, all symbols needing this are inputs; avoid the map lookup when non-input.
2092	if (symbol->getType().getQualifier().storage == glslang::EvqVaryingIn)
2093	id = translateForcedType(object: id);
2094	}
2095
2096	// Only process non-linkage-only nodes for generating actual static uses
2097	if (! linkageOnly || symbol->getQualifier().isSpecConstant()) {
2098	// Prepare to generate code for the access
2099
2100	// L-value chains will be computed left to right. We're on the symbol now,
2101	// which is the left-most part of the access chain, so now is "clear" time,
2102	// followed by setting the base.
2103	builder.clearAccessChain();
2104
2105	// For now, we consider all user variables as being in memory, so they are pointers,
2106	// except for
2107	// A) R-Value arguments to a function, which are an intermediate object.
2108	// See comments in handleUserFunctionCall().
2109	// B) Specialization constants (normal constants don't even come in as a variable),
2110	// These are also pure R-values.
2111	// C) R-Values from type translation, see above call to translateForcedType()
2112	glslang::TQualifier qualifier = symbol->getQualifier();
2113	if (qualifier.isSpecConstant() || rValueParameters.find(x: symbol->getId()) != rValueParameters.end() ||
2114	!builder.isPointerType(typeId: builder.getTypeId(resultId: id)))
2115	builder.setAccessChainRValue(id);
2116	else
2117	builder.setAccessChainLValue(id);
2118	}
2119
2120	#ifdef ENABLE_HLSL
2121	// Process linkage-only nodes for any special additional interface work.
2122	if (linkageOnly) {
2123	if (glslangIntermediate->getHlslFunctionality1()) {
2124	// Map implicit counter buffers to their originating buffers, which should have been
2125	// seen by now, given earlier pruning of unused counters, and preservation of order
2126	// of declaration.
2127	if (symbol->getType().getQualifier().isUniformOrBuffer()) {
2128	if (!glslangIntermediate->hasCounterBufferName(symbol->getName())) {
2129	// Save possible originating buffers for counter buffers, keyed by
2130	// making the potential counter-buffer name.
2131	std::string keyName = symbol->getName().c_str();
2132	keyName = glslangIntermediate->addCounterBufferName(keyName);
2133	counterOriginator[keyName] = symbol;
2134	} else {
2135	// Handle a counter buffer, by finding the saved originating buffer.
2136	std::string keyName = symbol->getName().c_str();
2137	auto it = counterOriginator.find(keyName);
2138	if (it != counterOriginator.end()) {
2139	id = getSymbolId(it->second);
2140	if (id != spv::NoResult) {
2141	spv::Id counterId = getSymbolId(symbol);
2142	if (counterId != spv::NoResult) {
2143	builder.addExtension("SPV_GOOGLE_hlsl_functionality1");
2144	builder.addDecorationId(id, spv::DecorationHlslCounterBufferGOOGLE, counterId);
2145	}
2146	}
2147	}
2148	}
2149	}
2150	}
2151	}
2152	#endif
2153	}
2154
2155	bool TGlslangToSpvTraverser::visitBinary(glslang::TVisit /* visit */, glslang::TIntermBinary* node)
2156	{
2157	builder.setDebugSourceLocation(line: node->getLoc().line, filename: node->getLoc().getFilename());
2158	if (node->getLeft()->getAsSymbolNode() != nullptr && node->getLeft()->getType().isStruct()) {
2159	glslangTypeToIdMap[node->getLeft()->getType().getStruct()] = node->getLeft()->getAsSymbolNode()->getId();
2160	}
2161	if (node->getRight()->getAsSymbolNode() != nullptr && node->getRight()->getType().isStruct()) {
2162	glslangTypeToIdMap[node->getRight()->getType().getStruct()] = node->getRight()->getAsSymbolNode()->getId();
2163	}
2164
2165	SpecConstantOpModeGuard spec_constant_op_mode_setter(&builder);
2166	if (node->getType().getQualifier().isSpecConstant())
2167	spec_constant_op_mode_setter.turnOnSpecConstantOpMode();
2168
2169	// First, handle special cases
2170	switch (node->getOp()) {
2171	case glslang::EOpAssign:
2172	case glslang::EOpAddAssign:
2173	case glslang::EOpSubAssign:
2174	case glslang::EOpMulAssign:
2175	case glslang::EOpVectorTimesMatrixAssign:
2176	case glslang::EOpVectorTimesScalarAssign:
2177	case glslang::EOpMatrixTimesScalarAssign:
2178	case glslang::EOpMatrixTimesMatrixAssign:
2179	case glslang::EOpDivAssign:
2180	case glslang::EOpModAssign:
2181	case glslang::EOpAndAssign:
2182	case glslang::EOpInclusiveOrAssign:
2183	case glslang::EOpExclusiveOrAssign:
2184	case glslang::EOpLeftShiftAssign:
2185	case glslang::EOpRightShiftAssign:
2186	// A bin-op assign "a += b" means the same thing as "a = a + b"
2187	// where a is evaluated before b. For a simple assignment, GLSL
2188	// says to evaluate the left before the right. So, always, left
2189	// node then right node.
2190	{
2191	// get the left l-value, save it away
2192	builder.clearAccessChain();
2193	node->getLeft()->traverse(this);
2194	spv::Builder::AccessChain lValue = builder.getAccessChain();
2195
2196	// evaluate the right
2197	builder.clearAccessChain();
2198	node->getRight()->traverse(this);
2199	spv::Id rValue = accessChainLoad(type: node->getRight()->getType());
2200
2201	// reset line number for assignment
2202	builder.setDebugSourceLocation(line: node->getLoc().line, filename: node->getLoc().getFilename());
2203
2204	if (node->getOp() != glslang::EOpAssign) {
2205	// the left is also an r-value
2206	builder.setAccessChain(lValue);
2207	spv::Id leftRValue = accessChainLoad(type: node->getLeft()->getType());
2208
2209	// do the operation
2210	spv::Builder::AccessChain::CoherentFlags coherentFlags = TranslateCoherent(type: node->getLeft()->getType());
2211	coherentFlags |= TranslateCoherent(type: node->getRight()->getType());
2212	OpDecorations decorations = { TranslatePrecisionDecoration(glslangPrecision: node->getOperationPrecision()),
2213	TranslateNoContractionDecoration(qualifier: node->getType().getQualifier()),
2214	TranslateNonUniformDecoration(coherentFlags) };
2215	rValue = createBinaryOperation(op: node->getOp(), decorations,
2216	typeId: convertGlslangToSpvType(type: node->getType()), left: leftRValue, right: rValue,
2217	typeProxy: node->getType().getBasicType());
2218
2219	// these all need their counterparts in createBinaryOperation()
2220	assert(rValue != spv::NoResult);
2221	}
2222
2223	// store the result
2224	builder.setAccessChain(lValue);
2225	multiTypeStore(node->getLeft()->getType(), rValue);
2226
2227	// assignments are expressions having an rValue after they are evaluated...
2228	builder.clearAccessChain();
2229	builder.setAccessChainRValue(rValue);
2230	}
2231	return false;
2232	case glslang::EOpIndexDirect:
2233	case glslang::EOpIndexDirectStruct:
2234	{
2235	// Structure, array, matrix, or vector indirection with statically known index.
2236	// Get the left part of the access chain.
2237	node->getLeft()->traverse(this);
2238
2239	// Add the next element in the chain
2240
2241	const int glslangIndex = node->getRight()->getAsConstantUnion()->getConstArray()[0].getIConst();
2242	if (! node->getLeft()->getType().isArray() &&
2243	node->getLeft()->getType().isVector() &&
2244	node->getOp() == glslang::EOpIndexDirect) {
2245	// Swizzle is uniform so propagate uniform into access chain
2246	spv::Builder::AccessChain::CoherentFlags coherentFlags = TranslateCoherent(type: node->getLeft()->getType());
2247	coherentFlags.nonUniform = 0;
2248	// This is essentially a hard-coded vector swizzle of size 1,
2249	// so short circuit the access-chain stuff with a swizzle.
2250	std::vector<unsigned> swizzle;
2251	swizzle.push_back(x: glslangIndex);
2252	int dummySize;
2253	builder.accessChainPushSwizzle(swizzle, preSwizzleBaseType: convertGlslangToSpvType(type: node->getLeft()->getType()),
2254	coherentFlags,
2255	alignment: glslangIntermediate->getBaseAlignmentScalar(
2256	node->getLeft()->getType(), size&: dummySize));
2257	} else {
2258
2259	// Load through a block reference is performed with a dot operator that
2260	// is mapped to EOpIndexDirectStruct. When we get to the actual reference,
2261	// do a load and reset the access chain.
2262	if (node->getLeft()->isReference() &&
2263	!node->getLeft()->getType().isArray() &&
2264	node->getOp() == glslang::EOpIndexDirectStruct)
2265	{
2266	spv::Id left = accessChainLoad(type: node->getLeft()->getType());
2267	builder.clearAccessChain();
2268	builder.setAccessChainLValue(left);
2269	}
2270
2271	int spvIndex = glslangIndex;
2272	if (node->getLeft()->getBasicType() == glslang::EbtBlock &&
2273	node->getOp() == glslang::EOpIndexDirectStruct)
2274	{
2275	// This may be, e.g., an anonymous block-member selection, which generally need
2276	// index remapping due to hidden members in anonymous blocks.
2277	long long glslangId = glslangTypeToIdMap[node->getLeft()->getType().getStruct()];
2278	if (memberRemapper.find(x: glslangId) != memberRemapper.end()) {
2279	std::vector<int>& remapper = memberRemapper[glslangId];
2280	assert(remapper.size() > 0);
2281	spvIndex = remapper[glslangIndex];
2282	}
2283	}
2284
2285	// Struct reference propagates uniform lvalue
2286	spv::Builder::AccessChain::CoherentFlags coherentFlags =
2287	TranslateCoherent(type: node->getLeft()->getType());
2288	coherentFlags.nonUniform = 0;
2289
2290	// normal case for indexing array or structure or block
2291	builder.accessChainPush(offset: builder.makeIntConstant(i: spvIndex),
2292	coherentFlags,
2293	alignment: node->getLeft()->getType().getBufferReferenceAlignment());
2294
2295	// Add capabilities here for accessing PointSize and clip/cull distance.
2296	// We have deferred generation of associated capabilities until now.
2297	if (node->getLeft()->getType().isStruct() && ! node->getLeft()->getType().isArray())
2298	declareUseOfStructMember(members: *(node->getLeft()->getType().getStruct()), glslangMember: glslangIndex);
2299	}
2300	}
2301	return false;
2302	case glslang::EOpIndexIndirect:
2303	{
2304	// Array, matrix, or vector indirection with variable index.
2305	// Will use native SPIR-V access-chain for and array indirection;
2306	// matrices are arrays of vectors, so will also work for a matrix.
2307	// Will use the access chain's 'component' for variable index into a vector.
2308
2309	// This adapter is building access chains left to right.
2310	// Set up the access chain to the left.
2311	node->getLeft()->traverse(this);
2312
2313	// save it so that computing the right side doesn't trash it
2314	spv::Builder::AccessChain partial = builder.getAccessChain();
2315
2316	// compute the next index in the chain
2317	builder.clearAccessChain();
2318	node->getRight()->traverse(this);
2319	spv::Id index = accessChainLoad(type: node->getRight()->getType());
2320
2321	addIndirectionIndexCapabilities(baseType: node->getLeft()->getType(), indexType: node->getRight()->getType());
2322
2323	// restore the saved access chain
2324	builder.setAccessChain(partial);
2325
2326	// Only if index is nonUniform should we propagate nonUniform into access chain
2327	spv::Builder::AccessChain::CoherentFlags index_flags = TranslateCoherent(type: node->getRight()->getType());
2328	spv::Builder::AccessChain::CoherentFlags coherent_flags = TranslateCoherent(type: node->getLeft()->getType());
2329	coherent_flags.nonUniform = index_flags.nonUniform;
2330
2331	if (! node->getLeft()->getType().isArray() && node->getLeft()->getType().isVector()) {
2332	int dummySize;
2333	builder.accessChainPushComponent(
2334	component: index, preSwizzleBaseType: convertGlslangToSpvType(type: node->getLeft()->getType()), coherentFlags: coherent_flags,
2335	alignment: glslangIntermediate->getBaseAlignmentScalar(node->getLeft()->getType(),
2336	size&: dummySize));
2337	} else
2338	builder.accessChainPush(offset: index, coherentFlags: coherent_flags,
2339	alignment: node->getLeft()->getType().getBufferReferenceAlignment());
2340	}
2341	return false;
2342	case glslang::EOpVectorSwizzle:
2343	{
2344	node->getLeft()->traverse(this);
2345	std::vector<unsigned> swizzle;
2346	convertSwizzle(*node->getRight()->getAsAggregate(), swizzle);
2347	int dummySize;
2348	builder.accessChainPushSwizzle(swizzle, preSwizzleBaseType: convertGlslangToSpvType(type: node->getLeft()->getType()),
2349	coherentFlags: TranslateCoherent(type: node->getLeft()->getType()),
2350	alignment: glslangIntermediate->getBaseAlignmentScalar(node->getLeft()->getType(),
2351	size&: dummySize));
2352	}
2353	return false;
2354	case glslang::EOpMatrixSwizzle:
2355	logger->missingFunctionality(f: "matrix swizzle");
2356	return true;
2357	case glslang::EOpLogicalOr:
2358	case glslang::EOpLogicalAnd:
2359	{
2360
2361	// These may require short circuiting, but can sometimes be done as straight
2362	// binary operations. The right operand must be short circuited if it has
2363	// side effects, and should probably be if it is complex.
2364	if (isTrivial(node: node->getRight()->getAsTyped()))
2365	break; // handle below as a normal binary operation
2366	// otherwise, we need to do dynamic short circuiting on the right operand
2367	spv::Id result = createShortCircuit(node->getOp(), left&: *node->getLeft()->getAsTyped(),
2368	right&: *node->getRight()->getAsTyped());
2369	builder.clearAccessChain();
2370	builder.setAccessChainRValue(result);
2371	}
2372	return false;
2373	default:
2374	break;
2375	}
2376
2377	// Assume generic binary op...
2378
2379	// get right operand
2380	builder.clearAccessChain();
2381	node->getLeft()->traverse(this);
2382	spv::Id left = accessChainLoad(type: node->getLeft()->getType());
2383
2384	// get left operand
2385	builder.clearAccessChain();
2386	node->getRight()->traverse(this);
2387	spv::Id right = accessChainLoad(type: node->getRight()->getType());
2388
2389	// get result
2390	OpDecorations decorations = { TranslatePrecisionDecoration(glslangPrecision: node->getOperationPrecision()),
2391	TranslateNoContractionDecoration(qualifier: node->getType().getQualifier()),
2392	TranslateNonUniformDecoration(qualifier: node->getType().getQualifier()) };
2393	spv::Id result = createBinaryOperation(op: node->getOp(), decorations,
2394	typeId: convertGlslangToSpvType(type: node->getType()), left, right,
2395	typeProxy: node->getLeft()->getType().getBasicType());
2396
2397	builder.clearAccessChain();
2398	if (! result) {
2399	logger->missingFunctionality(f: "unknown glslang binary operation");
2400	return true; // pick up a child as the place-holder result
2401	} else {
2402	builder.setAccessChainRValue(result);
2403	return false;
2404	}
2405	}
2406
2407	spv::Id TGlslangToSpvTraverser::convertLoadedBoolInUniformToUint(const glslang::TType& type,
2408	spv::Id nominalTypeId,
2409	spv::Id loadedId)
2410	{
2411	if (builder.isScalarType(typeId: nominalTypeId)) {
2412	// Conversion for bool
2413	spv::Id boolType = builder.makeBoolType();
2414	if (nominalTypeId != boolType)
2415	return builder.createBinOp(spv::OpINotEqual, typeId: boolType, operand1: loadedId, operand2: builder.makeUintConstant(u: 0));
2416	} else if (builder.isVectorType(typeId: nominalTypeId)) {
2417	// Conversion for bvec
2418	int vecSize = builder.getNumTypeComponents(typeId: nominalTypeId);
2419	spv::Id bvecType = builder.makeVectorType(component: builder.makeBoolType(), size: vecSize);
2420	if (nominalTypeId != bvecType)
2421	loadedId = builder.createBinOp(spv::OpINotEqual, typeId: bvecType, operand1: loadedId,
2422	operand2: makeSmearedConstant(constant: builder.makeUintConstant(u: 0), vectorSize: vecSize));
2423	} else if (builder.isArrayType(typeId: nominalTypeId)) {
2424	// Conversion for bool array
2425	spv::Id boolArrayTypeId = convertGlslangToSpvType(type);
2426	if (nominalTypeId != boolArrayTypeId)
2427	{
2428	// Use OpCopyLogical from SPIR-V 1.4 if available.
2429	if (glslangIntermediate->getSpv().spv >= glslang::EShTargetSpv_1_4)
2430	return builder.createUnaryOp(spv::OpCopyLogical, typeId: boolArrayTypeId, operand: loadedId);
2431
2432	glslang::TType glslangElementType(type, 0);
2433	spv::Id elementNominalTypeId = builder.getContainedTypeId(typeId: nominalTypeId);
2434	std::vector<spv::Id> constituents;
2435	for (int index = 0; index < type.getOuterArraySize(); ++index) {
2436	// get the element
2437	spv::Id elementValue = builder.createCompositeExtract(composite: loadedId, typeId: elementNominalTypeId, index);
2438
2439	// recursively convert it
2440	spv::Id elementConvertedValue = convertLoadedBoolInUniformToUint(type: glslangElementType, nominalTypeId: elementNominalTypeId, loadedId: elementValue);
2441	constituents.push_back(x: elementConvertedValue);
2442	}
2443	return builder.createCompositeConstruct(typeId: boolArrayTypeId, constituents);
2444	}
2445	}
2446
2447	return loadedId;
2448	}
2449
2450	// Figure out what, if any, type changes are needed when accessing a specific built-in.
2451	// Returns <the type SPIR-V requires for declarion, the type to translate to on use>.
2452	// Also see comment for 'forceType', regarding tracking SPIR-V-required types.
2453	std::pair<spv::Id, spv::Id> TGlslangToSpvTraverser::getForcedType(glslang::TBuiltInVariable glslangBuiltIn,
2454	const glslang::TType& glslangType)
2455	{
2456	switch(glslangBuiltIn)
2457	{
2458	case glslang::EbvSubGroupEqMask:
2459	case glslang::EbvSubGroupGeMask:
2460	case glslang::EbvSubGroupGtMask:
2461	case glslang::EbvSubGroupLeMask:
2462	case glslang::EbvSubGroupLtMask: {
2463	// these require changing a 64-bit scaler -> a vector of 32-bit components
2464	if (glslangType.isVector())
2465	break;
2466	spv::Id ivec4_type = builder.makeVectorType(component: builder.makeUintType(width: 32), size: 4);
2467	spv::Id uint64_type = builder.makeUintType(width: 64);
2468	std::pair<spv::Id, spv::Id> ret(ivec4_type, uint64_type);
2469	return ret;
2470	}
2471	// There are no SPIR-V builtins defined for these and map onto original non-transposed
2472	// builtins. During visitBinary we insert a transpose
2473	case glslang::EbvWorldToObject3x4:
2474	case glslang::EbvObjectToWorld3x4: {
2475	spv::Id mat43 = builder.makeMatrixType(component: builder.makeFloatType(width: 32), cols: 4, rows: 3);
2476	spv::Id mat34 = builder.makeMatrixType(component: builder.makeFloatType(width: 32), cols: 3, rows: 4);
2477	std::pair<spv::Id, spv::Id> ret(mat43, mat34);
2478	return ret;
2479	}
2480	default:
2481	break;
2482	}
2483
2484	std::pair<spv::Id, spv::Id> ret(spv::NoType, spv::NoType);
2485	return ret;
2486	}
2487
2488	// For an object previously identified (see getForcedType() and forceType)
2489	// as needing type translations, do the translation needed for a load, turning
2490	// an L-value into in R-value.
2491	spv::Id TGlslangToSpvTraverser::translateForcedType(spv::Id object)
2492	{
2493	const auto forceIt = forceType.find(x: object);
2494	if (forceIt == forceType.end())
2495	return object;
2496
2497	spv::Id desiredTypeId = forceIt->second;
2498	spv::Id objectTypeId = builder.getTypeId(resultId: object);
2499	assert(builder.isPointerType(objectTypeId));
2500	objectTypeId = builder.getContainedTypeId(typeId: objectTypeId);
2501	if (builder.isVectorType(typeId: objectTypeId) &&
2502	builder.getScalarTypeWidth(typeId: builder.getContainedTypeId(typeId: objectTypeId)) == 32) {
2503	if (builder.getScalarTypeWidth(typeId: desiredTypeId) == 64) {
2504	// handle 32-bit v.xy* -> 64-bit
2505	builder.clearAccessChain();
2506	builder.setAccessChainLValue(object);
2507	object = builder.accessChainLoad(precision: spv::NoPrecision, l_nonUniform: spv::DecorationMax, r_nonUniform: spv::DecorationMax, ResultType: objectTypeId);
2508	std::vector<spv::Id> components;
2509	components.push_back(x: builder.createCompositeExtract(composite: object, typeId: builder.getContainedTypeId(typeId: objectTypeId), index: 0));
2510	components.push_back(x: builder.createCompositeExtract(composite: object, typeId: builder.getContainedTypeId(typeId: objectTypeId), index: 1));
2511
2512	spv::Id vecType = builder.makeVectorType(component: builder.getContainedTypeId(typeId: objectTypeId), size: 2);
2513	return builder.createUnaryOp(spv::OpBitcast, typeId: desiredTypeId,
2514	operand: builder.createCompositeConstruct(typeId: vecType, constituents: components));
2515	} else {
2516	logger->missingFunctionality(f: "forcing 32-bit vector type to non 64-bit scalar");
2517	}
2518	} else if (builder.isMatrixType(typeId: objectTypeId)) {
2519	// There are no SPIR-V builtins defined for 3x4 variants of ObjectToWorld/WorldToObject
2520	// and we insert a transpose after loading the original non-transposed builtins
2521	builder.clearAccessChain();
2522	builder.setAccessChainLValue(object);
2523	object = builder.accessChainLoad(precision: spv::NoPrecision, l_nonUniform: spv::DecorationMax, r_nonUniform: spv::DecorationMax, ResultType: objectTypeId);
2524	return builder.createUnaryOp(spv::OpTranspose, typeId: desiredTypeId, operand: object);
2525
2526	} else {
2527	logger->missingFunctionality(f: "forcing non 32-bit vector type");
2528	}
2529
2530	return object;
2531	}
2532
2533	bool TGlslangToSpvTraverser::visitUnary(glslang::TVisit /* visit */, glslang::TIntermUnary* node)
2534	{
2535	builder.setDebugSourceLocation(line: node->getLoc().line, filename: node->getLoc().getFilename());
2536
2537	SpecConstantOpModeGuard spec_constant_op_mode_setter(&builder);
2538	if (node->getType().getQualifier().isSpecConstant())
2539	spec_constant_op_mode_setter.turnOnSpecConstantOpMode();
2540
2541	spv::Id result = spv::NoResult;
2542
2543	// try texturing first
2544	result = createImageTextureFunctionCall(node);
2545	if (result != spv::NoResult) {
2546	builder.clearAccessChain();
2547	builder.setAccessChainRValue(result);
2548
2549	return false; // done with this node
2550	}
2551
2552	// Non-texturing.
2553
2554	if (node->getOp() == glslang::EOpArrayLength) {
2555	// Quite special; won't want to evaluate the operand.
2556
2557	// Currently, the front-end does not allow .length() on an array until it is sized,
2558	// except for the last block membeor of an SSBO.
2559	// TODO: If this changes, link-time sized arrays might show up here, and need their
2560	// size extracted.
2561
2562	// Normal .length() would have been constant folded by the front-end.
2563	// So, this has to be block.lastMember.length().
2564	// SPV wants "block" and member number as the operands, go get them.
2565
2566	spv::Id length;
2567	if (node->getOperand()->getType().isCoopMat()) {
2568	spv::Id typeId = convertGlslangToSpvType(type: node->getOperand()->getType());
2569	assert(builder.isCooperativeMatrixType(typeId));
2570
2571	if (node->getOperand()->getType().isCoopMatKHR()) {
2572	length = builder.createCooperativeMatrixLengthKHR(type: typeId);
2573	} else {
2574	spec_constant_op_mode_setter.turnOnSpecConstantOpMode();
2575	length = builder.createCooperativeMatrixLengthNV(type: typeId);
2576	}
2577	} else {
2578	glslang::TIntermTyped* block = node->getOperand()->getAsBinaryNode()->getLeft();
2579	block->traverse(this);
2580	unsigned int member = node->getOperand()->getAsBinaryNode()->getRight()->getAsConstantUnion()
2581	->getConstArray()[0].getUConst();
2582	length = builder.createArrayLength(base: builder.accessChainGetLValue(), member);
2583	}
2584
2585	// GLSL semantics say the result of .length() is an int, while SPIR-V says
2586	// signedness must be 0. So, convert from SPIR-V unsigned back to GLSL's
2587	// AST expectation of a signed result.
2588	if (glslangIntermediate->getSource() == glslang::EShSourceGlsl) {
2589	if (builder.isInSpecConstCodeGenMode()) {
2590	length = builder.createBinOp(spv::OpIAdd, typeId: builder.makeIntType(width: 32), operand1: length, operand2: builder.makeIntConstant(i: 0));
2591	} else {
2592	length = builder.createUnaryOp(spv::OpBitcast, typeId: builder.makeIntType(width: 32), operand: length);
2593	}
2594	}
2595
2596	builder.clearAccessChain();
2597	builder.setAccessChainRValue(length);
2598
2599	return false;
2600	}
2601
2602	// Force variable declaration - Debug Mode Only
2603	if (node->getOp() == glslang::EOpDeclare) {
2604	builder.clearAccessChain();
2605	node->getOperand()->traverse(this);
2606	builder.clearAccessChain();
2607	return false;
2608	}
2609
2610	// Start by evaluating the operand
2611
2612	// Does it need a swizzle inversion? If so, evaluation is inverted;
2613	// operate first on the swizzle base, then apply the swizzle.
2614	spv::Id invertedType = spv::NoType;
2615	auto resultType = [&invertedType, &node, this](){ return invertedType != spv::NoType ?
2616	invertedType : convertGlslangToSpvType(type: node->getType()); };
2617	if (node->getOp() == glslang::EOpInterpolateAtCentroid)
2618	invertedType = getInvertedSwizzleType(*node->getOperand());
2619
2620	builder.clearAccessChain();
2621	TIntermNode *operandNode;
2622	if (invertedType != spv::NoType)
2623	operandNode = node->getOperand()->getAsBinaryNode()->getLeft();
2624	else
2625	operandNode = node->getOperand();
2626
2627	operandNode->traverse(this);
2628
2629	spv::Id operand = spv::NoResult;
2630
2631	spv::Builder::AccessChain::CoherentFlags lvalueCoherentFlags;
2632
2633	const auto hitObjectOpsWithLvalue = [](glslang::TOperator op) {
2634	switch(op) {
2635	case glslang::EOpReorderThreadNV:
2636	case glslang::EOpHitObjectGetCurrentTimeNV:
2637	case glslang::EOpHitObjectGetHitKindNV:
2638	case glslang::EOpHitObjectGetPrimitiveIndexNV:
2639	case glslang::EOpHitObjectGetGeometryIndexNV:
2640	case glslang::EOpHitObjectGetInstanceIdNV:
2641	case glslang::EOpHitObjectGetInstanceCustomIndexNV:
2642	case glslang::EOpHitObjectGetObjectRayDirectionNV:
2643	case glslang::EOpHitObjectGetObjectRayOriginNV:
2644	case glslang::EOpHitObjectGetWorldRayDirectionNV:
2645	case glslang::EOpHitObjectGetWorldRayOriginNV:
2646	case glslang::EOpHitObjectGetWorldToObjectNV:
2647	case glslang::EOpHitObjectGetObjectToWorldNV:
2648	case glslang::EOpHitObjectGetRayTMaxNV:
2649	case glslang::EOpHitObjectGetRayTMinNV:
2650	case glslang::EOpHitObjectIsEmptyNV:
2651	case glslang::EOpHitObjectIsHitNV:
2652	case glslang::EOpHitObjectIsMissNV:
2653	case glslang::EOpHitObjectRecordEmptyNV:
2654	case glslang::EOpHitObjectGetShaderBindingTableRecordIndexNV:
2655	case glslang::EOpHitObjectGetShaderRecordBufferHandleNV:
2656	return true;
2657	default:
2658	return false;
2659	}
2660	};
2661
2662	if (node->getOp() == glslang::EOpAtomicCounterIncrement ||
2663	node->getOp() == glslang::EOpAtomicCounterDecrement ||
2664	node->getOp() == glslang::EOpAtomicCounter ||
2665	(node->getOp() == glslang::EOpInterpolateAtCentroid &&
2666	glslangIntermediate->getSource() != glslang::EShSourceHlsl) ||
2667	node->getOp() == glslang::EOpRayQueryProceed ||
2668	node->getOp() == glslang::EOpRayQueryGetRayTMin ||
2669	node->getOp() == glslang::EOpRayQueryGetRayFlags ||
2670	node->getOp() == glslang::EOpRayQueryGetWorldRayOrigin ||
2671	node->getOp() == glslang::EOpRayQueryGetWorldRayDirection ||
2672	node->getOp() == glslang::EOpRayQueryGetIntersectionCandidateAABBOpaque ||
2673	node->getOp() == glslang::EOpRayQueryTerminate ||
2674	node->getOp() == glslang::EOpRayQueryConfirmIntersection ||
2675	(node->getOp() == glslang::EOpSpirvInst && operandNode->getAsTyped()->getQualifier().isSpirvByReference()) ||
2676	hitObjectOpsWithLvalue(node->getOp())) {
2677	operand = builder.accessChainGetLValue(); // Special case l-value operands
2678	lvalueCoherentFlags = builder.getAccessChain().coherentFlags;
2679	lvalueCoherentFlags |= TranslateCoherent(type: operandNode->getAsTyped()->getType());
2680	} else if (operandNode->getAsTyped()->getQualifier().isSpirvLiteral()) {
2681	// Will be translated to a literal value, make a placeholder here
2682	operand = spv::NoResult;
2683	} else {
2684	operand = accessChainLoad(type: node->getOperand()->getType());
2685	}
2686
2687	OpDecorations decorations = { TranslatePrecisionDecoration(glslangPrecision: node->getOperationPrecision()),
2688	TranslateNoContractionDecoration(qualifier: node->getType().getQualifier()),
2689	TranslateNonUniformDecoration(qualifier: node->getType().getQualifier()) };
2690
2691	// it could be a conversion
2692	if (! result) {
2693	result = createConversion(op: node->getOp(), decorations, destTypeId: resultType(), operand,
2694	resultBasicType: node->getType().getBasicType(), operandBasicType: node->getOperand()->getBasicType());
2695	if (result) {
2696	if (node->getType().isCoopMatKHR() && node->getOperand()->getAsTyped()->getType().isCoopMatKHR() &&
2697	!node->getAsTyped()->getType().sameCoopMatUse(right: node->getOperand()->getAsTyped()->getType())) {
2698	// Conversions that change use need CapabilityCooperativeMatrixConversionsNV
2699	builder.addCapability(cap: spv::CapabilityCooperativeMatrixConversionsNV);
2700	builder.addExtension(ext: spv::E_SPV_NV_cooperative_matrix2);
2701	}
2702	}
2703	}
2704
2705	// if not, then possibly an operation
2706	if (! result)
2707	result = createUnaryOperation(op: node->getOp(), decorations, typeId: resultType(), operand,
2708	typeProxy: node->getOperand()->getBasicType(), lvalueCoherentFlags, opType: node->getType());
2709
2710	// it could be attached to a SPIR-V intruction
2711	if (!result) {
2712	if (node->getOp() == glslang::EOpSpirvInst) {
2713	const auto& spirvInst = node->getSpirvInstruction();
2714	if (spirvInst.set == "") {
2715	spv::IdImmediate idImmOp = {true, operand};
2716	if (operandNode->getAsTyped()->getQualifier().isSpirvLiteral()) {
2717	// Translate the constant to a literal value
2718	std::vector<unsigned> literals;
2719	glslang::TVector<const glslang::TIntermConstantUnion*> constants;
2720	constants.push_back(x: operandNode->getAsConstantUnion());
2721	TranslateLiterals(constants, literals);
2722	idImmOp = {false, literals[0]};
2723	}
2724
2725	if (node->getBasicType() == glslang::EbtVoid)
2726	builder.createNoResultOp(static_cast<spv::Op>(spirvInst.id), operands: {idImmOp});
2727	else
2728	result = builder.createOp(static_cast<spv::Op>(spirvInst.id), typeId: resultType(), operands: {idImmOp});
2729	} else {
2730	result = builder.createBuiltinCall(
2731	resultType: resultType(), builtins: spirvInst.set == "GLSL.std.450" ? stdBuiltins : getExtBuiltins(name: spirvInst.set.c_str()),
2732	entryPoint: spirvInst.id, args: {operand});
2733	}
2734
2735	if (node->getBasicType() == glslang::EbtVoid)
2736	return false; // done with this node
2737	}
2738	}
2739
2740	if (result) {
2741	if (invertedType) {
2742	result = createInvertedSwizzle(precision: decorations.precision, *node->getOperand(), parentResult: result);
2743	decorations.addNonUniform(builder, t: result);
2744	}
2745
2746	builder.clearAccessChain();
2747	builder.setAccessChainRValue(result);
2748
2749	return false; // done with this node
2750	}
2751
2752	// it must be a special case, check...
2753	switch (node->getOp()) {
2754	case glslang::EOpPostIncrement:
2755	case glslang::EOpPostDecrement:
2756	case glslang::EOpPreIncrement:
2757	case glslang::EOpPreDecrement:
2758	{
2759	// we need the integer value "1" or the floating point "1.0" to add/subtract
2760	spv::Id one = 0;
2761	if (node->getBasicType() == glslang::EbtFloat)
2762	one = builder.makeFloatConstant(f: 1.0F);
2763	else if (node->getBasicType() == glslang::EbtDouble)
2764	one = builder.makeDoubleConstant(d: 1.0);
2765	else if (node->getBasicType() == glslang::EbtFloat16)
2766	one = builder.makeFloat16Constant(f16: 1.0F);
2767	else if (node->getBasicType() == glslang::EbtInt8 || node->getBasicType() == glslang::EbtUint8)
2768	one = builder.makeInt8Constant(i: 1);
2769	else if (node->getBasicType() == glslang::EbtInt16 || node->getBasicType() == glslang::EbtUint16)
2770	one = builder.makeInt16Constant(i: 1);
2771	else if (node->getBasicType() == glslang::EbtInt64 || node->getBasicType() == glslang::EbtUint64)
2772	one = builder.makeInt64Constant(i: 1);
2773	else
2774	one = builder.makeIntConstant(i: 1);
2775	glslang::TOperator op;
2776	if (node->getOp() == glslang::EOpPreIncrement ||
2777	node->getOp() == glslang::EOpPostIncrement)
2778	op = glslang::EOpAdd;
2779	else
2780	op = glslang::EOpSub;
2781
2782	spv::Id result = createBinaryOperation(op, decorations,
2783	typeId: convertGlslangToSpvType(type: node->getType()), left: operand, right: one,
2784	typeProxy: node->getType().getBasicType());
2785	assert(result != spv::NoResult);
2786
2787	// The result of operation is always stored, but conditionally the
2788	// consumed result. The consumed result is always an r-value.
2789	builder.accessChainStore(rvalue: result,
2790	nonUniform: TranslateNonUniformDecoration(coherentFlags: builder.getAccessChain().coherentFlags));
2791	builder.clearAccessChain();
2792	if (node->getOp() == glslang::EOpPreIncrement ||
2793	node->getOp() == glslang::EOpPreDecrement)
2794	builder.setAccessChainRValue(result);
2795	else
2796	builder.setAccessChainRValue(operand);
2797	}
2798
2799	return false;
2800
2801	case glslang::EOpAssumeEXT:
2802	builder.addCapability(cap: spv::CapabilityExpectAssumeKHR);
2803	builder.addExtension(ext: spv::E_SPV_KHR_expect_assume);
2804	builder.createNoResultOp(spv::OpAssumeTrueKHR, operand);
2805	return false;
2806	case glslang::EOpEmitStreamVertex:
2807	builder.createNoResultOp(spv::OpEmitStreamVertex, operand);
2808	return false;
2809	case glslang::EOpEndStreamPrimitive:
2810	builder.createNoResultOp(spv::OpEndStreamPrimitive, operand);
2811	return false;
2812	case glslang::EOpRayQueryTerminate:
2813	builder.createNoResultOp(spv::OpRayQueryTerminateKHR, operand);
2814	return false;
2815	case glslang::EOpRayQueryConfirmIntersection:
2816	builder.createNoResultOp(spv::OpRayQueryConfirmIntersectionKHR, operand);
2817	return false;
2818	case glslang::EOpReorderThreadNV:
2819	builder.createNoResultOp(spv::OpReorderThreadWithHitObjectNV, operand);
2820	return false;
2821	case glslang::EOpHitObjectRecordEmptyNV:
2822	builder.createNoResultOp(spv::OpHitObjectRecordEmptyNV, operand);
2823	return false;
2824
2825	case glslang::EOpCreateTensorLayoutNV:
2826	result = builder.createOp(spv::OpCreateTensorLayoutNV, typeId: resultType(), operands: std::vector<spv::Id>{});
2827	builder.clearAccessChain();
2828	builder.setAccessChainRValue(result);
2829	return false;
2830
2831	case glslang::EOpCreateTensorViewNV:
2832	result = builder.createOp(spv::OpCreateTensorViewNV, typeId: resultType(), operands: std::vector<spv::Id>{});
2833	builder.clearAccessChain();
2834	builder.setAccessChainRValue(result);
2835	return false;
2836
2837	default:
2838	logger->missingFunctionality(f: "unknown glslang unary");
2839	return true; // pick up operand as placeholder result
2840	}
2841	}
2842
2843	// Construct a composite object, recursively copying members if their types don't match
2844	spv::Id TGlslangToSpvTraverser::createCompositeConstruct(spv::Id resultTypeId, std::vector<spv::Id> constituents)
2845	{
2846	for (int c = 0; c < (int)constituents.size(); ++c) {
2847	spv::Id& constituent = constituents[c];
2848	spv::Id lType = builder.getContainedTypeId(typeId: resultTypeId, c);
2849	spv::Id rType = builder.getTypeId(resultId: constituent);
2850	if (lType != rType) {
2851	if (glslangIntermediate->getSpv().spv >= glslang::EShTargetSpv_1_4) {
2852	constituent = builder.createUnaryOp(spv::OpCopyLogical, typeId: lType, operand: constituent);
2853	} else if (builder.isStructType(typeId: rType)) {
2854	std::vector<spv::Id> rTypeConstituents;
2855	int numrTypeConstituents = builder.getNumTypeConstituents(typeId: rType);
2856	for (int i = 0; i < numrTypeConstituents; ++i) {
2857	rTypeConstituents.push_back(x: builder.createCompositeExtract(composite: constituent,
2858	typeId: builder.getContainedTypeId(typeId: rType, i), index: i));
2859	}
2860	constituents[c] = createCompositeConstruct(resultTypeId: lType, constituents: rTypeConstituents);
2861	} else {
2862	assert(builder.isArrayType(rType));
2863	std::vector<spv::Id> rTypeConstituents;
2864	int numrTypeConstituents = builder.getNumTypeConstituents(typeId: rType);
2865
2866	spv::Id elementRType = builder.getContainedTypeId(typeId: rType);
2867	for (int i = 0; i < numrTypeConstituents; ++i) {
2868	rTypeConstituents.push_back(x: builder.createCompositeExtract(composite: constituent, typeId: elementRType, index: i));
2869	}
2870	constituents[c] = createCompositeConstruct(resultTypeId: lType, constituents: rTypeConstituents);
2871	}
2872	}
2873	}
2874	return builder.createCompositeConstruct(typeId: resultTypeId, constituents);
2875	}
2876
2877	bool TGlslangToSpvTraverser::visitAggregate(glslang::TVisit visit, glslang::TIntermAggregate* node)
2878	{
2879	SpecConstantOpModeGuard spec_constant_op_mode_setter(&builder);
2880	if (node->getType().getQualifier().isSpecConstant())
2881	spec_constant_op_mode_setter.turnOnSpecConstantOpMode();
2882
2883	spv::Id result = spv::NoResult;
2884	spv::Id invertedType = spv::NoType; // to use to override the natural type of the node
2885	std::vector<spv::Builder::AccessChain> complexLvalues; // for holding swizzling l-values too complex for
2886	// SPIR-V, for an out parameter
2887	std::vector<spv::Id> temporaryLvalues; // temporaries to pass, as proxies for complexLValues
2888
2889	auto resultType = [&invertedType, &node, this](){
2890	if (invertedType != spv::NoType) {
2891	return invertedType;
2892	} else {
2893	auto ret = convertGlslangToSpvType(type: node->getType());
2894	// convertGlslangToSpvType may clobber the debug location, reset it
2895	builder.setDebugSourceLocation(line: node->getLoc().line, filename: node->getLoc().getFilename());
2896	return ret;
2897	}
2898	};
2899
2900	// try texturing
2901	result = createImageTextureFunctionCall(node);
2902	if (result != spv::NoResult) {
2903	builder.clearAccessChain();
2904	builder.setAccessChainRValue(result);
2905
2906	return false;
2907	} else if (node->getOp() == glslang::EOpImageStore ||
2908	node->getOp() == glslang::EOpImageStoreLod ||
2909	node->getOp() == glslang::EOpImageAtomicStore) {
2910	// "imageStore" is a special case, which has no result
2911	return false;
2912	}
2913
2914	glslang::TOperator binOp = glslang::EOpNull;
2915	bool reduceComparison = true;
2916	bool isMatrix = false;
2917	bool noReturnValue = false;
2918	bool atomic = false;
2919
2920	spv::Builder::AccessChain::CoherentFlags lvalueCoherentFlags;
2921
2922	assert(node->getOp());
2923
2924	spv::Decoration precision = TranslatePrecisionDecoration(glslangPrecision: node->getOperationPrecision());
2925
2926	switch (node->getOp()) {
2927	case glslang::EOpScope:
2928	case glslang::EOpSequence:
2929	{
2930	if (visit == glslang::EvPreVisit) {
2931	++sequenceDepth;
2932	if (sequenceDepth == 1) {
2933	// If this is the parent node of all the functions, we want to see them
2934	// early, so all call points have actual SPIR-V functions to reference.
2935	// In all cases, still let the traverser visit the children for us.
2936	makeFunctions(node->getAsAggregate()->getSequence());
2937
2938	// Global initializers is specific to the shader entry point, which does not exist in compile-only mode
2939	if (!options.compileOnly) {
2940	// Also, we want all globals initializers to go into the beginning of the entry point, before
2941	// anything else gets there, so visit out of order, doing them all now.
2942	makeGlobalInitializers(node->getAsAggregate()->getSequence());
2943	}
2944
2945	//Pre process linker objects for ray tracing stages
2946	if (glslangIntermediate->isRayTracingStage())
2947	collectRayTracingLinkerObjects();
2948
2949	// Initializers are done, don't want to visit again, but functions and link objects need to be processed,
2950	// so do them manually.
2951	visitFunctions(node->getAsAggregate()->getSequence());
2952
2953	return false;
2954	} else {
2955	if (node->getOp() == glslang::EOpScope) {
2956	auto loc = node->getLoc();
2957	builder.enterLexicalBlock(line: loc.line, column: loc.column);
2958	}
2959	}
2960	} else {
2961	if (sequenceDepth > 1 && node->getOp() == glslang::EOpScope)
2962	builder.leaveLexicalBlock();
2963	--sequenceDepth;
2964	}
2965
2966	return true;
2967	}
2968	case glslang::EOpLinkerObjects:
2969	{
2970	if (visit == glslang::EvPreVisit)
2971	linkageOnly = true;
2972	else
2973	linkageOnly = false;
2974
2975	return true;
2976	}
2977	case glslang::EOpComma:
2978	{
2979	// processing from left to right naturally leaves the right-most
2980	// lying around in the access chain
2981	glslang::TIntermSequence& glslangOperands = node->getSequence();
2982	for (int i = 0; i < (int)glslangOperands.size(); ++i)
2983	glslangOperands[i]->traverse(this);
2984
2985	return false;
2986	}
2987	case glslang::EOpFunction:
2988	if (visit == glslang::EvPreVisit) {
2989	if (options.generateDebugInfo) {
2990	builder.setDebugSourceLocation(line: node->getLoc().line, filename: node->getLoc().getFilename());
2991	}
2992	if (isShaderEntryPoint(node)) {
2993	inEntryPoint = true;
2994	builder.setBuildPoint(shaderEntry->getLastBlock());
2995	builder.enterFunction(function: shaderEntry);
2996	currentFunction = shaderEntry;
2997	} else {
2998	handleFunctionEntry(node);
2999	}
3000	if (options.generateDebugInfo && !options.emitNonSemanticShaderDebugInfo) {
3001	const auto& loc = node->getLoc();
3002	const char* sourceFileName = loc.getFilename();
3003	spv::Id sourceFileId = sourceFileName ? builder.getStringId(str: sourceFileName) : builder.getMainFileId();
3004	currentFunction->setDebugLineInfo(fileName: sourceFileId, line: loc.line, column: loc.column);
3005	}
3006	} else {
3007	if (options.generateDebugInfo) {
3008	if (glslangIntermediate->getSource() == glslang::EShSourceGlsl && node->getSequence().size() > 1) {
3009	auto endLoc = node->getSequence()[1]->getAsAggregate()->getEndLoc();
3010	builder.setDebugSourceLocation(line: endLoc.line, filename: endLoc.getFilename());
3011	}
3012	}
3013	if (inEntryPoint)
3014	entryPointTerminated = true;
3015	builder.leaveFunction();
3016	inEntryPoint = false;
3017	}
3018
3019	return true;
3020	case glslang::EOpParameters:
3021	// Parameters will have been consumed by EOpFunction processing, but not
3022	// the body, so we still visited the function node's children, making this
3023	// child redundant.
3024	return false;
3025	case glslang::EOpFunctionCall:
3026	{
3027	builder.setDebugSourceLocation(line: node->getLoc().line, filename: node->getLoc().getFilename());
3028	if (node->isUserDefined())
3029	result = handleUserFunctionCall(node);
3030	if (result) {
3031	builder.clearAccessChain();
3032	builder.setAccessChainRValue(result);
3033	} else
3034	logger->missingFunctionality(f: "missing user function; linker needs to catch that");
3035
3036	return false;
3037	}
3038	case glslang::EOpConstructMat2x2:
3039	case glslang::EOpConstructMat2x3:
3040	case glslang::EOpConstructMat2x4:
3041	case glslang::EOpConstructMat3x2:
3042	case glslang::EOpConstructMat3x3:
3043	case glslang::EOpConstructMat3x4:
3044	case glslang::EOpConstructMat4x2:
3045	case glslang::EOpConstructMat4x3:
3046	case glslang::EOpConstructMat4x4:
3047	case glslang::EOpConstructDMat2x2:
3048	case glslang::EOpConstructDMat2x3:
3049	case glslang::EOpConstructDMat2x4:
3050	case glslang::EOpConstructDMat3x2:
3051	case glslang::EOpConstructDMat3x3:
3052	case glslang::EOpConstructDMat3x4:
3053	case glslang::EOpConstructDMat4x2:
3054	case glslang::EOpConstructDMat4x3:
3055	case glslang::EOpConstructDMat4x4:
3056	case glslang::EOpConstructIMat2x2:
3057	case glslang::EOpConstructIMat2x3:
3058	case glslang::EOpConstructIMat2x4:
3059	case glslang::EOpConstructIMat3x2:
3060	case glslang::EOpConstructIMat3x3:
3061	case glslang::EOpConstructIMat3x4:
3062	case glslang::EOpConstructIMat4x2:
3063	case glslang::EOpConstructIMat4x3:
3064	case glslang::EOpConstructIMat4x4:
3065	case glslang::EOpConstructUMat2x2:
3066	case glslang::EOpConstructUMat2x3:
3067	case glslang::EOpConstructUMat2x4:
3068	case glslang::EOpConstructUMat3x2:
3069	case glslang::EOpConstructUMat3x3:
3070	case glslang::EOpConstructUMat3x4:
3071	case glslang::EOpConstructUMat4x2:
3072	case glslang::EOpConstructUMat4x3:
3073	case glslang::EOpConstructUMat4x4:
3074	case glslang::EOpConstructBMat2x2:
3075	case glslang::EOpConstructBMat2x3:
3076	case glslang::EOpConstructBMat2x4:
3077	case glslang::EOpConstructBMat3x2:
3078	case glslang::EOpConstructBMat3x3:
3079	case glslang::EOpConstructBMat3x4:
3080	case glslang::EOpConstructBMat4x2:
3081	case glslang::EOpConstructBMat4x3:
3082	case glslang::EOpConstructBMat4x4:
3083	case glslang::EOpConstructF16Mat2x2:
3084	case glslang::EOpConstructF16Mat2x3:
3085	case glslang::EOpConstructF16Mat2x4:
3086	case glslang::EOpConstructF16Mat3x2:
3087	case glslang::EOpConstructF16Mat3x3:
3088	case glslang::EOpConstructF16Mat3x4:
3089	case glslang::EOpConstructF16Mat4x2:
3090	case glslang::EOpConstructF16Mat4x3:
3091	case glslang::EOpConstructF16Mat4x4:
3092	isMatrix = true;
3093	[[fallthrough]];
3094	case glslang::EOpConstructFloat:
3095	case glslang::EOpConstructVec2:
3096	case glslang::EOpConstructVec3:
3097	case glslang::EOpConstructVec4:
3098	case glslang::EOpConstructDouble:
3099	case glslang::EOpConstructDVec2:
3100	case glslang::EOpConstructDVec3:
3101	case glslang::EOpConstructDVec4:
3102	case glslang::EOpConstructFloat16:
3103	case glslang::EOpConstructF16Vec2:
3104	case glslang::EOpConstructF16Vec3:
3105	case glslang::EOpConstructF16Vec4:
3106	case glslang::EOpConstructBool:
3107	case glslang::EOpConstructBVec2:
3108	case glslang::EOpConstructBVec3:
3109	case glslang::EOpConstructBVec4:
3110	case glslang::EOpConstructInt8:
3111	case glslang::EOpConstructI8Vec2:
3112	case glslang::EOpConstructI8Vec3:
3113	case glslang::EOpConstructI8Vec4:
3114	case glslang::EOpConstructUint8:
3115	case glslang::EOpConstructU8Vec2:
3116	case glslang::EOpConstructU8Vec3:
3117	case glslang::EOpConstructU8Vec4:
3118	case glslang::EOpConstructInt16:
3119	case glslang::EOpConstructI16Vec2:
3120	case glslang::EOpConstructI16Vec3:
3121	case glslang::EOpConstructI16Vec4:
3122	case glslang::EOpConstructUint16:
3123	case glslang::EOpConstructU16Vec2:
3124	case glslang::EOpConstructU16Vec3:
3125	case glslang::EOpConstructU16Vec4:
3126	case glslang::EOpConstructInt:
3127	case glslang::EOpConstructIVec2:
3128	case glslang::EOpConstructIVec3:
3129	case glslang::EOpConstructIVec4:
3130	case glslang::EOpConstructUint:
3131	case glslang::EOpConstructUVec2:
3132	case glslang::EOpConstructUVec3:
3133	case glslang::EOpConstructUVec4:
3134	case glslang::EOpConstructInt64:
3135	case glslang::EOpConstructI64Vec2:
3136	case glslang::EOpConstructI64Vec3:
3137	case glslang::EOpConstructI64Vec4:
3138	case glslang::EOpConstructUint64:
3139	case glslang::EOpConstructU64Vec2:
3140	case glslang::EOpConstructU64Vec3:
3141	case glslang::EOpConstructU64Vec4:
3142	case glslang::EOpConstructStruct:
3143	case glslang::EOpConstructTextureSampler:
3144	case glslang::EOpConstructReference:
3145	case glslang::EOpConstructCooperativeMatrixNV:
3146	case glslang::EOpConstructCooperativeMatrixKHR:
3147	{
3148	builder.setDebugSourceLocation(line: node->getLoc().line, filename: node->getLoc().getFilename());
3149	std::vector<spv::Id> arguments;
3150	translateArguments(node: *node, arguments, lvalueCoherentFlags);
3151	spv::Id constructed;
3152	if (node->getOp() == glslang::EOpConstructTextureSampler) {
3153	const glslang::TType& texType = node->getSequence()[0]->getAsTyped()->getType();
3154	if (glslangIntermediate->getSpv().spv >= glslang::EShTargetSpv_1_6 &&
3155	texType.getSampler().isBuffer()) {
3156	// SamplerBuffer is not supported in spirv1.6 so
3157	// `samplerBuffer(textureBuffer, sampler)` is a no-op
3158	// and textureBuffer is the result going forward
3159	constructed = arguments[0];
3160	} else
3161	constructed = builder.createOp(spv::OpSampledImage, typeId: resultType(), operands: arguments);
3162	} else if (node->getOp() == glslang::EOpConstructCooperativeMatrixKHR &&
3163	node->getType().isCoopMatKHR() && node->getSequence()[0]->getAsTyped()->getType().isCoopMatKHR()) {
3164	builder.addCapability(cap: spv::CapabilityCooperativeMatrixConversionsNV);
3165	builder.addExtension(ext: spv::E_SPV_NV_cooperative_matrix2);
3166	constructed = builder.createCooperativeMatrixConversion(typeId: resultType(), source: arguments[0]);
3167	} else if (node->getOp() == glslang::EOpConstructStruct ||
3168	node->getOp() == glslang::EOpConstructCooperativeMatrixNV ||
3169	node->getOp() == glslang::EOpConstructCooperativeMatrixKHR ||
3170	node->getType().isArray()) {
3171	std::vector<spv::Id> constituents;
3172	for (int c = 0; c < (int)arguments.size(); ++c)
3173	constituents.push_back(x: arguments[c]);
3174	constructed = createCompositeConstruct(resultTypeId: resultType(), constituents);
3175	} else if (isMatrix)
3176	constructed = builder.createMatrixConstructor(precision, sources: arguments, constructee: resultType());
3177	else
3178	constructed = builder.createConstructor(precision, sources: arguments, resultTypeId: resultType());
3179
3180	if (node->getType().getQualifier().isNonUniform()) {
3181	builder.addDecoration(constructed, spv::DecorationNonUniformEXT);
3182	}
3183
3184	builder.clearAccessChain();
3185	builder.setAccessChainRValue(constructed);
3186
3187	return false;
3188	}
3189
3190	// These six are component-wise compares with component-wise results.
3191	// Forward on to createBinaryOperation(), requesting a vector result.
3192	case glslang::EOpLessThan:
3193	case glslang::EOpGreaterThan:
3194	case glslang::EOpLessThanEqual:
3195	case glslang::EOpGreaterThanEqual:
3196	case glslang::EOpVectorEqual:
3197	case glslang::EOpVectorNotEqual:
3198	{
3199	// Map the operation to a binary
3200	binOp = node->getOp();
3201	reduceComparison = false;
3202	switch (node->getOp()) {
3203	case glslang::EOpVectorEqual: binOp = glslang::EOpVectorEqual; break;
3204	case glslang::EOpVectorNotEqual: binOp = glslang::EOpVectorNotEqual; break;
3205	default: binOp = node->getOp(); break;
3206	}
3207
3208	break;
3209	}
3210	case glslang::EOpMul:
3211	// component-wise matrix multiply
3212	binOp = glslang::EOpMul;
3213	break;
3214	case glslang::EOpOuterProduct:
3215	// two vectors multiplied to make a matrix
3216	binOp = glslang::EOpOuterProduct;
3217	break;
3218	case glslang::EOpDot:
3219	{
3220	// for scalar dot product, use multiply
3221	glslang::TIntermSequence& glslangOperands = node->getSequence();
3222	if (glslangOperands[0]->getAsTyped()->getVectorSize() == 1)
3223	binOp = glslang::EOpMul;
3224	break;
3225	}
3226	case glslang::EOpMod:
3227	// when an aggregate, this is the floating-point mod built-in function,
3228	// which can be emitted by the one in createBinaryOperation()
3229	binOp = glslang::EOpMod;
3230	break;
3231
3232	case glslang::EOpEmitVertex:
3233	case glslang::EOpEndPrimitive:
3234	case glslang::EOpBarrier:
3235	case glslang::EOpMemoryBarrier:
3236	case glslang::EOpMemoryBarrierAtomicCounter:
3237	case glslang::EOpMemoryBarrierBuffer:
3238	case glslang::EOpMemoryBarrierImage:
3239	case glslang::EOpMemoryBarrierShared:
3240	case glslang::EOpGroupMemoryBarrier:
3241	case glslang::EOpDeviceMemoryBarrier:
3242	case glslang::EOpAllMemoryBarrierWithGroupSync:
3243	case glslang::EOpDeviceMemoryBarrierWithGroupSync:
3244	case glslang::EOpWorkgroupMemoryBarrier:
3245	case glslang::EOpWorkgroupMemoryBarrierWithGroupSync:
3246	case glslang::EOpSubgroupBarrier:
3247	case glslang::EOpSubgroupMemoryBarrier:
3248	case glslang::EOpSubgroupMemoryBarrierBuffer:
3249	case glslang::EOpSubgroupMemoryBarrierImage:
3250	case glslang::EOpSubgroupMemoryBarrierShared:
3251	noReturnValue = true;
3252	// These all have 0 operands and will naturally finish up in the code below for 0 operands
3253	break;
3254
3255	case glslang::EOpAtomicAdd:
3256	case glslang::EOpAtomicSubtract:
3257	case glslang::EOpAtomicMin:
3258	case glslang::EOpAtomicMax:
3259	case glslang::EOpAtomicAnd:
3260	case glslang::EOpAtomicOr:
3261	case glslang::EOpAtomicXor:
3262	case glslang::EOpAtomicExchange:
3263	case glslang::EOpAtomicCompSwap:
3264	atomic = true;
3265	break;
3266
3267	case glslang::EOpAtomicStore:
3268	noReturnValue = true;
3269	[[fallthrough]];
3270	case glslang::EOpAtomicLoad:
3271	atomic = true;
3272	break;
3273
3274	case glslang::EOpAtomicCounterAdd:
3275	case glslang::EOpAtomicCounterSubtract:
3276	case glslang::EOpAtomicCounterMin:
3277	case glslang::EOpAtomicCounterMax:
3278	case glslang::EOpAtomicCounterAnd:
3279	case glslang::EOpAtomicCounterOr:
3280	case glslang::EOpAtomicCounterXor:
3281	case glslang::EOpAtomicCounterExchange:
3282	case glslang::EOpAtomicCounterCompSwap:
3283	builder.addExtension(ext: "SPV_KHR_shader_atomic_counter_ops");
3284	builder.addCapability(cap: spv::CapabilityAtomicStorageOps);
3285	atomic = true;
3286	break;
3287
3288	case glslang::EOpAbsDifference:
3289	case glslang::EOpAddSaturate:
3290	case glslang::EOpSubSaturate:
3291	case glslang::EOpAverage:
3292	case glslang::EOpAverageRounded:
3293	case glslang::EOpMul32x16:
3294	builder.addCapability(cap: spv::CapabilityIntegerFunctions2INTEL);
3295	builder.addExtension(ext: "SPV_INTEL_shader_integer_functions2");
3296	binOp = node->getOp();
3297	break;
3298
3299	case glslang::EOpExpectEXT:
3300	builder.addCapability(cap: spv::CapabilityExpectAssumeKHR);
3301	builder.addExtension(ext: spv::E_SPV_KHR_expect_assume);
3302	binOp = node->getOp();
3303	break;
3304
3305	case glslang::EOpIgnoreIntersectionNV:
3306	case glslang::EOpTerminateRayNV:
3307	case glslang::EOpTraceNV:
3308	case glslang::EOpTraceRayMotionNV:
3309	case glslang::EOpTraceKHR:
3310	case glslang::EOpExecuteCallableNV:
3311	case glslang::EOpExecuteCallableKHR:
3312	case glslang::EOpWritePackedPrimitiveIndices4x8NV:
3313	case glslang::EOpEmitMeshTasksEXT:
3314	case glslang::EOpSetMeshOutputsEXT:
3315	noReturnValue = true;
3316	break;
3317	case glslang::EOpRayQueryInitialize:
3318	case glslang::EOpRayQueryTerminate:
3319	case glslang::EOpRayQueryGenerateIntersection:
3320	case glslang::EOpRayQueryConfirmIntersection:
3321	builder.addExtension(ext: "SPV_KHR_ray_query");
3322	builder.addCapability(cap: spv::CapabilityRayQueryKHR);
3323	noReturnValue = true;
3324	break;
3325	case glslang::EOpRayQueryProceed:
3326	case glslang::EOpRayQueryGetIntersectionType:
3327	case glslang::EOpRayQueryGetRayTMin:
3328	case glslang::EOpRayQueryGetRayFlags:
3329	case glslang::EOpRayQueryGetIntersectionT:
3330	case glslang::EOpRayQueryGetIntersectionInstanceCustomIndex:
3331	case glslang::EOpRayQueryGetIntersectionInstanceId:
3332	case glslang::EOpRayQueryGetIntersectionInstanceShaderBindingTableRecordOffset:
3333	case glslang::EOpRayQueryGetIntersectionGeometryIndex:
3334	case glslang::EOpRayQueryGetIntersectionPrimitiveIndex:
3335	case glslang::EOpRayQueryGetIntersectionBarycentrics:
3336	case glslang::EOpRayQueryGetIntersectionFrontFace:
3337	case glslang::EOpRayQueryGetIntersectionCandidateAABBOpaque:
3338	case glslang::EOpRayQueryGetIntersectionObjectRayDirection:
3339	case glslang::EOpRayQueryGetIntersectionObjectRayOrigin:
3340	case glslang::EOpRayQueryGetWorldRayDirection:
3341	case glslang::EOpRayQueryGetWorldRayOrigin:
3342	case glslang::EOpRayQueryGetIntersectionObjectToWorld:
3343	case glslang::EOpRayQueryGetIntersectionWorldToObject:
3344	builder.addExtension(ext: "SPV_KHR_ray_query");
3345	builder.addCapability(cap: spv::CapabilityRayQueryKHR);
3346	break;
3347	case glslang::EOpCooperativeMatrixLoad:
3348	case glslang::EOpCooperativeMatrixStore:
3349	case glslang::EOpCooperativeMatrixLoadNV:
3350	case glslang::EOpCooperativeMatrixStoreNV:
3351	case glslang::EOpCooperativeMatrixLoadTensorNV:
3352	case glslang::EOpCooperativeMatrixStoreTensorNV:
3353	case glslang::EOpCooperativeMatrixReduceNV:
3354	case glslang::EOpCooperativeMatrixPerElementOpNV:
3355	case glslang::EOpCooperativeMatrixTransposeNV:
3356	noReturnValue = true;
3357	break;
3358	case glslang::EOpBeginInvocationInterlock:
3359	case glslang::EOpEndInvocationInterlock:
3360	builder.addExtension(ext: spv::E_SPV_EXT_fragment_shader_interlock);
3361	noReturnValue = true;
3362	break;
3363
3364	case glslang::EOpHitObjectTraceRayNV:
3365	case glslang::EOpHitObjectTraceRayMotionNV:
3366	case glslang::EOpHitObjectGetAttributesNV:
3367	case glslang::EOpHitObjectExecuteShaderNV:
3368	case glslang::EOpHitObjectRecordEmptyNV:
3369	case glslang::EOpHitObjectRecordMissNV:
3370	case glslang::EOpHitObjectRecordMissMotionNV:
3371	case glslang::EOpHitObjectRecordHitNV:
3372	case glslang::EOpHitObjectRecordHitMotionNV:
3373	case glslang::EOpHitObjectRecordHitWithIndexNV:
3374	case glslang::EOpHitObjectRecordHitWithIndexMotionNV:
3375	case glslang::EOpReorderThreadNV:
3376	noReturnValue = true;
3377	[[fallthrough]];
3378	case glslang::EOpHitObjectIsEmptyNV:
3379	case glslang::EOpHitObjectIsMissNV:
3380	case glslang::EOpHitObjectIsHitNV:
3381	case glslang::EOpHitObjectGetRayTMinNV:
3382	case glslang::EOpHitObjectGetRayTMaxNV:
3383	case glslang::EOpHitObjectGetObjectRayOriginNV:
3384	case glslang::EOpHitObjectGetObjectRayDirectionNV:
3385	case glslang::EOpHitObjectGetWorldRayOriginNV:
3386	case glslang::EOpHitObjectGetWorldRayDirectionNV:
3387	case glslang::EOpHitObjectGetObjectToWorldNV:
3388	case glslang::EOpHitObjectGetWorldToObjectNV:
3389	case glslang::EOpHitObjectGetInstanceCustomIndexNV:
3390	case glslang::EOpHitObjectGetInstanceIdNV:
3391	case glslang::EOpHitObjectGetGeometryIndexNV:
3392	case glslang::EOpHitObjectGetPrimitiveIndexNV:
3393	case glslang::EOpHitObjectGetHitKindNV:
3394	case glslang::EOpHitObjectGetCurrentTimeNV:
3395	case glslang::EOpHitObjectGetShaderBindingTableRecordIndexNV:
3396	case glslang::EOpHitObjectGetShaderRecordBufferHandleNV:
3397	builder.addExtension(ext: spv::E_SPV_NV_shader_invocation_reorder);
3398	builder.addCapability(cap: spv::CapabilityShaderInvocationReorderNV);
3399	break;
3400	case glslang::EOpRayQueryGetIntersectionTriangleVertexPositionsEXT:
3401	builder.addExtension(ext: spv::E_SPV_KHR_ray_tracing_position_fetch);
3402	builder.addCapability(cap: spv::CapabilityRayQueryPositionFetchKHR);
3403	noReturnValue = true;
3404	break;
3405
3406	case glslang::EOpImageSampleWeightedQCOM:
3407	builder.addCapability(cap: spv::CapabilityTextureSampleWeightedQCOM);
3408	builder.addExtension(ext: spv::E_SPV_QCOM_image_processing);
3409	break;
3410	case glslang::EOpImageBoxFilterQCOM:
3411	builder.addCapability(cap: spv::CapabilityTextureBoxFilterQCOM);
3412	builder.addExtension(ext: spv::E_SPV_QCOM_image_processing);
3413	break;
3414	case glslang::EOpImageBlockMatchSADQCOM:
3415	case glslang::EOpImageBlockMatchSSDQCOM:
3416	builder.addCapability(cap: spv::CapabilityTextureBlockMatchQCOM);
3417	builder.addExtension(ext: spv::E_SPV_QCOM_image_processing);
3418	break;
3419
3420	case glslang::EOpImageBlockMatchWindowSSDQCOM:
3421	case glslang::EOpImageBlockMatchWindowSADQCOM:
3422	builder.addCapability(cap: spv::CapabilityTextureBlockMatchQCOM);
3423	builder.addExtension(ext: spv::E_SPV_QCOM_image_processing);
3424	builder.addCapability(cap: spv::CapabilityTextureBlockMatch2QCOM);
3425	builder.addExtension(ext: spv::E_SPV_QCOM_image_processing2);
3426	break;
3427
3428	case glslang::EOpImageBlockMatchGatherSSDQCOM:
3429	case glslang::EOpImageBlockMatchGatherSADQCOM:
3430	builder.addCapability(cap: spv::CapabilityTextureBlockMatchQCOM);
3431	builder.addExtension(ext: spv::E_SPV_QCOM_image_processing);
3432	builder.addCapability(cap: spv::CapabilityTextureBlockMatch2QCOM);
3433	builder.addExtension(ext: spv::E_SPV_QCOM_image_processing2);
3434	break;
3435
3436	case glslang::EOpFetchMicroTriangleVertexPositionNV:
3437	case glslang::EOpFetchMicroTriangleVertexBarycentricNV:
3438	builder.addExtension(ext: spv::E_SPV_NV_displacement_micromap);
3439	builder.addCapability(cap: spv::CapabilityDisplacementMicromapNV);
3440	break;
3441
3442	case glslang::EOpDebugPrintf:
3443	noReturnValue = true;
3444	break;
3445
3446	default:
3447	break;
3448	}
3449
3450	//
3451	// See if it maps to a regular operation.
3452	//
3453	if (binOp != glslang::EOpNull) {
3454	glslang::TIntermTyped* left = node->getSequence()[0]->getAsTyped();
3455	glslang::TIntermTyped* right = node->getSequence()[1]->getAsTyped();
3456	assert(left && right);
3457
3458	builder.clearAccessChain();
3459	left->traverse(this);
3460	spv::Id leftId = accessChainLoad(type: left->getType());
3461
3462	builder.clearAccessChain();
3463	right->traverse(this);
3464	spv::Id rightId = accessChainLoad(type: right->getType());
3465
3466	builder.setDebugSourceLocation(line: node->getLoc().line, filename: node->getLoc().getFilename());
3467	OpDecorations decorations = { precision,
3468	TranslateNoContractionDecoration(qualifier: node->getType().getQualifier()),
3469	TranslateNonUniformDecoration(qualifier: node->getType().getQualifier()) };
3470	result = createBinaryOperation(op: binOp, decorations,
3471	typeId: resultType(), left: leftId, right: rightId,
3472	typeProxy: left->getType().getBasicType(), reduceComparison);
3473
3474	// code above should only make binOp that exists in createBinaryOperation
3475	assert(result != spv::NoResult);
3476	builder.clearAccessChain();
3477	builder.setAccessChainRValue(result);
3478
3479	return false;
3480	}
3481
3482	//
3483	// Create the list of operands.
3484	//
3485	glslang::TIntermSequence& glslangOperands = node->getSequence();
3486	std::vector<spv::Id> operands;
3487	std::vector<spv::IdImmediate> memoryAccessOperands;
3488	for (int arg = 0; arg < (int)glslangOperands.size(); ++arg) {
3489	// special case l-value operands; there are just a few
3490	bool lvalue = false;
3491	switch (node->getOp()) {
3492	case glslang::EOpModf:
3493	if (arg == 1)
3494	lvalue = true;
3495	break;
3496
3497
3498
3499	case glslang::EOpHitObjectRecordHitNV:
3500	case glslang::EOpHitObjectRecordHitMotionNV:
3501	case glslang::EOpHitObjectRecordHitWithIndexNV:
3502	case glslang::EOpHitObjectRecordHitWithIndexMotionNV:
3503	case glslang::EOpHitObjectTraceRayNV:
3504	case glslang::EOpHitObjectTraceRayMotionNV:
3505	case glslang::EOpHitObjectExecuteShaderNV:
3506	case glslang::EOpHitObjectRecordMissNV:
3507	case glslang::EOpHitObjectRecordMissMotionNV:
3508	case glslang::EOpHitObjectGetAttributesNV:
3509	if (arg == 0)
3510	lvalue = true;
3511	break;
3512
3513	case glslang::EOpRayQueryInitialize:
3514	case glslang::EOpRayQueryTerminate:
3515	case glslang::EOpRayQueryConfirmIntersection:
3516	case glslang::EOpRayQueryProceed:
3517	case glslang::EOpRayQueryGenerateIntersection:
3518	case glslang::EOpRayQueryGetIntersectionType:
3519	case glslang::EOpRayQueryGetIntersectionT:
3520	case glslang::EOpRayQueryGetIntersectionInstanceCustomIndex:
3521	case glslang::EOpRayQueryGetIntersectionInstanceId:
3522	case glslang::EOpRayQueryGetIntersectionInstanceShaderBindingTableRecordOffset:
3523	case glslang::EOpRayQueryGetIntersectionGeometryIndex:
3524	case glslang::EOpRayQueryGetIntersectionPrimitiveIndex:
3525	case glslang::EOpRayQueryGetIntersectionBarycentrics:
3526	case glslang::EOpRayQueryGetIntersectionFrontFace:
3527	case glslang::EOpRayQueryGetIntersectionObjectRayDirection:
3528	case glslang::EOpRayQueryGetIntersectionObjectRayOrigin:
3529	case glslang::EOpRayQueryGetIntersectionObjectToWorld:
3530	case glslang::EOpRayQueryGetIntersectionWorldToObject:
3531	if (arg == 0)
3532	lvalue = true;
3533	break;
3534
3535	case glslang::EOpAtomicAdd:
3536	case glslang::EOpAtomicSubtract:
3537	case glslang::EOpAtomicMin:
3538	case glslang::EOpAtomicMax:
3539	case glslang::EOpAtomicAnd:
3540	case glslang::EOpAtomicOr:
3541	case glslang::EOpAtomicXor:
3542	case glslang::EOpAtomicExchange:
3543	case glslang::EOpAtomicCompSwap:
3544	if (arg == 0)
3545	lvalue = true;
3546	break;
3547
3548	case glslang::EOpFrexp:
3549	if (arg == 1)
3550	lvalue = true;
3551	break;
3552	case glslang::EOpInterpolateAtSample:
3553	case glslang::EOpInterpolateAtOffset:
3554	case glslang::EOpInterpolateAtVertex:
3555	if (arg == 0) {
3556	// If GLSL, use the address of the interpolant argument.
3557	// If HLSL, use an internal version of OpInterolates that takes
3558	// the rvalue of the interpolant. A fixup pass in spirv-opt
3559	// legalization will remove the OpLoad and convert to an lvalue.
3560	// Had to do this because legalization will only propagate a
3561	// builtin into an rvalue.
3562	lvalue = glslangIntermediate->getSource() != glslang::EShSourceHlsl;
3563
3564	// Does it need a swizzle inversion? If so, evaluation is inverted;
3565	// operate first on the swizzle base, then apply the swizzle.
3566	// That is, we transform
3567	//
3568	// interpolate(v.zy) -> interpolate(v).zy
3569	//
3570	if (glslangOperands[0]->getAsOperator() &&
3571	glslangOperands[0]->getAsOperator()->getOp() == glslang::EOpVectorSwizzle)
3572	invertedType = convertGlslangToSpvType(
3573	type: glslangOperands[0]->getAsBinaryNode()->getLeft()->getType());
3574	}
3575	break;
3576	case glslang::EOpAtomicLoad:
3577	case glslang::EOpAtomicStore:
3578	case glslang::EOpAtomicCounterAdd:
3579	case glslang::EOpAtomicCounterSubtract:
3580	case glslang::EOpAtomicCounterMin:
3581	case glslang::EOpAtomicCounterMax:
3582	case glslang::EOpAtomicCounterAnd:
3583	case glslang::EOpAtomicCounterOr:
3584	case glslang::EOpAtomicCounterXor:
3585	case glslang::EOpAtomicCounterExchange:
3586	case glslang::EOpAtomicCounterCompSwap:
3587	if (arg == 0)
3588	lvalue = true;
3589	break;
3590	case glslang::EOpAddCarry:
3591	case glslang::EOpSubBorrow:
3592	if (arg == 2)
3593	lvalue = true;
3594	break;
3595	case glslang::EOpUMulExtended:
3596	case glslang::EOpIMulExtended:
3597	if (arg >= 2)
3598	lvalue = true;
3599	break;
3600	case glslang::EOpCooperativeMatrixLoad:
3601	case glslang::EOpCooperativeMatrixLoadNV:
3602	case glslang::EOpCooperativeMatrixLoadTensorNV:
3603	if (arg == 0 || arg == 1)
3604	lvalue = true;
3605	break;
3606	case glslang::EOpCooperativeMatrixStore:
3607	case glslang::EOpCooperativeMatrixStoreNV:
3608	case glslang::EOpCooperativeMatrixStoreTensorNV:
3609	if (arg == 1)
3610	lvalue = true;
3611	break;
3612	case glslang::EOpCooperativeMatrixReduceNV:
3613	case glslang::EOpCooperativeMatrixPerElementOpNV:
3614	case glslang::EOpCooperativeMatrixTransposeNV:
3615	if (arg == 0)
3616	lvalue = true;
3617	break;
3618	case glslang::EOpSpirvInst:
3619	if (glslangOperands[arg]->getAsTyped()->getQualifier().isSpirvByReference())
3620	lvalue = true;
3621	break;
3622	case glslang::EOpReorderThreadNV:
3623	//Three variants of reorderThreadNV, two of them use hitObjectNV
3624	if (arg == 0 && glslangOperands.size() != 2)
3625	lvalue = true;
3626	break;
3627	case glslang::EOpRayQueryGetIntersectionTriangleVertexPositionsEXT:
3628	if (arg == 0 || arg == 2)
3629	lvalue = true;
3630	break;
3631	default:
3632	break;
3633	}
3634	builder.clearAccessChain();
3635	if (invertedType != spv::NoType && arg == 0)
3636	glslangOperands[0]->getAsBinaryNode()->getLeft()->traverse(this);
3637	else
3638	glslangOperands[arg]->traverse(this);
3639
3640	if (node->getOp() == glslang::EOpCooperativeMatrixLoad ||
3641	node->getOp() == glslang::EOpCooperativeMatrixStore ||
3642	node->getOp() == glslang::EOpCooperativeMatrixLoadNV ||
3643	node->getOp() == glslang::EOpCooperativeMatrixStoreNV ||
3644	node->getOp() == glslang::EOpCooperativeMatrixLoadTensorNV ||
3645	node->getOp() == glslang::EOpCooperativeMatrixStoreTensorNV) {
3646
3647	if (arg == 1) {
3648	// fold "element" parameter into the access chain
3649	spv::Builder::AccessChain save = builder.getAccessChain();
3650	builder.clearAccessChain();
3651	glslangOperands[2]->traverse(this);
3652
3653	spv::Id elementId = accessChainLoad(type: glslangOperands[2]->getAsTyped()->getType());
3654
3655	builder.setAccessChain(save);
3656
3657	// Point to the first element of the array.
3658	builder.accessChainPush(offset: elementId,
3659	coherentFlags: TranslateCoherent(type: glslangOperands[arg]->getAsTyped()->getType()),
3660	alignment: glslangOperands[arg]->getAsTyped()->getType().getBufferReferenceAlignment());
3661
3662	spv::Builder::AccessChain::CoherentFlags coherentFlags = builder.getAccessChain().coherentFlags;
3663	unsigned int alignment = builder.getAccessChain().alignment;
3664
3665	int memoryAccess = TranslateMemoryAccess(coherentFlags);
3666	if (node->getOp() == glslang::EOpCooperativeMatrixLoad ||
3667	node->getOp() == glslang::EOpCooperativeMatrixLoadNV ||
3668	node->getOp() == glslang::EOpCooperativeMatrixLoadTensorNV)
3669	memoryAccess &= ~spv::MemoryAccessMakePointerAvailableKHRMask;
3670	if (node->getOp() == glslang::EOpCooperativeMatrixStore ||
3671	node->getOp() == glslang::EOpCooperativeMatrixStoreNV ||
3672	node->getOp() == glslang::EOpCooperativeMatrixStoreTensorNV)
3673	memoryAccess &= ~spv::MemoryAccessMakePointerVisibleKHRMask;
3674	if (builder.getStorageClass(resultId: builder.getAccessChain().base) ==
3675	spv::StorageClassPhysicalStorageBufferEXT) {
3676	memoryAccess = (spv::MemoryAccessMask)(memoryAccess | spv::MemoryAccessAlignedMask);
3677	}
3678
3679	memoryAccessOperands.push_back(x: spv::IdImmediate(false, memoryAccess));
3680
3681	if (memoryAccess & spv::MemoryAccessAlignedMask) {
3682	memoryAccessOperands.push_back(x: spv::IdImmediate(false, alignment));
3683	}
3684
3685	if (memoryAccess &
3686	(spv::MemoryAccessMakePointerAvailableKHRMask | spv::MemoryAccessMakePointerVisibleKHRMask)) {
3687	memoryAccessOperands.push_back(x: spv::IdImmediate(true,
3688	builder.makeUintConstant(u: TranslateMemoryScope(coherentFlags))));
3689	}
3690	} else if (arg == 2) {
3691	continue;
3692	}
3693	}
3694
3695	// for l-values, pass the address, for r-values, pass the value
3696	if (lvalue) {
3697	if (invertedType == spv::NoType && !builder.isSpvLvalue()) {
3698	// SPIR-V cannot represent an l-value containing a swizzle that doesn't
3699	// reduce to a simple access chain. So, we need a temporary vector to
3700	// receive the result, and must later swizzle that into the original
3701	// l-value.
3702	complexLvalues.push_back(x: builder.getAccessChain());
3703	temporaryLvalues.push_back(x: builder.createVariable(
3704	precision: spv::NoPrecision, storageClass: spv::StorageClassFunction,
3705	type: builder.accessChainGetInferredType(), name: "swizzleTemp"));
3706	operands.push_back(x: temporaryLvalues.back());
3707	} else {
3708	operands.push_back(x: builder.accessChainGetLValue());
3709	}
3710	lvalueCoherentFlags = builder.getAccessChain().coherentFlags;
3711	lvalueCoherentFlags |= TranslateCoherent(type: glslangOperands[arg]->getAsTyped()->getType());
3712	} else {
3713	builder.setDebugSourceLocation(line: node->getLoc().line, filename: node->getLoc().getFilename());
3714	glslang::TOperator glslangOp = node->getOp();
3715	if (arg == 1 &&
3716	(glslangOp == glslang::EOpRayQueryGetIntersectionType ||
3717	glslangOp == glslang::EOpRayQueryGetIntersectionT ||
3718	glslangOp == glslang::EOpRayQueryGetIntersectionInstanceCustomIndex ||
3719	glslangOp == glslang::EOpRayQueryGetIntersectionInstanceId ||
3720	glslangOp == glslang::EOpRayQueryGetIntersectionInstanceShaderBindingTableRecordOffset ||
3721	glslangOp == glslang::EOpRayQueryGetIntersectionGeometryIndex ||
3722	glslangOp == glslang::EOpRayQueryGetIntersectionPrimitiveIndex ||
3723	glslangOp == glslang::EOpRayQueryGetIntersectionBarycentrics ||
3724	glslangOp == glslang::EOpRayQueryGetIntersectionFrontFace ||
3725	glslangOp == glslang::EOpRayQueryGetIntersectionObjectRayDirection ||
3726	glslangOp == glslang::EOpRayQueryGetIntersectionObjectRayOrigin ||
3727	glslangOp == glslang::EOpRayQueryGetIntersectionObjectToWorld ||
3728	glslangOp == glslang::EOpRayQueryGetIntersectionWorldToObject ||
3729	glslangOp == glslang::EOpRayQueryGetIntersectionTriangleVertexPositionsEXT
3730	)) {
3731	bool cond = glslangOperands[arg]->getAsConstantUnion()->getConstArray()[0].getBConst();
3732	operands.push_back(x: builder.makeIntConstant(i: cond ? 1 : 0));
3733	} else if ((arg == 10 && glslangOp == glslang::EOpTraceKHR) ||
3734	(arg == 11 && glslangOp == glslang::EOpTraceRayMotionNV) ||
3735	(arg == 1 && glslangOp == glslang::EOpExecuteCallableKHR) ||
3736	(arg == 1 && glslangOp == glslang::EOpHitObjectExecuteShaderNV) ||
3737	(arg == 11 && glslangOp == glslang::EOpHitObjectTraceRayNV) ||
3738	(arg == 12 && glslangOp == glslang::EOpHitObjectTraceRayMotionNV)) {
3739	const int set = glslangOp == glslang::EOpExecuteCallableKHR ? 1 : 0;
3740	const int location = glslangOperands[arg]->getAsConstantUnion()->getConstArray()[0].getUConst();
3741	auto itNode = locationToSymbol[set].find(x: location);
3742	visitSymbol(symbol: itNode->second);
3743	spv::Id symId = getSymbolId(node: itNode->second);
3744	operands.push_back(x: symId);
3745	} else if ((arg == 12 && glslangOp == glslang::EOpHitObjectRecordHitNV) ||
3746	(arg == 13 && glslangOp == glslang::EOpHitObjectRecordHitMotionNV) ||
3747	(arg == 11 && glslangOp == glslang::EOpHitObjectRecordHitWithIndexNV) ||
3748	(arg == 12 && glslangOp == glslang::EOpHitObjectRecordHitWithIndexMotionNV) ||
3749	(arg == 1 && glslangOp == glslang::EOpHitObjectGetAttributesNV)) {
3750	const int location = glslangOperands[arg]->getAsConstantUnion()->getConstArray()[0].getUConst();
3751	const int set = 2;
3752	auto itNode = locationToSymbol[set].find(x: location);
3753	visitSymbol(symbol: itNode->second);
3754	spv::Id symId = getSymbolId(node: itNode->second);
3755	operands.push_back(x: symId);
3756	} else if (glslangOperands[arg]->getAsTyped()->getQualifier().isSpirvLiteral()) {
3757	// Will be translated to a literal value, make a placeholder here
3758	operands.push_back(x: spv::NoResult);
3759	} else if (glslangOperands[arg]->getAsTyped()->getBasicType() == glslang::EbtFunction) {
3760	spv::Function* function = functionMap[glslangOperands[arg]->getAsSymbolNode()->getMangledName().c_str()];
3761	assert(function);
3762	operands.push_back(x: function->getId());
3763	} else {
3764	operands.push_back(x: accessChainLoad(type: glslangOperands[arg]->getAsTyped()->getType()));
3765	}
3766	}
3767	}
3768
3769	builder.setDebugSourceLocation(line: node->getLoc().line, filename: node->getLoc().getFilename());
3770	if (node->getOp() == glslang::EOpCooperativeMatrixLoadTensorNV) {
3771	std::vector<spv::IdImmediate> idImmOps;
3772
3773	builder.addCapability(cap: spv::CapabilityCooperativeMatrixTensorAddressingNV);
3774	builder.addExtension(ext: spv::E_SPV_NV_cooperative_matrix2);
3775
3776	spv::Id object = builder.createLoad(lValue: operands[0], precision: spv::NoPrecision);
3777
3778	idImmOps.push_back(x: spv::IdImmediate(true, operands[1])); // Pointer
3779	idImmOps.push_back(x: spv::IdImmediate(true, object)); // Object
3780	idImmOps.push_back(x: spv::IdImmediate(true, operands[2])); // tensorLayout
3781
3782	idImmOps.insert(position: idImmOps.end(), first: memoryAccessOperands.begin(), last: memoryAccessOperands.end()); // memoryaccess
3783
3784	// initialize tensor operands to zero, then OR in flags based on the operands
3785	size_t tensorOpIdx = idImmOps.size();
3786	idImmOps.push_back(x: spv::IdImmediate(false, 0));
3787
3788	for (uint32_t i = 3; i < operands.size(); ++i) {
3789	if (builder.isTensorView(resultId: operands[i])) {
3790	idImmOps[tensorOpIdx].word |= spv::TensorAddressingOperandsTensorViewMask;
3791	} else {
3792	// must be the decode func
3793	idImmOps[tensorOpIdx].word |= spv::TensorAddressingOperandsDecodeFuncMask;
3794	builder.addCapability(cap: spv::CapabilityCooperativeMatrixBlockLoadsNV);
3795	}
3796	idImmOps.push_back(x: spv::IdImmediate(true, operands[i])); // tensorView or decodeFunc
3797	}
3798
3799	// get the pointee type
3800	spv::Id typeId = builder.getContainedTypeId(typeId: builder.getTypeId(resultId: operands[0]));
3801	assert(builder.isCooperativeMatrixType(typeId));
3802	// do the op
3803	spv::Id result = builder.createOp(spv::OpCooperativeMatrixLoadTensorNV, typeId, operands: idImmOps);
3804	// store the result to the pointer (out param 'm')
3805	builder.createStore(rValue: result, lValue: operands[0]);
3806	result = 0;
3807	} else if (node->getOp() == glslang::EOpCooperativeMatrixLoad ||
3808	node->getOp() == glslang::EOpCooperativeMatrixLoadNV) {
3809	std::vector<spv::IdImmediate> idImmOps;
3810
3811	idImmOps.push_back(x: spv::IdImmediate(true, operands[1])); // buf
3812	if (node->getOp() == glslang::EOpCooperativeMatrixLoad) {
3813	idImmOps.push_back(x: spv::IdImmediate(true, operands[3])); // matrixLayout
3814	auto layout = builder.getConstantScalar(resultId: operands[3]);
3815	if (layout == spv::CooperativeMatrixLayoutRowBlockedInterleavedARM ||
3816	layout == spv::CooperativeMatrixLayoutColumnBlockedInterleavedARM) {
3817	builder.addExtension(ext: spv::E_SPV_ARM_cooperative_matrix_layouts);
3818	builder.addCapability(cap: spv::CapabilityCooperativeMatrixLayoutsARM);
3819	}
3820	idImmOps.push_back(x: spv::IdImmediate(true, operands[2])); // stride
3821	} else {
3822	idImmOps.push_back(x: spv::IdImmediate(true, operands[2])); // stride
3823	idImmOps.push_back(x: spv::IdImmediate(true, operands[3])); // colMajor
3824	}
3825	idImmOps.insert(position: idImmOps.end(), first: memoryAccessOperands.begin(), last: memoryAccessOperands.end());
3826	// get the pointee type
3827	spv::Id typeId = builder.getContainedTypeId(typeId: builder.getTypeId(resultId: operands[0]));
3828	assert(builder.isCooperativeMatrixType(typeId));
3829	// do the op
3830	spv::Id result = node->getOp() == glslang::EOpCooperativeMatrixLoad
3831	? builder.createOp(spv::OpCooperativeMatrixLoadKHR, typeId, operands: idImmOps)
3832	: builder.createOp(spv::OpCooperativeMatrixLoadNV, typeId, operands: idImmOps);
3833	// store the result to the pointer (out param 'm')
3834	builder.createStore(rValue: result, lValue: operands[0]);
3835	result = 0;
3836	} else if (node->getOp() == glslang::EOpCooperativeMatrixStoreTensorNV) {
3837	std::vector<spv::IdImmediate> idImmOps;
3838
3839	idImmOps.push_back(x: spv::IdImmediate(true, operands[1])); // buf
3840	idImmOps.push_back(x: spv::IdImmediate(true, operands[0])); // object
3841
3842	builder.addCapability(cap: spv::CapabilityCooperativeMatrixTensorAddressingNV);
3843	builder.addExtension(ext: spv::E_SPV_NV_cooperative_matrix2);
3844
3845	idImmOps.push_back(x: spv::IdImmediate(true, operands[2])); // tensorLayout
3846
3847	idImmOps.insert(position: idImmOps.end(), first: memoryAccessOperands.begin(), last: memoryAccessOperands.end()); // memoryaccess
3848
3849	if (operands.size() > 3) {
3850	idImmOps.push_back(x: spv::IdImmediate(false, spv::TensorAddressingOperandsTensorViewMask));
3851	idImmOps.push_back(x: spv::IdImmediate(true, operands[3])); // tensorView
3852	} else {
3853	idImmOps.push_back(x: spv::IdImmediate(false, 0));
3854	}
3855
3856	builder.createNoResultOp(spv::OpCooperativeMatrixStoreTensorNV, operands: idImmOps);
3857	result = 0;
3858	} else if (node->getOp() == glslang::EOpCooperativeMatrixStore ||
3859	node->getOp() == glslang::EOpCooperativeMatrixStoreNV) {
3860	std::vector<spv::IdImmediate> idImmOps;
3861
3862	idImmOps.push_back(x: spv::IdImmediate(true, operands[1])); // buf
3863	idImmOps.push_back(x: spv::IdImmediate(true, operands[0])); // object
3864	if (node->getOp() == glslang::EOpCooperativeMatrixStore) {
3865	idImmOps.push_back(x: spv::IdImmediate(true, operands[3])); // matrixLayout
3866	auto layout = builder.getConstantScalar(resultId: operands[3]);
3867	if (layout == spv::CooperativeMatrixLayoutRowBlockedInterleavedARM ||
3868	layout == spv::CooperativeMatrixLayoutColumnBlockedInterleavedARM) {
3869	builder.addExtension(ext: spv::E_SPV_ARM_cooperative_matrix_layouts);
3870	builder.addCapability(cap: spv::CapabilityCooperativeMatrixLayoutsARM);
3871	}
3872	idImmOps.push_back(x: spv::IdImmediate(true, operands[2])); // stride
3873	} else {
3874	idImmOps.push_back(x: spv::IdImmediate(true, operands[2])); // stride
3875	idImmOps.push_back(x: spv::IdImmediate(true, operands[3])); // colMajor
3876	}
3877	idImmOps.insert(position: idImmOps.end(), first: memoryAccessOperands.begin(), last: memoryAccessOperands.end());
3878
3879	if (node->getOp() == glslang::EOpCooperativeMatrixStore)
3880	builder.createNoResultOp(spv::OpCooperativeMatrixStoreKHR, operands: idImmOps);
3881	else
3882	builder.createNoResultOp(spv::OpCooperativeMatrixStoreNV, operands: idImmOps);
3883	result = 0;
3884	} else if (node->getOp() == glslang::EOpRayQueryGetIntersectionTriangleVertexPositionsEXT) {
3885	std::vector<spv::IdImmediate> idImmOps;
3886
3887	idImmOps.push_back(x: spv::IdImmediate(true, operands[0])); // q
3888	idImmOps.push_back(x: spv::IdImmediate(true, operands[1])); // committed
3889
3890	spv::Id typeId = builder.makeArrayType(element: builder.makeVectorType(component: builder.makeFloatType(width: 32), size: 3),
3891	sizeId: builder.makeUintConstant(u: 3), stride: 0);
3892	// do the op
3893
3894	spv::Op spvOp = spv::OpRayQueryGetIntersectionTriangleVertexPositionsKHR;
3895
3896	spv::Id result = builder.createOp(spvOp, typeId, operands: idImmOps);
3897	// store the result to the pointer (out param 'm')
3898	builder.createStore(rValue: result, lValue: operands[2]);
3899	result = 0;
3900	} else if (node->getOp() == glslang::EOpCooperativeMatrixMulAdd) {
3901	uint32_t matrixOperands = 0;
3902
3903	// If the optional operand is present, initialize matrixOperands to that value.
3904	if (glslangOperands.size() == 4 && glslangOperands[3]->getAsConstantUnion()) {
3905	matrixOperands = glslangOperands[3]->getAsConstantUnion()->getConstArray()[0].getIConst();
3906	}
3907
3908	// Determine Cooperative Matrix Operands bits from the signedness of the types.
3909	if (isTypeSignedInt(type: glslangOperands[0]->getAsTyped()->getBasicType()))
3910	matrixOperands |= spv::CooperativeMatrixOperandsMatrixASignedComponentsKHRMask;
3911	if (isTypeSignedInt(type: glslangOperands[1]->getAsTyped()->getBasicType()))
3912	matrixOperands |= spv::CooperativeMatrixOperandsMatrixBSignedComponentsKHRMask;
3913	if (isTypeSignedInt(type: glslangOperands[2]->getAsTyped()->getBasicType()))
3914	matrixOperands |= spv::CooperativeMatrixOperandsMatrixCSignedComponentsKHRMask;
3915	if (isTypeSignedInt(type: node->getBasicType()))
3916	matrixOperands |= spv::CooperativeMatrixOperandsMatrixResultSignedComponentsKHRMask;
3917
3918	std::vector<spv::IdImmediate> idImmOps;
3919	idImmOps.push_back(x: spv::IdImmediate(true, operands[0]));
3920	idImmOps.push_back(x: spv::IdImmediate(true, operands[1]));
3921	idImmOps.push_back(x: spv::IdImmediate(true, operands[2]));
3922	if (matrixOperands != 0)
3923	idImmOps.push_back(x: spv::IdImmediate(false, matrixOperands));
3924
3925	result = builder.createOp(spv::OpCooperativeMatrixMulAddKHR, typeId: resultType(), operands: idImmOps);
3926	} else if (node->getOp() == glslang::EOpCooperativeMatrixReduceNV) {
3927	builder.addCapability(cap: spv::CapabilityCooperativeMatrixReductionsNV);
3928	builder.addExtension(ext: spv::E_SPV_NV_cooperative_matrix2);
3929
3930	spv::Op opcode = spv::OpCooperativeMatrixReduceNV;
3931	unsigned mask = glslangOperands[2]->getAsConstantUnion()->getConstArray()[0].getUConst();
3932
3933	spv::Id typeId = builder.getContainedTypeId(typeId: builder.getTypeId(resultId: operands[0]));
3934	assert(builder.isCooperativeMatrixType(typeId));
3935
3936	result = builder.createCooperativeMatrixReduce(opcode, typeId, source: operands[1], mask, func: operands[3]);
3937	// store the result to the pointer (out param 'm')
3938	builder.createStore(rValue: result, lValue: operands[0]);
3939	result = 0;
3940	} else if (node->getOp() == glslang::EOpCooperativeMatrixPerElementOpNV) {
3941	builder.addCapability(cap: spv::CapabilityCooperativeMatrixPerElementOperationsNV);
3942	builder.addExtension(ext: spv::E_SPV_NV_cooperative_matrix2);
3943
3944	spv::Id typeId = builder.getContainedTypeId(typeId: builder.getTypeId(resultId: operands[0]));
3945	assert(builder.isCooperativeMatrixType(typeId));
3946
3947	result = builder.createCooperativeMatrixPerElementOp(typeId, operands);
3948	// store the result to the pointer
3949	builder.createStore(rValue: result, lValue: operands[0]);
3950	result = 0;
3951	} else if (node->getOp() == glslang::EOpCooperativeMatrixTransposeNV) {
3952
3953	builder.addCapability(cap: spv::CapabilityCooperativeMatrixConversionsNV);
3954	builder.addExtension(ext: spv::E_SPV_NV_cooperative_matrix2);
3955
3956	spv::Id typeId = builder.getContainedTypeId(typeId: builder.getTypeId(resultId: operands[0]));
3957	assert(builder.isCooperativeMatrixType(typeId));
3958
3959	result = builder.createUnaryOp(spv::OpCooperativeMatrixTransposeNV, typeId, operand: operands[1]);
3960	// store the result to the pointer
3961	builder.createStore(rValue: result, lValue: operands[0]);
3962	result = 0;
3963	} else if (atomic) {
3964	// Handle all atomics
3965	glslang::TBasicType typeProxy = (node->getOp() == glslang::EOpAtomicStore)
3966	? node->getSequence()[0]->getAsTyped()->getBasicType() : node->getBasicType();
3967	result = createAtomicOperation(op: node->getOp(), precision, typeId: resultType(), operands, typeProxy,
3968	lvalueCoherentFlags, opType: node->getType());
3969	} else if (node->getOp() == glslang::EOpSpirvInst) {
3970	const auto& spirvInst = node->getSpirvInstruction();
3971	if (spirvInst.set == "") {
3972	std::vector<spv::IdImmediate> idImmOps;
3973	for (unsigned int i = 0; i < glslangOperands.size(); ++i) {
3974	if (glslangOperands[i]->getAsTyped()->getQualifier().isSpirvLiteral()) {
3975	// Translate the constant to a literal value
3976	std::vector<unsigned> literals;
3977	glslang::TVector<const glslang::TIntermConstantUnion*> constants;
3978	constants.push_back(x: glslangOperands[i]->getAsConstantUnion());
3979	TranslateLiterals(constants, literals);
3980	idImmOps.push_back(x: {false, literals[0]});
3981	} else
3982	idImmOps.push_back(x: {true, operands[i]});
3983	}
3984
3985	if (node->getBasicType() == glslang::EbtVoid)
3986	builder.createNoResultOp(static_cast<spv::Op>(spirvInst.id), operands: idImmOps);
3987	else
3988	result = builder.createOp(static_cast<spv::Op>(spirvInst.id), typeId: resultType(), operands: idImmOps);
3989	} else {
3990	result = builder.createBuiltinCall(
3991	resultType: resultType(), builtins: spirvInst.set == "GLSL.std.450" ? stdBuiltins : getExtBuiltins(name: spirvInst.set.c_str()),
3992	entryPoint: spirvInst.id, args: operands);
3993	}
3994	noReturnValue = node->getBasicType() == glslang::EbtVoid;
3995	} else if (node->getOp() == glslang::EOpDebugPrintf) {
3996	if (!nonSemanticDebugPrintf) {
3997	nonSemanticDebugPrintf = builder.import("NonSemantic.DebugPrintf");
3998	}
3999	result = builder.createBuiltinCall(resultType: builder.makeVoidType(), builtins: nonSemanticDebugPrintf, entryPoint: spv::NonSemanticDebugPrintfDebugPrintf, args: operands);
4000	builder.addExtension(ext: spv::E_SPV_KHR_non_semantic_info);
4001	} else {
4002	// Pass through to generic operations.
4003	switch (glslangOperands.size()) {
4004	case 0:
4005	result = createNoArgOperation(op: node->getOp(), precision, typeId: resultType());
4006	break;
4007	case 1:
4008	{
4009	OpDecorations decorations = { precision,
4010	TranslateNoContractionDecoration(qualifier: node->getType().getQualifier()),
4011	TranslateNonUniformDecoration(qualifier: node->getType().getQualifier()) };
4012	result = createUnaryOperation(
4013	op: node->getOp(), decorations,
4014	typeId: resultType(), operand: operands.front(),
4015	typeProxy: glslangOperands[0]->getAsTyped()->getBasicType(), lvalueCoherentFlags, opType: node->getType());
4016	}
4017	break;
4018	default:
4019	result = createMiscOperation(op: node->getOp(), precision, typeId: resultType(), operands, typeProxy: node->getBasicType());
4020	break;
4021	}
4022
4023	if (invertedType != spv::NoResult)
4024	result = createInvertedSwizzle(precision, *glslangOperands[0]->getAsBinaryNode(), parentResult: result);
4025
4026	for (unsigned int i = 0; i < temporaryLvalues.size(); ++i) {
4027	builder.setAccessChain(complexLvalues[i]);
4028	builder.accessChainStore(rvalue: builder.createLoad(lValue: temporaryLvalues[i], precision: spv::NoPrecision),
4029	nonUniform: TranslateNonUniformDecoration(coherentFlags: complexLvalues[i].coherentFlags));
4030	}
4031	}
4032
4033	if (noReturnValue)
4034	return false;
4035
4036	if (! result) {
4037	logger->missingFunctionality(f: "unknown glslang aggregate");
4038	return true; // pick up a child as a placeholder operand
4039	} else {
4040	builder.clearAccessChain();
4041	builder.setAccessChainRValue(result);
4042	return false;
4043	}
4044	}
4045
4046	// This path handles both if-then-else and ?:
4047	// The if-then-else has a node type of void, while
4048	// ?: has either a void or a non-void node type
4049	//
4050	// Leaving the result, when not void:
4051	// GLSL only has r-values as the result of a :?, but
4052	// if we have an l-value, that can be more efficient if it will
4053	// become the base of a complex r-value expression, because the
4054	// next layer copies r-values into memory to use the access-chain mechanism
4055	bool TGlslangToSpvTraverser::visitSelection(glslang::TVisit /* visit */, glslang::TIntermSelection* node)
4056	{
4057	// see if OpSelect can handle it
4058	const auto isOpSelectable = [&]() {
4059	if (node->getBasicType() == glslang::EbtVoid)
4060	return false;
4061	// OpSelect can do all other types starting with SPV 1.4
4062	if (glslangIntermediate->getSpv().spv < glslang::EShTargetSpv_1_4) {
4063	// pre-1.4, only scalars and vectors can be handled
4064	if ((!node->getType().isScalar() && !node->getType().isVector()))
4065	return false;
4066	}
4067	return true;
4068	};
4069
4070	// See if it simple and safe, or required, to execute both sides.
4071	// Crucially, side effects must be either semantically required or avoided,
4072	// and there are performance trade-offs.
4073	// Return true if required or a good idea (and safe) to execute both sides,
4074	// false otherwise.
4075	const auto bothSidesPolicy = [&]() -> bool {
4076	// do we have both sides?
4077	if (node->getTrueBlock() == nullptr ||
4078	node->getFalseBlock() == nullptr)
4079	return false;
4080
4081	// required? (unless we write additional code to look for side effects
4082	// and make performance trade-offs if none are present)
4083	if (!node->getShortCircuit())
4084	return true;
4085
4086	// if not required to execute both, decide based on performance/practicality...
4087
4088	if (!isOpSelectable())
4089	return false;
4090
4091	assert(node->getType() == node->getTrueBlock() ->getAsTyped()->getType() &&
4092	node->getType() == node->getFalseBlock()->getAsTyped()->getType());
4093
4094	// return true if a single operand to ? : is okay for OpSelect
4095	const auto operandOkay = [](glslang::TIntermTyped* node) {
4096	return node->getAsSymbolNode() || node->getType().getQualifier().isConstant();
4097	};
4098
4099	return operandOkay(node->getTrueBlock() ->getAsTyped()) &&
4100	operandOkay(node->getFalseBlock()->getAsTyped());
4101	};
4102
4103	spv::Id result = spv::NoResult; // upcoming result selecting between trueValue and falseValue
4104	// emit the condition before doing anything with selection
4105	node->getCondition()->traverse(this);
4106	spv::Id condition = accessChainLoad(type: node->getCondition()->getType());
4107
4108	// Find a way of executing both sides and selecting the right result.
4109	const auto executeBothSides = [&]() -> void {
4110	// execute both sides
4111	spv::Id resultType = convertGlslangToSpvType(type: node->getType());
4112	node->getTrueBlock()->traverse(this);
4113	spv::Id trueValue = accessChainLoad(type: node->getTrueBlock()->getAsTyped()->getType());
4114	node->getFalseBlock()->traverse(this);
4115	spv::Id falseValue = accessChainLoad(type: node->getFalseBlock()->getAsTyped()->getType());
4116
4117	builder.setDebugSourceLocation(line: node->getLoc().line, filename: node->getLoc().getFilename());
4118
4119	// done if void
4120	if (node->getBasicType() == glslang::EbtVoid)
4121	return;
4122
4123	// emit code to select between trueValue and falseValue
4124	// see if OpSelect can handle the result type, and that the SPIR-V types
4125	// of the inputs match the result type.
4126	if (isOpSelectable()) {
4127	// Emit OpSelect for this selection.
4128
4129	// smear condition to vector, if necessary (AST is always scalar)
4130	// Before 1.4, smear like for mix(), starting with 1.4, keep it scalar
4131	if (glslangIntermediate->getSpv().spv < glslang::EShTargetSpv_1_4 && builder.isVector(resultId: trueValue)) {
4132	condition = builder.smearScalar(precision: spv::NoPrecision, scalarVal: condition,
4133	vectorType: builder.makeVectorType(component: builder.makeBoolType(),
4134	size: builder.getNumComponents(resultId: trueValue)));
4135	}
4136
4137	// If the types do not match, it is because of mismatched decorations on aggregates.
4138	// Since isOpSelectable only lets us get here for SPIR-V >= 1.4, we can use OpCopyObject
4139	// to get matching types.
4140	if (builder.getTypeId(resultId: trueValue) != resultType) {
4141	trueValue = builder.createUnaryOp(spv::OpCopyLogical, typeId: resultType, operand: trueValue);
4142	}
4143	if (builder.getTypeId(resultId: falseValue) != resultType) {
4144	falseValue = builder.createUnaryOp(spv::OpCopyLogical, typeId: resultType, operand: falseValue);
4145	}
4146
4147	// OpSelect
4148	result = builder.createTriOp(spv::OpSelect, typeId: resultType, operand1: condition, operand2: trueValue, operand3: falseValue);
4149
4150	builder.clearAccessChain();
4151	builder.setAccessChainRValue(result);
4152	} else {
4153	// We need control flow to select the result.
4154	// TODO: Once SPIR-V OpSelect allows arbitrary types, eliminate this path.
4155	result = builder.createVariable(precision: TranslatePrecisionDecoration(type: node->getType()),
4156	storageClass: spv::StorageClassFunction, type: resultType);
4157
4158	// Selection control:
4159	const spv::SelectionControlMask control = TranslateSelectionControl(selectionNode: *node);
4160
4161	// make an "if" based on the value created by the condition
4162	spv::Builder::If ifBuilder(condition, control, builder);
4163
4164	// emit the "then" statement
4165	builder.clearAccessChain();
4166	builder.setAccessChainLValue(result);
4167	multiTypeStore(node->getType(), rValue: trueValue);
4168
4169	ifBuilder.makeBeginElse();
4170	// emit the "else" statement
4171	builder.clearAccessChain();
4172	builder.setAccessChainLValue(result);
4173	multiTypeStore(node->getType(), rValue: falseValue);
4174
4175	// finish off the control flow
4176	ifBuilder.makeEndIf();
4177
4178	builder.clearAccessChain();
4179	builder.setAccessChainLValue(result);
4180	}
4181	};
4182
4183	// Execute the one side needed, as per the condition
4184	const auto executeOneSide = [&]() {
4185	// Always emit control flow.
4186	if (node->getBasicType() != glslang::EbtVoid) {
4187	result = builder.createVariable(precision: TranslatePrecisionDecoration(type: node->getType()), storageClass: spv::StorageClassFunction,
4188	type: convertGlslangToSpvType(type: node->getType()));
4189	}
4190
4191	// Selection control:
4192	const spv::SelectionControlMask control = TranslateSelectionControl(selectionNode: *node);
4193
4194	// make an "if" based on the value created by the condition
4195	spv::Builder::If ifBuilder(condition, control, builder);
4196
4197	// emit the "then" statement
4198	if (node->getTrueBlock() != nullptr) {
4199	node->getTrueBlock()->traverse(this);
4200	if (result != spv::NoResult) {
4201	spv::Id load = accessChainLoad(type: node->getTrueBlock()->getAsTyped()->getType());
4202
4203	builder.clearAccessChain();
4204	builder.setAccessChainLValue(result);
4205	multiTypeStore(node->getType(), rValue: load);
4206	}
4207	}
4208
4209	if (node->getFalseBlock() != nullptr) {
4210	ifBuilder.makeBeginElse();
4211	// emit the "else" statement
4212	node->getFalseBlock()->traverse(this);
4213	if (result != spv::NoResult) {
4214	spv::Id load = accessChainLoad(type: node->getFalseBlock()->getAsTyped()->getType());
4215
4216	builder.clearAccessChain();
4217	builder.setAccessChainLValue(result);
4218	multiTypeStore(node->getType(), rValue: load);
4219	}
4220	}
4221
4222	// finish off the control flow
4223	ifBuilder.makeEndIf();
4224
4225	if (result != spv::NoResult) {
4226	builder.clearAccessChain();
4227	builder.setAccessChainLValue(result);
4228	}
4229	};
4230
4231	// Try for OpSelect (or a requirement to execute both sides)
4232	if (bothSidesPolicy()) {
4233	SpecConstantOpModeGuard spec_constant_op_mode_setter(&builder);
4234	if (node->getType().getQualifier().isSpecConstant())
4235	spec_constant_op_mode_setter.turnOnSpecConstantOpMode();
4236	executeBothSides();
4237	} else
4238	executeOneSide();
4239
4240	return false;
4241	}
4242
4243	bool TGlslangToSpvTraverser::visitSwitch(glslang::TVisit /* visit */, glslang::TIntermSwitch* node)
4244	{
4245	// emit and get the condition before doing anything with switch
4246	node->getCondition()->traverse(this);
4247	spv::Id selector = accessChainLoad(type: node->getCondition()->getAsTyped()->getType());
4248
4249	// Selection control:
4250	const spv::SelectionControlMask control = TranslateSwitchControl(switchNode: *node);
4251
4252	// browse the children to sort out code segments
4253	int defaultSegment = -1;
4254	std::vector<TIntermNode*> codeSegments;
4255	glslang::TIntermSequence& sequence = node->getBody()->getSequence();
4256	std::vector<int> caseValues;
4257	std::vector<int> valueIndexToSegment(sequence.size()); // note: probably not all are used, it is an overestimate
4258	for (glslang::TIntermSequence::iterator c = sequence.begin(); c != sequence.end(); ++c) {
4259	TIntermNode* child = *c;
4260	if (child->getAsBranchNode() && child->getAsBranchNode()->getFlowOp() == glslang::EOpDefault)
4261	defaultSegment = (int)codeSegments.size();
4262	else if (child->getAsBranchNode() && child->getAsBranchNode()->getFlowOp() == glslang::EOpCase) {
4263	valueIndexToSegment[caseValues.size()] = (int)codeSegments.size();
4264	caseValues.push_back(x: child->getAsBranchNode()->getExpression()->getAsConstantUnion()
4265	->getConstArray()[0].getIConst());
4266	} else
4267	codeSegments.push_back(x: child);
4268	}
4269
4270	// handle the case where the last code segment is missing, due to no code
4271	// statements between the last case and the end of the switch statement
4272	if ((caseValues.size() && (int)codeSegments.size() == valueIndexToSegment[caseValues.size() - 1]) ||
4273	(int)codeSegments.size() == defaultSegment)
4274	codeSegments.push_back(x: nullptr);
4275
4276	// make the switch statement
4277	std::vector<spv::Block*> segmentBlocks; // returned, as the blocks allocated in the call
4278	builder.makeSwitch(condition: selector, control, numSegments: (int)codeSegments.size(), caseValues, valueToSegment: valueIndexToSegment, defaultSegment,
4279	segmentBB&: segmentBlocks);
4280
4281	// emit all the code in the segments
4282	breakForLoop.push(x: false);
4283	for (unsigned int s = 0; s < codeSegments.size(); ++s) {
4284	builder.nextSwitchSegment(segmentBB&: segmentBlocks, segment: s);
4285	if (codeSegments[s])
4286	codeSegments[s]->traverse(this);
4287	else
4288	builder.addSwitchBreak(implicit: true);
4289	}
4290	breakForLoop.pop();
4291
4292	builder.endSwitch(segmentBB&: segmentBlocks);
4293
4294	return false;
4295	}
4296
4297	void TGlslangToSpvTraverser::visitConstantUnion(glslang::TIntermConstantUnion* node)
4298	{
4299	if (node->getQualifier().isSpirvLiteral())
4300	return; // Translated to a literal value, skip further processing
4301
4302	int nextConst = 0;
4303	spv::Id constant = createSpvConstantFromConstUnionArray(type: node->getType(), node->getConstArray(), nextConst, specConstant: false);
4304
4305	builder.clearAccessChain();
4306	builder.setAccessChainRValue(constant);
4307	}
4308
4309	bool TGlslangToSpvTraverser::visitLoop(glslang::TVisit /* visit */, glslang::TIntermLoop* node)
4310	{
4311	auto blocks = builder.makeNewLoop();
4312	builder.createBranch(implicit: true, block: &blocks.head);
4313
4314	// Loop control:
4315	std::vector<unsigned int> operands;
4316	const spv::LoopControlMask control = TranslateLoopControl(loopNode: *node, operands);
4317
4318	// Spec requires back edges to target header blocks, and every header block
4319	// must dominate its merge block. Make a header block first to ensure these
4320	// conditions are met. By definition, it will contain OpLoopMerge, followed
4321	// by a block-ending branch. But we don't want to put any other body/test
4322	// instructions in it, since the body/test may have arbitrary instructions,
4323	// including merges of its own.
4324	builder.setBuildPoint(&blocks.head);
4325	builder.setDebugSourceLocation(line: node->getLoc().line, filename: node->getLoc().getFilename());
4326	builder.createLoopMerge(mergeBlock: &blocks.merge, continueBlock: &blocks.continue_target, control, operands);
4327	if (node->testFirst() && node->getTest()) {
4328	spv::Block& test = builder.makeNewBlock();
4329	builder.createBranch(implicit: true, block: &test);
4330
4331	builder.setBuildPoint(&test);
4332	node->getTest()->traverse(this);
4333	spv::Id condition = accessChainLoad(type: node->getTest()->getType());
4334	builder.createConditionalBranch(condition, thenBlock: &blocks.body, elseBlock: &blocks.merge);
4335
4336	builder.setBuildPoint(&blocks.body);
4337	breakForLoop.push(x: true);
4338	if (node->getBody())
4339	node->getBody()->traverse(this);
4340	builder.createBranch(implicit: true, block: &blocks.continue_target);
4341	breakForLoop.pop();
4342
4343	builder.setBuildPoint(&blocks.continue_target);
4344	if (node->getTerminal())
4345	node->getTerminal()->traverse(this);
4346	builder.createBranch(implicit: true, block: &blocks.head);
4347	} else {
4348	builder.setDebugSourceLocation(line: node->getLoc().line, filename: node->getLoc().getFilename());
4349	builder.createBranch(implicit: true, block: &blocks.body);
4350
4351	breakForLoop.push(x: true);
4352	builder.setBuildPoint(&blocks.body);
4353	if (node->getBody())
4354	node->getBody()->traverse(this);
4355	builder.createBranch(implicit: true, block: &blocks.continue_target);
4356	breakForLoop.pop();
4357
4358	builder.setBuildPoint(&blocks.continue_target);
4359	if (node->getTerminal())
4360	node->getTerminal()->traverse(this);
4361	if (node->getTest()) {
4362	node->getTest()->traverse(this);
4363	spv::Id condition =
4364	accessChainLoad(type: node->getTest()->getType());
4365	builder.createConditionalBranch(condition, thenBlock: &blocks.head, elseBlock: &blocks.merge);
4366	} else {
4367	// TODO: unless there was a break/return/discard instruction
4368	// somewhere in the body, this is an infinite loop, so we should
4369	// issue a warning.
4370	builder.createBranch(implicit: true, block: &blocks.head);
4371	}
4372	}
4373	builder.setBuildPoint(&blocks.merge);
4374	builder.closeLoop();
4375	return false;
4376	}
4377
4378	bool TGlslangToSpvTraverser::visitBranch(glslang::TVisit /* visit */, glslang::TIntermBranch* node)
4379	{
4380	if (node->getExpression())
4381	node->getExpression()->traverse(this);
4382
4383	builder.setDebugSourceLocation(line: node->getLoc().line, filename: node->getLoc().getFilename());
4384
4385	switch (node->getFlowOp()) {
4386	case glslang::EOpKill:
4387	if (glslangIntermediate->getSpv().spv >= glslang::EShTargetSpv_1_6) {
4388	if (glslangIntermediate->getSource() == glslang::EShSourceHlsl) {
4389	builder.addCapability(cap: spv::CapabilityDemoteToHelperInvocation);
4390	builder.createNoResultOp(spv::OpDemoteToHelperInvocationEXT);
4391	} else {
4392	builder.makeStatementTerminator(opcode: spv::OpTerminateInvocation, name: "post-terminate-invocation");
4393	}
4394	} else {
4395	builder.makeStatementTerminator(opcode: spv::OpKill, name: "post-discard");
4396	}
4397	break;
4398	case glslang::EOpTerminateInvocation:
4399	builder.addExtension(ext: spv::E_SPV_KHR_terminate_invocation);
4400	builder.makeStatementTerminator(opcode: spv::OpTerminateInvocation, name: "post-terminate-invocation");
4401	break;
4402	case glslang::EOpBreak:
4403	if (breakForLoop.top())
4404	builder.createLoopExit();
4405	else
4406	builder.addSwitchBreak(implicit: false);
4407	break;
4408	case glslang::EOpContinue:
4409	builder.createLoopContinue();
4410	break;
4411	case glslang::EOpReturn:
4412	if (node->getExpression() != nullptr) {
4413	const glslang::TType& glslangReturnType = node->getExpression()->getType();
4414	spv::Id returnId = accessChainLoad(type: glslangReturnType);
4415	if (builder.getTypeId(resultId: returnId) != currentFunction->getReturnType() ||
4416	TranslatePrecisionDecoration(type: glslangReturnType) != currentFunction->getReturnPrecision()) {
4417	builder.clearAccessChain();
4418	spv::Id copyId = builder.createVariable(precision: currentFunction->getReturnPrecision(),
4419	storageClass: spv::StorageClassFunction, type: currentFunction->getReturnType());
4420	builder.setAccessChainLValue(copyId);
4421	multiTypeStore(glslangReturnType, rValue: returnId);
4422	returnId = builder.createLoad(lValue: copyId, precision: currentFunction->getReturnPrecision());
4423	}
4424	builder.makeReturn(implicit: false, retVal: returnId);
4425	} else
4426	builder.makeReturn(implicit: false);
4427
4428	builder.clearAccessChain();
4429	break;
4430
4431	case glslang::EOpDemote:
4432	builder.createNoResultOp(spv::OpDemoteToHelperInvocationEXT);
4433	builder.addExtension(ext: spv::E_SPV_EXT_demote_to_helper_invocation);
4434	builder.addCapability(cap: spv::CapabilityDemoteToHelperInvocationEXT);
4435	break;
4436	case glslang::EOpTerminateRayKHR:
4437	builder.makeStatementTerminator(opcode: spv::OpTerminateRayKHR, name: "post-terminateRayKHR");
4438	break;
4439	case glslang::EOpIgnoreIntersectionKHR:
4440	builder.makeStatementTerminator(opcode: spv::OpIgnoreIntersectionKHR, name: "post-ignoreIntersectionKHR");
4441	break;
4442
4443	default:
4444	assert(0);
4445	break;
4446	}
4447
4448	return false;
4449	}
4450
4451	spv::Id TGlslangToSpvTraverser::createSpvVariable(const glslang::TIntermSymbol* node, spv::Id forcedType)
4452	{
4453	// First, steer off constants, which are not SPIR-V variables, but
4454	// can still have a mapping to a SPIR-V Id.
4455	// This includes specialization constants.
4456	if (node->getQualifier().isConstant()) {
4457	spv::Id result = createSpvConstant(*node);
4458	if (result != spv::NoResult)
4459	return result;
4460	}
4461
4462	// Now, handle actual variables
4463	spv::StorageClass storageClass = TranslateStorageClass(type: node->getType());
4464	spv::Id spvType = forcedType == spv::NoType ? convertGlslangToSpvType(type: node->getType())
4465	: forcedType;
4466
4467	const bool contains16BitType = node->getType().contains16BitFloat() ||
4468	node->getType().contains16BitInt();
4469	if (contains16BitType) {
4470	switch (storageClass) {
4471	case spv::StorageClassInput:
4472	case spv::StorageClassOutput:
4473	builder.addIncorporatedExtension(ext: spv::E_SPV_KHR_16bit_storage, incorporatedVersion: spv::Spv_1_3);
4474	builder.addCapability(cap: spv::CapabilityStorageInputOutput16);
4475	break;
4476	case spv::StorageClassUniform:
4477	builder.addIncorporatedExtension(ext: spv::E_SPV_KHR_16bit_storage, incorporatedVersion: spv::Spv_1_3);
4478	if (node->getType().getQualifier().storage == glslang::EvqBuffer)
4479	builder.addCapability(cap: spv::CapabilityStorageUniformBufferBlock16);
4480	else
4481	builder.addCapability(cap: spv::CapabilityStorageUniform16);
4482	break;
4483	case spv::StorageClassPushConstant:
4484	builder.addIncorporatedExtension(ext: spv::E_SPV_KHR_16bit_storage, incorporatedVersion: spv::Spv_1_3);
4485	builder.addCapability(cap: spv::CapabilityStoragePushConstant16);
4486	break;
4487	case spv::StorageClassStorageBuffer:
4488	case spv::StorageClassPhysicalStorageBufferEXT:
4489	builder.addIncorporatedExtension(ext: spv::E_SPV_KHR_16bit_storage, incorporatedVersion: spv::Spv_1_3);
4490	builder.addCapability(cap: spv::CapabilityStorageUniformBufferBlock16);
4491	break;
4492	default:
4493	if (storageClass == spv::StorageClassWorkgroup &&
4494	node->getType().getBasicType() == glslang::EbtBlock) {
4495	builder.addCapability(cap: spv::CapabilityWorkgroupMemoryExplicitLayout16BitAccessKHR);
4496	break;
4497	}
4498	if (node->getType().contains16BitFloat())
4499	builder.addCapability(cap: spv::CapabilityFloat16);
4500	if (node->getType().contains16BitInt())
4501	builder.addCapability(cap: spv::CapabilityInt16);
4502	break;
4503	}
4504	}
4505
4506	if (node->getType().contains8BitInt()) {
4507	if (storageClass == spv::StorageClassPushConstant) {
4508	builder.addIncorporatedExtension(ext: spv::E_SPV_KHR_8bit_storage, incorporatedVersion: spv::Spv_1_5);
4509	builder.addCapability(cap: spv::CapabilityStoragePushConstant8);
4510	} else if (storageClass == spv::StorageClassUniform) {
4511	builder.addIncorporatedExtension(ext: spv::E_SPV_KHR_8bit_storage, incorporatedVersion: spv::Spv_1_5);
4512	builder.addCapability(cap: spv::CapabilityUniformAndStorageBuffer8BitAccess);
4513	} else if (storageClass == spv::StorageClassStorageBuffer) {
4514	builder.addIncorporatedExtension(ext: spv::E_SPV_KHR_8bit_storage, incorporatedVersion: spv::Spv_1_5);
4515	builder.addCapability(cap: spv::CapabilityStorageBuffer8BitAccess);
4516	} else if (storageClass == spv::StorageClassWorkgroup &&
4517	node->getType().getBasicType() == glslang::EbtBlock) {
4518	builder.addCapability(cap: spv::CapabilityWorkgroupMemoryExplicitLayout8BitAccessKHR);
4519	} else {
4520	builder.addCapability(cap: spv::CapabilityInt8);
4521	}
4522	}
4523
4524	const char* name = node->getName().c_str();
4525	if (glslang::IsAnonymous(name))
4526	name = "";
4527
4528	spv::Id initializer = spv::NoResult;
4529
4530	if (node->getType().getQualifier().storage == glslang::EvqUniform && !node->getConstArray().empty()) {
4531	int nextConst = 0;
4532	initializer = createSpvConstantFromConstUnionArray(type: node->getType(),
4533	node->getConstArray(),
4534	nextConst,
4535	specConstant: false /* specConst */);
4536	} else if (node->getType().getQualifier().isNullInit()) {
4537	initializer = builder.makeNullConstant(typeId: spvType);
4538	}
4539
4540	spv::Id var = builder.createVariable(precision: spv::NoPrecision, storageClass, type: spvType, name, initializer, compilerGenerated: false);
4541	std::vector<spv::Decoration> topLevelDecorations;
4542	glslang::TQualifier typeQualifier = node->getType().getQualifier();
4543	TranslateMemoryDecoration(qualifier: typeQualifier, memory&: topLevelDecorations, useVulkanMemoryModel: glslangIntermediate->usingVulkanMemoryModel());
4544	for (auto deco : topLevelDecorations) {
4545	builder.addDecoration(var, deco);
4546	}
4547	return var;
4548	}
4549
4550	// Return type Id of the sampled type.
4551	spv::Id TGlslangToSpvTraverser::getSampledType(const glslang::TSampler& sampler)
4552	{
4553	switch (sampler.type) {
4554	case glslang::EbtInt: return builder.makeIntType(width: 32);
4555	case glslang::EbtUint: return builder.makeUintType(width: 32);
4556	case glslang::EbtFloat: return builder.makeFloatType(width: 32);
4557	case glslang::EbtFloat16:
4558	builder.addExtension(ext: spv::E_SPV_AMD_gpu_shader_half_float_fetch);
4559	builder.addCapability(cap: spv::CapabilityFloat16ImageAMD);
4560	return builder.makeFloatType(width: 16);
4561	case glslang::EbtInt64:
4562	builder.addExtension(ext: spv::E_SPV_EXT_shader_image_int64);
4563	builder.addCapability(cap: spv::CapabilityInt64ImageEXT);
4564	return builder.makeIntType(width: 64);
4565	case glslang::EbtUint64:
4566	builder.addExtension(ext: spv::E_SPV_EXT_shader_image_int64);
4567	builder.addCapability(cap: spv::CapabilityInt64ImageEXT);
4568	return builder.makeUintType(width: 64);
4569	default:
4570	assert(0);
4571	return builder.makeFloatType(width: 32);
4572	}
4573	}
4574
4575	// If node is a swizzle operation, return the type that should be used if
4576	// the swizzle base is first consumed by another operation, before the swizzle
4577	// is applied.
4578	spv::Id TGlslangToSpvTraverser::getInvertedSwizzleType(const glslang::TIntermTyped& node)
4579	{
4580	if (node.getAsOperator() &&
4581	node.getAsOperator()->getOp() == glslang::EOpVectorSwizzle)
4582	return convertGlslangToSpvType(type: node.getAsBinaryNode()->getLeft()->getType());
4583	else
4584	return spv::NoType;
4585	}
4586
4587	// When inverting a swizzle with a parent op, this function
4588	// will apply the swizzle operation to a completed parent operation.
4589	spv::Id TGlslangToSpvTraverser::createInvertedSwizzle(spv::Decoration precision, const glslang::TIntermTyped& node,
4590	spv::Id parentResult)
4591	{
4592	std::vector<unsigned> swizzle;
4593	convertSwizzle(*node.getAsBinaryNode()->getRight()->getAsAggregate(), swizzle);
4594	return builder.createRvalueSwizzle(precision, typeId: convertGlslangToSpvType(type: node.getType()), source: parentResult, channels: swizzle);
4595	}
4596
4597	// Convert a glslang AST swizzle node to a swizzle vector for building SPIR-V.
4598	void TGlslangToSpvTraverser::convertSwizzle(const glslang::TIntermAggregate& node, std::vector<unsigned>& swizzle)
4599	{
4600	const glslang::TIntermSequence& swizzleSequence = node.getSequence();
4601	for (int i = 0; i < (int)swizzleSequence.size(); ++i)
4602	swizzle.push_back(x: swizzleSequence[i]->getAsConstantUnion()->getConstArray()[0].getIConst());
4603	}
4604
4605	// Convert from a glslang type to an SPV type, by calling into a
4606	// recursive version of this function. This establishes the inherited
4607	// layout state rooted from the top-level type.
4608	spv::Id TGlslangToSpvTraverser::convertGlslangToSpvType(const glslang::TType& type, bool forwardReferenceOnly)
4609	{
4610	return convertGlslangToSpvType(type, getExplicitLayout(type), type.getQualifier(), lastBufferBlockMember: false, forwardReferenceOnly);
4611	}
4612
4613	spv::LinkageType TGlslangToSpvTraverser::convertGlslangLinkageToSpv(glslang::TLinkType linkType)
4614	{
4615	switch (linkType) {
4616	case glslang::ELinkExport:
4617	return spv::LinkageTypeExport;
4618	default:
4619	return spv::LinkageTypeMax;
4620	}
4621	}
4622
4623	// Do full recursive conversion of an arbitrary glslang type to a SPIR-V Id.
4624	// explicitLayout can be kept the same throughout the hierarchical recursive walk.
4625	// Mutually recursive with convertGlslangStructToSpvType().
4626	spv::Id TGlslangToSpvTraverser::convertGlslangToSpvType(const glslang::TType& type,
4627	glslang::TLayoutPacking explicitLayout, const glslang::TQualifier& qualifier,
4628	bool lastBufferBlockMember, bool forwardReferenceOnly)
4629	{
4630	spv::Id spvType = spv::NoResult;
4631
4632	switch (type.getBasicType()) {
4633	case glslang::EbtVoid:
4634	spvType = builder.makeVoidType();
4635	assert (! type.isArray());
4636	break;
4637	case glslang::EbtBool:
4638	// "transparent" bool doesn't exist in SPIR-V. The GLSL convention is
4639	// a 32-bit int where non-0 means true.
4640	if (explicitLayout != glslang::ElpNone)
4641	spvType = builder.makeUintType(width: 32);
4642	else
4643	spvType = builder.makeBoolType();
4644	break;
4645	case glslang::EbtInt:
4646	spvType = builder.makeIntType(width: 32);
4647	break;
4648	case glslang::EbtUint:
4649	spvType = builder.makeUintType(width: 32);
4650	break;
4651	case glslang::EbtFloat:
4652	spvType = builder.makeFloatType(width: 32);
4653	break;
4654	case glslang::EbtDouble:
4655	spvType = builder.makeFloatType(width: 64);
4656	break;
4657	case glslang::EbtFloat16:
4658	spvType = builder.makeFloatType(width: 16);
4659	break;
4660	case glslang::EbtInt8:
4661	spvType = builder.makeIntType(width: 8);
4662	break;
4663	case glslang::EbtUint8:
4664	spvType = builder.makeUintType(width: 8);
4665	break;
4666	case glslang::EbtInt16:
4667	spvType = builder.makeIntType(width: 16);
4668	break;
4669	case glslang::EbtUint16:
4670	spvType = builder.makeUintType(width: 16);
4671	break;
4672	case glslang::EbtInt64:
4673	spvType = builder.makeIntType(width: 64);
4674	break;
4675	case glslang::EbtUint64:
4676	spvType = builder.makeUintType(width: 64);
4677	break;
4678	case glslang::EbtAtomicUint:
4679	builder.addCapability(cap: spv::CapabilityAtomicStorage);
4680	spvType = builder.makeUintType(width: 32);
4681	break;
4682	case glslang::EbtAccStruct:
4683	switch (glslangIntermediate->getStage()) {
4684	case EShLangRayGen:
4685	case EShLangIntersect:
4686	case EShLangAnyHit:
4687	case EShLangClosestHit:
4688	case EShLangMiss:
4689	case EShLangCallable:
4690	// these all should have the RayTracingNV/KHR capability already
4691	break;
4692	default:
4693	{
4694	auto& extensions = glslangIntermediate->getRequestedExtensions();
4695	if (extensions.find(x: "GL_EXT_ray_query") != extensions.end()) {
4696	builder.addExtension(ext: spv::E_SPV_KHR_ray_query);
4697	builder.addCapability(cap: spv::CapabilityRayQueryKHR);
4698	}
4699	}
4700	break;
4701	}
4702	spvType = builder.makeAccelerationStructureType();
4703	break;
4704	case glslang::EbtRayQuery:
4705	{
4706	auto& extensions = glslangIntermediate->getRequestedExtensions();
4707	if (extensions.find(x: "GL_EXT_ray_query") != extensions.end()) {
4708	builder.addExtension(ext: spv::E_SPV_KHR_ray_query);
4709	builder.addCapability(cap: spv::CapabilityRayQueryKHR);
4710	}
4711	spvType = builder.makeRayQueryType();
4712	}
4713	break;
4714	case glslang::EbtReference:
4715	{
4716	// Make the forward pointer, then recurse to convert the structure type, then
4717	// patch up the forward pointer with a real pointer type.
4718	if (forwardPointers.find(x: type.getReferentType()) == forwardPointers.end()) {
4719	spv::Id forwardId = builder.makeForwardPointer(spv::StorageClassPhysicalStorageBufferEXT);
4720	forwardPointers[type.getReferentType()] = forwardId;
4721	}
4722	spvType = forwardPointers[type.getReferentType()];
4723	if (!forwardReferenceOnly) {
4724	spv::Id referentType = convertGlslangToSpvType(type: *type.getReferentType());
4725	builder.makePointerFromForwardPointer(spv::StorageClassPhysicalStorageBufferEXT,
4726	forwardPointerType: forwardPointers[type.getReferentType()],
4727	pointee: referentType);
4728	}
4729	}
4730	break;
4731	case glslang::EbtSampler:
4732	{
4733	const glslang::TSampler& sampler = type.getSampler();
4734	if (sampler.isPureSampler()) {
4735	spvType = builder.makeSamplerType();
4736	} else {
4737	// an image is present, make its type
4738	spvType = builder.makeImageType(sampledType: getSampledType(sampler), TranslateDimensionality(sampler),
4739	depth: sampler.isShadow(), arrayed: sampler.isArrayed(), ms: sampler.isMultiSample(),
4740	sampled: sampler.isImageClass() ? 2 : 1, format: TranslateImageFormat(type));
4741	if (sampler.isCombined() &&
4742	(!sampler.isBuffer() || glslangIntermediate->getSpv().spv < glslang::EShTargetSpv_1_6)) {
4743	// Already has both image and sampler, make the combined type. Only combine sampler to
4744	// buffer if before SPIR-V 1.6.
4745	spvType = builder.makeSampledImageType(imageType: spvType);
4746	}
4747	}
4748	}
4749	break;
4750	case glslang::EbtStruct:
4751	case glslang::EbtBlock:
4752	{
4753	// If we've seen this struct type, return it
4754	const glslang::TTypeList* glslangMembers = type.getStruct();
4755
4756	// Try to share structs for different layouts, but not yet for other
4757	// kinds of qualification (primarily not yet including interpolant qualification).
4758	if (! HasNonLayoutQualifiers(type, qualifier))
4759	spvType = structMap[explicitLayout][qualifier.layoutMatrix][glslangMembers];
4760	if (spvType != spv::NoResult)
4761	break;
4762
4763	// else, we haven't seen it...
4764	if (type.getBasicType() == glslang::EbtBlock)
4765	memberRemapper[glslangTypeToIdMap[glslangMembers]].resize(new_size: glslangMembers->size());
4766	spvType = convertGlslangStructToSpvType(type, glslangStruct: glslangMembers, explicitLayout, qualifier);
4767	}
4768	break;
4769	case glslang::EbtString:
4770	// no type used for OpString
4771	return 0;
4772
4773	case glslang::EbtHitObjectNV: {
4774	builder.addExtension(ext: spv::E_SPV_NV_shader_invocation_reorder);
4775	builder.addCapability(cap: spv::CapabilityShaderInvocationReorderNV);
4776	spvType = builder.makeHitObjectNVType();
4777	}
4778	break;
4779	case glslang::EbtSpirvType: {
4780	// GL_EXT_spirv_intrinsics
4781	const auto& spirvType = type.getSpirvType();
4782	const auto& spirvInst = spirvType.spirvInst;
4783
4784	std::vector<spv::IdImmediate> operands;
4785	for (const auto& typeParam : spirvType.typeParams) {
4786	if (typeParam.getAsConstant() != nullptr) {
4787	// Constant expression
4788	auto constant = typeParam.getAsConstant();
4789	if (constant->isLiteral()) {
4790	if (constant->getBasicType() == glslang::EbtFloat) {
4791	float floatValue = static_cast<float>(constant->getConstArray()[0].getDConst());
4792	unsigned literal;
4793	static_assert(sizeof(literal) == sizeof(floatValue), "sizeof(unsigned) != sizeof(float)");
4794	memcpy(dest: &literal, src: &floatValue, n: sizeof(literal));
4795	operands.push_back(x: {false, literal});
4796	} else if (constant->getBasicType() == glslang::EbtInt) {
4797	unsigned literal = constant->getConstArray()[0].getIConst();
4798	operands.push_back(x: {false, literal});
4799	} else if (constant->getBasicType() == glslang::EbtUint) {
4800	unsigned literal = constant->getConstArray()[0].getUConst();
4801	operands.push_back(x: {false, literal});
4802	} else if (constant->getBasicType() == glslang::EbtBool) {
4803	unsigned literal = constant->getConstArray()[0].getBConst();
4804	operands.push_back(x: {false, literal});
4805	} else if (constant->getBasicType() == glslang::EbtString) {
4806	auto str = constant->getConstArray()[0].getSConst()->c_str();
4807	unsigned literal = 0;
4808	char* literalPtr = reinterpret_cast<char*>(&literal);
4809	unsigned charCount = 0;
4810	char ch = 0;
4811	do {
4812	ch = *(str++);
4813	*(literalPtr++) = ch;
4814	++charCount;
4815	if (charCount == 4) {
4816	operands.push_back(x: {false, literal});
4817	literalPtr = reinterpret_cast<char*>(&literal);
4818	charCount = 0;
4819	}
4820	} while (ch != 0);
4821
4822	// Partial literal is padded with 0
4823	if (charCount > 0) {
4824	for (; charCount < 4; ++charCount)
4825	*(literalPtr++) = 0;
4826	operands.push_back(x: {false, literal});
4827	}
4828	} else
4829	assert(0); // Unexpected type
4830	} else
4831	operands.push_back(x: {true, createSpvConstant(*constant)});
4832	} else {
4833	// Type specifier
4834	assert(typeParam.getAsType() != nullptr);
4835	operands.push_back(x: {true, convertGlslangToSpvType(type: *typeParam.getAsType())});
4836	}
4837	}
4838
4839	assert(spirvInst.set == ""); // Currently, couldn't be extended instructions.
4840	spvType = builder.makeGenericType(opcode: static_cast<spv::Op>(spirvInst.id), operands);
4841
4842	break;
4843	}
4844	case glslang::EbtTensorLayoutNV:
4845	{
4846	builder.addCapability(cap: spv::CapabilityTensorAddressingNV);
4847	builder.addExtension(ext: spv::E_SPV_NV_tensor_addressing);
4848
4849	std::vector<spv::IdImmediate> operands;
4850	for (uint32_t i = 0; i < 2; ++i) {
4851	operands.push_back(x: {true, makeArraySizeId(*type.getTypeParameters()->arraySizes, dim: i, allowZero: true)});
4852	}
4853	spvType = builder.makeGenericType(opcode: spv::OpTypeTensorLayoutNV, operands);
4854	break;
4855	}
4856	case glslang::EbtTensorViewNV:
4857	{
4858	builder.addCapability(cap: spv::CapabilityTensorAddressingNV);
4859	builder.addExtension(ext: spv::E_SPV_NV_tensor_addressing);
4860
4861	uint32_t dim = type.getTypeParameters()->arraySizes->getDimSize(dim: 0);
4862	assert(dim >= 1 && dim <= 5);
4863	std::vector<spv::IdImmediate> operands;
4864	for (uint32_t i = 0; i < dim + 2; ++i) {
4865	operands.push_back(x: {true, makeArraySizeId(*type.getTypeParameters()->arraySizes, dim: i, allowZero: true, boolType: i==1)});
4866	}
4867	spvType = builder.makeGenericType(opcode: spv::OpTypeTensorViewNV, operands);
4868	break;
4869	}
4870	default:
4871	assert(0);
4872	break;
4873	}
4874
4875	if (type.isMatrix())
4876	spvType = builder.makeMatrixType(component: spvType, cols: type.getMatrixCols(), rows: type.getMatrixRows());
4877	else {
4878	// If this variable has a vector element count greater than 1, create a SPIR-V vector
4879	if (type.getVectorSize() > 1)
4880	spvType = builder.makeVectorType(component: spvType, size: type.getVectorSize());
4881	}
4882
4883	if (type.isCoopMatNV()) {
4884	builder.addCapability(cap: spv::CapabilityCooperativeMatrixNV);
4885	builder.addExtension(ext: spv::E_SPV_NV_cooperative_matrix);
4886
4887	if (type.getBasicType() == glslang::EbtFloat16)
4888	builder.addCapability(cap: spv::CapabilityFloat16);
4889	if (type.getBasicType() == glslang::EbtUint8 ||
4890	type.getBasicType() == glslang::EbtInt8) {
4891	builder.addCapability(cap: spv::CapabilityInt8);
4892	}
4893
4894	spv::Id scope = makeArraySizeId(*type.getTypeParameters()->arraySizes, dim: 1);
4895	spv::Id rows = makeArraySizeId(*type.getTypeParameters()->arraySizes, dim: 2);
4896	spv::Id cols = makeArraySizeId(*type.getTypeParameters()->arraySizes, dim: 3);
4897
4898	spvType = builder.makeCooperativeMatrixTypeNV(component: spvType, scope, rows, cols);
4899	}
4900
4901	if (type.isCoopMatKHR()) {
4902	builder.addCapability(cap: spv::CapabilityCooperativeMatrixKHR);
4903	builder.addExtension(ext: spv::E_SPV_KHR_cooperative_matrix);
4904
4905	if (type.getBasicType() == glslang::EbtFloat16)
4906	builder.addCapability(cap: spv::CapabilityFloat16);
4907	if (type.getBasicType() == glslang::EbtUint8 || type.getBasicType() == glslang::EbtInt8) {
4908	builder.addCapability(cap: spv::CapabilityInt8);
4909	}
4910
4911	spv::Id scope = makeArraySizeId(*type.getTypeParameters()->arraySizes, dim: 0);
4912	spv::Id rows = makeArraySizeId(*type.getTypeParameters()->arraySizes, dim: 1);
4913	spv::Id cols = makeArraySizeId(*type.getTypeParameters()->arraySizes, dim: 2);
4914	spv::Id use = builder.makeUintConstant(u: type.getCoopMatKHRuse());
4915
4916	spvType = builder.makeCooperativeMatrixTypeKHR(component: spvType, scope, rows, cols, use);
4917	}
4918
4919	if (type.isArray()) {
4920	int stride = 0; // keep this 0 unless doing an explicit layout; 0 will mean no decoration, no stride
4921
4922	// Do all but the outer dimension
4923	if (type.getArraySizes()->getNumDims() > 1) {
4924	// We need to decorate array strides for types needing explicit layout, except blocks.
4925	if (explicitLayout != glslang::ElpNone && type.getBasicType() != glslang::EbtBlock) {
4926	// Use a dummy glslang type for querying internal strides of
4927	// arrays of arrays, but using just a one-dimensional array.
4928	glslang::TType simpleArrayType(type, 0); // deference type of the array
4929	while (simpleArrayType.getArraySizes()->getNumDims() > 1)
4930	simpleArrayType.getArraySizes()->dereference();
4931
4932	// Will compute the higher-order strides here, rather than making a whole
4933	// pile of types and doing repetitive recursion on their contents.
4934	stride = getArrayStride(arrayType: simpleArrayType, explicitLayout, qualifier.layoutMatrix);
4935	}
4936
4937	// make the arrays
4938	for (int dim = type.getArraySizes()->getNumDims() - 1; dim > 0; --dim) {
4939	spvType = builder.makeArrayType(element: spvType, sizeId: makeArraySizeId(*type.getArraySizes(), dim), stride);
4940	if (stride > 0)
4941	builder.addDecoration(spvType, spv::DecorationArrayStride, num: stride);
4942	stride *= type.getArraySizes()->getDimSize(dim);
4943	}
4944	} else {
4945	// single-dimensional array, and don't yet have stride
4946
4947	// We need to decorate array strides for types needing explicit layout, except blocks.
4948	if (explicitLayout != glslang::ElpNone && type.getBasicType() != glslang::EbtBlock)
4949	stride = getArrayStride(arrayType: type, explicitLayout, qualifier.layoutMatrix);
4950	}
4951
4952	// Do the outer dimension, which might not be known for a runtime-sized array.
4953	// (Unsized arrays that survive through linking will be runtime-sized arrays)
4954	if (type.isSizedArray())
4955	spvType = builder.makeArrayType(element: spvType, sizeId: makeArraySizeId(*type.getArraySizes(), dim: 0), stride);
4956	else {
4957	if (!lastBufferBlockMember) {
4958	builder.addIncorporatedExtension(ext: "SPV_EXT_descriptor_indexing", incorporatedVersion: spv::Spv_1_5);
4959	builder.addCapability(cap: spv::CapabilityRuntimeDescriptorArrayEXT);
4960	}
4961	spvType = builder.makeRuntimeArray(element: spvType);
4962	}
4963	if (stride > 0)
4964	builder.addDecoration(spvType, spv::DecorationArrayStride, num: stride);
4965	}
4966
4967	return spvType;
4968	}
4969
4970	// Apply SPIR-V decorations to the SPIR-V object (provided by SPIR-V ID). If member index is provided, the
4971	// decorations are applied to this member.
4972	void TGlslangToSpvTraverser::applySpirvDecorate(const glslang::TType& type, spv::Id id, std::optional<int> member)
4973	{
4974	assert(type.getQualifier().hasSpirvDecorate());
4975
4976	const glslang::TSpirvDecorate& spirvDecorate = type.getQualifier().getSpirvDecorate();
4977
4978	// Add spirv_decorate
4979	for (auto& decorate : spirvDecorate.decorates) {
4980	if (!decorate.second.empty()) {
4981	std::vector<unsigned> literals;
4982	TranslateLiterals(constants: decorate.second, literals);
4983	if (member.has_value())
4984	builder.addMemberDecoration(id, member: *member, static_cast<spv::Decoration>(decorate.first), literals);
4985	else
4986	builder.addDecoration(id, static_cast<spv::Decoration>(decorate.first), literals);
4987	} else {
4988	if (member.has_value())
4989	builder.addMemberDecoration(id, member: *member, static_cast<spv::Decoration>(decorate.first));
4990	else
4991	builder.addDecoration(id, static_cast<spv::Decoration>(decorate.first));
4992	}
4993	}
4994
4995	// Add spirv_decorate_id
4996	if (member.has_value()) {
4997	// spirv_decorate_id not applied to members
4998	assert(spirvDecorate.decorateIds.empty());
4999	} else {
5000	for (auto& decorateId : spirvDecorate.decorateIds) {
5001	std::vector<spv::Id> operandIds;
5002	assert(!decorateId.second.empty());
5003	for (auto extraOperand : decorateId.second) {
5004	if (extraOperand->getQualifier().isFrontEndConstant())
5005	operandIds.push_back(x: createSpvConstant(*extraOperand));
5006	else
5007	operandIds.push_back(x: getSymbolId(node: extraOperand->getAsSymbolNode()));
5008	}
5009	builder.addDecorationId(id, static_cast<spv::Decoration>(decorateId.first), operandIds);
5010	}
5011	}
5012
5013	// Add spirv_decorate_string
5014	for (auto& decorateString : spirvDecorate.decorateStrings) {
5015	std::vector<const char*> strings;
5016	assert(!decorateString.second.empty());
5017	for (auto extraOperand : decorateString.second) {
5018	const char* string = extraOperand->getConstArray()[0].getSConst()->c_str();
5019	strings.push_back(x: string);
5020	}
5021	if (member.has_value())
5022	builder.addMemberDecoration(id, member: *member, static_cast<spv::Decoration>(decorateString.first), strings);
5023	else
5024	builder.addDecoration(id, static_cast<spv::Decoration>(decorateString.first), strings);
5025	}
5026	}
5027
5028	// TODO: this functionality should exist at a higher level, in creating the AST
5029	//
5030	// Identify interface members that don't have their required extension turned on.
5031	//
5032	bool TGlslangToSpvTraverser::filterMember(const glslang::TType& member)
5033	{
5034	auto& extensions = glslangIntermediate->getRequestedExtensions();
5035
5036	if (member.getFieldName() == "gl_SecondaryViewportMaskNV" &&
5037	extensions.find(x: "GL_NV_stereo_view_rendering") == extensions.end())
5038	return true;
5039	if (member.getFieldName() == "gl_SecondaryPositionNV" &&
5040	extensions.find(x: "GL_NV_stereo_view_rendering") == extensions.end())
5041	return true;
5042
5043	if (glslangIntermediate->getStage() == EShLangMesh) {
5044	if (member.getFieldName() == "gl_PrimitiveShadingRateEXT" &&
5045	extensions.find(x: "GL_EXT_fragment_shading_rate") == extensions.end())
5046	return true;
5047	}
5048
5049	if (glslangIntermediate->getStage() != EShLangMesh) {
5050	if (member.getFieldName() == "gl_ViewportMask" &&
5051	extensions.find(x: "GL_NV_viewport_array2") == extensions.end())
5052	return true;
5053	if (member.getFieldName() == "gl_PositionPerViewNV" &&
5054	extensions.find(x: "GL_NVX_multiview_per_view_attributes") == extensions.end())
5055	return true;
5056	if (member.getFieldName() == "gl_ViewportMaskPerViewNV" &&
5057	extensions.find(x: "GL_NVX_multiview_per_view_attributes") == extensions.end())
5058	return true;
5059	}
5060
5061	return false;
5062	}
5063
5064	// Do full recursive conversion of a glslang structure (or block) type to a SPIR-V Id.
5065	// explicitLayout can be kept the same throughout the hierarchical recursive walk.
5066	// Mutually recursive with convertGlslangToSpvType().
5067	spv::Id TGlslangToSpvTraverser::convertGlslangStructToSpvType(const glslang::TType& type,
5068	const glslang::TTypeList* glslangMembers,
5069	glslang::TLayoutPacking explicitLayout,
5070	const glslang::TQualifier& qualifier)
5071	{
5072	// Create a vector of struct types for SPIR-V to consume
5073	std::vector<spv::Id> spvMembers;
5074	int memberDelta = 0; // how much the member's index changes from glslang to SPIR-V, normally 0,
5075	// except sometimes for blocks
5076	std::vector<std::pair<glslang::TType*, glslang::TQualifier> > deferredForwardPointers;
5077	for (int i = 0; i < (int)glslangMembers->size(); i++) {
5078	auto& glslangMember = (*glslangMembers)[i];
5079	if (glslangMember.type->hiddenMember()) {
5080	++memberDelta;
5081	if (type.getBasicType() == glslang::EbtBlock)
5082	memberRemapper[glslangTypeToIdMap[glslangMembers]][i] = -1;
5083	} else {
5084	if (type.getBasicType() == glslang::EbtBlock) {
5085	if (filterMember(member: *glslangMember.type)) {
5086	memberDelta++;
5087	memberRemapper[glslangTypeToIdMap[glslangMembers]][i] = -1;
5088	continue;
5089	}
5090	memberRemapper[glslangTypeToIdMap[glslangMembers]][i] = i - memberDelta;
5091	}
5092	// modify just this child's view of the qualifier
5093	glslang::TQualifier memberQualifier = glslangMember.type->getQualifier();
5094	InheritQualifiers(child&: memberQualifier, parent: qualifier);
5095
5096	// manually inherit location
5097	if (! memberQualifier.hasLocation() && qualifier.hasLocation())
5098	memberQualifier.layoutLocation = qualifier.layoutLocation;
5099
5100	// recurse
5101	bool lastBufferBlockMember = qualifier.storage == glslang::EvqBuffer &&
5102	i == (int)glslangMembers->size() - 1;
5103
5104	// Make forward pointers for any pointer members.
5105	if (glslangMember.type->isReference() &&
5106	forwardPointers.find(x: glslangMember.type->getReferentType()) == forwardPointers.end()) {
5107	deferredForwardPointers.push_back(x: std::make_pair(x: glslangMember.type, y&: memberQualifier));
5108	}
5109
5110	// Create the member type.
5111	auto const spvMember = convertGlslangToSpvType(type: *glslangMember.type, explicitLayout, qualifier: memberQualifier, lastBufferBlockMember,
5112	forwardReferenceOnly: glslangMember.type->isReference());
5113	spvMembers.push_back(x: spvMember);
5114
5115	// Update the builder with the type's location so that we can create debug types for the structure members.
5116	// There doesn't exist a "clean" entry point for this information to be passed along to the builder so, for now,
5117	// it is stored in the builder and consumed during the construction of composite debug types.
5118	// TODO: This probably warrants further investigation. This approach was decided to be the least ugly of the
5119	// quick and dirty approaches that were tried.
5120	// Advantages of this approach:
5121	// + Relatively clean. No direct calls into debug type system.
5122	// + Handles nested recursive structures.
5123	// Disadvantages of this approach:
5124	// + Not as clean as desired. Traverser queries/sets persistent state. This is fragile.
5125	// + Table lookup during creation of composite debug types. This really shouldn't be necessary.
5126	if(options.emitNonSemanticShaderDebugInfo) {
5127	builder.debugTypeLocs[spvMember].name = glslangMember.type->getFieldName().c_str();
5128	builder.debugTypeLocs[spvMember].line = glslangMember.loc.line;
5129	builder.debugTypeLocs[spvMember].column = glslangMember.loc.column;
5130	}
5131	}
5132	}
5133
5134	// Make the SPIR-V type
5135	spv::Id spvType = builder.makeStructType(members: spvMembers, name: type.getTypeName().c_str(), compilerGenerated: false);
5136	if (! HasNonLayoutQualifiers(type, qualifier))
5137	structMap[explicitLayout][qualifier.layoutMatrix][glslangMembers] = spvType;
5138
5139	// Decorate it
5140	decorateStructType(type, glslangStruct: glslangMembers, explicitLayout, qualifier, spvType, spvMembers);
5141
5142	for (int i = 0; i < (int)deferredForwardPointers.size(); ++i) {
5143	auto it = deferredForwardPointers[i];
5144	convertGlslangToSpvType(type: *it.first, explicitLayout, qualifier: it.second, lastBufferBlockMember: false);
5145	}
5146
5147	return spvType;
5148	}
5149
5150	void TGlslangToSpvTraverser::decorateStructType(const glslang::TType& type,
5151	const glslang::TTypeList* glslangMembers,
5152	glslang::TLayoutPacking explicitLayout,
5153	const glslang::TQualifier& qualifier,
5154	spv::Id spvType,
5155	const std::vector<spv::Id>& spvMembers)
5156	{
5157	// Name and decorate the non-hidden members
5158	int offset = -1;
5159	bool memberLocationInvalid = type.isArrayOfArrays() ||
5160	(type.isArray() && (type.getQualifier().isArrayedIo(language: glslangIntermediate->getStage()) == false));
5161	for (int i = 0; i < (int)glslangMembers->size(); i++) {
5162	glslang::TType& glslangMember = *(*glslangMembers)[i].type;
5163	int member = i;
5164	if (type.getBasicType() == glslang::EbtBlock) {
5165	member = memberRemapper[glslangTypeToIdMap[glslangMembers]][i];
5166	if (filterMember(member: glslangMember))
5167	continue;
5168	}
5169
5170	// modify just this child's view of the qualifier
5171	glslang::TQualifier memberQualifier = glslangMember.getQualifier();
5172	InheritQualifiers(child&: memberQualifier, parent: qualifier);
5173
5174	// using -1 above to indicate a hidden member
5175	if (member < 0)
5176	continue;
5177
5178	builder.addMemberName(spvType, member, name: glslangMember.getFieldName().c_str());
5179	builder.addMemberDecoration(spvType, member,
5180	TranslateLayoutDecoration(type: glslangMember, matrixLayout: memberQualifier.layoutMatrix));
5181	builder.addMemberDecoration(spvType, member, TranslatePrecisionDecoration(type: glslangMember));
5182	// Add interpolation and auxiliary storage decorations only to
5183	// top-level members of Input and Output storage classes
5184	if (type.getQualifier().storage == glslang::EvqVaryingIn ||
5185	type.getQualifier().storage == glslang::EvqVaryingOut) {
5186	if (type.getBasicType() == glslang::EbtBlock ||
5187	glslangIntermediate->getSource() == glslang::EShSourceHlsl) {
5188	builder.addMemberDecoration(spvType, member, TranslateInterpolationDecoration(qualifier: memberQualifier));
5189	builder.addMemberDecoration(spvType, member, TranslateAuxiliaryStorageDecoration(qualifier: memberQualifier));
5190	addMeshNVDecoration(id: spvType, member, qualifier: memberQualifier);
5191	}
5192	}
5193	builder.addMemberDecoration(spvType, member, TranslateInvariantDecoration(qualifier: memberQualifier));
5194
5195	if (type.getBasicType() == glslang::EbtBlock &&
5196	qualifier.storage == glslang::EvqBuffer) {
5197	// Add memory decorations only to top-level members of shader storage block
5198	std::vector<spv::Decoration> memory;
5199	TranslateMemoryDecoration(qualifier: memberQualifier, memory, useVulkanMemoryModel: glslangIntermediate->usingVulkanMemoryModel());
5200	for (unsigned int i = 0; i < memory.size(); ++i)
5201	builder.addMemberDecoration(spvType, member, memory[i]);
5202	}
5203
5204	// Location assignment was already completed correctly by the front end,
5205	// just track whether a member needs to be decorated.
5206	// Ignore member locations if the container is an array, as that's
5207	// ill-specified and decisions have been made to not allow this.
5208	if (!memberLocationInvalid && memberQualifier.hasLocation())
5209	builder.addMemberDecoration(spvType, member, spv::DecorationLocation, num: memberQualifier.layoutLocation);
5210
5211	// component, XFB, others
5212	if (glslangMember.getQualifier().hasComponent())
5213	builder.addMemberDecoration(spvType, member, spv::DecorationComponent,
5214	num: glslangMember.getQualifier().layoutComponent);
5215	if (glslangMember.getQualifier().hasXfbOffset())
5216	builder.addMemberDecoration(spvType, member, spv::DecorationOffset,
5217	num: glslangMember.getQualifier().layoutXfbOffset);
5218	else if (explicitLayout != glslang::ElpNone) {
5219	// figure out what to do with offset, which is accumulating
5220	int nextOffset;
5221	updateMemberOffset(structType: type, memberType: glslangMember, currentOffset&: offset, nextOffset, explicitLayout, memberQualifier.layoutMatrix);
5222	if (offset >= 0)
5223	builder.addMemberDecoration(spvType, member, spv::DecorationOffset, num: offset);
5224	offset = nextOffset;
5225	}
5226
5227	if (glslangMember.isMatrix() && explicitLayout != glslang::ElpNone)
5228	builder.addMemberDecoration(spvType, member, spv::DecorationMatrixStride,
5229	num: getMatrixStride(matrixType: glslangMember, explicitLayout, memberQualifier.layoutMatrix));
5230
5231	// built-in variable decorations
5232	spv::BuiltIn builtIn = TranslateBuiltInDecoration(builtIn: glslangMember.getQualifier().builtIn, memberDeclaration: true);
5233	if (builtIn != spv::BuiltInMax)
5234	builder.addMemberDecoration(spvType, member, spv::DecorationBuiltIn, num: (int)builtIn);
5235
5236	// nonuniform
5237	builder.addMemberDecoration(spvType, member, TranslateNonUniformDecoration(qualifier: glslangMember.getQualifier()));
5238
5239	if (glslangIntermediate->getHlslFunctionality1() && memberQualifier.semanticName != nullptr) {
5240	builder.addExtension(ext: "SPV_GOOGLE_hlsl_functionality1");
5241	builder.addMemberDecoration(spvType, member, (spv::Decoration)spv::DecorationHlslSemanticGOOGLE,
5242	memberQualifier.semanticName);
5243	}
5244
5245	if (builtIn == spv::BuiltInLayer) {
5246	// SPV_NV_viewport_array2 extension
5247	if (glslangMember.getQualifier().layoutViewportRelative){
5248	builder.addMemberDecoration(spvType, member, (spv::Decoration)spv::DecorationViewportRelativeNV);
5249	builder.addCapability(cap: spv::CapabilityShaderViewportMaskNV);
5250	builder.addExtension(ext: spv::E_SPV_NV_viewport_array2);
5251	}
5252	if (glslangMember.getQualifier().layoutSecondaryViewportRelativeOffset != -2048){
5253	builder.addMemberDecoration(spvType, member,
5254	(spv::Decoration)spv::DecorationSecondaryViewportRelativeNV,
5255	num: glslangMember.getQualifier().layoutSecondaryViewportRelativeOffset);
5256	builder.addCapability(cap: spv::CapabilityShaderStereoViewNV);
5257	builder.addExtension(ext: spv::E_SPV_NV_stereo_view_rendering);
5258	}
5259	}
5260	if (glslangMember.getQualifier().layoutPassthrough) {
5261	builder.addMemberDecoration(spvType, member, (spv::Decoration)spv::DecorationPassthroughNV);
5262	builder.addCapability(cap: spv::CapabilityGeometryShaderPassthroughNV);
5263	builder.addExtension(ext: spv::E_SPV_NV_geometry_shader_passthrough);
5264	}
5265
5266	// Add SPIR-V decorations (GL_EXT_spirv_intrinsics)
5267	if (glslangMember.getQualifier().hasSpirvDecorate())
5268	applySpirvDecorate(type: glslangMember, id: spvType, member);
5269	}
5270
5271	// Decorate the structure
5272	builder.addDecoration(spvType, TranslateLayoutDecoration(type, matrixLayout: qualifier.layoutMatrix));
5273	const auto basicType = type.getBasicType();
5274	const auto typeStorageQualifier = type.getQualifier().storage;
5275	if (basicType == glslang::EbtBlock) {
5276	builder.addDecoration(spvType, TranslateBlockDecoration(storage: typeStorageQualifier, useStorageBuffer: glslangIntermediate->usingStorageBuffer()));
5277	} else if (basicType == glslang::EbtStruct && glslangIntermediate->getSpv().vulkan > 0) {
5278	const auto hasRuntimeArray = !spvMembers.empty() && builder.getOpCode(id: spvMembers.back()) == spv::OpTypeRuntimeArray;
5279	if (hasRuntimeArray) {
5280	builder.addDecoration(spvType, TranslateBlockDecoration(storage: typeStorageQualifier, useStorageBuffer: glslangIntermediate->usingStorageBuffer()));
5281	}
5282	}
5283
5284	if (qualifier.hasHitObjectShaderRecordNV())
5285	builder.addDecoration(spvType, spv::DecorationHitObjectShaderRecordBufferNV);
5286	}
5287
5288	// Turn the expression forming the array size into an id.
5289	// This is not quite trivial, because of specialization constants.
5290	// Sometimes, a raw constant is turned into an Id, and sometimes
5291	// a specialization constant expression is.
5292	spv::Id TGlslangToSpvTraverser::makeArraySizeId(const glslang::TArraySizes& arraySizes, int dim, bool allowZero, bool boolType)
5293	{
5294	// First, see if this is sized with a node, meaning a specialization constant:
5295	glslang::TIntermTyped* specNode = arraySizes.getDimNode(dim);
5296	if (specNode != nullptr) {
5297	builder.clearAccessChain();
5298	SpecConstantOpModeGuard spec_constant_op_mode_setter(&builder);
5299	spec_constant_op_mode_setter.turnOnSpecConstantOpMode();
5300	specNode->traverse(this);
5301	return accessChainLoad(type: specNode->getAsTyped()->getType());
5302	}
5303
5304	// Otherwise, need a compile-time (front end) size, get it:
5305	int size = arraySizes.getDimSize(dim);
5306
5307	if (!allowZero)
5308	assert(size > 0);
5309
5310	if (boolType) {
5311	return builder.makeBoolConstant(b: size);
5312	} else {
5313	return builder.makeUintConstant(u: size);
5314	}
5315	}
5316
5317	// Wrap the builder's accessChainLoad to:
5318	// - localize handling of RelaxedPrecision
5319	// - use the SPIR-V inferred type instead of another conversion of the glslang type
5320	// (avoids unnecessary work and possible type punning for structures)
5321	// - do conversion of concrete to abstract type
5322	spv::Id TGlslangToSpvTraverser::accessChainLoad(const glslang::TType& type)
5323	{
5324	spv::Id nominalTypeId = builder.accessChainGetInferredType();
5325
5326	spv::Builder::AccessChain::CoherentFlags coherentFlags = builder.getAccessChain().coherentFlags;
5327	coherentFlags |= TranslateCoherent(type);
5328
5329	spv::MemoryAccessMask accessMask = spv::MemoryAccessMask(TranslateMemoryAccess(coherentFlags) & ~spv::MemoryAccessMakePointerAvailableKHRMask);
5330	// If the value being loaded is HelperInvocation, SPIR-V 1.6 is being generated (so that
5331	// SPV_EXT_demote_to_helper_invocation is in core) and the memory model is in use, add
5332	// the Volatile MemoryAccess semantic.
5333	if (type.getQualifier().builtIn == glslang::EbvHelperInvocation &&
5334	glslangIntermediate->usingVulkanMemoryModel() &&
5335	glslangIntermediate->getSpv().spv >= glslang::EShTargetSpv_1_6) {
5336	accessMask = spv::MemoryAccessMask(accessMask | spv::MemoryAccessVolatileMask);
5337	}
5338
5339	unsigned int alignment = builder.getAccessChain().alignment;
5340	alignment |= type.getBufferReferenceAlignment();
5341
5342	spv::Id loadedId = builder.accessChainLoad(precision: TranslatePrecisionDecoration(type),
5343	l_nonUniform: TranslateNonUniformDecoration(coherentFlags: builder.getAccessChain().coherentFlags),
5344	r_nonUniform: TranslateNonUniformDecoration(qualifier: type.getQualifier()),
5345	ResultType: nominalTypeId,
5346	memoryAccess: accessMask,
5347	scope: TranslateMemoryScope(coherentFlags),
5348	alignment);
5349
5350	// Need to convert to abstract types when necessary
5351	if (type.getBasicType() == glslang::EbtBool) {
5352	loadedId = convertLoadedBoolInUniformToUint(type, nominalTypeId, loadedId);
5353	}
5354
5355	return loadedId;
5356	}
5357
5358	// Wrap the builder's accessChainStore to:
5359	// - do conversion of concrete to abstract type
5360	//
5361	// Implicitly uses the existing builder.accessChain as the storage target.
5362	void TGlslangToSpvTraverser::accessChainStore(const glslang::TType& type, spv::Id rvalue)
5363	{
5364	// Need to convert to abstract types when necessary
5365	if (type.getBasicType() == glslang::EbtBool) {
5366	spv::Id nominalTypeId = builder.accessChainGetInferredType();
5367
5368	if (builder.isScalarType(typeId: nominalTypeId)) {
5369	// Conversion for bool
5370	spv::Id boolType = builder.makeBoolType();
5371	if (nominalTypeId != boolType) {
5372	// keep these outside arguments, for determinant order-of-evaluation
5373	spv::Id one = builder.makeUintConstant(u: 1);
5374	spv::Id zero = builder.makeUintConstant(u: 0);
5375	rvalue = builder.createTriOp(spv::OpSelect, typeId: nominalTypeId, operand1: rvalue, operand2: one, operand3: zero);
5376	} else if (builder.getTypeId(resultId: rvalue) != boolType)
5377	rvalue = builder.createBinOp(spv::OpINotEqual, typeId: boolType, operand1: rvalue, operand2: builder.makeUintConstant(u: 0));
5378	} else if (builder.isVectorType(typeId: nominalTypeId)) {
5379	// Conversion for bvec
5380	int vecSize = builder.getNumTypeComponents(typeId: nominalTypeId);
5381	spv::Id bvecType = builder.makeVectorType(component: builder.makeBoolType(), size: vecSize);
5382	if (nominalTypeId != bvecType) {
5383	// keep these outside arguments, for determinant order-of-evaluation
5384	spv::Id one = makeSmearedConstant(constant: builder.makeUintConstant(u: 1), vectorSize: vecSize);
5385	spv::Id zero = makeSmearedConstant(constant: builder.makeUintConstant(u: 0), vectorSize: vecSize);
5386	rvalue = builder.createTriOp(spv::OpSelect, typeId: nominalTypeId, operand1: rvalue, operand2: one, operand3: zero);
5387	} else if (builder.getTypeId(resultId: rvalue) != bvecType)
5388	rvalue = builder.createBinOp(spv::OpINotEqual, typeId: bvecType, operand1: rvalue,
5389	operand2: makeSmearedConstant(constant: builder.makeUintConstant(u: 0), vectorSize: vecSize));
5390	}
5391	}
5392
5393	spv::Builder::AccessChain::CoherentFlags coherentFlags = builder.getAccessChain().coherentFlags;
5394	coherentFlags |= TranslateCoherent(type);
5395
5396	unsigned int alignment = builder.getAccessChain().alignment;
5397	alignment |= type.getBufferReferenceAlignment();
5398
5399	builder.accessChainStore(rvalue, nonUniform: TranslateNonUniformDecoration(coherentFlags: builder.getAccessChain().coherentFlags),
5400	memoryAccess: spv::MemoryAccessMask(TranslateMemoryAccess(coherentFlags) &
5401	~spv::MemoryAccessMakePointerVisibleKHRMask),
5402	scope: TranslateMemoryScope(coherentFlags), alignment);
5403	}
5404
5405	// For storing when types match at the glslang level, but not might match at the
5406	// SPIR-V level.
5407	//
5408	// This especially happens when a single glslang type expands to multiple
5409	// SPIR-V types, like a struct that is used in a member-undecorated way as well
5410	// as in a member-decorated way.
5411	//
5412	// NOTE: This function can handle any store request; if it's not special it
5413	// simplifies to a simple OpStore.
5414	//
5415	// Implicitly uses the existing builder.accessChain as the storage target.
5416	void TGlslangToSpvTraverser::multiTypeStore(const glslang::TType& type, spv::Id rValue)
5417	{
5418	// we only do the complex path here if it's an aggregate
5419	if (! type.isStruct() && ! type.isArray()) {
5420	accessChainStore(type, rvalue: rValue);
5421	return;
5422	}
5423
5424	// and, it has to be a case of type aliasing
5425	spv::Id rType = builder.getTypeId(resultId: rValue);
5426	spv::Id lValue = builder.accessChainGetLValue();
5427	spv::Id lType = builder.getContainedTypeId(typeId: builder.getTypeId(resultId: lValue));
5428	if (lType == rType) {
5429	accessChainStore(type, rvalue: rValue);
5430	return;
5431	}
5432
5433	// Recursively (as needed) copy an aggregate type to a different aggregate type,
5434	// where the two types were the same type in GLSL. This requires member
5435	// by member copy, recursively.
5436
5437	// SPIR-V 1.4 added an instruction to do help do this.
5438	if (glslangIntermediate->getSpv().spv >= glslang::EShTargetSpv_1_4) {
5439	// However, bool in uniform space is changed to int, so
5440	// OpCopyLogical does not work for that.
5441	// TODO: It would be more robust to do a full recursive verification of the types satisfying SPIR-V rules.
5442	bool rBool = builder.containsType(typeId: builder.getTypeId(resultId: rValue), typeOp: spv::OpTypeBool, width: 0);
5443	bool lBool = builder.containsType(typeId: lType, typeOp: spv::OpTypeBool, width: 0);
5444	if (lBool == rBool) {
5445	spv::Id logicalCopy = builder.createUnaryOp(spv::OpCopyLogical, typeId: lType, operand: rValue);
5446	accessChainStore(type, rvalue: logicalCopy);
5447	return;
5448	}
5449	}
5450
5451	// If an array, copy element by element.
5452	if (type.isArray()) {
5453	glslang::TType glslangElementType(type, 0);
5454	spv::Id elementRType = builder.getContainedTypeId(typeId: rType);
5455	for (int index = 0; index < type.getOuterArraySize(); ++index) {
5456	// get the source member
5457	spv::Id elementRValue = builder.createCompositeExtract(composite: rValue, typeId: elementRType, index);
5458
5459	// set up the target storage
5460	builder.clearAccessChain();
5461	builder.setAccessChainLValue(lValue);
5462	builder.accessChainPush(offset: builder.makeIntConstant(i: index), coherentFlags: TranslateCoherent(type),
5463	alignment: type.getBufferReferenceAlignment());
5464
5465	// store the member
5466	multiTypeStore(type: glslangElementType, rValue: elementRValue);
5467	}
5468	} else {
5469	assert(type.isStruct());
5470
5471	// loop over structure members
5472	const glslang::TTypeList& members = *type.getStruct();
5473	for (int m = 0; m < (int)members.size(); ++m) {
5474	const glslang::TType& glslangMemberType = *members[m].type;
5475
5476	// get the source member
5477	spv::Id memberRType = builder.getContainedTypeId(typeId: rType, m);
5478	spv::Id memberRValue = builder.createCompositeExtract(composite: rValue, typeId: memberRType, index: m);
5479
5480	// set up the target storage
5481	builder.clearAccessChain();
5482	builder.setAccessChainLValue(lValue);
5483	builder.accessChainPush(offset: builder.makeIntConstant(i: m), coherentFlags: TranslateCoherent(type),
5484	alignment: type.getBufferReferenceAlignment());
5485
5486	// store the member
5487	multiTypeStore(type: glslangMemberType, rValue: memberRValue);
5488	}
5489	}
5490	}
5491
5492	// Decide whether or not this type should be
5493	// decorated with offsets and strides, and if so
5494	// whether std140 or std430 rules should be applied.
5495	glslang::TLayoutPacking TGlslangToSpvTraverser::getExplicitLayout(const glslang::TType& type) const
5496	{
5497	// has to be a block
5498	if (type.getBasicType() != glslang::EbtBlock)
5499	return glslang::ElpNone;
5500
5501	// has to be a uniform or buffer block or task in/out blocks
5502	if (type.getQualifier().storage != glslang::EvqUniform &&
5503	type.getQualifier().storage != glslang::EvqBuffer &&
5504	type.getQualifier().storage != glslang::EvqShared &&
5505	!type.getQualifier().isTaskMemory())
5506	return glslang::ElpNone;
5507
5508	// return the layout to use
5509	switch (type.getQualifier().layoutPacking) {
5510	case glslang::ElpStd140:
5511	case glslang::ElpStd430:
5512	case glslang::ElpScalar:
5513	return type.getQualifier().layoutPacking;
5514	default:
5515	return glslang::ElpNone;
5516	}
5517	}
5518
5519	// Given an array type, returns the integer stride required for that array
5520	int TGlslangToSpvTraverser::getArrayStride(const glslang::TType& arrayType, glslang::TLayoutPacking explicitLayout,
5521	glslang::TLayoutMatrix matrixLayout)
5522	{
5523	int size;
5524	int stride;
5525	glslangIntermediate->getMemberAlignment(arrayType, size, stride, layoutPacking: explicitLayout,
5526	rowMajor: matrixLayout == glslang::ElmRowMajor);
5527
5528	return stride;
5529	}
5530
5531	// Given a matrix type, or array (of array) of matrixes type, returns the integer stride required for that matrix
5532	// when used as a member of an interface block
5533	int TGlslangToSpvTraverser::getMatrixStride(const glslang::TType& matrixType, glslang::TLayoutPacking explicitLayout,
5534	glslang::TLayoutMatrix matrixLayout)
5535	{
5536	glslang::TType elementType;
5537	elementType.shallowCopy(copyOf: matrixType);
5538	elementType.clearArraySizes();
5539
5540	int size;
5541	int stride;
5542	glslangIntermediate->getMemberAlignment(elementType, size, stride, layoutPacking: explicitLayout,
5543	rowMajor: matrixLayout == glslang::ElmRowMajor);
5544
5545	return stride;
5546	}
5547
5548	// Given a member type of a struct, realign the current offset for it, and compute
5549	// the next (not yet aligned) offset for the next member, which will get aligned
5550	// on the next call.
5551	// 'currentOffset' should be passed in already initialized, ready to modify, and reflecting
5552	// the migration of data from nextOffset -> currentOffset. It should be -1 on the first call.
5553	// -1 means a non-forced member offset (no decoration needed).
5554	void TGlslangToSpvTraverser::updateMemberOffset(const glslang::TType& structType, const glslang::TType& memberType,
5555	int& currentOffset, int& nextOffset, glslang::TLayoutPacking explicitLayout, glslang::TLayoutMatrix matrixLayout)
5556	{
5557	// this will get a positive value when deemed necessary
5558	nextOffset = -1;
5559
5560	// override anything in currentOffset with user-set offset
5561	if (memberType.getQualifier().hasOffset())
5562	currentOffset = memberType.getQualifier().layoutOffset;
5563
5564	// It could be that current linker usage in glslang updated all the layoutOffset,
5565	// in which case the following code does not matter. But, that's not quite right
5566	// once cross-compilation unit GLSL validation is done, as the original user
5567	// settings are needed in layoutOffset, and then the following will come into play.
5568
5569	if (explicitLayout == glslang::ElpNone) {
5570	if (! memberType.getQualifier().hasOffset())
5571	currentOffset = -1;
5572
5573	return;
5574	}
5575
5576	// Getting this far means we need explicit offsets
5577	if (currentOffset < 0)
5578	currentOffset = 0;
5579
5580	// Now, currentOffset is valid (either 0, or from a previous nextOffset),
5581	// but possibly not yet correctly aligned.
5582
5583	int memberSize;
5584	int dummyStride;
5585	int memberAlignment = glslangIntermediate->getMemberAlignment(memberType, size&: memberSize, stride&: dummyStride, layoutPacking: explicitLayout,
5586	rowMajor: matrixLayout == glslang::ElmRowMajor);
5587
5588	bool isVectorLike = memberType.isVector();
5589	if (memberType.isMatrix()) {
5590	if (matrixLayout == glslang::ElmRowMajor)
5591	isVectorLike = memberType.getMatrixRows() == 1;
5592	else
5593	isVectorLike = memberType.getMatrixCols() == 1;
5594	}
5595
5596	// Adjust alignment for HLSL rules
5597	// TODO: make this consistent in early phases of code:
5598	// adjusting this late means inconsistencies with earlier code, which for reflection is an issue
5599	// Until reflection is brought in sync with these adjustments, don't apply to $Global,
5600	// which is the most likely to rely on reflection, and least likely to rely implicit layouts
5601	if (glslangIntermediate->usingHlslOffsets() &&
5602	! memberType.isStruct() && structType.getTypeName().compare(s: "$Global") != 0) {
5603	int componentSize;
5604	int componentAlignment = glslangIntermediate->getBaseAlignmentScalar(memberType, size&: componentSize);
5605	if (! memberType.isArray() && isVectorLike && componentAlignment <= 4)
5606	memberAlignment = componentAlignment;
5607
5608	// Don't add unnecessary padding after this member
5609	// (undo std140 bumping size to a mutliple of vec4)
5610	if (explicitLayout == glslang::ElpStd140) {
5611	if (memberType.isMatrix()) {
5612	if (matrixLayout == glslang::ElmRowMajor)
5613	memberSize -= componentSize * (4 - memberType.getMatrixCols());
5614	else
5615	memberSize -= componentSize * (4 - memberType.getMatrixRows());
5616	} else if (memberType.isArray())
5617	memberSize -= componentSize * (4 - memberType.getVectorSize());
5618	}
5619	}
5620
5621	// Bump up to member alignment
5622	glslang::RoundToPow2(number&: currentOffset, powerOf2: memberAlignment);
5623
5624	// Bump up to vec4 if there is a bad straddle
5625	if (explicitLayout != glslang::ElpScalar && glslangIntermediate->improperStraddle(type: memberType, size: memberSize,
5626	offset: currentOffset, vectorLike: isVectorLike))
5627	glslang::RoundToPow2(number&: currentOffset, powerOf2: 16);
5628
5629	nextOffset = currentOffset + memberSize;
5630	}
5631
5632	void TGlslangToSpvTraverser::declareUseOfStructMember(const glslang::TTypeList& members, int glslangMember)
5633	{
5634	const glslang::TBuiltInVariable glslangBuiltIn = members[glslangMember].type->getQualifier().builtIn;
5635	switch (glslangBuiltIn)
5636	{
5637	case glslang::EbvPointSize:
5638	case glslang::EbvClipDistance:
5639	case glslang::EbvCullDistance:
5640	case glslang::EbvViewportMaskNV:
5641	case glslang::EbvSecondaryPositionNV:
5642	case glslang::EbvSecondaryViewportMaskNV:
5643	case glslang::EbvPositionPerViewNV:
5644	case glslang::EbvViewportMaskPerViewNV:
5645	case glslang::EbvTaskCountNV:
5646	case glslang::EbvPrimitiveCountNV:
5647	case glslang::EbvPrimitiveIndicesNV:
5648	case glslang::EbvClipDistancePerViewNV:
5649	case glslang::EbvCullDistancePerViewNV:
5650	case glslang::EbvLayerPerViewNV:
5651	case glslang::EbvMeshViewCountNV:
5652	case glslang::EbvMeshViewIndicesNV:
5653	// Generate the associated capability. Delegate to TranslateBuiltInDecoration.
5654	// Alternately, we could just call this for any glslang built-in, since the
5655	// capability already guards against duplicates.
5656	TranslateBuiltInDecoration(builtIn: glslangBuiltIn, memberDeclaration: false);
5657	break;
5658	default:
5659	// Capabilities were already generated when the struct was declared.
5660	break;
5661	}
5662	}
5663
5664	bool TGlslangToSpvTraverser::isShaderEntryPoint(const glslang::TIntermAggregate* node)
5665	{
5666	return node->getName().compare(s: glslangIntermediate->getEntryPointMangledName().c_str()) == 0;
5667	}
5668
5669	// Does parameter need a place to keep writes, separate from the original?
5670	// Assumes called after originalParam(), which filters out block/buffer/opaque-based
5671	// qualifiers such that we should have only in/out/inout/constreadonly here.
5672	bool TGlslangToSpvTraverser::writableParam(glslang::TStorageQualifier qualifier) const
5673	{
5674	assert(qualifier == glslang::EvqIn ||
5675	qualifier == glslang::EvqOut ||
5676	qualifier == glslang::EvqInOut ||
5677	qualifier == glslang::EvqUniform ||
5678	qualifier == glslang::EvqConstReadOnly);
5679	return qualifier != glslang::EvqConstReadOnly &&
5680	qualifier != glslang::EvqUniform;
5681	}
5682
5683	// Is parameter pass-by-original?
5684	bool TGlslangToSpvTraverser::originalParam(glslang::TStorageQualifier qualifier, const glslang::TType& paramType,
5685	bool implicitThisParam)
5686	{
5687	if (implicitThisParam) // implicit this
5688	return true;
5689	if (glslangIntermediate->getSource() == glslang::EShSourceHlsl)
5690	return paramType.getBasicType() == glslang::EbtBlock;
5691	return (paramType.containsOpaque() && !glslangIntermediate->getBindlessMode()) || // sampler, etc.
5692	paramType.getQualifier().isSpirvByReference() || // spirv_by_reference
5693	(paramType.getBasicType() == glslang::EbtBlock && qualifier == glslang::EvqBuffer); // SSBO
5694	}
5695
5696	// Make all the functions, skeletally, without actually visiting their bodies.
5697	void TGlslangToSpvTraverser::makeFunctions(const glslang::TIntermSequence& glslFunctions)
5698	{
5699	const auto getParamDecorations = [&](std::vector<spv::Decoration>& decorations, const glslang::TType& type,
5700	bool useVulkanMemoryModel) {
5701	spv::Decoration paramPrecision = TranslatePrecisionDecoration(type);
5702	if (paramPrecision != spv::NoPrecision)
5703	decorations.push_back(x: paramPrecision);
5704	TranslateMemoryDecoration(qualifier: type.getQualifier(), memory&: decorations, useVulkanMemoryModel);
5705	if (type.isReference()) {
5706	// Original and non-writable params pass the pointer directly and
5707	// use restrict/aliased, others are stored to a pointer in Function
5708	// memory and use RestrictPointer/AliasedPointer.
5709	if (originalParam(qualifier: type.getQualifier().storage, paramType: type, implicitThisParam: false) ||
5710	!writableParam(qualifier: type.getQualifier().storage)) {
5711	// TranslateMemoryDecoration added Restrict decoration already.
5712	if (!type.getQualifier().isRestrict()) {
5713	decorations.push_back(x: spv::DecorationAliased);
5714	}
5715	} else {
5716	decorations.push_back(x: type.getQualifier().isRestrict() ? spv::DecorationRestrictPointerEXT :
5717	spv::DecorationAliasedPointerEXT);
5718	}
5719	}
5720	};
5721
5722	for (int f = 0; f < (int)glslFunctions.size(); ++f) {
5723	glslang::TIntermAggregate* glslFunction = glslFunctions[f]->getAsAggregate();
5724	if (! glslFunction || glslFunction->getOp() != glslang::EOpFunction)
5725	continue;
5726
5727	builder.setDebugSourceLocation(line: glslFunction->getLoc().line, filename: glslFunction->getLoc().getFilename());
5728
5729	if (isShaderEntryPoint(node: glslFunction)) {
5730	// For HLSL, the entry function is actually a compiler generated function to resolve the difference of
5731	// entry function signature between HLSL and SPIR-V. So we don't emit debug information for that.
5732	if (glslangIntermediate->getSource() != glslang::EShSourceHlsl) {
5733	builder.setupFunctionDebugInfo(function: shaderEntry, name: glslangIntermediate->getEntryPointMangledName().c_str(),
5734	paramTypes: std::vector<spv::Id>(), // main function has no param
5735	paramNames: std::vector<char const*>());
5736	}
5737	continue;
5738	}
5739	// We're on a user function. Set up the basic interface for the function now,
5740	// so that it's available to call. Translating the body will happen later.
5741	//
5742	// Typically (except for a "const in" parameter), an address will be passed to the
5743	// function. What it is an address of varies:
5744	//
5745	// - "in" parameters not marked as "const" can be written to without modifying the calling
5746	// argument so that write needs to be to a copy, hence the address of a copy works.
5747	//
5748	// - "const in" parameters can just be the r-value, as no writes need occur.
5749	//
5750	// - "out" and "inout" arguments can't be done as pointers to the calling argument, because
5751	// GLSL has copy-in/copy-out semantics. They can be handled though with a pointer to a copy.
5752
5753	std::vector<spv::Id> paramTypes;
5754	std::vector<char const*> paramNames;
5755	std::vector<std::vector<spv::Decoration>> paramDecorations; // list of decorations per parameter
5756	glslang::TIntermSequence& parameters = glslFunction->getSequence()[0]->getAsAggregate()->getSequence();
5757
5758	#ifdef ENABLE_HLSL
5759	bool implicitThis = (int)parameters.size() > 0 && parameters[0]->getAsSymbolNode()->getName() ==
5760	glslangIntermediate->implicitThisName;
5761	#else
5762	bool implicitThis = false;
5763	#endif
5764
5765	paramDecorations.resize(new_size: parameters.size());
5766	for (int p = 0; p < (int)parameters.size(); ++p) {
5767	const glslang::TType& paramType = parameters[p]->getAsTyped()->getType();
5768	spv::Id typeId = convertGlslangToSpvType(type: paramType);
5769	if (originalParam(qualifier: paramType.getQualifier().storage, paramType, implicitThisParam: implicitThis && p == 0))
5770	typeId = builder.makePointer(TranslateStorageClass(type: paramType), pointee: typeId);
5771	else if (writableParam(qualifier: paramType.getQualifier().storage))
5772	typeId = builder.makePointer(spv::StorageClassFunction, pointee: typeId);
5773	else
5774	rValueParameters.insert(x: parameters[p]->getAsSymbolNode()->getId());
5775	getParamDecorations(paramDecorations[p], paramType, glslangIntermediate->usingVulkanMemoryModel());
5776	paramTypes.push_back(x: typeId);
5777	}
5778
5779	for (auto const parameter:parameters) {
5780	paramNames.push_back(x: parameter->getAsSymbolNode()->getName().c_str());
5781	}
5782
5783	spv::Block* functionBlock;
5784	spv::Function* function = builder.makeFunctionEntry(
5785	precision: TranslatePrecisionDecoration(type: glslFunction->getType()), returnType: convertGlslangToSpvType(type: glslFunction->getType()),
5786	name: glslFunction->getName().c_str(), linkType: convertGlslangLinkageToSpv(linkType: glslFunction->getLinkType()), paramTypes,
5787	precisions: paramDecorations, entry: &functionBlock);
5788	builder.setupFunctionDebugInfo(function, name: glslFunction->getName().c_str(), paramTypes, paramNames);
5789	if (implicitThis)
5790	function->setImplicitThis();
5791
5792	// Track function to emit/call later
5793	functionMap[glslFunction->getName().c_str()] = function;
5794
5795	// Set the parameter id's
5796	for (int p = 0; p < (int)parameters.size(); ++p) {
5797	symbolValues[parameters[p]->getAsSymbolNode()->getId()] = function->getParamId(p);
5798	// give a name too
5799	builder.addName(function->getParamId(p), name: parameters[p]->getAsSymbolNode()->getName().c_str());
5800
5801	const glslang::TType& paramType = parameters[p]->getAsTyped()->getType();
5802	if (paramType.contains8BitInt())
5803	builder.addCapability(cap: spv::CapabilityInt8);
5804	if (paramType.contains16BitInt())
5805	builder.addCapability(cap: spv::CapabilityInt16);
5806	if (paramType.contains16BitFloat())
5807	builder.addCapability(cap: spv::CapabilityFloat16);
5808	}
5809	}
5810	}
5811
5812	// Process all the initializers, while skipping the functions and link objects
5813	void TGlslangToSpvTraverser::makeGlobalInitializers(const glslang::TIntermSequence& initializers)
5814	{
5815	builder.setBuildPoint(shaderEntry->getLastBlock());
5816	for (int i = 0; i < (int)initializers.size(); ++i) {
5817	glslang::TIntermAggregate* initializer = initializers[i]->getAsAggregate();
5818	if (initializer && initializer->getOp() != glslang::EOpFunction && initializer->getOp() !=
5819	glslang::EOpLinkerObjects) {
5820
5821	// We're on a top-level node that's not a function. Treat as an initializer, whose
5822	// code goes into the beginning of the entry point.
5823	initializer->traverse(this);
5824	}
5825	}
5826	}
5827	// Walk over all linker objects to create a map for payload and callable data linker objects
5828	// and their location to be used during codegen for OpTraceKHR and OpExecuteCallableKHR
5829	// This is done here since it is possible that these linker objects are not be referenced in the AST
5830	void TGlslangToSpvTraverser::collectRayTracingLinkerObjects()
5831	{
5832	glslang::TIntermAggregate* linkerObjects = glslangIntermediate->findLinkerObjects();
5833	for (auto& objSeq : linkerObjects->getSequence()) {
5834	auto objNode = objSeq->getAsSymbolNode();
5835	if (objNode != nullptr) {
5836	if (objNode->getQualifier().hasLocation()) {
5837	unsigned int location = objNode->getQualifier().layoutLocation;
5838	auto st = objNode->getQualifier().storage;
5839	int set;
5840	switch (st)
5841	{
5842	case glslang::EvqPayload:
5843	case glslang::EvqPayloadIn:
5844	set = 0;
5845	break;
5846	case glslang::EvqCallableData:
5847	case glslang::EvqCallableDataIn:
5848	set = 1;
5849	break;
5850
5851	case glslang::EvqHitObjectAttrNV:
5852	set = 2;
5853	break;
5854
5855	default:
5856	set = -1;
5857	}
5858	if (set != -1)
5859	locationToSymbol[set].insert(x: std::make_pair(x&: location, y&: objNode));
5860	}
5861	}
5862	}
5863	}
5864	// Process all the functions, while skipping initializers.
5865	void TGlslangToSpvTraverser::visitFunctions(const glslang::TIntermSequence& glslFunctions)
5866	{
5867	for (int f = 0; f < (int)glslFunctions.size(); ++f) {
5868	glslang::TIntermAggregate* node = glslFunctions[f]->getAsAggregate();
5869	if (node && (node->getOp() == glslang::EOpFunction || node->getOp() == glslang::EOpLinkerObjects))
5870	node->traverse(this);
5871	}
5872	}
5873
5874	void TGlslangToSpvTraverser::handleFunctionEntry(const glslang::TIntermAggregate* node)
5875	{
5876	// SPIR-V functions should already be in the functionMap from the prepass
5877	// that called makeFunctions().
5878	currentFunction = functionMap[node->getName().c_str()];
5879	spv::Block* functionBlock = currentFunction->getEntryBlock();
5880	builder.setBuildPoint(functionBlock);
5881	builder.enterFunction(function: currentFunction);
5882	}
5883
5884	void TGlslangToSpvTraverser::translateArguments(const glslang::TIntermAggregate& node, std::vector<spv::Id>& arguments,
5885	spv::Builder::AccessChain::CoherentFlags &lvalueCoherentFlags)
5886	{
5887	const glslang::TIntermSequence& glslangArguments = node.getSequence();
5888
5889	glslang::TSampler sampler = {};
5890	bool cubeCompare = false;
5891	bool f16ShadowCompare = false;
5892	if (node.isTexture() || node.isImage()) {
5893	sampler = glslangArguments[0]->getAsTyped()->getType().getSampler();
5894	cubeCompare = sampler.dim == glslang::EsdCube && sampler.arrayed && sampler.shadow;
5895	f16ShadowCompare = sampler.shadow &&
5896	glslangArguments[1]->getAsTyped()->getType().getBasicType() == glslang::EbtFloat16;
5897	}
5898
5899	for (int i = 0; i < (int)glslangArguments.size(); ++i) {
5900	builder.clearAccessChain();
5901	glslangArguments[i]->traverse(this);
5902
5903	// Special case l-value operands
5904	bool lvalue = false;
5905	switch (node.getOp()) {
5906	case glslang::EOpImageAtomicAdd:
5907	case glslang::EOpImageAtomicMin:
5908	case glslang::EOpImageAtomicMax:
5909	case glslang::EOpImageAtomicAnd:
5910	case glslang::EOpImageAtomicOr:
5911	case glslang::EOpImageAtomicXor:
5912	case glslang::EOpImageAtomicExchange:
5913	case glslang::EOpImageAtomicCompSwap:
5914	case glslang::EOpImageAtomicLoad:
5915	case glslang::EOpImageAtomicStore:
5916	if (i == 0)
5917	lvalue = true;
5918	break;
5919	case glslang::EOpSparseImageLoad:
5920	if ((sampler.ms && i == 3) || (! sampler.ms && i == 2))
5921	lvalue = true;
5922	break;
5923	case glslang::EOpSparseTexture:
5924	if (((cubeCompare || f16ShadowCompare) && i == 3) || (! (cubeCompare || f16ShadowCompare) && i == 2))
5925	lvalue = true;
5926	break;
5927	case glslang::EOpSparseTextureClamp:
5928	if (((cubeCompare || f16ShadowCompare) && i == 4) || (! (cubeCompare || f16ShadowCompare) && i == 3))
5929	lvalue = true;
5930	break;
5931	case glslang::EOpSparseTextureLod:
5932	case glslang::EOpSparseTextureOffset:
5933	if ((f16ShadowCompare && i == 4) || (! f16ShadowCompare && i == 3))
5934	lvalue = true;
5935	break;
5936	case glslang::EOpSparseTextureFetch:
5937	if ((sampler.dim != glslang::EsdRect && i == 3) || (sampler.dim == glslang::EsdRect && i == 2))
5938	lvalue = true;
5939	break;
5940	case glslang::EOpSparseTextureFetchOffset:
5941	if ((sampler.dim != glslang::EsdRect && i == 4) || (sampler.dim == glslang::EsdRect && i == 3))
5942	lvalue = true;
5943	break;
5944	case glslang::EOpSparseTextureLodOffset:
5945	case glslang::EOpSparseTextureGrad:
5946	case glslang::EOpSparseTextureOffsetClamp:
5947	if ((f16ShadowCompare && i == 5) || (! f16ShadowCompare && i == 4))
5948	lvalue = true;
5949	break;
5950	case glslang::EOpSparseTextureGradOffset:
5951	case glslang::EOpSparseTextureGradClamp:
5952	if ((f16ShadowCompare && i == 6) || (! f16ShadowCompare && i == 5))
5953	lvalue = true;
5954	break;
5955	case glslang::EOpSparseTextureGradOffsetClamp:
5956	if ((f16ShadowCompare && i == 7) || (! f16ShadowCompare && i == 6))
5957	lvalue = true;
5958	break;
5959	case glslang::EOpSparseTextureGather:
5960	if ((sampler.shadow && i == 3) || (! sampler.shadow && i == 2))
5961	lvalue = true;
5962	break;
5963	case glslang::EOpSparseTextureGatherOffset:
5964	case glslang::EOpSparseTextureGatherOffsets:
5965	if ((sampler.shadow && i == 4) || (! sampler.shadow && i == 3))
5966	lvalue = true;
5967	break;
5968	case glslang::EOpSparseTextureGatherLod:
5969	if (i == 3)
5970	lvalue = true;
5971	break;
5972	case glslang::EOpSparseTextureGatherLodOffset:
5973	case glslang::EOpSparseTextureGatherLodOffsets:
5974	if (i == 4)
5975	lvalue = true;
5976	break;
5977	case glslang::EOpSparseImageLoadLod:
5978	if (i == 3)
5979	lvalue = true;
5980	break;
5981	case glslang::EOpImageSampleFootprintNV:
5982	if (i == 4)
5983	lvalue = true;
5984	break;
5985	case glslang::EOpImageSampleFootprintClampNV:
5986	case glslang::EOpImageSampleFootprintLodNV:
5987	if (i == 5)
5988	lvalue = true;
5989	break;
5990	case glslang::EOpImageSampleFootprintGradNV:
5991	if (i == 6)
5992	lvalue = true;
5993	break;
5994	case glslang::EOpImageSampleFootprintGradClampNV:
5995	if (i == 7)
5996	lvalue = true;
5997	break;
5998	case glslang::EOpRayQueryGetIntersectionTriangleVertexPositionsEXT:
5999	if (i == 2)
6000	lvalue = true;
6001	break;
6002	default:
6003	break;
6004	}
6005
6006	if (lvalue) {
6007	spv::Id lvalue_id = builder.accessChainGetLValue();
6008	arguments.push_back(x: lvalue_id);
6009	lvalueCoherentFlags = builder.getAccessChain().coherentFlags;
6010	builder.addDecoration(lvalue_id, TranslateNonUniformDecoration(coherentFlags: lvalueCoherentFlags));
6011	lvalueCoherentFlags |= TranslateCoherent(type: glslangArguments[i]->getAsTyped()->getType());
6012	} else {
6013	if (i > 0 &&
6014	glslangArguments[i]->getAsSymbolNode() && glslangArguments[i-1]->getAsSymbolNode() &&
6015	glslangArguments[i]->getAsSymbolNode()->getId() == glslangArguments[i-1]->getAsSymbolNode()->getId()) {
6016	// Reuse the id if possible
6017	arguments.push_back(x: arguments[i-1]);
6018	} else {
6019	arguments.push_back(x: accessChainLoad(type: glslangArguments[i]->getAsTyped()->getType()));
6020	}
6021	}
6022	}
6023	}
6024
6025	void TGlslangToSpvTraverser::translateArguments(glslang::TIntermUnary& node, std::vector<spv::Id>& arguments)
6026	{
6027	builder.clearAccessChain();
6028	node.getOperand()->traverse(this);
6029	arguments.push_back(x: accessChainLoad(type: node.getOperand()->getType()));
6030	}
6031
6032	spv::Id TGlslangToSpvTraverser::createImageTextureFunctionCall(glslang::TIntermOperator* node)
6033	{
6034	if (! node->isImage() && ! node->isTexture())
6035	return spv::NoResult;
6036
6037	builder.setDebugSourceLocation(line: node->getLoc().line, filename: node->getLoc().getFilename());
6038
6039	// Process a GLSL texturing op (will be SPV image)
6040
6041	const glslang::TType &imageType = node->getAsAggregate()
6042	? node->getAsAggregate()->getSequence()[0]->getAsTyped()->getType()
6043	: node->getAsUnaryNode()->getOperand()->getAsTyped()->getType();
6044	const glslang::TSampler sampler = imageType.getSampler();
6045	bool f16ShadowCompare = (sampler.shadow && node->getAsAggregate())
6046	? node->getAsAggregate()->getSequence()[1]->getAsTyped()->getType().getBasicType() == glslang::EbtFloat16
6047	: false;
6048
6049	const auto signExtensionMask = [&]() {
6050	if (builder.getSpvVersion() >= spv::Spv_1_4) {
6051	if (sampler.type == glslang::EbtUint)
6052	return spv::ImageOperandsZeroExtendMask;
6053	else if (sampler.type == glslang::EbtInt)
6054	return spv::ImageOperandsSignExtendMask;
6055	}
6056	return spv::ImageOperandsMaskNone;
6057	};
6058
6059	spv::Builder::AccessChain::CoherentFlags lvalueCoherentFlags;
6060
6061	std::vector<spv::Id> arguments;
6062	if (node->getAsAggregate())
6063	translateArguments(node: *node->getAsAggregate(), arguments, lvalueCoherentFlags);
6064	else
6065	translateArguments(node&: *node->getAsUnaryNode(), arguments);
6066	spv::Decoration precision = TranslatePrecisionDecoration(type: node->getType());
6067
6068	spv::Builder::TextureParameters params = { };
6069	params.sampler = arguments[0];
6070
6071	glslang::TCrackedTextureOp cracked;
6072	node->crackTexture(sampler, cracked);
6073
6074	const bool isUnsignedResult = node->getType().getBasicType() == glslang::EbtUint;
6075
6076	if (builder.isSampledImage(resultId: params.sampler) &&
6077	((cracked.query && node->getOp() != glslang::EOpTextureQueryLod) || cracked.fragMask || cracked.fetch)) {
6078	params.sampler = builder.createUnaryOp(spv::OpImage, typeId: builder.getImageType(resultId: params.sampler), operand: params.sampler);
6079	if (imageType.getQualifier().isNonUniform()) {
6080	builder.addDecoration(params.sampler, spv::DecorationNonUniformEXT);
6081	}
6082	}
6083	// Check for queries
6084	if (cracked.query) {
6085	switch (node->getOp()) {
6086	case glslang::EOpImageQuerySize:
6087	case glslang::EOpTextureQuerySize:
6088	if (arguments.size() > 1) {
6089	params.lod = arguments[1];
6090	return builder.createTextureQueryCall(spv::OpImageQuerySizeLod, params, isUnsignedResult);
6091	} else
6092	return builder.createTextureQueryCall(spv::OpImageQuerySize, params, isUnsignedResult);
6093	case glslang::EOpImageQuerySamples:
6094	case glslang::EOpTextureQuerySamples:
6095	return builder.createTextureQueryCall(spv::OpImageQuerySamples, params, isUnsignedResult);
6096	case glslang::EOpTextureQueryLod:
6097	params.coords = arguments[1];
6098	return builder.createTextureQueryCall(spv::OpImageQueryLod, params, isUnsignedResult);
6099	case glslang::EOpTextureQueryLevels:
6100	return builder.createTextureQueryCall(spv::OpImageQueryLevels, params, isUnsignedResult);
6101	case glslang::EOpSparseTexelsResident:
6102	return builder.createUnaryOp(spv::OpImageSparseTexelsResident, typeId: builder.makeBoolType(), operand: arguments[0]);
6103	default:
6104	assert(0);
6105	break;
6106	}
6107	}
6108
6109	int components = node->getType().getVectorSize();
6110
6111	if (node->getOp() == glslang::EOpImageLoad ||
6112	node->getOp() == glslang::EOpImageLoadLod ||
6113	node->getOp() == glslang::EOpTextureFetch ||
6114	node->getOp() == glslang::EOpTextureFetchOffset) {
6115	// These must produce 4 components, per SPIR-V spec. We'll add a conversion constructor if needed.
6116	// This will only happen through the HLSL path for operator[], so we do not have to handle e.g.
6117	// the EOpTexture/Proj/Lod/etc family. It would be harmless to do so, but would need more logic
6118	// here around e.g. which ones return scalars or other types.
6119	components = 4;
6120	}
6121
6122	glslang::TType returnType(node->getType().getBasicType(), glslang::EvqTemporary, components);
6123
6124	auto resultType = [&returnType,this]{ return convertGlslangToSpvType(type: returnType); };
6125
6126	// Check for image functions other than queries
6127	if (node->isImage()) {
6128	std::vector<spv::IdImmediate> operands;
6129	auto opIt = arguments.begin();
6130	spv::IdImmediate image = { true, *(opIt++) };
6131	operands.push_back(x: image);
6132
6133	// Handle subpass operations
6134	// TODO: GLSL should change to have the "MS" only on the type rather than the
6135	// built-in function.
6136	if (cracked.subpass) {
6137	// add on the (0,0) coordinate
6138	spv::Id zero = builder.makeIntConstant(i: 0);
6139	std::vector<spv::Id> comps;
6140	comps.push_back(x: zero);
6141	comps.push_back(x: zero);
6142	spv::IdImmediate coord = { true,
6143	builder.makeCompositeConstant(type: builder.makeVectorType(component: builder.makeIntType(width: 32), size: 2), comps) };
6144	operands.push_back(x: coord);
6145	spv::IdImmediate imageOperands = { false, spv::ImageOperandsMaskNone };
6146	imageOperands.word = imageOperands.word | signExtensionMask();
6147	if (sampler.isMultiSample()) {
6148	imageOperands.word = imageOperands.word | spv::ImageOperandsSampleMask;
6149	}
6150	if (imageOperands.word != spv::ImageOperandsMaskNone) {
6151	operands.push_back(x: imageOperands);
6152	if (sampler.isMultiSample()) {
6153	spv::IdImmediate imageOperand = { true, *(opIt++) };
6154	operands.push_back(x: imageOperand);
6155	}
6156	}
6157	spv::Id result = builder.createOp(spv::OpImageRead, typeId: resultType(), operands);
6158	builder.setPrecision(id: result, precision);
6159	return result;
6160	}
6161
6162	if (cracked.attachmentEXT) {
6163	if (opIt != arguments.end()) {
6164	spv::IdImmediate sample = { true, *opIt };
6165	operands.push_back(x: sample);
6166	}
6167	spv::Id result = builder.createOp(spv::OpColorAttachmentReadEXT, typeId: resultType(), operands);
6168	builder.addExtension(ext: spv::E_SPV_EXT_shader_tile_image);
6169	builder.setPrecision(id: result, precision);
6170	return result;
6171	}
6172
6173	spv::IdImmediate coord = { true, *(opIt++) };
6174	operands.push_back(x: coord);
6175	if (node->getOp() == glslang::EOpImageLoad || node->getOp() == glslang::EOpImageLoadLod) {
6176	spv::ImageOperandsMask mask = spv::ImageOperandsMaskNone;
6177	if (sampler.isMultiSample()) {
6178	mask = mask | spv::ImageOperandsSampleMask;
6179	}
6180	if (cracked.lod) {
6181	builder.addExtension(ext: spv::E_SPV_AMD_shader_image_load_store_lod);
6182	builder.addCapability(cap: spv::CapabilityImageReadWriteLodAMD);
6183	mask = mask | spv::ImageOperandsLodMask;
6184	}
6185	mask = mask | TranslateImageOperands(coherentFlags: TranslateCoherent(type: imageType));
6186	mask = (spv::ImageOperandsMask)(mask & ~spv::ImageOperandsMakeTexelAvailableKHRMask);
6187	mask = mask | signExtensionMask();
6188	if (mask != spv::ImageOperandsMaskNone) {
6189	spv::IdImmediate imageOperands = { false, (unsigned int)mask };
6190	operands.push_back(x: imageOperands);
6191	}
6192	if (mask & spv::ImageOperandsSampleMask) {
6193	spv::IdImmediate imageOperand = { true, *opIt++ };
6194	operands.push_back(x: imageOperand);
6195	}
6196	if (mask & spv::ImageOperandsLodMask) {
6197	spv::IdImmediate imageOperand = { true, *opIt++ };
6198	operands.push_back(x: imageOperand);
6199	}
6200	if (mask & spv::ImageOperandsMakeTexelVisibleKHRMask) {
6201	spv::IdImmediate imageOperand = { true,
6202	builder.makeUintConstant(u: TranslateMemoryScope(coherentFlags: TranslateCoherent(type: imageType))) };
6203	operands.push_back(x: imageOperand);
6204	}
6205
6206	if (builder.getImageTypeFormat(typeId: builder.getImageType(resultId: operands.front().word)) == spv::ImageFormatUnknown)
6207	builder.addCapability(cap: spv::CapabilityStorageImageReadWithoutFormat);
6208
6209	std::vector<spv::Id> result(1, builder.createOp(spv::OpImageRead, typeId: resultType(), operands));
6210	builder.setPrecision(id: result[0], precision);
6211
6212	// If needed, add a conversion constructor to the proper size.
6213	if (components != node->getType().getVectorSize())
6214	result[0] = builder.createConstructor(precision, sources: result, resultTypeId: convertGlslangToSpvType(type: node->getType()));
6215
6216	return result[0];
6217	} else if (node->getOp() == glslang::EOpImageStore || node->getOp() == glslang::EOpImageStoreLod) {
6218
6219	// Push the texel value before the operands
6220	if (sampler.isMultiSample() || cracked.lod) {
6221	spv::IdImmediate texel = { true, *(opIt + 1) };
6222	operands.push_back(x: texel);
6223	} else {
6224	spv::IdImmediate texel = { true, *opIt };
6225	operands.push_back(x: texel);
6226	}
6227
6228	spv::ImageOperandsMask mask = spv::ImageOperandsMaskNone;
6229	if (sampler.isMultiSample()) {
6230	mask = mask | spv::ImageOperandsSampleMask;
6231	}
6232	if (cracked.lod) {
6233	builder.addExtension(ext: spv::E_SPV_AMD_shader_image_load_store_lod);
6234	builder.addCapability(cap: spv::CapabilityImageReadWriteLodAMD);
6235	mask = mask | spv::ImageOperandsLodMask;
6236	}
6237	mask = mask | TranslateImageOperands(coherentFlags: TranslateCoherent(type: imageType));
6238	mask = (spv::ImageOperandsMask)(mask & ~spv::ImageOperandsMakeTexelVisibleKHRMask);
6239	mask = mask | signExtensionMask();
6240	if (mask != spv::ImageOperandsMaskNone) {
6241	spv::IdImmediate imageOperands = { false, (unsigned int)mask };
6242	operands.push_back(x: imageOperands);
6243	}
6244	if (mask & spv::ImageOperandsSampleMask) {
6245	spv::IdImmediate imageOperand = { true, *opIt++ };
6246	operands.push_back(x: imageOperand);
6247	}
6248	if (mask & spv::ImageOperandsLodMask) {
6249	spv::IdImmediate imageOperand = { true, *opIt++ };
6250	operands.push_back(x: imageOperand);
6251	}
6252	if (mask & spv::ImageOperandsMakeTexelAvailableKHRMask) {
6253	spv::IdImmediate imageOperand = { true,
6254	builder.makeUintConstant(u: TranslateMemoryScope(coherentFlags: TranslateCoherent(type: imageType))) };
6255	operands.push_back(x: imageOperand);
6256	}
6257
6258	builder.createNoResultOp(spv::OpImageWrite, operands);
6259	if (builder.getImageTypeFormat(typeId: builder.getImageType(resultId: operands.front().word)) == spv::ImageFormatUnknown)
6260	builder.addCapability(cap: spv::CapabilityStorageImageWriteWithoutFormat);
6261	return spv::NoResult;
6262	} else if (node->getOp() == glslang::EOpSparseImageLoad ||
6263	node->getOp() == glslang::EOpSparseImageLoadLod) {
6264	builder.addCapability(cap: spv::CapabilitySparseResidency);
6265	if (builder.getImageTypeFormat(typeId: builder.getImageType(resultId: operands.front().word)) == spv::ImageFormatUnknown)
6266	builder.addCapability(cap: spv::CapabilityStorageImageReadWithoutFormat);
6267
6268	spv::ImageOperandsMask mask = spv::ImageOperandsMaskNone;
6269	if (sampler.isMultiSample()) {
6270	mask = mask | spv::ImageOperandsSampleMask;
6271	}
6272	if (cracked.lod) {
6273	builder.addExtension(ext: spv::E_SPV_AMD_shader_image_load_store_lod);
6274	builder.addCapability(cap: spv::CapabilityImageReadWriteLodAMD);
6275
6276	mask = mask | spv::ImageOperandsLodMask;
6277	}
6278	mask = mask | TranslateImageOperands(coherentFlags: TranslateCoherent(type: imageType));
6279	mask = (spv::ImageOperandsMask)(mask & ~spv::ImageOperandsMakeTexelAvailableKHRMask);
6280	mask = mask | signExtensionMask();
6281	if (mask != spv::ImageOperandsMaskNone) {
6282	spv::IdImmediate imageOperands = { false, (unsigned int)mask };
6283	operands.push_back(x: imageOperands);
6284	}
6285	if (mask & spv::ImageOperandsSampleMask) {
6286	spv::IdImmediate imageOperand = { true, *opIt++ };
6287	operands.push_back(x: imageOperand);
6288	}
6289	if (mask & spv::ImageOperandsLodMask) {
6290	spv::IdImmediate imageOperand = { true, *opIt++ };
6291	operands.push_back(x: imageOperand);
6292	}
6293	if (mask & spv::ImageOperandsMakeTexelVisibleKHRMask) {
6294	spv::IdImmediate imageOperand = { true, builder.makeUintConstant(u: TranslateMemoryScope(
6295	coherentFlags: TranslateCoherent(type: imageType))) };
6296	operands.push_back(x: imageOperand);
6297	}
6298
6299	// Create the return type that was a special structure
6300	spv::Id texelOut = *opIt;
6301	spv::Id typeId0 = resultType();
6302	spv::Id typeId1 = builder.getDerefTypeId(resultId: texelOut);
6303	spv::Id resultTypeId = builder.makeStructResultType(type0: typeId0, type1: typeId1);
6304
6305	spv::Id resultId = builder.createOp(spv::OpImageSparseRead, typeId: resultTypeId, operands);
6306
6307	// Decode the return type
6308	builder.createStore(rValue: builder.createCompositeExtract(composite: resultId, typeId: typeId1, index: 1), lValue: texelOut);
6309	return builder.createCompositeExtract(composite: resultId, typeId: typeId0, index: 0);
6310	} else {
6311	// Process image atomic operations
6312
6313	// GLSL "IMAGE_PARAMS" will involve in constructing an image texel pointer and this pointer,
6314	// as the first source operand, is required by SPIR-V atomic operations.
6315	// For non-MS, the sample value should be 0
6316	spv::IdImmediate sample = { true, sampler.isMultiSample() ? *(opIt++) : builder.makeUintConstant(u: 0) };
6317	operands.push_back(x: sample);
6318
6319	spv::Id resultTypeId;
6320	glslang::TBasicType typeProxy = node->getBasicType();
6321	// imageAtomicStore has a void return type so base the pointer type on
6322	// the type of the value operand.
6323	if (node->getOp() == glslang::EOpImageAtomicStore) {
6324	resultTypeId = builder.makePointer(spv::StorageClassImage, pointee: builder.getTypeId(resultId: *opIt));
6325	typeProxy = node->getAsAggregate()->getSequence()[0]->getAsTyped()->getType().getSampler().type;
6326	} else {
6327	resultTypeId = builder.makePointer(spv::StorageClassImage, pointee: resultType());
6328	}
6329	spv::Id pointer = builder.createOp(spv::OpImageTexelPointer, typeId: resultTypeId, operands);
6330	if (imageType.getQualifier().nonUniform) {
6331	builder.addDecoration(pointer, spv::DecorationNonUniformEXT);
6332	}
6333
6334	std::vector<spv::Id> operands;
6335	operands.push_back(x: pointer);
6336	for (; opIt != arguments.end(); ++opIt)
6337	operands.push_back(x: *opIt);
6338
6339	return createAtomicOperation(op: node->getOp(), precision, typeId: resultType(), operands, typeProxy,
6340	lvalueCoherentFlags, opType: node->getType());
6341	}
6342	}
6343
6344	// Check for fragment mask functions other than queries
6345	if (cracked.fragMask) {
6346	assert(sampler.ms);
6347
6348	auto opIt = arguments.begin();
6349	std::vector<spv::Id> operands;
6350
6351	operands.push_back(x: params.sampler);
6352	++opIt;
6353
6354	if (sampler.isSubpass()) {
6355	// add on the (0,0) coordinate
6356	spv::Id zero = builder.makeIntConstant(i: 0);
6357	std::vector<spv::Id> comps;
6358	comps.push_back(x: zero);
6359	comps.push_back(x: zero);
6360	operands.push_back(x: builder.makeCompositeConstant(
6361	type: builder.makeVectorType(component: builder.makeIntType(width: 32), size: 2), comps));
6362	}
6363
6364	for (; opIt != arguments.end(); ++opIt)
6365	operands.push_back(x: *opIt);
6366
6367	spv::Op fragMaskOp = spv::OpNop;
6368	if (node->getOp() == glslang::EOpFragmentMaskFetch)
6369	fragMaskOp = spv::OpFragmentMaskFetchAMD;
6370	else if (node->getOp() == glslang::EOpFragmentFetch)
6371	fragMaskOp = spv::OpFragmentFetchAMD;
6372
6373	builder.addExtension(ext: spv::E_SPV_AMD_shader_fragment_mask);
6374	builder.addCapability(cap: spv::CapabilityFragmentMaskAMD);
6375	return builder.createOp(fragMaskOp, typeId: resultType(), operands);
6376	}
6377
6378	// Check for texture functions other than queries
6379	bool sparse = node->isSparseTexture();
6380	bool imageFootprint = node->isImageFootprint();
6381	bool cubeCompare = sampler.dim == glslang::EsdCube && sampler.isArrayed() && sampler.isShadow();
6382
6383	// check for bias argument
6384	bool bias = false;
6385	if (! cracked.lod && ! cracked.grad && ! cracked.fetch && ! cubeCompare) {
6386	int nonBiasArgCount = 2;
6387	if (cracked.gather)
6388	++nonBiasArgCount; // comp argument should be present when bias argument is present
6389
6390	if (f16ShadowCompare)
6391	++nonBiasArgCount;
6392	if (cracked.offset)
6393	++nonBiasArgCount;
6394	else if (cracked.offsets)
6395	++nonBiasArgCount;
6396	if (cracked.grad)
6397	nonBiasArgCount += 2;
6398	if (cracked.lodClamp)
6399	++nonBiasArgCount;
6400	if (sparse)
6401	++nonBiasArgCount;
6402	if (imageFootprint)
6403	//Following three extra arguments
6404	// int granularity, bool coarse, out gl_TextureFootprint2DNV footprint
6405	nonBiasArgCount += 3;
6406	if ((int)arguments.size() > nonBiasArgCount)
6407	bias = true;
6408	}
6409
6410	if (cracked.gather) {
6411	const auto& sourceExtensions = glslangIntermediate->getRequestedExtensions();
6412	if (bias || cracked.lod ||
6413	sourceExtensions.find(x: glslang::E_GL_AMD_texture_gather_bias_lod) != sourceExtensions.end()) {
6414	builder.addExtension(ext: spv::E_SPV_AMD_texture_gather_bias_lod);
6415	builder.addCapability(cap: spv::CapabilityImageGatherBiasLodAMD);
6416	}
6417	}
6418
6419	// set the rest of the arguments
6420
6421	params.coords = arguments[1];
6422	int extraArgs = 0;
6423	bool noImplicitLod = false;
6424
6425	// sort out where Dref is coming from
6426	if (cubeCompare || f16ShadowCompare) {
6427	params.Dref = arguments[2];
6428	++extraArgs;
6429	} else if (sampler.shadow && cracked.gather) {
6430	params.Dref = arguments[2];
6431	++extraArgs;
6432	} else if (sampler.shadow) {
6433	std::vector<spv::Id> indexes;
6434	int dRefComp;
6435	if (cracked.proj)
6436	dRefComp = 2; // "The resulting 3rd component of P in the shadow forms is used as Dref"
6437	else
6438	dRefComp = builder.getNumComponents(resultId: params.coords) - 1;
6439	indexes.push_back(x: dRefComp);
6440	params.Dref = builder.createCompositeExtract(composite: params.coords,
6441	typeId: builder.getScalarTypeId(typeId: builder.getTypeId(resultId: params.coords)), indexes);
6442	}
6443
6444	// lod
6445	if (cracked.lod) {
6446	params.lod = arguments[2 + extraArgs];
6447	++extraArgs;
6448	} else if (glslangIntermediate->getStage() != EShLangFragment &&
6449	!(glslangIntermediate->getStage() == EShLangCompute &&
6450	glslangIntermediate->hasLayoutDerivativeModeNone())) {
6451	// we need to invent the default lod for an explicit lod instruction for a non-fragment stage
6452	noImplicitLod = true;
6453	}
6454
6455	// multisample
6456	if (sampler.isMultiSample()) {
6457	params.sample = arguments[2 + extraArgs]; // For MS, "sample" should be specified
6458	++extraArgs;
6459	}
6460
6461	// gradient
6462	if (cracked.grad) {
6463	params.gradX = arguments[2 + extraArgs];
6464	params.gradY = arguments[3 + extraArgs];
6465	extraArgs += 2;
6466	}
6467
6468	// offset and offsets
6469	if (cracked.offset) {
6470	params.offset = arguments[2 + extraArgs];
6471	++extraArgs;
6472	} else if (cracked.offsets) {
6473	params.offsets = arguments[2 + extraArgs];
6474	++extraArgs;
6475	}
6476
6477	// lod clamp
6478	if (cracked.lodClamp) {
6479	params.lodClamp = arguments[2 + extraArgs];
6480	++extraArgs;
6481	}
6482	// sparse
6483	if (sparse) {
6484	params.texelOut = arguments[2 + extraArgs];
6485	++extraArgs;
6486	}
6487	// gather component
6488	if (cracked.gather && ! sampler.shadow) {
6489	// default component is 0, if missing, otherwise an argument
6490	if (2 + extraArgs < (int)arguments.size()) {
6491	params.component = arguments[2 + extraArgs];
6492	++extraArgs;
6493	} else
6494	params.component = builder.makeIntConstant(i: 0);
6495	}
6496	spv::Id resultStruct = spv::NoResult;
6497	if (imageFootprint) {
6498	//Following three extra arguments
6499	// int granularity, bool coarse, out gl_TextureFootprint2DNV footprint
6500	params.granularity = arguments[2 + extraArgs];
6501	params.coarse = arguments[3 + extraArgs];
6502	resultStruct = arguments[4 + extraArgs];
6503	extraArgs += 3;
6504	}
6505
6506	// bias
6507	if (bias) {
6508	params.bias = arguments[2 + extraArgs];
6509	++extraArgs;
6510	}
6511
6512	if (imageFootprint) {
6513	builder.addExtension(ext: spv::E_SPV_NV_shader_image_footprint);
6514	builder.addCapability(cap: spv::CapabilityImageFootprintNV);
6515
6516
6517	//resultStructType(OpenGL type) contains 5 elements:
6518	//struct gl_TextureFootprint2DNV {
6519	// uvec2 anchor;
6520	// uvec2 offset;
6521	// uvec2 mask;
6522	// uint lod;
6523	// uint granularity;
6524	//};
6525	//or
6526	//struct gl_TextureFootprint3DNV {
6527	// uvec3 anchor;
6528	// uvec3 offset;
6529	// uvec2 mask;
6530	// uint lod;
6531	// uint granularity;
6532	//};
6533	spv::Id resultStructType = builder.getContainedTypeId(typeId: builder.getTypeId(resultId: resultStruct));
6534	assert(builder.isStructType(resultStructType));
6535
6536	//resType (SPIR-V type) contains 6 elements:
6537	//Member 0 must be a Boolean type scalar(LOD),
6538	//Member 1 must be a vector of integer type, whose Signedness operand is 0(anchor),
6539	//Member 2 must be a vector of integer type, whose Signedness operand is 0(offset),
6540	//Member 3 must be a vector of integer type, whose Signedness operand is 0(mask),
6541	//Member 4 must be a scalar of integer type, whose Signedness operand is 0(lod),
6542	//Member 5 must be a scalar of integer type, whose Signedness operand is 0(granularity).
6543	std::vector<spv::Id> members;
6544	members.push_back(x: resultType());
6545	for (int i = 0; i < 5; i++) {
6546	members.push_back(x: builder.getContainedTypeId(typeId: resultStructType, i));
6547	}
6548	spv::Id resType = builder.makeStructType(members, name: "ResType");
6549
6550	//call ImageFootprintNV
6551	spv::Id res = builder.createTextureCall(precision, resultType: resType, sparse, fetch: cracked.fetch, proj: cracked.proj,
6552	gather: cracked.gather, noImplicit: noImplicitLod, params, signExtensionMask());
6553
6554	//copy resType (SPIR-V type) to resultStructType(OpenGL type)
6555	for (int i = 0; i < 5; i++) {
6556	builder.clearAccessChain();
6557	builder.setAccessChainLValue(resultStruct);
6558
6559	//Accessing to a struct we created, no coherent flag is set
6560	spv::Builder::AccessChain::CoherentFlags flags;
6561	flags.clear();
6562
6563	builder.accessChainPush(offset: builder.makeIntConstant(i), coherentFlags: flags, alignment: 0);
6564	builder.accessChainStore(rvalue: builder.createCompositeExtract(composite: res, typeId: builder.getContainedTypeId(typeId: resType, i+1),
6565	index: i+1), nonUniform: TranslateNonUniformDecoration(qualifier: imageType.getQualifier()));
6566	}
6567	return builder.createCompositeExtract(composite: res, typeId: resultType(), index: 0);
6568	}
6569
6570	// projective component (might not to move)
6571	// GLSL: "The texture coordinates consumed from P, not including the last component of P,
6572	// are divided by the last component of P."
6573	// SPIR-V: "... (u [, v] [, w], q)... It may be a vector larger than needed, but all
6574	// unused components will appear after all used components."
6575	if (cracked.proj) {
6576	int projSourceComp = builder.getNumComponents(resultId: params.coords) - 1;
6577	int projTargetComp;
6578	switch (sampler.dim) {
6579	case glslang::Esd1D: projTargetComp = 1; break;
6580	case glslang::Esd2D: projTargetComp = 2; break;
6581	case glslang::EsdRect: projTargetComp = 2; break;
6582	default: projTargetComp = projSourceComp; break;
6583	}
6584	// copy the projective coordinate if we have to
6585	if (projTargetComp != projSourceComp) {
6586	spv::Id projComp = builder.createCompositeExtract(composite: params.coords,
6587	typeId: builder.getScalarTypeId(typeId: builder.getTypeId(resultId: params.coords)), index: projSourceComp);
6588	params.coords = builder.createCompositeInsert(object: projComp, composite: params.coords,
6589	typeId: builder.getTypeId(resultId: params.coords), index: projTargetComp);
6590	}
6591	}
6592
6593	// nonprivate
6594	if (imageType.getQualifier().nonprivate) {
6595	params.nonprivate = true;
6596	}
6597
6598	// volatile
6599	if (imageType.getQualifier().volatil) {
6600	params.volatil = true;
6601	}
6602
6603	std::vector<spv::Id> result( 1,
6604	builder.createTextureCall(precision, resultType: resultType(), sparse, fetch: cracked.fetch, proj: cracked.proj, gather: cracked.gather,
6605	noImplicit: noImplicitLod, params, signExtensionMask())
6606	);
6607
6608	if (components != node->getType().getVectorSize())
6609	result[0] = builder.createConstructor(precision, sources: result, resultTypeId: convertGlslangToSpvType(type: node->getType()));
6610
6611	return result[0];
6612	}
6613
6614	spv::Id TGlslangToSpvTraverser::handleUserFunctionCall(const glslang::TIntermAggregate* node)
6615	{
6616	// Grab the function's pointer from the previously created function
6617	spv::Function* function = functionMap[node->getName().c_str()];
6618	if (! function)
6619	return 0;
6620
6621	const glslang::TIntermSequence& glslangArgs = node->getSequence();
6622	const glslang::TQualifierList& qualifiers = node->getQualifierList();
6623
6624	// See comments in makeFunctions() for details about the semantics for parameter passing.
6625	//
6626	// These imply we need a four step process:
6627	// 1. Evaluate the arguments
6628	// 2. Allocate and make copies of in, out, and inout arguments
6629	// 3. Make the call
6630	// 4. Copy back the results
6631
6632	// 1. Evaluate the arguments and their types
6633	std::vector<spv::Builder::AccessChain> lValues;
6634	std::vector<spv::Id> rValues;
6635	std::vector<const glslang::TType*> argTypes;
6636	for (int a = 0; a < (int)glslangArgs.size(); ++a) {
6637	argTypes.push_back(x: &glslangArgs[a]->getAsTyped()->getType());
6638	// build l-value
6639	builder.clearAccessChain();
6640	glslangArgs[a]->traverse(this);
6641	// keep outputs and pass-by-originals as l-values, evaluate others as r-values
6642	if (originalParam(qualifier: qualifiers[a], paramType: *argTypes[a], implicitThisParam: function->hasImplicitThis() && a == 0) ||
6643	writableParam(qualifier: qualifiers[a])) {
6644	// save l-value
6645	lValues.push_back(x: builder.getAccessChain());
6646	} else {
6647	// process r-value
6648	rValues.push_back(x: accessChainLoad(type: *argTypes.back()));
6649	}
6650	}
6651
6652	// Reset source location to the function call location after argument evaluation
6653	builder.setDebugSourceLocation(line: node->getLoc().line, filename: node->getLoc().getFilename());
6654
6655	// 2. Allocate space for anything needing a copy, and if it's "in" or "inout"
6656	// copy the original into that space.
6657	//
6658	// Also, build up the list of actual arguments to pass in for the call
6659	int lValueCount = 0;
6660	int rValueCount = 0;
6661	std::vector<spv::Id> spvArgs;
6662	for (int a = 0; a < (int)glslangArgs.size(); ++a) {
6663	spv::Id arg;
6664	if (originalParam(qualifier: qualifiers[a], paramType: *argTypes[a], implicitThisParam: function->hasImplicitThis() && a == 0)) {
6665	builder.setAccessChain(lValues[lValueCount]);
6666	arg = builder.accessChainGetLValue();
6667	++lValueCount;
6668	} else if (writableParam(qualifier: qualifiers[a])) {
6669	// need space to hold the copy
6670	arg = builder.createVariable(precision: function->getParamPrecision(param: a), storageClass: spv::StorageClassFunction,
6671	type: builder.getContainedTypeId(typeId: function->getParamType(p: a)), name: "param");
6672	if (qualifiers[a] == glslang::EvqIn || qualifiers[a] == glslang::EvqInOut) {
6673	// need to copy the input into output space
6674	builder.setAccessChain(lValues[lValueCount]);
6675	spv::Id copy = accessChainLoad(type: *argTypes[a]);
6676	builder.clearAccessChain();
6677	builder.setAccessChainLValue(arg);
6678	multiTypeStore(type: *argTypes[a], rValue: copy);
6679	}
6680	++lValueCount;
6681	} else {
6682	// process r-value, which involves a copy for a type mismatch
6683	if (function->getParamType(p: a) != builder.getTypeId(resultId: rValues[rValueCount]) ||
6684	TranslatePrecisionDecoration(type: *argTypes[a]) != function->getParamPrecision(param: a))
6685	{
6686	spv::Id argCopy = builder.createVariable(precision: function->getParamPrecision(param: a), storageClass: spv::StorageClassFunction, type: function->getParamType(p: a), name: "arg");
6687	builder.clearAccessChain();
6688	builder.setAccessChainLValue(argCopy);
6689	multiTypeStore(type: *argTypes[a], rValue: rValues[rValueCount]);
6690	arg = builder.createLoad(lValue: argCopy, precision: function->getParamPrecision(param: a));
6691	} else
6692	arg = rValues[rValueCount];
6693	++rValueCount;
6694	}
6695	spvArgs.push_back(x: arg);
6696	}
6697
6698	// 3. Make the call.
6699	spv::Id result = builder.createFunctionCall(function, spvArgs);
6700	builder.setPrecision(id: result, precision: TranslatePrecisionDecoration(type: node->getType()));
6701	builder.addDecoration(result, TranslateNonUniformDecoration(qualifier: node->getType().getQualifier()));
6702
6703	// 4. Copy back out an "out" arguments.
6704	lValueCount = 0;
6705	for (int a = 0; a < (int)glslangArgs.size(); ++a) {
6706	if (originalParam(qualifier: qualifiers[a], paramType: *argTypes[a], implicitThisParam: function->hasImplicitThis() && a == 0))
6707	++lValueCount;
6708	else if (writableParam(qualifier: qualifiers[a])) {
6709	if (qualifiers[a] == glslang::EvqOut || qualifiers[a] == glslang::EvqInOut) {
6710	spv::Id copy = builder.createLoad(lValue: spvArgs[a], precision: spv::NoPrecision);
6711	builder.addDecoration(copy, TranslateNonUniformDecoration(qualifier: argTypes[a]->getQualifier()));
6712	builder.setAccessChain(lValues[lValueCount]);
6713	multiTypeStore(type: *argTypes[a], rValue: copy);
6714	}
6715	++lValueCount;
6716	}
6717	}
6718
6719	return result;
6720	}
6721
6722	// Translate AST operation to SPV operation, already having SPV-based operands/types.
6723	spv::Id TGlslangToSpvTraverser::createBinaryOperation(glslang::TOperator op, OpDecorations& decorations,
6724	spv::Id typeId, spv::Id left, spv::Id right,
6725	glslang::TBasicType typeProxy, bool reduceComparison)
6726	{
6727	bool isUnsigned = isTypeUnsignedInt(type: typeProxy);
6728	bool isFloat = isTypeFloat(type: typeProxy);
6729	bool isBool = typeProxy == glslang::EbtBool;
6730
6731	spv::Op binOp = spv::OpNop;
6732	bool needMatchingVectors = true; // for non-matrix ops, would a scalar need to smear to match a vector?
6733	bool comparison = false;
6734
6735	switch (op) {
6736	case glslang::EOpAdd:
6737	case glslang::EOpAddAssign:
6738	if (isFloat)
6739	binOp = spv::OpFAdd;
6740	else
6741	binOp = spv::OpIAdd;
6742	break;
6743	case glslang::EOpSub:
6744	case glslang::EOpSubAssign:
6745	if (isFloat)
6746	binOp = spv::OpFSub;
6747	else
6748	binOp = spv::OpISub;
6749	break;
6750	case glslang::EOpMul:
6751	case glslang::EOpMulAssign:
6752	if (isFloat)
6753	binOp = spv::OpFMul;
6754	else
6755	binOp = spv::OpIMul;
6756	break;
6757	case glslang::EOpVectorTimesScalar:
6758	case glslang::EOpVectorTimesScalarAssign:
6759	if (isFloat && (builder.isVector(resultId: left) || builder.isVector(resultId: right))) {
6760	if (builder.isVector(resultId: right))
6761	std::swap(a&: left, b&: right);
6762	assert(builder.isScalar(right));
6763	needMatchingVectors = false;
6764	binOp = spv::OpVectorTimesScalar;
6765	} else if (isFloat)
6766	binOp = spv::OpFMul;
6767	else
6768	binOp = spv::OpIMul;
6769	break;
6770	case glslang::EOpVectorTimesMatrix:
6771	case glslang::EOpVectorTimesMatrixAssign:
6772	binOp = spv::OpVectorTimesMatrix;
6773	break;
6774	case glslang::EOpMatrixTimesVector:
6775	binOp = spv::OpMatrixTimesVector;
6776	break;
6777	case glslang::EOpMatrixTimesScalar:
6778	case glslang::EOpMatrixTimesScalarAssign:
6779	binOp = spv::OpMatrixTimesScalar;
6780	break;
6781	case glslang::EOpMatrixTimesMatrix:
6782	case glslang::EOpMatrixTimesMatrixAssign:
6783	binOp = spv::OpMatrixTimesMatrix;
6784	break;
6785	case glslang::EOpOuterProduct:
6786	binOp = spv::OpOuterProduct;
6787	needMatchingVectors = false;
6788	break;
6789
6790	case glslang::EOpDiv:
6791	case glslang::EOpDivAssign:
6792	if (isFloat)
6793	binOp = spv::OpFDiv;
6794	else if (isUnsigned)
6795	binOp = spv::OpUDiv;
6796	else
6797	binOp = spv::OpSDiv;
6798	break;
6799	case glslang::EOpMod:
6800	case glslang::EOpModAssign:
6801	if (isFloat)
6802	binOp = spv::OpFMod;
6803	else if (isUnsigned)
6804	binOp = spv::OpUMod;
6805	else
6806	binOp = spv::OpSMod;
6807	break;
6808	case glslang::EOpRightShift:
6809	case glslang::EOpRightShiftAssign:
6810	if (isUnsigned)
6811	binOp = spv::OpShiftRightLogical;
6812	else
6813	binOp = spv::OpShiftRightArithmetic;
6814	break;
6815	case glslang::EOpLeftShift:
6816	case glslang::EOpLeftShiftAssign:
6817	binOp = spv::OpShiftLeftLogical;
6818	break;
6819	case glslang::EOpAnd:
6820	case glslang::EOpAndAssign:
6821	binOp = spv::OpBitwiseAnd;
6822	break;
6823	case glslang::EOpLogicalAnd:
6824	needMatchingVectors = false;
6825	binOp = spv::OpLogicalAnd;
6826	break;
6827	case glslang::EOpInclusiveOr:
6828	case glslang::EOpInclusiveOrAssign:
6829	binOp = spv::OpBitwiseOr;
6830	break;
6831	case glslang::EOpLogicalOr:
6832	needMatchingVectors = false;
6833	binOp = spv::OpLogicalOr;
6834	break;
6835	case glslang::EOpExclusiveOr:
6836	case glslang::EOpExclusiveOrAssign:
6837	binOp = spv::OpBitwiseXor;
6838	break;
6839	case glslang::EOpLogicalXor:
6840	needMatchingVectors = false;
6841	binOp = spv::OpLogicalNotEqual;
6842	break;
6843
6844	case glslang::EOpAbsDifference:
6845	binOp = isUnsigned ? spv::OpAbsUSubINTEL : spv::OpAbsISubINTEL;
6846	break;
6847
6848	case glslang::EOpAddSaturate:
6849	binOp = isUnsigned ? spv::OpUAddSatINTEL : spv::OpIAddSatINTEL;
6850	break;
6851
6852	case glslang::EOpSubSaturate:
6853	binOp = isUnsigned ? spv::OpUSubSatINTEL : spv::OpISubSatINTEL;
6854	break;
6855
6856	case glslang::EOpAverage:
6857	binOp = isUnsigned ? spv::OpUAverageINTEL : spv::OpIAverageINTEL;
6858	break;
6859
6860	case glslang::EOpAverageRounded:
6861	binOp = isUnsigned ? spv::OpUAverageRoundedINTEL : spv::OpIAverageRoundedINTEL;
6862	break;
6863
6864	case glslang::EOpMul32x16:
6865	binOp = isUnsigned ? spv::OpUMul32x16INTEL : spv::OpIMul32x16INTEL;
6866	break;
6867
6868	case glslang::EOpExpectEXT:
6869	binOp = spv::OpExpectKHR;
6870	break;
6871
6872	case glslang::EOpLessThan:
6873	case glslang::EOpGreaterThan:
6874	case glslang::EOpLessThanEqual:
6875	case glslang::EOpGreaterThanEqual:
6876	case glslang::EOpEqual:
6877	case glslang::EOpNotEqual:
6878	case glslang::EOpVectorEqual:
6879	case glslang::EOpVectorNotEqual:
6880	comparison = true;
6881	break;
6882	default:
6883	break;
6884	}
6885
6886	// handle mapped binary operations (should be non-comparison)
6887	if (binOp != spv::OpNop) {
6888	assert(comparison == false);
6889	if (builder.isMatrix(resultId: left) || builder.isMatrix(resultId: right) ||
6890	builder.isCooperativeMatrix(resultId: left) || builder.isCooperativeMatrix(resultId: right))
6891	return createBinaryMatrixOperation(binOp, decorations, typeId, left, right);
6892
6893	// No matrix involved; make both operands be the same number of components, if needed
6894	if (needMatchingVectors)
6895	builder.promoteScalar(precision: decorations.precision, left, right);
6896
6897	spv::Id result = builder.createBinOp(binOp, typeId, operand1: left, operand2: right);
6898	decorations.addNoContraction(builder, t: result);
6899	decorations.addNonUniform(builder, t: result);
6900	return builder.setPrecision(id: result, precision: decorations.precision);
6901	}
6902
6903	if (! comparison)
6904	return 0;
6905
6906	// Handle comparison instructions
6907
6908	if (reduceComparison && (op == glslang::EOpEqual || op == glslang::EOpNotEqual)
6909	&& (builder.isVector(resultId: left) || builder.isMatrix(resultId: left) || builder.isAggregate(resultId: left))) {
6910	spv::Id result = builder.createCompositeCompare(precision: decorations.precision, left, right, op == glslang::EOpEqual);
6911	decorations.addNonUniform(builder, t: result);
6912	return result;
6913	}
6914
6915	switch (op) {
6916	case glslang::EOpLessThan:
6917	if (isFloat)
6918	binOp = spv::OpFOrdLessThan;
6919	else if (isUnsigned)
6920	binOp = spv::OpULessThan;
6921	else
6922	binOp = spv::OpSLessThan;
6923	break;
6924	case glslang::EOpGreaterThan:
6925	if (isFloat)
6926	binOp = spv::OpFOrdGreaterThan;
6927	else if (isUnsigned)
6928	binOp = spv::OpUGreaterThan;
6929	else
6930	binOp = spv::OpSGreaterThan;
6931	break;
6932	case glslang::EOpLessThanEqual:
6933	if (isFloat)
6934	binOp = spv::OpFOrdLessThanEqual;
6935	else if (isUnsigned)
6936	binOp = spv::OpULessThanEqual;
6937	else
6938	binOp = spv::OpSLessThanEqual;
6939	break;
6940	case glslang::EOpGreaterThanEqual:
6941	if (isFloat)
6942	binOp = spv::OpFOrdGreaterThanEqual;
6943	else if (isUnsigned)
6944	binOp = spv::OpUGreaterThanEqual;
6945	else
6946	binOp = spv::OpSGreaterThanEqual;
6947	break;
6948	case glslang::EOpEqual:
6949	case glslang::EOpVectorEqual:
6950	if (isFloat)
6951	binOp = spv::OpFOrdEqual;
6952	else if (isBool)
6953	binOp = spv::OpLogicalEqual;
6954	else
6955	binOp = spv::OpIEqual;
6956	break;
6957	case glslang::EOpNotEqual:
6958	case glslang::EOpVectorNotEqual:
6959	if (isFloat)
6960	binOp = spv::OpFUnordNotEqual;
6961	else if (isBool)
6962	binOp = spv::OpLogicalNotEqual;
6963	else
6964	binOp = spv::OpINotEqual;
6965	break;
6966	default:
6967	break;
6968	}
6969
6970	if (binOp != spv::OpNop) {
6971	spv::Id result = builder.createBinOp(binOp, typeId, operand1: left, operand2: right);
6972	decorations.addNoContraction(builder, t: result);
6973	decorations.addNonUniform(builder, t: result);
6974	return builder.setPrecision(id: result, precision: decorations.precision);
6975	}
6976
6977	return 0;
6978	}
6979
6980	//
6981	// Translate AST matrix operation to SPV operation, already having SPV-based operands/types.
6982	// These can be any of:
6983	//
6984	// matrix * scalar
6985	// scalar * matrix
6986	// matrix * matrix linear algebraic
6987	// matrix * vector
6988	// vector * matrix
6989	// matrix * matrix componentwise
6990	// matrix op matrix op in {+, -, /}
6991	// matrix op scalar op in {+, -, /}
6992	// scalar op matrix op in {+, -, /}
6993	//
6994	spv::Id TGlslangToSpvTraverser::createBinaryMatrixOperation(spv::Op op, OpDecorations& decorations, spv::Id typeId,
6995	spv::Id left, spv::Id right)
6996	{
6997	bool firstClass = true;
6998
6999	// First, handle first-class matrix operations (* and matrix/scalar)
7000	switch (op) {
7001	case spv::OpFDiv:
7002	if (builder.isMatrix(resultId: left) && builder.isScalar(resultId: right)) {
7003	// turn matrix / scalar into a multiply...
7004	spv::Id resultType = builder.getTypeId(resultId: right);
7005	right = builder.createBinOp(spv::OpFDiv, typeId: resultType, operand1: builder.makeFpConstant(type: resultType, d: 1.0), operand2: right);
7006	op = spv::OpMatrixTimesScalar;
7007	} else
7008	firstClass = false;
7009	break;
7010	case spv::OpMatrixTimesScalar:
7011	if (builder.isMatrix(resultId: right) || builder.isCooperativeMatrix(resultId: right))
7012	std::swap(a&: left, b&: right);
7013	assert(builder.isScalar(right));
7014	break;
7015	case spv::OpVectorTimesMatrix:
7016	assert(builder.isVector(left));
7017	assert(builder.isMatrix(right));
7018	break;
7019	case spv::OpMatrixTimesVector:
7020	assert(builder.isMatrix(left));
7021	assert(builder.isVector(right));
7022	break;
7023	case spv::OpMatrixTimesMatrix:
7024	assert(builder.isMatrix(left));
7025	assert(builder.isMatrix(right));
7026	break;
7027	default:
7028	firstClass = false;
7029	break;
7030	}
7031
7032	if (builder.isCooperativeMatrix(resultId: left) || builder.isCooperativeMatrix(resultId: right))
7033	firstClass = true;
7034
7035	if (firstClass) {
7036	spv::Id result = builder.createBinOp(op, typeId, operand1: left, operand2: right);
7037	decorations.addNoContraction(builder, t: result);
7038	decorations.addNonUniform(builder, t: result);
7039	return builder.setPrecision(id: result, precision: decorations.precision);
7040	}
7041
7042	// Handle component-wise +, -, *, %, and / for all combinations of type.
7043	// The result type of all of them is the same type as the (a) matrix operand.
7044	// The algorithm is to:
7045	// - break the matrix(es) into vectors
7046	// - smear any scalar to a vector
7047	// - do vector operations
7048	// - make a matrix out the vector results
7049	switch (op) {
7050	case spv::OpFAdd:
7051	case spv::OpFSub:
7052	case spv::OpFDiv:
7053	case spv::OpFMod:
7054	case spv::OpFMul:
7055	{
7056	// one time set up...
7057	bool leftMat = builder.isMatrix(resultId: left);
7058	bool rightMat = builder.isMatrix(resultId: right);
7059	unsigned int numCols = leftMat ? builder.getNumColumns(resultId: left) : builder.getNumColumns(resultId: right);
7060	int numRows = leftMat ? builder.getNumRows(resultId: left) : builder.getNumRows(resultId: right);
7061	spv::Id scalarType = builder.getScalarTypeId(typeId);
7062	spv::Id vecType = builder.makeVectorType(component: scalarType, size: numRows);
7063	std::vector<spv::Id> results;
7064	spv::Id smearVec = spv::NoResult;
7065	if (builder.isScalar(resultId: left))
7066	smearVec = builder.smearScalar(precision: decorations.precision, scalarVal: left, vectorType: vecType);
7067	else if (builder.isScalar(resultId: right))
7068	smearVec = builder.smearScalar(precision: decorations.precision, scalarVal: right, vectorType: vecType);
7069
7070	// do each vector op
7071	for (unsigned int c = 0; c < numCols; ++c) {
7072	std::vector<unsigned int> indexes;
7073	indexes.push_back(x: c);
7074	spv::Id leftVec = leftMat ? builder.createCompositeExtract( composite: left, typeId: vecType, indexes) : smearVec;
7075	spv::Id rightVec = rightMat ? builder.createCompositeExtract(composite: right, typeId: vecType, indexes) : smearVec;
7076	spv::Id result = builder.createBinOp(op, typeId: vecType, operand1: leftVec, operand2: rightVec);
7077	decorations.addNoContraction(builder, t: result);
7078	decorations.addNonUniform(builder, t: result);
7079	results.push_back(x: builder.setPrecision(id: result, precision: decorations.precision));
7080	}
7081
7082	// put the pieces together
7083	spv::Id result = builder.setPrecision(id: builder.createCompositeConstruct(typeId, constituents: results), precision: decorations.precision);
7084	decorations.addNonUniform(builder, t: result);
7085	return result;
7086	}
7087	default:
7088	assert(0);
7089	return spv::NoResult;
7090	}
7091	}
7092
7093	spv::Id TGlslangToSpvTraverser::createUnaryOperation(glslang::TOperator op, OpDecorations& decorations, spv::Id typeId,
7094	spv::Id operand, glslang::TBasicType typeProxy, const spv::Builder::AccessChain::CoherentFlags &lvalueCoherentFlags,
7095	const glslang::TType &opType)
7096	{
7097	spv::Op unaryOp = spv::OpNop;
7098	int extBuiltins = -1;
7099	int libCall = -1;
7100	bool isUnsigned = isTypeUnsignedInt(type: typeProxy);
7101	bool isFloat = isTypeFloat(type: typeProxy);
7102
7103	switch (op) {
7104	case glslang::EOpNegative:
7105	if (isFloat) {
7106	unaryOp = spv::OpFNegate;
7107	if (builder.isMatrixType(typeId))
7108	return createUnaryMatrixOperation(op: unaryOp, decorations, typeId, operand, typeProxy);
7109	} else
7110	unaryOp = spv::OpSNegate;
7111	break;
7112
7113	case glslang::EOpLogicalNot:
7114	case glslang::EOpVectorLogicalNot:
7115	unaryOp = spv::OpLogicalNot;
7116	break;
7117	case glslang::EOpBitwiseNot:
7118	unaryOp = spv::OpNot;
7119	break;
7120
7121	case glslang::EOpDeterminant:
7122	libCall = spv::GLSLstd450Determinant;
7123	break;
7124	case glslang::EOpMatrixInverse:
7125	libCall = spv::GLSLstd450MatrixInverse;
7126	break;
7127	case glslang::EOpTranspose:
7128	unaryOp = spv::OpTranspose;
7129	break;
7130
7131	case glslang::EOpRadians:
7132	libCall = spv::GLSLstd450Radians;
7133	break;
7134	case glslang::EOpDegrees:
7135	libCall = spv::GLSLstd450Degrees;
7136	break;
7137	case glslang::EOpSin:
7138	libCall = spv::GLSLstd450Sin;
7139	break;
7140	case glslang::EOpCos:
7141	libCall = spv::GLSLstd450Cos;
7142	break;
7143	case glslang::EOpTan:
7144	libCall = spv::GLSLstd450Tan;
7145	break;
7146	case glslang::EOpAcos:
7147	libCall = spv::GLSLstd450Acos;
7148	break;
7149	case glslang::EOpAsin:
7150	libCall = spv::GLSLstd450Asin;
7151	break;
7152	case glslang::EOpAtan:
7153	libCall = spv::GLSLstd450Atan;
7154	break;
7155
7156	case glslang::EOpAcosh:
7157	libCall = spv::GLSLstd450Acosh;
7158	break;
7159	case glslang::EOpAsinh:
7160	libCall = spv::GLSLstd450Asinh;
7161	break;
7162	case glslang::EOpAtanh:
7163	libCall = spv::GLSLstd450Atanh;
7164	break;
7165	case glslang::EOpTanh:
7166	libCall = spv::GLSLstd450Tanh;
7167	break;
7168	case glslang::EOpCosh:
7169	libCall = spv::GLSLstd450Cosh;
7170	break;
7171	case glslang::EOpSinh:
7172	libCall = spv::GLSLstd450Sinh;
7173	break;
7174
7175	case glslang::EOpLength:
7176	libCall = spv::GLSLstd450Length;
7177	break;
7178	case glslang::EOpNormalize:
7179	libCall = spv::GLSLstd450Normalize;
7180	break;
7181
7182	case glslang::EOpExp:
7183	libCall = spv::GLSLstd450Exp;
7184	break;
7185	case glslang::EOpLog:
7186	libCall = spv::GLSLstd450Log;
7187	break;
7188	case glslang::EOpExp2:
7189	libCall = spv::GLSLstd450Exp2;
7190	break;
7191	case glslang::EOpLog2:
7192	libCall = spv::GLSLstd450Log2;
7193	break;
7194	case glslang::EOpSqrt:
7195	libCall = spv::GLSLstd450Sqrt;
7196	break;
7197	case glslang::EOpInverseSqrt:
7198	libCall = spv::GLSLstd450InverseSqrt;
7199	break;
7200
7201	case glslang::EOpFloor:
7202	libCall = spv::GLSLstd450Floor;
7203	break;
7204	case glslang::EOpTrunc:
7205	libCall = spv::GLSLstd450Trunc;
7206	break;
7207	case glslang::EOpRound:
7208	libCall = spv::GLSLstd450Round;
7209	break;
7210	case glslang::EOpRoundEven:
7211	libCall = spv::GLSLstd450RoundEven;
7212	break;
7213	case glslang::EOpCeil:
7214	libCall = spv::GLSLstd450Ceil;
7215	break;
7216	case glslang::EOpFract:
7217	libCall = spv::GLSLstd450Fract;
7218	break;
7219
7220	case glslang::EOpIsNan:
7221	unaryOp = spv::OpIsNan;
7222	break;
7223	case glslang::EOpIsInf:
7224	unaryOp = spv::OpIsInf;
7225	break;
7226	case glslang::EOpIsFinite:
7227	unaryOp = spv::OpIsFinite;
7228	break;
7229
7230	case glslang::EOpFloatBitsToInt:
7231	case glslang::EOpFloatBitsToUint:
7232	case glslang::EOpIntBitsToFloat:
7233	case glslang::EOpUintBitsToFloat:
7234	case glslang::EOpDoubleBitsToInt64:
7235	case glslang::EOpDoubleBitsToUint64:
7236	case glslang::EOpInt64BitsToDouble:
7237	case glslang::EOpUint64BitsToDouble:
7238	case glslang::EOpFloat16BitsToInt16:
7239	case glslang::EOpFloat16BitsToUint16:
7240	case glslang::EOpInt16BitsToFloat16:
7241	case glslang::EOpUint16BitsToFloat16:
7242	unaryOp = spv::OpBitcast;
7243	break;
7244
7245	case glslang::EOpPackSnorm2x16:
7246	libCall = spv::GLSLstd450PackSnorm2x16;
7247	break;
7248	case glslang::EOpUnpackSnorm2x16:
7249	libCall = spv::GLSLstd450UnpackSnorm2x16;
7250	break;
7251	case glslang::EOpPackUnorm2x16:
7252	libCall = spv::GLSLstd450PackUnorm2x16;
7253	break;
7254	case glslang::EOpUnpackUnorm2x16:
7255	libCall = spv::GLSLstd450UnpackUnorm2x16;
7256	break;
7257	case glslang::EOpPackHalf2x16:
7258	libCall = spv::GLSLstd450PackHalf2x16;
7259	break;
7260	case glslang::EOpUnpackHalf2x16:
7261	libCall = spv::GLSLstd450UnpackHalf2x16;
7262	break;
7263	case glslang::EOpPackSnorm4x8:
7264	libCall = spv::GLSLstd450PackSnorm4x8;
7265	break;
7266	case glslang::EOpUnpackSnorm4x8:
7267	libCall = spv::GLSLstd450UnpackSnorm4x8;
7268	break;
7269	case glslang::EOpPackUnorm4x8:
7270	libCall = spv::GLSLstd450PackUnorm4x8;
7271	break;
7272	case glslang::EOpUnpackUnorm4x8:
7273	libCall = spv::GLSLstd450UnpackUnorm4x8;
7274	break;
7275	case glslang::EOpPackDouble2x32:
7276	libCall = spv::GLSLstd450PackDouble2x32;
7277	break;
7278	case glslang::EOpUnpackDouble2x32:
7279	libCall = spv::GLSLstd450UnpackDouble2x32;
7280	break;
7281
7282	case glslang::EOpPackInt2x32:
7283	case glslang::EOpUnpackInt2x32:
7284	case glslang::EOpPackUint2x32:
7285	case glslang::EOpUnpackUint2x32:
7286	case glslang::EOpPack16:
7287	case glslang::EOpPack32:
7288	case glslang::EOpPack64:
7289	case glslang::EOpUnpack32:
7290	case glslang::EOpUnpack16:
7291	case glslang::EOpUnpack8:
7292	case glslang::EOpPackInt2x16:
7293	case glslang::EOpUnpackInt2x16:
7294	case glslang::EOpPackUint2x16:
7295	case glslang::EOpUnpackUint2x16:
7296	case glslang::EOpPackInt4x16:
7297	case glslang::EOpUnpackInt4x16:
7298	case glslang::EOpPackUint4x16:
7299	case glslang::EOpUnpackUint4x16:
7300	case glslang::EOpPackFloat2x16:
7301	case glslang::EOpUnpackFloat2x16:
7302	unaryOp = spv::OpBitcast;
7303	break;
7304
7305	case glslang::EOpDPdx:
7306	unaryOp = spv::OpDPdx;
7307	break;
7308	case glslang::EOpDPdy:
7309	unaryOp = spv::OpDPdy;
7310	break;
7311	case glslang::EOpFwidth:
7312	unaryOp = spv::OpFwidth;
7313	break;
7314
7315	case glslang::EOpAny:
7316	unaryOp = spv::OpAny;
7317	break;
7318	case glslang::EOpAll:
7319	unaryOp = spv::OpAll;
7320	break;
7321
7322	case glslang::EOpAbs:
7323	if (isFloat)
7324	libCall = spv::GLSLstd450FAbs;
7325	else
7326	libCall = spv::GLSLstd450SAbs;
7327	break;
7328	case glslang::EOpSign:
7329	if (isFloat)
7330	libCall = spv::GLSLstd450FSign;
7331	else
7332	libCall = spv::GLSLstd450SSign;
7333	break;
7334
7335	case glslang::EOpDPdxFine:
7336	unaryOp = spv::OpDPdxFine;
7337	break;
7338	case glslang::EOpDPdyFine:
7339	unaryOp = spv::OpDPdyFine;
7340	break;
7341	case glslang::EOpFwidthFine:
7342	unaryOp = spv::OpFwidthFine;
7343	break;
7344	case glslang::EOpDPdxCoarse:
7345	unaryOp = spv::OpDPdxCoarse;
7346	break;
7347	case glslang::EOpDPdyCoarse:
7348	unaryOp = spv::OpDPdyCoarse;
7349	break;
7350	case glslang::EOpFwidthCoarse:
7351	unaryOp = spv::OpFwidthCoarse;
7352	break;
7353	case glslang::EOpRayQueryProceed:
7354	unaryOp = spv::OpRayQueryProceedKHR;
7355	break;
7356	case glslang::EOpRayQueryGetRayTMin:
7357	unaryOp = spv::OpRayQueryGetRayTMinKHR;
7358	break;
7359	case glslang::EOpRayQueryGetRayFlags:
7360	unaryOp = spv::OpRayQueryGetRayFlagsKHR;
7361	break;
7362	case glslang::EOpRayQueryGetWorldRayOrigin:
7363	unaryOp = spv::OpRayQueryGetWorldRayOriginKHR;
7364	break;
7365	case glslang::EOpRayQueryGetWorldRayDirection:
7366	unaryOp = spv::OpRayQueryGetWorldRayDirectionKHR;
7367	break;
7368	case glslang::EOpRayQueryGetIntersectionCandidateAABBOpaque:
7369	unaryOp = spv::OpRayQueryGetIntersectionCandidateAABBOpaqueKHR;
7370	break;
7371	case glslang::EOpInterpolateAtCentroid:
7372	if (typeProxy == glslang::EbtFloat16)
7373	builder.addExtension(ext: spv::E_SPV_AMD_gpu_shader_half_float);
7374	libCall = spv::GLSLstd450InterpolateAtCentroid;
7375	break;
7376	case glslang::EOpAtomicCounterIncrement:
7377	case glslang::EOpAtomicCounterDecrement:
7378	case glslang::EOpAtomicCounter:
7379	{
7380	// Handle all of the atomics in one place, in createAtomicOperation()
7381	std::vector<spv::Id> operands;
7382	operands.push_back(x: operand);
7383	return createAtomicOperation(op, precision: decorations.precision, typeId, operands, typeProxy, lvalueCoherentFlags, opType);
7384	}
7385
7386	case glslang::EOpBitFieldReverse:
7387	unaryOp = spv::OpBitReverse;
7388	break;
7389	case glslang::EOpBitCount:
7390	unaryOp = spv::OpBitCount;
7391	break;
7392	case glslang::EOpFindLSB:
7393	libCall = spv::GLSLstd450FindILsb;
7394	break;
7395	case glslang::EOpFindMSB:
7396	if (isUnsigned)
7397	libCall = spv::GLSLstd450FindUMsb;
7398	else
7399	libCall = spv::GLSLstd450FindSMsb;
7400	break;
7401
7402	case glslang::EOpCountLeadingZeros:
7403	builder.addCapability(cap: spv::CapabilityIntegerFunctions2INTEL);
7404	builder.addExtension(ext: "SPV_INTEL_shader_integer_functions2");
7405	unaryOp = spv::OpUCountLeadingZerosINTEL;
7406	break;
7407
7408	case glslang::EOpCountTrailingZeros:
7409	builder.addCapability(cap: spv::CapabilityIntegerFunctions2INTEL);
7410	builder.addExtension(ext: "SPV_INTEL_shader_integer_functions2");
7411	unaryOp = spv::OpUCountTrailingZerosINTEL;
7412	break;
7413
7414	case glslang::EOpBallot:
7415	case glslang::EOpReadFirstInvocation:
7416	case glslang::EOpAnyInvocation:
7417	case glslang::EOpAllInvocations:
7418	case glslang::EOpAllInvocationsEqual:
7419	case glslang::EOpMinInvocations:
7420	case glslang::EOpMaxInvocations:
7421	case glslang::EOpAddInvocations:
7422	case glslang::EOpMinInvocationsNonUniform:
7423	case glslang::EOpMaxInvocationsNonUniform:
7424	case glslang::EOpAddInvocationsNonUniform:
7425	case glslang::EOpMinInvocationsInclusiveScan:
7426	case glslang::EOpMaxInvocationsInclusiveScan:
7427	case glslang::EOpAddInvocationsInclusiveScan:
7428	case glslang::EOpMinInvocationsInclusiveScanNonUniform:
7429	case glslang::EOpMaxInvocationsInclusiveScanNonUniform:
7430	case glslang::EOpAddInvocationsInclusiveScanNonUniform:
7431	case glslang::EOpMinInvocationsExclusiveScan:
7432	case glslang::EOpMaxInvocationsExclusiveScan:
7433	case glslang::EOpAddInvocationsExclusiveScan:
7434	case glslang::EOpMinInvocationsExclusiveScanNonUniform:
7435	case glslang::EOpMaxInvocationsExclusiveScanNonUniform:
7436	case glslang::EOpAddInvocationsExclusiveScanNonUniform:
7437	{
7438	std::vector<spv::Id> operands;
7439	operands.push_back(x: operand);
7440	return createInvocationsOperation(op, typeId, operands, typeProxy);
7441	}
7442	case glslang::EOpSubgroupAll:
7443	case glslang::EOpSubgroupAny:
7444	case glslang::EOpSubgroupAllEqual:
7445	case glslang::EOpSubgroupBroadcastFirst:
7446	case glslang::EOpSubgroupBallot:
7447	case glslang::EOpSubgroupInverseBallot:
7448	case glslang::EOpSubgroupBallotBitCount:
7449	case glslang::EOpSubgroupBallotInclusiveBitCount:
7450	case glslang::EOpSubgroupBallotExclusiveBitCount:
7451	case glslang::EOpSubgroupBallotFindLSB:
7452	case glslang::EOpSubgroupBallotFindMSB:
7453	case glslang::EOpSubgroupAdd:
7454	case glslang::EOpSubgroupMul:
7455	case glslang::EOpSubgroupMin:
7456	case glslang::EOpSubgroupMax:
7457	case glslang::EOpSubgroupAnd:
7458	case glslang::EOpSubgroupOr:
7459	case glslang::EOpSubgroupXor:
7460	case glslang::EOpSubgroupInclusiveAdd:
7461	case glslang::EOpSubgroupInclusiveMul:
7462	case glslang::EOpSubgroupInclusiveMin:
7463	case glslang::EOpSubgroupInclusiveMax:
7464	case glslang::EOpSubgroupInclusiveAnd:
7465	case glslang::EOpSubgroupInclusiveOr:
7466	case glslang::EOpSubgroupInclusiveXor:
7467	case glslang::EOpSubgroupExclusiveAdd:
7468	case glslang::EOpSubgroupExclusiveMul:
7469	case glslang::EOpSubgroupExclusiveMin:
7470	case glslang::EOpSubgroupExclusiveMax:
7471	case glslang::EOpSubgroupExclusiveAnd:
7472	case glslang::EOpSubgroupExclusiveOr:
7473	case glslang::EOpSubgroupExclusiveXor:
7474	case glslang::EOpSubgroupQuadSwapHorizontal:
7475	case glslang::EOpSubgroupQuadSwapVertical:
7476	case glslang::EOpSubgroupQuadSwapDiagonal:
7477	case glslang::EOpSubgroupQuadAll:
7478	case glslang::EOpSubgroupQuadAny: {
7479	std::vector<spv::Id> operands;
7480	operands.push_back(x: operand);
7481	return createSubgroupOperation(op, typeId, operands, typeProxy);
7482	}
7483	case glslang::EOpMbcnt:
7484	extBuiltins = getExtBuiltins(name: spv::E_SPV_AMD_shader_ballot);
7485	libCall = spv::MbcntAMD;
7486	break;
7487
7488	case glslang::EOpCubeFaceIndex:
7489	extBuiltins = getExtBuiltins(name: spv::E_SPV_AMD_gcn_shader);
7490	libCall = spv::CubeFaceIndexAMD;
7491	break;
7492
7493	case glslang::EOpCubeFaceCoord:
7494	extBuiltins = getExtBuiltins(name: spv::E_SPV_AMD_gcn_shader);
7495	libCall = spv::CubeFaceCoordAMD;
7496	break;
7497	case glslang::EOpSubgroupPartition:
7498	unaryOp = spv::OpGroupNonUniformPartitionNV;
7499	break;
7500	case glslang::EOpConstructReference:
7501	unaryOp = spv::OpBitcast;
7502	break;
7503
7504	case glslang::EOpConvUint64ToAccStruct:
7505	case glslang::EOpConvUvec2ToAccStruct:
7506	unaryOp = spv::OpConvertUToAccelerationStructureKHR;
7507	break;
7508
7509	case glslang::EOpHitObjectIsEmptyNV:
7510	unaryOp = spv::OpHitObjectIsEmptyNV;
7511	break;
7512
7513	case glslang::EOpHitObjectIsMissNV:
7514	unaryOp = spv::OpHitObjectIsMissNV;
7515	break;
7516
7517	case glslang::EOpHitObjectIsHitNV:
7518	unaryOp = spv::OpHitObjectIsHitNV;
7519	break;
7520
7521	case glslang::EOpHitObjectGetObjectRayOriginNV:
7522	unaryOp = spv::OpHitObjectGetObjectRayOriginNV;
7523	break;
7524
7525	case glslang::EOpHitObjectGetObjectRayDirectionNV:
7526	unaryOp = spv::OpHitObjectGetObjectRayDirectionNV;
7527	break;
7528
7529	case glslang::EOpHitObjectGetWorldRayOriginNV:
7530	unaryOp = spv::OpHitObjectGetWorldRayOriginNV;
7531	break;
7532
7533	case glslang::EOpHitObjectGetWorldRayDirectionNV:
7534	unaryOp = spv::OpHitObjectGetWorldRayDirectionNV;
7535	break;
7536
7537	case glslang::EOpHitObjectGetObjectToWorldNV:
7538	unaryOp = spv::OpHitObjectGetObjectToWorldNV;
7539	break;
7540
7541	case glslang::EOpHitObjectGetWorldToObjectNV:
7542	unaryOp = spv::OpHitObjectGetWorldToObjectNV;
7543	break;
7544
7545	case glslang::EOpHitObjectGetRayTMinNV:
7546	unaryOp = spv::OpHitObjectGetRayTMinNV;
7547	break;
7548
7549	case glslang::EOpHitObjectGetRayTMaxNV:
7550	unaryOp = spv::OpHitObjectGetRayTMaxNV;
7551	break;
7552
7553	case glslang::EOpHitObjectGetPrimitiveIndexNV:
7554	unaryOp = spv::OpHitObjectGetPrimitiveIndexNV;
7555	break;
7556
7557	case glslang::EOpHitObjectGetInstanceIdNV:
7558	unaryOp = spv::OpHitObjectGetInstanceIdNV;
7559	break;
7560
7561	case glslang::EOpHitObjectGetInstanceCustomIndexNV:
7562	unaryOp = spv::OpHitObjectGetInstanceCustomIndexNV;
7563	break;
7564
7565	case glslang::EOpHitObjectGetGeometryIndexNV:
7566	unaryOp = spv::OpHitObjectGetGeometryIndexNV;
7567	break;
7568
7569	case glslang::EOpHitObjectGetHitKindNV:
7570	unaryOp = spv::OpHitObjectGetHitKindNV;
7571	break;
7572
7573	case glslang::EOpHitObjectGetCurrentTimeNV:
7574	unaryOp = spv::OpHitObjectGetCurrentTimeNV;
7575	break;
7576
7577	case glslang::EOpHitObjectGetShaderBindingTableRecordIndexNV:
7578	unaryOp = spv::OpHitObjectGetShaderBindingTableRecordIndexNV;
7579	break;
7580
7581	case glslang::EOpHitObjectGetShaderRecordBufferHandleNV:
7582	unaryOp = spv::OpHitObjectGetShaderRecordBufferHandleNV;
7583	break;
7584
7585	case glslang::EOpFetchMicroTriangleVertexPositionNV:
7586	unaryOp = spv::OpFetchMicroTriangleVertexPositionNV;
7587	break;
7588
7589	case glslang::EOpFetchMicroTriangleVertexBarycentricNV:
7590	unaryOp = spv::OpFetchMicroTriangleVertexBarycentricNV;
7591	break;
7592
7593	case glslang::EOpCopyObject:
7594	unaryOp = spv::OpCopyObject;
7595	break;
7596
7597	case glslang::EOpDepthAttachmentReadEXT:
7598	builder.addExtension(ext: spv::E_SPV_EXT_shader_tile_image);
7599	builder.addCapability(cap: spv::CapabilityTileImageDepthReadAccessEXT);
7600	unaryOp = spv::OpDepthAttachmentReadEXT;
7601	decorations.precision = spv::NoPrecision;
7602	break;
7603	case glslang::EOpStencilAttachmentReadEXT:
7604	builder.addExtension(ext: spv::E_SPV_EXT_shader_tile_image);
7605	builder.addCapability(cap: spv::CapabilityTileImageStencilReadAccessEXT);
7606	unaryOp = spv::OpStencilAttachmentReadEXT;
7607	decorations.precision = spv::DecorationRelaxedPrecision;
7608	break;
7609
7610	default:
7611	return 0;
7612	}
7613
7614	spv::Id id;
7615	if (libCall >= 0) {
7616	std::vector<spv::Id> args;
7617	args.push_back(x: operand);
7618	id = builder.createBuiltinCall(resultType: typeId, builtins: extBuiltins >= 0 ? extBuiltins : stdBuiltins, entryPoint: libCall, args);
7619	} else {
7620	id = builder.createUnaryOp(unaryOp, typeId, operand);
7621	}
7622
7623	decorations.addNoContraction(builder, t: id);
7624	decorations.addNonUniform(builder, t: id);
7625	return builder.setPrecision(id, precision: decorations.precision);
7626	}
7627
7628	// Create a unary operation on a matrix
7629	spv::Id TGlslangToSpvTraverser::createUnaryMatrixOperation(spv::Op op, OpDecorations& decorations, spv::Id typeId,
7630	spv::Id operand, glslang::TBasicType /* typeProxy */)
7631	{
7632	// Handle unary operations vector by vector.
7633	// The result type is the same type as the original type.
7634	// The algorithm is to:
7635	// - break the matrix into vectors
7636	// - apply the operation to each vector
7637	// - make a matrix out the vector results
7638
7639	// get the types sorted out
7640	int numCols = builder.getNumColumns(resultId: operand);
7641	int numRows = builder.getNumRows(resultId: operand);
7642	spv::Id srcVecType = builder.makeVectorType(component: builder.getScalarTypeId(typeId: builder.getTypeId(resultId: operand)), size: numRows);
7643	spv::Id destVecType = builder.makeVectorType(component: builder.getScalarTypeId(typeId), size: numRows);
7644	std::vector<spv::Id> results;
7645
7646	// do each vector op
7647	for (int c = 0; c < numCols; ++c) {
7648	std::vector<unsigned int> indexes;
7649	indexes.push_back(x: c);
7650	spv::Id srcVec = builder.createCompositeExtract(composite: operand, typeId: srcVecType, indexes);
7651	spv::Id destVec = builder.createUnaryOp(op, typeId: destVecType, operand: srcVec);
7652	decorations.addNoContraction(builder, t: destVec);
7653	decorations.addNonUniform(builder, t: destVec);
7654	results.push_back(x: builder.setPrecision(id: destVec, precision: decorations.precision));
7655	}
7656
7657	// put the pieces together
7658	spv::Id result = builder.setPrecision(id: builder.createCompositeConstruct(typeId, constituents: results), precision: decorations.precision);
7659	decorations.addNonUniform(builder, t: result);
7660	return result;
7661	}
7662
7663	// For converting integers where both the bitwidth and the signedness could
7664	// change, but only do the width change here. The caller is still responsible
7665	// for the signedness conversion.
7666	// destType is the final type that will be converted to, but this function
7667	// may only be doing part of that conversion.
7668	spv::Id TGlslangToSpvTraverser::createIntWidthConversion(spv::Id operand, int vectorSize, spv::Id destType,
7669	glslang::TBasicType resultBasicType, glslang::TBasicType operandBasicType)
7670	{
7671	// Get the result type width, based on the type to convert to.
7672	int width = GetNumBits(type: resultBasicType);
7673
7674	// Get the conversion operation and result type,
7675	// based on the target width, but the source type.
7676	spv::Id type = spv::NoType;
7677	spv::Op convOp = spv::OpNop;
7678	if (isTypeSignedInt(type: operandBasicType)) {
7679	convOp = spv::OpSConvert;
7680	type = builder.makeIntType(width);
7681	} else {
7682	convOp = spv::OpUConvert;
7683	type = builder.makeUintType(width);
7684	}
7685
7686	if (vectorSize > 0)
7687	type = builder.makeVectorType(component: type, size: vectorSize);
7688	else if (builder.getOpCode(id: destType) == spv::OpTypeCooperativeMatrixKHR ||
7689	builder.getOpCode(id: destType) == spv::OpTypeCooperativeMatrixNV) {
7690
7691	type = builder.makeCooperativeMatrixTypeWithSameShape(component: type, otherType: destType);
7692	}
7693
7694	return builder.createUnaryOp(convOp, typeId: type, operand);
7695	}
7696
7697	spv::Id TGlslangToSpvTraverser::createConversion(glslang::TOperator op, OpDecorations& decorations, spv::Id destType,
7698	spv::Id operand, glslang::TBasicType resultBasicType, glslang::TBasicType operandBasicType)
7699	{
7700	spv::Op convOp = spv::OpNop;
7701	spv::Id zero = 0;
7702	spv::Id one = 0;
7703
7704	int vectorSize = builder.isVectorType(typeId: destType) ? builder.getNumTypeComponents(typeId: destType) : 0;
7705
7706	if (IsOpNumericConv(op)) {
7707	if (isTypeSignedInt(type: operandBasicType) && isTypeFloat(type: resultBasicType)) {
7708	convOp = spv::OpConvertSToF;
7709	}
7710	if (isTypeUnsignedInt(type: operandBasicType) && isTypeFloat(type: resultBasicType)) {
7711	convOp = spv::OpConvertUToF;
7712	}
7713	if (isTypeFloat(type: operandBasicType) && isTypeSignedInt(type: resultBasicType)) {
7714	convOp = spv::OpConvertFToS;
7715	}
7716	if (isTypeFloat(type: operandBasicType) && isTypeUnsignedInt(type: resultBasicType)) {
7717	convOp = spv::OpConvertFToU;
7718	}
7719	if (isTypeSignedInt(type: operandBasicType) && isTypeSignedInt(type: resultBasicType)) {
7720	convOp = spv::OpSConvert;
7721	}
7722	if (isTypeUnsignedInt(type: operandBasicType) && isTypeUnsignedInt(type: resultBasicType)) {
7723	convOp = spv::OpUConvert;
7724	}
7725	if (isTypeFloat(type: operandBasicType) && isTypeFloat(type: resultBasicType)) {
7726	convOp = spv::OpFConvert;
7727	if (builder.isMatrixType(typeId: destType))
7728	return createUnaryMatrixOperation(op: convOp, decorations, typeId: destType, operand, operandBasicType);
7729	}
7730	if (isTypeInt(type: operandBasicType) && isTypeInt(type: resultBasicType) &&
7731	isTypeUnsignedInt(type: operandBasicType) != isTypeUnsignedInt(type: resultBasicType)) {
7732
7733	if (GetNumBits(type: operandBasicType) != GetNumBits(type: resultBasicType)) {
7734	// OpSConvert/OpUConvert + OpBitCast
7735	operand = createIntWidthConversion(operand, vectorSize, destType, resultBasicType, operandBasicType);
7736	}
7737
7738	if (builder.isInSpecConstCodeGenMode()) {
7739	uint32_t bits = GetNumBits(type: resultBasicType);
7740	spv::Id zeroType = builder.makeUintType(width: bits);
7741	if (bits == 64) {
7742	zero = builder.makeInt64Constant(typeId: zeroType, value: 0, specConstant: false);
7743	} else {
7744	zero = builder.makeIntConstant(typeId: zeroType, value: 0, specConstant: false);
7745	}
7746	zero = makeSmearedConstant(constant: zero, vectorSize);
7747	// Use OpIAdd, instead of OpBitcast to do the conversion when
7748	// generating for OpSpecConstantOp instruction.
7749	return builder.createBinOp(spv::OpIAdd, typeId: destType, operand1: operand, operand2: zero);
7750	}
7751	// For normal run-time conversion instruction, use OpBitcast.
7752	convOp = spv::OpBitcast;
7753	}
7754	if (resultBasicType == glslang::EbtBool) {
7755	uint32_t bits = GetNumBits(type: operandBasicType);
7756	if (isTypeInt(type: operandBasicType)) {
7757	spv::Id zeroType = builder.makeUintType(width: bits);
7758	if (bits == 64) {
7759	zero = builder.makeInt64Constant(typeId: zeroType, value: 0, specConstant: false);
7760	} else {
7761	zero = builder.makeIntConstant(typeId: zeroType, value: 0, specConstant: false);
7762	}
7763	zero = makeSmearedConstant(constant: zero, vectorSize);
7764	return builder.createBinOp(spv::OpINotEqual, typeId: destType, operand1: operand, operand2: zero);
7765	} else {
7766	assert(isTypeFloat(operandBasicType));
7767	if (bits == 64) {
7768	zero = builder.makeDoubleConstant(d: 0.0);
7769	} else if (bits == 32) {
7770	zero = builder.makeFloatConstant(f: 0.0);
7771	} else {
7772	assert(bits == 16);
7773	zero = builder.makeFloat16Constant(f16: 0.0);
7774	}
7775	zero = makeSmearedConstant(constant: zero, vectorSize);
7776	return builder.createBinOp(spv::OpFUnordNotEqual, typeId: destType, operand1: operand, operand2: zero);
7777	}
7778	}
7779	if (operandBasicType == glslang::EbtBool) {
7780	uint32_t bits = GetNumBits(type: resultBasicType);
7781	convOp = spv::OpSelect;
7782	if (isTypeInt(type: resultBasicType)) {
7783	spv::Id zeroType = isTypeSignedInt(type: resultBasicType) ? builder.makeIntType(width: bits) : builder.makeUintType(width: bits);
7784	if (bits == 64) {
7785	zero = builder.makeInt64Constant(typeId: zeroType, value: 0, specConstant: false);
7786	one = builder.makeInt64Constant(typeId: zeroType, value: 1, specConstant: false);
7787	} else {
7788	zero = builder.makeIntConstant(typeId: zeroType, value: 0, specConstant: false);
7789	one = builder.makeIntConstant(typeId: zeroType, value: 1, specConstant: false);
7790	}
7791	} else {
7792	assert(isTypeFloat(resultBasicType));
7793	if (bits == 64) {
7794	zero = builder.makeDoubleConstant(d: 0.0);
7795	one = builder.makeDoubleConstant(d: 1.0);
7796	} else if (bits == 32) {
7797	zero = builder.makeFloatConstant(f: 0.0);
7798	one = builder.makeFloatConstant(f: 1.0);
7799	} else {
7800	assert(bits == 16);
7801	zero = builder.makeFloat16Constant(f16: 0.0);
7802	one = builder.makeFloat16Constant(f16: 1.0);
7803	}
7804	}
7805	}
7806	}
7807
7808	if (convOp == spv::OpNop) {
7809	switch (op) {
7810	case glslang::EOpConvUint64ToPtr:
7811	convOp = spv::OpConvertUToPtr;
7812	break;
7813	case glslang::EOpConvPtrToUint64:
7814	convOp = spv::OpConvertPtrToU;
7815	break;
7816	case glslang::EOpConvPtrToUvec2:
7817	case glslang::EOpConvUvec2ToPtr:
7818	convOp = spv::OpBitcast;
7819	break;
7820
7821	default:
7822	break;
7823	}
7824	}
7825
7826	spv::Id result = 0;
7827	if (convOp == spv::OpNop)
7828	return result;
7829
7830	if (convOp == spv::OpSelect) {
7831	zero = makeSmearedConstant(constant: zero, vectorSize);
7832	one = makeSmearedConstant(constant: one, vectorSize);
7833	result = builder.createTriOp(convOp, typeId: destType, operand1: operand, operand2: one, operand3: zero);
7834	} else
7835	result = builder.createUnaryOp(convOp, typeId: destType, operand);
7836
7837	result = builder.setPrecision(id: result, precision: decorations.precision);
7838	decorations.addNonUniform(builder, t: result);
7839	return result;
7840	}
7841
7842	spv::Id TGlslangToSpvTraverser::makeSmearedConstant(spv::Id constant, int vectorSize)
7843	{
7844	if (vectorSize == 0)
7845	return constant;
7846
7847	spv::Id vectorTypeId = builder.makeVectorType(component: builder.getTypeId(resultId: constant), size: vectorSize);
7848	std::vector<spv::Id> components;
7849	for (int c = 0; c < vectorSize; ++c)
7850	components.push_back(x: constant);
7851	return builder.makeCompositeConstant(type: vectorTypeId, comps: components);
7852	}
7853
7854	// For glslang ops that map to SPV atomic opCodes
7855	spv::Id TGlslangToSpvTraverser::createAtomicOperation(glslang::TOperator op, spv::Decoration /*precision*/,
7856	spv::Id typeId, std::vector<spv::Id>& operands, glslang::TBasicType typeProxy,
7857	const spv::Builder::AccessChain::CoherentFlags &lvalueCoherentFlags, const glslang::TType &opType)
7858	{
7859	spv::Op opCode = spv::OpNop;
7860
7861	switch (op) {
7862	case glslang::EOpAtomicAdd:
7863	case glslang::EOpImageAtomicAdd:
7864	case glslang::EOpAtomicCounterAdd:
7865	opCode = spv::OpAtomicIAdd;
7866	if (typeProxy == glslang::EbtFloat16 || typeProxy == glslang::EbtFloat || typeProxy == glslang::EbtDouble) {
7867	opCode = spv::OpAtomicFAddEXT;
7868	if (typeProxy == glslang::EbtFloat16 &&
7869	(opType.getVectorSize() == 2 || opType.getVectorSize() == 4)) {
7870	builder.addExtension(ext: spv::E_SPV_NV_shader_atomic_fp16_vector);
7871	builder.addCapability(cap: spv::CapabilityAtomicFloat16VectorNV);
7872	} else {
7873	builder.addExtension(ext: spv::E_SPV_EXT_shader_atomic_float_add);
7874	if (typeProxy == glslang::EbtFloat16) {
7875	builder.addExtension(ext: spv::E_SPV_EXT_shader_atomic_float16_add);
7876	builder.addCapability(cap: spv::CapabilityAtomicFloat16AddEXT);
7877	} else if (typeProxy == glslang::EbtFloat) {
7878	builder.addCapability(cap: spv::CapabilityAtomicFloat32AddEXT);
7879	} else {
7880	builder.addCapability(cap: spv::CapabilityAtomicFloat64AddEXT);
7881	}
7882	}
7883	}
7884	break;
7885	case glslang::EOpAtomicSubtract:
7886	case glslang::EOpAtomicCounterSubtract:
7887	opCode = spv::OpAtomicISub;
7888	break;
7889	case glslang::EOpAtomicMin:
7890	case glslang::EOpImageAtomicMin:
7891	case glslang::EOpAtomicCounterMin:
7892	if (typeProxy == glslang::EbtFloat16 || typeProxy == glslang::EbtFloat || typeProxy == glslang::EbtDouble) {
7893	opCode = spv::OpAtomicFMinEXT;
7894	if (typeProxy == glslang::EbtFloat16 &&
7895	(opType.getVectorSize() == 2 || opType.getVectorSize() == 4)) {
7896	builder.addExtension(ext: spv::E_SPV_NV_shader_atomic_fp16_vector);
7897	builder.addCapability(cap: spv::CapabilityAtomicFloat16VectorNV);
7898	} else {
7899	builder.addExtension(ext: spv::E_SPV_EXT_shader_atomic_float_min_max);
7900	if (typeProxy == glslang::EbtFloat16)
7901	builder.addCapability(cap: spv::CapabilityAtomicFloat16MinMaxEXT);
7902	else if (typeProxy == glslang::EbtFloat)
7903	builder.addCapability(cap: spv::CapabilityAtomicFloat32MinMaxEXT);
7904	else
7905	builder.addCapability(cap: spv::CapabilityAtomicFloat64MinMaxEXT);
7906	}
7907	} else if (typeProxy == glslang::EbtUint || typeProxy == glslang::EbtUint64) {
7908	opCode = spv::OpAtomicUMin;
7909	} else {
7910	opCode = spv::OpAtomicSMin;
7911	}
7912	break;
7913	case glslang::EOpAtomicMax:
7914	case glslang::EOpImageAtomicMax:
7915	case glslang::EOpAtomicCounterMax:
7916	if (typeProxy == glslang::EbtFloat16 || typeProxy == glslang::EbtFloat || typeProxy == glslang::EbtDouble) {
7917	opCode = spv::OpAtomicFMaxEXT;
7918	if (typeProxy == glslang::EbtFloat16 &&
7919	(opType.getVectorSize() == 2 || opType.getVectorSize() == 4)) {
7920	builder.addExtension(ext: spv::E_SPV_NV_shader_atomic_fp16_vector);
7921	builder.addCapability(cap: spv::CapabilityAtomicFloat16VectorNV);
7922	} else {
7923	builder.addExtension(ext: spv::E_SPV_EXT_shader_atomic_float_min_max);
7924	if (typeProxy == glslang::EbtFloat16)
7925	builder.addCapability(cap: spv::CapabilityAtomicFloat16MinMaxEXT);
7926	else if (typeProxy == glslang::EbtFloat)
7927	builder.addCapability(cap: spv::CapabilityAtomicFloat32MinMaxEXT);
7928	else
7929	builder.addCapability(cap: spv::CapabilityAtomicFloat64MinMaxEXT);
7930	}
7931	} else if (typeProxy == glslang::EbtUint || typeProxy == glslang::EbtUint64) {
7932	opCode = spv::OpAtomicUMax;
7933	} else {
7934	opCode = spv::OpAtomicSMax;
7935	}
7936	break;
7937	case glslang::EOpAtomicAnd:
7938	case glslang::EOpImageAtomicAnd:
7939	case glslang::EOpAtomicCounterAnd:
7940	opCode = spv::OpAtomicAnd;
7941	break;
7942	case glslang::EOpAtomicOr:
7943	case glslang::EOpImageAtomicOr:
7944	case glslang::EOpAtomicCounterOr:
7945	opCode = spv::OpAtomicOr;
7946	break;
7947	case glslang::EOpAtomicXor:
7948	case glslang::EOpImageAtomicXor:
7949	case glslang::EOpAtomicCounterXor:
7950	opCode = spv::OpAtomicXor;
7951	break;
7952	case glslang::EOpAtomicExchange:
7953	case glslang::EOpImageAtomicExchange:
7954	case glslang::EOpAtomicCounterExchange:
7955	if ((typeProxy == glslang::EbtFloat16) &&
7956	(opType.getVectorSize() == 2 || opType.getVectorSize() == 4)) {
7957	builder.addExtension(ext: spv::E_SPV_NV_shader_atomic_fp16_vector);
7958	builder.addCapability(cap: spv::CapabilityAtomicFloat16VectorNV);
7959	}
7960
7961	opCode = spv::OpAtomicExchange;
7962	break;
7963	case glslang::EOpAtomicCompSwap:
7964	case glslang::EOpImageAtomicCompSwap:
7965	case glslang::EOpAtomicCounterCompSwap:
7966	opCode = spv::OpAtomicCompareExchange;
7967	break;
7968	case glslang::EOpAtomicCounterIncrement:
7969	opCode = spv::OpAtomicIIncrement;
7970	break;
7971	case glslang::EOpAtomicCounterDecrement:
7972	opCode = spv::OpAtomicIDecrement;
7973	break;
7974	case glslang::EOpAtomicCounter:
7975	case glslang::EOpImageAtomicLoad:
7976	case glslang::EOpAtomicLoad:
7977	opCode = spv::OpAtomicLoad;
7978	break;
7979	case glslang::EOpAtomicStore:
7980	case glslang::EOpImageAtomicStore:
7981	opCode = spv::OpAtomicStore;
7982	break;
7983	default:
7984	assert(0);
7985	break;
7986	}
7987
7988	if (typeProxy == glslang::EbtInt64 || typeProxy == glslang::EbtUint64)
7989	builder.addCapability(cap: spv::CapabilityInt64Atomics);
7990
7991	// Sort out the operands
7992	// - mapping from glslang -> SPV
7993	// - there are extra SPV operands that are optional in glslang
7994	// - compare-exchange swaps the value and comparator
7995	// - compare-exchange has an extra memory semantics
7996	// - EOpAtomicCounterDecrement needs a post decrement
7997	spv::Id pointerId = 0, compareId = 0, valueId = 0;
7998	// scope defaults to Device in the old model, QueueFamilyKHR in the new model
7999	spv::Id scopeId;
8000	if (glslangIntermediate->usingVulkanMemoryModel()) {
8001	scopeId = builder.makeUintConstant(u: spv::ScopeQueueFamilyKHR);
8002	} else {
8003	scopeId = builder.makeUintConstant(u: spv::ScopeDevice);
8004	}
8005	// semantics default to relaxed
8006	spv::Id semanticsId = builder.makeUintConstant(u: lvalueCoherentFlags.isVolatile() &&
8007	glslangIntermediate->usingVulkanMemoryModel() ?
8008	spv::MemorySemanticsVolatileMask :
8009	spv::MemorySemanticsMaskNone);
8010	spv::Id semanticsId2 = semanticsId;
8011
8012	pointerId = operands[0];
8013	if (opCode == spv::OpAtomicIIncrement || opCode == spv::OpAtomicIDecrement) {
8014	// no additional operands
8015	} else if (opCode == spv::OpAtomicCompareExchange) {
8016	compareId = operands[1];
8017	valueId = operands[2];
8018	if (operands.size() > 3) {
8019	scopeId = operands[3];
8020	semanticsId = builder.makeUintConstant(
8021	u: builder.getConstantScalar(resultId: operands[4]) | builder.getConstantScalar(resultId: operands[5]));
8022	semanticsId2 = builder.makeUintConstant(
8023	u: builder.getConstantScalar(resultId: operands[6]) | builder.getConstantScalar(resultId: operands[7]));
8024	}
8025	} else if (opCode == spv::OpAtomicLoad) {
8026	if (operands.size() > 1) {
8027	scopeId = operands[1];
8028	semanticsId = builder.makeUintConstant(
8029	u: builder.getConstantScalar(resultId: operands[2]) | builder.getConstantScalar(resultId: operands[3]));
8030	}
8031	} else {
8032	// atomic store or RMW
8033	valueId = operands[1];
8034	if (operands.size() > 2) {
8035	scopeId = operands[2];
8036	semanticsId = builder.makeUintConstant
8037	(u: builder.getConstantScalar(resultId: operands[3]) | builder.getConstantScalar(resultId: operands[4]));
8038	}
8039	}
8040
8041	// Check for capabilities
8042	unsigned semanticsImmediate = builder.getConstantScalar(resultId: semanticsId) | builder.getConstantScalar(resultId: semanticsId2);
8043	if (semanticsImmediate & (spv::MemorySemanticsMakeAvailableKHRMask |
8044	spv::MemorySemanticsMakeVisibleKHRMask |
8045	spv::MemorySemanticsOutputMemoryKHRMask |
8046	spv::MemorySemanticsVolatileMask)) {
8047	builder.addCapability(cap: spv::CapabilityVulkanMemoryModelKHR);
8048	}
8049
8050	if (builder.getConstantScalar(resultId: scopeId) == spv::ScopeQueueFamily) {
8051	builder.addCapability(cap: spv::CapabilityVulkanMemoryModelKHR);
8052	}
8053
8054	if (glslangIntermediate->usingVulkanMemoryModel() && builder.getConstantScalar(resultId: scopeId) == spv::ScopeDevice) {
8055	builder.addCapability(cap: spv::CapabilityVulkanMemoryModelDeviceScopeKHR);
8056	}
8057
8058	std::vector<spv::Id> spvAtomicOperands; // hold the spv operands
8059	spvAtomicOperands.reserve(n: 6);
8060	spvAtomicOperands.push_back(x: pointerId);
8061	spvAtomicOperands.push_back(x: scopeId);
8062	spvAtomicOperands.push_back(x: semanticsId);
8063	if (opCode == spv::OpAtomicCompareExchange) {
8064	spvAtomicOperands.push_back(x: semanticsId2);
8065	spvAtomicOperands.push_back(x: valueId);
8066	spvAtomicOperands.push_back(x: compareId);
8067	} else if (opCode != spv::OpAtomicLoad && opCode != spv::OpAtomicIIncrement && opCode != spv::OpAtomicIDecrement) {
8068	spvAtomicOperands.push_back(x: valueId);
8069	}
8070
8071	if (opCode == spv::OpAtomicStore) {
8072	builder.createNoResultOp(opCode, operands: spvAtomicOperands);
8073	return 0;
8074	} else {
8075	spv::Id resultId = builder.createOp(opCode, typeId, operands: spvAtomicOperands);
8076
8077	// GLSL and HLSL atomic-counter decrement return post-decrement value,
8078	// while SPIR-V returns pre-decrement value. Translate between these semantics.
8079	if (op == glslang::EOpAtomicCounterDecrement)
8080	resultId = builder.createBinOp(spv::OpISub, typeId, operand1: resultId, operand2: builder.makeIntConstant(i: 1));
8081
8082	return resultId;
8083	}
8084	}
8085
8086	// Create group invocation operations.
8087	spv::Id TGlslangToSpvTraverser::createInvocationsOperation(glslang::TOperator op, spv::Id typeId,
8088	std::vector<spv::Id>& operands, glslang::TBasicType typeProxy)
8089	{
8090	bool isUnsigned = isTypeUnsignedInt(type: typeProxy);
8091	bool isFloat = isTypeFloat(type: typeProxy);
8092
8093	spv::Op opCode = spv::OpNop;
8094	std::vector<spv::IdImmediate> spvGroupOperands;
8095	spv::GroupOperation groupOperation = spv::GroupOperationMax;
8096
8097	if (op == glslang::EOpBallot || op == glslang::EOpReadFirstInvocation ||
8098	op == glslang::EOpReadInvocation) {
8099	builder.addExtension(ext: spv::E_SPV_KHR_shader_ballot);
8100	builder.addCapability(cap: spv::CapabilitySubgroupBallotKHR);
8101	} else if (op == glslang::EOpAnyInvocation ||
8102	op == glslang::EOpAllInvocations ||
8103	op == glslang::EOpAllInvocationsEqual) {
8104	builder.addExtension(ext: spv::E_SPV_KHR_subgroup_vote);
8105	builder.addCapability(cap: spv::CapabilitySubgroupVoteKHR);
8106	} else {
8107	builder.addCapability(cap: spv::CapabilityGroups);
8108	if (op == glslang::EOpMinInvocationsNonUniform ||
8109	op == glslang::EOpMaxInvocationsNonUniform ||
8110	op == glslang::EOpAddInvocationsNonUniform ||
8111	op == glslang::EOpMinInvocationsInclusiveScanNonUniform ||
8112	op == glslang::EOpMaxInvocationsInclusiveScanNonUniform ||
8113	op == glslang::EOpAddInvocationsInclusiveScanNonUniform ||
8114	op == glslang::EOpMinInvocationsExclusiveScanNonUniform ||
8115	op == glslang::EOpMaxInvocationsExclusiveScanNonUniform ||
8116	op == glslang::EOpAddInvocationsExclusiveScanNonUniform)
8117	builder.addExtension(ext: spv::E_SPV_AMD_shader_ballot);
8118
8119	switch (op) {
8120	case glslang::EOpMinInvocations:
8121	case glslang::EOpMaxInvocations:
8122	case glslang::EOpAddInvocations:
8123	case glslang::EOpMinInvocationsNonUniform:
8124	case glslang::EOpMaxInvocationsNonUniform:
8125	case glslang::EOpAddInvocationsNonUniform:
8126	groupOperation = spv::GroupOperationReduce;
8127	break;
8128	case glslang::EOpMinInvocationsInclusiveScan:
8129	case glslang::EOpMaxInvocationsInclusiveScan:
8130	case glslang::EOpAddInvocationsInclusiveScan:
8131	case glslang::EOpMinInvocationsInclusiveScanNonUniform:
8132	case glslang::EOpMaxInvocationsInclusiveScanNonUniform:
8133	case glslang::EOpAddInvocationsInclusiveScanNonUniform:
8134	groupOperation = spv::GroupOperationInclusiveScan;
8135	break;
8136	case glslang::EOpMinInvocationsExclusiveScan:
8137	case glslang::EOpMaxInvocationsExclusiveScan:
8138	case glslang::EOpAddInvocationsExclusiveScan:
8139	case glslang::EOpMinInvocationsExclusiveScanNonUniform:
8140	case glslang::EOpMaxInvocationsExclusiveScanNonUniform:
8141	case glslang::EOpAddInvocationsExclusiveScanNonUniform:
8142	groupOperation = spv::GroupOperationExclusiveScan;
8143	break;
8144	default:
8145	break;
8146	}
8147	spv::IdImmediate scope = { true, builder.makeUintConstant(u: spv::ScopeSubgroup) };
8148	spvGroupOperands.push_back(x: scope);
8149	if (groupOperation != spv::GroupOperationMax) {
8150	spv::IdImmediate groupOp = { false, (unsigned)groupOperation };
8151	spvGroupOperands.push_back(x: groupOp);
8152	}
8153	}
8154
8155	for (auto opIt = operands.begin(); opIt != operands.end(); ++opIt) {
8156	spv::IdImmediate op = { true, *opIt };
8157	spvGroupOperands.push_back(x: op);
8158	}
8159
8160	switch (op) {
8161	case glslang::EOpAnyInvocation:
8162	opCode = spv::OpSubgroupAnyKHR;
8163	break;
8164	case glslang::EOpAllInvocations:
8165	opCode = spv::OpSubgroupAllKHR;
8166	break;
8167	case glslang::EOpAllInvocationsEqual:
8168	opCode = spv::OpSubgroupAllEqualKHR;
8169	break;
8170	case glslang::EOpReadInvocation:
8171	opCode = spv::OpSubgroupReadInvocationKHR;
8172	if (builder.isVectorType(typeId))
8173	return CreateInvocationsVectorOperation(op: opCode, groupOperation, typeId, operands);
8174	break;
8175	case glslang::EOpReadFirstInvocation:
8176	opCode = spv::OpSubgroupFirstInvocationKHR;
8177	if (builder.isVectorType(typeId))
8178	return CreateInvocationsVectorOperation(op: opCode, groupOperation, typeId, operands);
8179	break;
8180	case glslang::EOpBallot:
8181	{
8182	// NOTE: According to the spec, the result type of "OpSubgroupBallotKHR" must be a 4 component vector of 32
8183	// bit integer types. The GLSL built-in function "ballotARB()" assumes the maximum number of invocations in
8184	// a subgroup is 64. Thus, we have to convert uvec4.xy to uint64_t as follow:
8185	//
8186	// result = Bitcast(SubgroupBallotKHR(Predicate).xy)
8187	//
8188	spv::Id uintType = builder.makeUintType(width: 32);
8189	spv::Id uvec4Type = builder.makeVectorType(component: uintType, size: 4);
8190	spv::Id result = builder.createOp(spv::OpSubgroupBallotKHR, typeId: uvec4Type, operands: spvGroupOperands);
8191
8192	std::vector<spv::Id> components;
8193	components.push_back(x: builder.createCompositeExtract(composite: result, typeId: uintType, index: 0));
8194	components.push_back(x: builder.createCompositeExtract(composite: result, typeId: uintType, index: 1));
8195
8196	spv::Id uvec2Type = builder.makeVectorType(component: uintType, size: 2);
8197	return builder.createUnaryOp(spv::OpBitcast, typeId,
8198	operand: builder.createCompositeConstruct(typeId: uvec2Type, constituents: components));
8199	}
8200
8201	case glslang::EOpMinInvocations:
8202	case glslang::EOpMaxInvocations:
8203	case glslang::EOpAddInvocations:
8204	case glslang::EOpMinInvocationsInclusiveScan:
8205	case glslang::EOpMaxInvocationsInclusiveScan:
8206	case glslang::EOpAddInvocationsInclusiveScan:
8207	case glslang::EOpMinInvocationsExclusiveScan:
8208	case glslang::EOpMaxInvocationsExclusiveScan:
8209	case glslang::EOpAddInvocationsExclusiveScan:
8210	if (op == glslang::EOpMinInvocations ||
8211	op == glslang::EOpMinInvocationsInclusiveScan ||
8212	op == glslang::EOpMinInvocationsExclusiveScan) {
8213	if (isFloat)
8214	opCode = spv::OpGroupFMin;
8215	else {
8216	if (isUnsigned)
8217	opCode = spv::OpGroupUMin;
8218	else
8219	opCode = spv::OpGroupSMin;
8220	}
8221	} else if (op == glslang::EOpMaxInvocations ||
8222	op == glslang::EOpMaxInvocationsInclusiveScan ||
8223	op == glslang::EOpMaxInvocationsExclusiveScan) {
8224	if (isFloat)
8225	opCode = spv::OpGroupFMax;
8226	else {
8227	if (isUnsigned)
8228	opCode = spv::OpGroupUMax;
8229	else
8230	opCode = spv::OpGroupSMax;
8231	}
8232	} else {
8233	if (isFloat)
8234	opCode = spv::OpGroupFAdd;
8235	else
8236	opCode = spv::OpGroupIAdd;
8237	}
8238
8239	if (builder.isVectorType(typeId))
8240	return CreateInvocationsVectorOperation(op: opCode, groupOperation, typeId, operands);
8241
8242	break;
8243	case glslang::EOpMinInvocationsNonUniform:
8244	case glslang::EOpMaxInvocationsNonUniform:
8245	case glslang::EOpAddInvocationsNonUniform:
8246	case glslang::EOpMinInvocationsInclusiveScanNonUniform:
8247	case glslang::EOpMaxInvocationsInclusiveScanNonUniform:
8248	case glslang::EOpAddInvocationsInclusiveScanNonUniform:
8249	case glslang::EOpMinInvocationsExclusiveScanNonUniform:
8250	case glslang::EOpMaxInvocationsExclusiveScanNonUniform:
8251	case glslang::EOpAddInvocationsExclusiveScanNonUniform:
8252	if (op == glslang::EOpMinInvocationsNonUniform ||
8253	op == glslang::EOpMinInvocationsInclusiveScanNonUniform ||
8254	op == glslang::EOpMinInvocationsExclusiveScanNonUniform) {
8255	if (isFloat)
8256	opCode = spv::OpGroupFMinNonUniformAMD;
8257	else {
8258	if (isUnsigned)
8259	opCode = spv::OpGroupUMinNonUniformAMD;
8260	else
8261	opCode = spv::OpGroupSMinNonUniformAMD;
8262	}
8263	}
8264	else if (op == glslang::EOpMaxInvocationsNonUniform ||
8265	op == glslang::EOpMaxInvocationsInclusiveScanNonUniform ||
8266	op == glslang::EOpMaxInvocationsExclusiveScanNonUniform) {
8267	if (isFloat)
8268	opCode = spv::OpGroupFMaxNonUniformAMD;
8269	else {
8270	if (isUnsigned)
8271	opCode = spv::OpGroupUMaxNonUniformAMD;
8272	else
8273	opCode = spv::OpGroupSMaxNonUniformAMD;
8274	}
8275	}
8276	else {
8277	if (isFloat)
8278	opCode = spv::OpGroupFAddNonUniformAMD;
8279	else
8280	opCode = spv::OpGroupIAddNonUniformAMD;
8281	}
8282
8283	if (builder.isVectorType(typeId))
8284	return CreateInvocationsVectorOperation(op: opCode, groupOperation, typeId, operands);
8285
8286	break;
8287	default:
8288	logger->missingFunctionality(f: "invocation operation");
8289	return spv::NoResult;
8290	}
8291
8292	assert(opCode != spv::OpNop);
8293	return builder.createOp(opCode, typeId, operands: spvGroupOperands);
8294	}
8295
8296	// Create group invocation operations on a vector
8297	spv::Id TGlslangToSpvTraverser::CreateInvocationsVectorOperation(spv::Op op, spv::GroupOperation groupOperation,
8298	spv::Id typeId, std::vector<spv::Id>& operands)
8299	{
8300	assert(op == spv::OpGroupFMin || op == spv::OpGroupUMin || op == spv::OpGroupSMin ||
8301	op == spv::OpGroupFMax || op == spv::OpGroupUMax || op == spv::OpGroupSMax ||
8302	op == spv::OpGroupFAdd || op == spv::OpGroupIAdd || op == spv::OpGroupBroadcast ||
8303	op == spv::OpSubgroupReadInvocationKHR || op == spv::OpSubgroupFirstInvocationKHR ||
8304	op == spv::OpGroupFMinNonUniformAMD || op == spv::OpGroupUMinNonUniformAMD ||
8305	op == spv::OpGroupSMinNonUniformAMD ||
8306	op == spv::OpGroupFMaxNonUniformAMD || op == spv::OpGroupUMaxNonUniformAMD ||
8307	op == spv::OpGroupSMaxNonUniformAMD ||
8308	op == spv::OpGroupFAddNonUniformAMD || op == spv::OpGroupIAddNonUniformAMD);
8309
8310	// Handle group invocation operations scalar by scalar.
8311	// The result type is the same type as the original type.
8312	// The algorithm is to:
8313	// - break the vector into scalars
8314	// - apply the operation to each scalar
8315	// - make a vector out the scalar results
8316
8317	// get the types sorted out
8318	int numComponents = builder.getNumComponents(resultId: operands[0]);
8319	spv::Id scalarType = builder.getScalarTypeId(typeId: builder.getTypeId(resultId: operands[0]));
8320	std::vector<spv::Id> results;
8321
8322	// do each scalar op
8323	for (int comp = 0; comp < numComponents; ++comp) {
8324	std::vector<unsigned int> indexes;
8325	indexes.push_back(x: comp);
8326	spv::IdImmediate scalar = { true, builder.createCompositeExtract(composite: operands[0], typeId: scalarType, indexes) };
8327	std::vector<spv::IdImmediate> spvGroupOperands;
8328	if (op == spv::OpSubgroupReadInvocationKHR) {
8329	spvGroupOperands.push_back(x: scalar);
8330	spv::IdImmediate operand = { true, operands[1] };
8331	spvGroupOperands.push_back(x: operand);
8332	} else if (op == spv::OpSubgroupFirstInvocationKHR) {
8333	spvGroupOperands.push_back(x: scalar);
8334	} else if (op == spv::OpGroupBroadcast) {
8335	spv::IdImmediate scope = { true, builder.makeUintConstant(u: spv::ScopeSubgroup) };
8336	spvGroupOperands.push_back(x: scope);
8337	spvGroupOperands.push_back(x: scalar);
8338	spv::IdImmediate operand = { true, operands[1] };
8339	spvGroupOperands.push_back(x: operand);
8340	} else {
8341	spv::IdImmediate scope = { true, builder.makeUintConstant(u: spv::ScopeSubgroup) };
8342	spvGroupOperands.push_back(x: scope);
8343	spv::IdImmediate groupOp = { false, (unsigned)groupOperation };
8344	spvGroupOperands.push_back(x: groupOp);
8345	spvGroupOperands.push_back(x: scalar);
8346	}
8347
8348	results.push_back(x: builder.createOp(op, typeId: scalarType, operands: spvGroupOperands));
8349	}
8350
8351	// put the pieces together
8352	return builder.createCompositeConstruct(typeId, constituents: results);
8353	}
8354
8355	// Create subgroup invocation operations.
8356	spv::Id TGlslangToSpvTraverser::createSubgroupOperation(glslang::TOperator op, spv::Id typeId,
8357	std::vector<spv::Id>& operands, glslang::TBasicType typeProxy)
8358	{
8359	// Add the required capabilities.
8360	switch (op) {
8361	case glslang::EOpSubgroupElect:
8362	builder.addCapability(cap: spv::CapabilityGroupNonUniform);
8363	break;
8364	case glslang::EOpSubgroupQuadAll:
8365	case glslang::EOpSubgroupQuadAny:
8366	builder.addExtension(ext: spv::E_SPV_KHR_quad_control);
8367	builder.addCapability(cap: spv::CapabilityQuadControlKHR);
8368	[[fallthrough]];
8369	case glslang::EOpSubgroupAll:
8370	case glslang::EOpSubgroupAny:
8371	case glslang::EOpSubgroupAllEqual:
8372	builder.addCapability(cap: spv::CapabilityGroupNonUniform);
8373	builder.addCapability(cap: spv::CapabilityGroupNonUniformVote);
8374	break;
8375	case glslang::EOpSubgroupBroadcast:
8376	case glslang::EOpSubgroupBroadcastFirst:
8377	case glslang::EOpSubgroupBallot:
8378	case glslang::EOpSubgroupInverseBallot:
8379	case glslang::EOpSubgroupBallotBitExtract:
8380	case glslang::EOpSubgroupBallotBitCount:
8381	case glslang::EOpSubgroupBallotInclusiveBitCount:
8382	case glslang::EOpSubgroupBallotExclusiveBitCount:
8383	case glslang::EOpSubgroupBallotFindLSB:
8384	case glslang::EOpSubgroupBallotFindMSB:
8385	builder.addCapability(cap: spv::CapabilityGroupNonUniform);
8386	builder.addCapability(cap: spv::CapabilityGroupNonUniformBallot);
8387	break;
8388	case glslang::EOpSubgroupRotate:
8389	case glslang::EOpSubgroupClusteredRotate:
8390	builder.addExtension(ext: spv::E_SPV_KHR_subgroup_rotate);
8391	builder.addCapability(cap: spv::CapabilityGroupNonUniformRotateKHR);
8392	break;
8393	case glslang::EOpSubgroupShuffle:
8394	case glslang::EOpSubgroupShuffleXor:
8395	builder.addCapability(cap: spv::CapabilityGroupNonUniform);
8396	builder.addCapability(cap: spv::CapabilityGroupNonUniformShuffle);
8397	break;
8398	case glslang::EOpSubgroupShuffleUp:
8399	case glslang::EOpSubgroupShuffleDown:
8400	builder.addCapability(cap: spv::CapabilityGroupNonUniform);
8401	builder.addCapability(cap: spv::CapabilityGroupNonUniformShuffleRelative);
8402	break;
8403	case glslang::EOpSubgroupAdd:
8404	case glslang::EOpSubgroupMul:
8405	case glslang::EOpSubgroupMin:
8406	case glslang::EOpSubgroupMax:
8407	case glslang::EOpSubgroupAnd:
8408	case glslang::EOpSubgroupOr:
8409	case glslang::EOpSubgroupXor:
8410	case glslang::EOpSubgroupInclusiveAdd:
8411	case glslang::EOpSubgroupInclusiveMul:
8412	case glslang::EOpSubgroupInclusiveMin:
8413	case glslang::EOpSubgroupInclusiveMax:
8414	case glslang::EOpSubgroupInclusiveAnd:
8415	case glslang::EOpSubgroupInclusiveOr:
8416	case glslang::EOpSubgroupInclusiveXor:
8417	case glslang::EOpSubgroupExclusiveAdd:
8418	case glslang::EOpSubgroupExclusiveMul:
8419	case glslang::EOpSubgroupExclusiveMin:
8420	case glslang::EOpSubgroupExclusiveMax:
8421	case glslang::EOpSubgroupExclusiveAnd:
8422	case glslang::EOpSubgroupExclusiveOr:
8423	case glslang::EOpSubgroupExclusiveXor:
8424	builder.addCapability(cap: spv::CapabilityGroupNonUniform);
8425	builder.addCapability(cap: spv::CapabilityGroupNonUniformArithmetic);
8426	break;
8427	case glslang::EOpSubgroupClusteredAdd:
8428	case glslang::EOpSubgroupClusteredMul:
8429	case glslang::EOpSubgroupClusteredMin:
8430	case glslang::EOpSubgroupClusteredMax:
8431	case glslang::EOpSubgroupClusteredAnd:
8432	case glslang::EOpSubgroupClusteredOr:
8433	case glslang::EOpSubgroupClusteredXor:
8434	builder.addCapability(cap: spv::CapabilityGroupNonUniform);
8435	builder.addCapability(cap: spv::CapabilityGroupNonUniformClustered);
8436	break;
8437	case glslang::EOpSubgroupQuadBroadcast:
8438	case glslang::EOpSubgroupQuadSwapHorizontal:
8439	case glslang::EOpSubgroupQuadSwapVertical:
8440	case glslang::EOpSubgroupQuadSwapDiagonal:
8441	builder.addCapability(cap: spv::CapabilityGroupNonUniform);
8442	builder.addCapability(cap: spv::CapabilityGroupNonUniformQuad);
8443	break;
8444	case glslang::EOpSubgroupPartitionedAdd:
8445	case glslang::EOpSubgroupPartitionedMul:
8446	case glslang::EOpSubgroupPartitionedMin:
8447	case glslang::EOpSubgroupPartitionedMax:
8448	case glslang::EOpSubgroupPartitionedAnd:
8449	case glslang::EOpSubgroupPartitionedOr:
8450	case glslang::EOpSubgroupPartitionedXor:
8451	case glslang::EOpSubgroupPartitionedInclusiveAdd:
8452	case glslang::EOpSubgroupPartitionedInclusiveMul:
8453	case glslang::EOpSubgroupPartitionedInclusiveMin:
8454	case glslang::EOpSubgroupPartitionedInclusiveMax:
8455	case glslang::EOpSubgroupPartitionedInclusiveAnd:
8456	case glslang::EOpSubgroupPartitionedInclusiveOr:
8457	case glslang::EOpSubgroupPartitionedInclusiveXor:
8458	case glslang::EOpSubgroupPartitionedExclusiveAdd:
8459	case glslang::EOpSubgroupPartitionedExclusiveMul:
8460	case glslang::EOpSubgroupPartitionedExclusiveMin:
8461	case glslang::EOpSubgroupPartitionedExclusiveMax:
8462	case glslang::EOpSubgroupPartitionedExclusiveAnd:
8463	case glslang::EOpSubgroupPartitionedExclusiveOr:
8464	case glslang::EOpSubgroupPartitionedExclusiveXor:
8465	builder.addExtension(ext: spv::E_SPV_NV_shader_subgroup_partitioned);
8466	builder.addCapability(cap: spv::CapabilityGroupNonUniformPartitionedNV);
8467	break;
8468	default: assert(0 && "Unhandled subgroup operation!");
8469	}
8470
8471
8472	const bool isUnsigned = isTypeUnsignedInt(type: typeProxy);
8473	const bool isFloat = isTypeFloat(type: typeProxy);
8474	const bool isBool = typeProxy == glslang::EbtBool;
8475
8476	spv::Op opCode = spv::OpNop;
8477
8478	// Figure out which opcode to use.
8479	switch (op) {
8480	case glslang::EOpSubgroupElect: opCode = spv::OpGroupNonUniformElect; break;
8481	case glslang::EOpSubgroupQuadAll: opCode = spv::OpGroupNonUniformQuadAllKHR; break;
8482	case glslang::EOpSubgroupAll: opCode = spv::OpGroupNonUniformAll; break;
8483	case glslang::EOpSubgroupQuadAny: opCode = spv::OpGroupNonUniformQuadAnyKHR; break;
8484	case glslang::EOpSubgroupAny: opCode = spv::OpGroupNonUniformAny; break;
8485	case glslang::EOpSubgroupAllEqual: opCode = spv::OpGroupNonUniformAllEqual; break;
8486	case glslang::EOpSubgroupBroadcast: opCode = spv::OpGroupNonUniformBroadcast; break;
8487	case glslang::EOpSubgroupBroadcastFirst: opCode = spv::OpGroupNonUniformBroadcastFirst; break;
8488	case glslang::EOpSubgroupBallot: opCode = spv::OpGroupNonUniformBallot; break;
8489	case glslang::EOpSubgroupInverseBallot: opCode = spv::OpGroupNonUniformInverseBallot; break;
8490	case glslang::EOpSubgroupBallotBitExtract: opCode = spv::OpGroupNonUniformBallotBitExtract; break;
8491	case glslang::EOpSubgroupBallotBitCount:
8492	case glslang::EOpSubgroupBallotInclusiveBitCount:
8493	case glslang::EOpSubgroupBallotExclusiveBitCount: opCode = spv::OpGroupNonUniformBallotBitCount; break;
8494	case glslang::EOpSubgroupBallotFindLSB: opCode = spv::OpGroupNonUniformBallotFindLSB; break;
8495	case glslang::EOpSubgroupBallotFindMSB: opCode = spv::OpGroupNonUniformBallotFindMSB; break;
8496	case glslang::EOpSubgroupShuffle: opCode = spv::OpGroupNonUniformShuffle; break;
8497	case glslang::EOpSubgroupShuffleXor: opCode = spv::OpGroupNonUniformShuffleXor; break;
8498	case glslang::EOpSubgroupShuffleUp: opCode = spv::OpGroupNonUniformShuffleUp; break;
8499	case glslang::EOpSubgroupShuffleDown: opCode = spv::OpGroupNonUniformShuffleDown; break;
8500	case glslang::EOpSubgroupRotate:
8501	case glslang::EOpSubgroupClusteredRotate: opCode = spv::OpGroupNonUniformRotateKHR; break;
8502	case glslang::EOpSubgroupAdd:
8503	case glslang::EOpSubgroupInclusiveAdd:
8504	case glslang::EOpSubgroupExclusiveAdd:
8505	case glslang::EOpSubgroupClusteredAdd:
8506	case glslang::EOpSubgroupPartitionedAdd:
8507	case glslang::EOpSubgroupPartitionedInclusiveAdd:
8508	case glslang::EOpSubgroupPartitionedExclusiveAdd:
8509	if (isFloat) {
8510	opCode = spv::OpGroupNonUniformFAdd;
8511	} else {
8512	opCode = spv::OpGroupNonUniformIAdd;
8513	}
8514	break;
8515	case glslang::EOpSubgroupMul:
8516	case glslang::EOpSubgroupInclusiveMul:
8517	case glslang::EOpSubgroupExclusiveMul:
8518	case glslang::EOpSubgroupClusteredMul:
8519	case glslang::EOpSubgroupPartitionedMul:
8520	case glslang::EOpSubgroupPartitionedInclusiveMul:
8521	case glslang::EOpSubgroupPartitionedExclusiveMul:
8522	if (isFloat) {
8523	opCode = spv::OpGroupNonUniformFMul;
8524	} else {
8525	opCode = spv::OpGroupNonUniformIMul;
8526	}
8527	break;
8528	case glslang::EOpSubgroupMin:
8529	case glslang::EOpSubgroupInclusiveMin:
8530	case glslang::EOpSubgroupExclusiveMin:
8531	case glslang::EOpSubgroupClusteredMin:
8532	case glslang::EOpSubgroupPartitionedMin:
8533	case glslang::EOpSubgroupPartitionedInclusiveMin:
8534	case glslang::EOpSubgroupPartitionedExclusiveMin:
8535	if (isFloat) {
8536	opCode = spv::OpGroupNonUniformFMin;
8537	} else if (isUnsigned) {
8538	opCode = spv::OpGroupNonUniformUMin;
8539	} else {
8540	opCode = spv::OpGroupNonUniformSMin;
8541	}
8542	break;
8543	case glslang::EOpSubgroupMax:
8544	case glslang::EOpSubgroupInclusiveMax:
8545	case glslang::EOpSubgroupExclusiveMax:
8546	case glslang::EOpSubgroupClusteredMax:
8547	case glslang::EOpSubgroupPartitionedMax:
8548	case glslang::EOpSubgroupPartitionedInclusiveMax:
8549	case glslang::EOpSubgroupPartitionedExclusiveMax:
8550	if (isFloat) {
8551	opCode = spv::OpGroupNonUniformFMax;
8552	} else if (isUnsigned) {
8553	opCode = spv::OpGroupNonUniformUMax;
8554	} else {
8555	opCode = spv::OpGroupNonUniformSMax;
8556	}
8557	break;
8558	case glslang::EOpSubgroupAnd:
8559	case glslang::EOpSubgroupInclusiveAnd:
8560	case glslang::EOpSubgroupExclusiveAnd:
8561	case glslang::EOpSubgroupClusteredAnd:
8562	case glslang::EOpSubgroupPartitionedAnd:
8563	case glslang::EOpSubgroupPartitionedInclusiveAnd:
8564	case glslang::EOpSubgroupPartitionedExclusiveAnd:
8565	if (isBool) {
8566	opCode = spv::OpGroupNonUniformLogicalAnd;
8567	} else {
8568	opCode = spv::OpGroupNonUniformBitwiseAnd;
8569	}
8570	break;
8571	case glslang::EOpSubgroupOr:
8572	case glslang::EOpSubgroupInclusiveOr:
8573	case glslang::EOpSubgroupExclusiveOr:
8574	case glslang::EOpSubgroupClusteredOr:
8575	case glslang::EOpSubgroupPartitionedOr:
8576	case glslang::EOpSubgroupPartitionedInclusiveOr:
8577	case glslang::EOpSubgroupPartitionedExclusiveOr:
8578	if (isBool) {
8579	opCode = spv::OpGroupNonUniformLogicalOr;
8580	} else {
8581	opCode = spv::OpGroupNonUniformBitwiseOr;
8582	}
8583	break;
8584	case glslang::EOpSubgroupXor:
8585	case glslang::EOpSubgroupInclusiveXor:
8586	case glslang::EOpSubgroupExclusiveXor:
8587	case glslang::EOpSubgroupClusteredXor:
8588	case glslang::EOpSubgroupPartitionedXor:
8589	case glslang::EOpSubgroupPartitionedInclusiveXor:
8590	case glslang::EOpSubgroupPartitionedExclusiveXor:
8591	if (isBool) {
8592	opCode = spv::OpGroupNonUniformLogicalXor;
8593	} else {
8594	opCode = spv::OpGroupNonUniformBitwiseXor;
8595	}
8596	break;
8597	case glslang::EOpSubgroupQuadBroadcast: opCode = spv::OpGroupNonUniformQuadBroadcast; break;
8598	case glslang::EOpSubgroupQuadSwapHorizontal:
8599	case glslang::EOpSubgroupQuadSwapVertical:
8600	case glslang::EOpSubgroupQuadSwapDiagonal: opCode = spv::OpGroupNonUniformQuadSwap; break;
8601	default: assert(0 && "Unhandled subgroup operation!");
8602	}
8603
8604	// get the right Group Operation
8605	spv::GroupOperation groupOperation = spv::GroupOperationMax;
8606	switch (op) {
8607	default:
8608	break;
8609	case glslang::EOpSubgroupBallotBitCount:
8610	case glslang::EOpSubgroupAdd:
8611	case glslang::EOpSubgroupMul:
8612	case glslang::EOpSubgroupMin:
8613	case glslang::EOpSubgroupMax:
8614	case glslang::EOpSubgroupAnd:
8615	case glslang::EOpSubgroupOr:
8616	case glslang::EOpSubgroupXor:
8617	groupOperation = spv::GroupOperationReduce;
8618	break;
8619	case glslang::EOpSubgroupBallotInclusiveBitCount:
8620	case glslang::EOpSubgroupInclusiveAdd:
8621	case glslang::EOpSubgroupInclusiveMul:
8622	case glslang::EOpSubgroupInclusiveMin:
8623	case glslang::EOpSubgroupInclusiveMax:
8624	case glslang::EOpSubgroupInclusiveAnd:
8625	case glslang::EOpSubgroupInclusiveOr:
8626	case glslang::EOpSubgroupInclusiveXor:
8627	groupOperation = spv::GroupOperationInclusiveScan;
8628	break;
8629	case glslang::EOpSubgroupBallotExclusiveBitCount:
8630	case glslang::EOpSubgroupExclusiveAdd:
8631	case glslang::EOpSubgroupExclusiveMul:
8632	case glslang::EOpSubgroupExclusiveMin:
8633	case glslang::EOpSubgroupExclusiveMax:
8634	case glslang::EOpSubgroupExclusiveAnd:
8635	case glslang::EOpSubgroupExclusiveOr:
8636	case glslang::EOpSubgroupExclusiveXor:
8637	groupOperation = spv::GroupOperationExclusiveScan;
8638	break;
8639	case glslang::EOpSubgroupClusteredAdd:
8640	case glslang::EOpSubgroupClusteredMul:
8641	case glslang::EOpSubgroupClusteredMin:
8642	case glslang::EOpSubgroupClusteredMax:
8643	case glslang::EOpSubgroupClusteredAnd:
8644	case glslang::EOpSubgroupClusteredOr:
8645	case glslang::EOpSubgroupClusteredXor:
8646	groupOperation = spv::GroupOperationClusteredReduce;
8647	break;
8648	case glslang::EOpSubgroupPartitionedAdd:
8649	case glslang::EOpSubgroupPartitionedMul:
8650	case glslang::EOpSubgroupPartitionedMin:
8651	case glslang::EOpSubgroupPartitionedMax:
8652	case glslang::EOpSubgroupPartitionedAnd:
8653	case glslang::EOpSubgroupPartitionedOr:
8654	case glslang::EOpSubgroupPartitionedXor:
8655	groupOperation = spv::GroupOperationPartitionedReduceNV;
8656	break;
8657	case glslang::EOpSubgroupPartitionedInclusiveAdd:
8658	case glslang::EOpSubgroupPartitionedInclusiveMul:
8659	case glslang::EOpSubgroupPartitionedInclusiveMin:
8660	case glslang::EOpSubgroupPartitionedInclusiveMax:
8661	case glslang::EOpSubgroupPartitionedInclusiveAnd:
8662	case glslang::EOpSubgroupPartitionedInclusiveOr:
8663	case glslang::EOpSubgroupPartitionedInclusiveXor:
8664	groupOperation = spv::GroupOperationPartitionedInclusiveScanNV;
8665	break;
8666	case glslang::EOpSubgroupPartitionedExclusiveAdd:
8667	case glslang::EOpSubgroupPartitionedExclusiveMul:
8668	case glslang::EOpSubgroupPartitionedExclusiveMin:
8669	case glslang::EOpSubgroupPartitionedExclusiveMax:
8670	case glslang::EOpSubgroupPartitionedExclusiveAnd:
8671	case glslang::EOpSubgroupPartitionedExclusiveOr:
8672	case glslang::EOpSubgroupPartitionedExclusiveXor:
8673	groupOperation = spv::GroupOperationPartitionedExclusiveScanNV;
8674	break;
8675	}
8676
8677	// build the instruction
8678	std::vector<spv::IdImmediate> spvGroupOperands;
8679
8680	// Every operation begins with the Execution Scope operand.
8681	spv::IdImmediate executionScope = { true, builder.makeUintConstant(u: spv::ScopeSubgroup) };
8682	// All other ops need the execution scope. Quad Control Ops don't need scope, it's always Quad.
8683	if (opCode != spv::OpGroupNonUniformQuadAllKHR && opCode != spv::OpGroupNonUniformQuadAnyKHR) {
8684	spvGroupOperands.push_back(x: executionScope);
8685	}
8686
8687	// Next, for all operations that use a Group Operation, push that as an operand.
8688	if (groupOperation != spv::GroupOperationMax) {
8689	spv::IdImmediate groupOperand = { false, (unsigned)groupOperation };
8690	spvGroupOperands.push_back(x: groupOperand);
8691	}
8692
8693	// Push back the operands next.
8694	for (auto opIt = operands.cbegin(); opIt != operands.cend(); ++opIt) {
8695	spv::IdImmediate operand = { true, *opIt };
8696	spvGroupOperands.push_back(x: operand);
8697	}
8698
8699	// Some opcodes have additional operands.
8700	spv::Id directionId = spv::NoResult;
8701	switch (op) {
8702	default: break;
8703	case glslang::EOpSubgroupQuadSwapHorizontal: directionId = builder.makeUintConstant(u: 0); break;
8704	case glslang::EOpSubgroupQuadSwapVertical: directionId = builder.makeUintConstant(u: 1); break;
8705	case glslang::EOpSubgroupQuadSwapDiagonal: directionId = builder.makeUintConstant(u: 2); break;
8706	}
8707	if (directionId != spv::NoResult) {
8708	spv::IdImmediate direction = { true, directionId };
8709	spvGroupOperands.push_back(x: direction);
8710	}
8711
8712	return builder.createOp(opCode, typeId, operands: spvGroupOperands);
8713	}
8714
8715	spv::Id TGlslangToSpvTraverser::createMiscOperation(glslang::TOperator op, spv::Decoration precision,
8716	spv::Id typeId, std::vector<spv::Id>& operands, glslang::TBasicType typeProxy)
8717	{
8718	bool isUnsigned = isTypeUnsignedInt(type: typeProxy);
8719	bool isFloat = isTypeFloat(type: typeProxy);
8720
8721	spv::Op opCode = spv::OpNop;
8722	int extBuiltins = -1;
8723	int libCall = -1;
8724	size_t consumedOperands = operands.size();
8725	spv::Id typeId0 = 0;
8726	if (consumedOperands > 0)
8727	typeId0 = builder.getTypeId(resultId: operands[0]);
8728	spv::Id typeId1 = 0;
8729	if (consumedOperands > 1)
8730	typeId1 = builder.getTypeId(resultId: operands[1]);
8731	spv::Id frexpIntType = 0;
8732
8733	switch (op) {
8734	case glslang::EOpMin:
8735	if (isFloat)
8736	libCall = nanMinMaxClamp ? spv::GLSLstd450NMin : spv::GLSLstd450FMin;
8737	else if (isUnsigned)
8738	libCall = spv::GLSLstd450UMin;
8739	else
8740	libCall = spv::GLSLstd450SMin;
8741	builder.promoteScalar(precision, left&: operands.front(), right&: operands.back());
8742	break;
8743	case glslang::EOpModf:
8744	{
8745	libCall = spv::GLSLstd450ModfStruct;
8746	assert(builder.isFloatType(builder.getScalarTypeId(typeId0)));
8747	int width = builder.getScalarTypeWidth(typeId: typeId0);
8748	if (width == 16)
8749	builder.addExtension(ext: spv::E_SPV_AMD_gpu_shader_half_float);
8750	// The returned struct has two members of the same type as the first argument
8751	typeId = builder.makeStructResultType(type0: typeId0, type1: typeId0);
8752	consumedOperands = 1;
8753	}
8754	break;
8755	case glslang::EOpMax:
8756	if (isFloat)
8757	libCall = nanMinMaxClamp ? spv::GLSLstd450NMax : spv::GLSLstd450FMax;
8758	else if (isUnsigned)
8759	libCall = spv::GLSLstd450UMax;
8760	else
8761	libCall = spv::GLSLstd450SMax;
8762	builder.promoteScalar(precision, left&: operands.front(), right&: operands.back());
8763	break;
8764	case glslang::EOpPow:
8765	libCall = spv::GLSLstd450Pow;
8766	break;
8767	case glslang::EOpDot:
8768	opCode = spv::OpDot;
8769	break;
8770	case glslang::EOpAtan:
8771	libCall = spv::GLSLstd450Atan2;
8772	break;
8773
8774	case glslang::EOpClamp:
8775	if (isFloat)
8776	libCall = nanMinMaxClamp ? spv::GLSLstd450NClamp : spv::GLSLstd450FClamp;
8777	else if (isUnsigned)
8778	libCall = spv::GLSLstd450UClamp;
8779	else
8780	libCall = spv::GLSLstd450SClamp;
8781	builder.promoteScalar(precision, left&: operands.front(), right&: operands[1]);
8782	builder.promoteScalar(precision, left&: operands.front(), right&: operands[2]);
8783	break;
8784	case glslang::EOpMix:
8785	if (! builder.isBoolType(typeId: builder.getScalarTypeId(typeId: builder.getTypeId(resultId: operands.back())))) {
8786	assert(isFloat);
8787	libCall = spv::GLSLstd450FMix;
8788	} else {
8789	opCode = spv::OpSelect;
8790	std::swap(a&: operands.front(), b&: operands.back());
8791	}
8792	builder.promoteScalar(precision, left&: operands.front(), right&: operands.back());
8793	break;
8794	case glslang::EOpStep:
8795	libCall = spv::GLSLstd450Step;
8796	builder.promoteScalar(precision, left&: operands.front(), right&: operands.back());
8797	break;
8798	case glslang::EOpSmoothStep:
8799	libCall = spv::GLSLstd450SmoothStep;
8800	builder.promoteScalar(precision, left&: operands[0], right&: operands[2]);
8801	builder.promoteScalar(precision, left&: operands[1], right&: operands[2]);
8802	break;
8803
8804	case glslang::EOpDistance:
8805	libCall = spv::GLSLstd450Distance;
8806	break;
8807	case glslang::EOpCross:
8808	libCall = spv::GLSLstd450Cross;
8809	break;
8810	case glslang::EOpFaceForward:
8811	libCall = spv::GLSLstd450FaceForward;
8812	break;
8813	case glslang::EOpReflect:
8814	libCall = spv::GLSLstd450Reflect;
8815	break;
8816	case glslang::EOpRefract:
8817	libCall = spv::GLSLstd450Refract;
8818	break;
8819	case glslang::EOpBarrier:
8820	{
8821	// This is for the extended controlBarrier function, with four operands.
8822	// The unextended barrier() goes through createNoArgOperation.
8823	assert(operands.size() == 4);
8824	unsigned int executionScope = builder.getConstantScalar(resultId: operands[0]);
8825	unsigned int memoryScope = builder.getConstantScalar(resultId: operands[1]);
8826	unsigned int semantics = builder.getConstantScalar(resultId: operands[2]) | builder.getConstantScalar(resultId: operands[3]);
8827	builder.createControlBarrier(execution: (spv::Scope)executionScope, memory: (spv::Scope)memoryScope,
8828	(spv::MemorySemanticsMask)semantics);
8829	if (semantics & (spv::MemorySemanticsMakeAvailableKHRMask |
8830	spv::MemorySemanticsMakeVisibleKHRMask |
8831	spv::MemorySemanticsOutputMemoryKHRMask |
8832	spv::MemorySemanticsVolatileMask)) {
8833	builder.addCapability(cap: spv::CapabilityVulkanMemoryModelKHR);
8834	}
8835	if (glslangIntermediate->usingVulkanMemoryModel() && (executionScope == spv::ScopeDevice ||
8836	memoryScope == spv::ScopeDevice)) {
8837	builder.addCapability(cap: spv::CapabilityVulkanMemoryModelDeviceScopeKHR);
8838	}
8839	return 0;
8840	}
8841	break;
8842	case glslang::EOpMemoryBarrier:
8843	{
8844	// This is for the extended memoryBarrier function, with three operands.
8845	// The unextended memoryBarrier() goes through createNoArgOperation.
8846	assert(operands.size() == 3);
8847	unsigned int memoryScope = builder.getConstantScalar(resultId: operands[0]);
8848	unsigned int semantics = builder.getConstantScalar(resultId: operands[1]) | builder.getConstantScalar(resultId: operands[2]);
8849	builder.createMemoryBarrier(executionScope: (spv::Scope)memoryScope, memorySemantics: (spv::MemorySemanticsMask)semantics);
8850	if (semantics & (spv::MemorySemanticsMakeAvailableKHRMask |
8851	spv::MemorySemanticsMakeVisibleKHRMask |
8852	spv::MemorySemanticsOutputMemoryKHRMask |
8853	spv::MemorySemanticsVolatileMask)) {
8854	builder.addCapability(cap: spv::CapabilityVulkanMemoryModelKHR);
8855	}
8856	if (glslangIntermediate->usingVulkanMemoryModel() && memoryScope == spv::ScopeDevice) {
8857	builder.addCapability(cap: spv::CapabilityVulkanMemoryModelDeviceScopeKHR);
8858	}
8859	return 0;
8860	}
8861	break;
8862
8863	case glslang::EOpInterpolateAtSample:
8864	if (typeProxy == glslang::EbtFloat16)
8865	builder.addExtension(ext: spv::E_SPV_AMD_gpu_shader_half_float);
8866	libCall = spv::GLSLstd450InterpolateAtSample;
8867	break;
8868	case glslang::EOpInterpolateAtOffset:
8869	if (typeProxy == glslang::EbtFloat16)
8870	builder.addExtension(ext: spv::E_SPV_AMD_gpu_shader_half_float);
8871	libCall = spv::GLSLstd450InterpolateAtOffset;
8872	break;
8873	case glslang::EOpAddCarry:
8874	opCode = spv::OpIAddCarry;
8875	typeId = builder.makeStructResultType(type0: typeId0, type1: typeId0);
8876	consumedOperands = 2;
8877	break;
8878	case glslang::EOpSubBorrow:
8879	opCode = spv::OpISubBorrow;
8880	typeId = builder.makeStructResultType(type0: typeId0, type1: typeId0);
8881	consumedOperands = 2;
8882	break;
8883	case glslang::EOpUMulExtended:
8884	opCode = spv::OpUMulExtended;
8885	typeId = builder.makeStructResultType(type0: typeId0, type1: typeId0);
8886	consumedOperands = 2;
8887	break;
8888	case glslang::EOpIMulExtended:
8889	opCode = spv::OpSMulExtended;
8890	typeId = builder.makeStructResultType(type0: typeId0, type1: typeId0);
8891	consumedOperands = 2;
8892	break;
8893	case glslang::EOpBitfieldExtract:
8894	if (isUnsigned)
8895	opCode = spv::OpBitFieldUExtract;
8896	else
8897	opCode = spv::OpBitFieldSExtract;
8898	break;
8899	case glslang::EOpBitfieldInsert:
8900	opCode = spv::OpBitFieldInsert;
8901	break;
8902
8903	case glslang::EOpFma:
8904	libCall = spv::GLSLstd450Fma;
8905	break;
8906	case glslang::EOpFrexp:
8907	{
8908	libCall = spv::GLSLstd450FrexpStruct;
8909	assert(builder.isPointerType(typeId1));
8910	typeId1 = builder.getContainedTypeId(typeId: typeId1);
8911	int width = builder.getScalarTypeWidth(typeId: typeId1);
8912	if (width == 16)
8913	// Using 16-bit exp operand, enable extension SPV_AMD_gpu_shader_int16
8914	builder.addExtension(ext: spv::E_SPV_AMD_gpu_shader_int16);
8915	if (builder.getNumComponents(resultId: operands[0]) == 1)
8916	frexpIntType = builder.makeIntegerType(width, hasSign: true);
8917	else
8918	frexpIntType = builder.makeVectorType(component: builder.makeIntegerType(width, hasSign: true),
8919	size: builder.getNumComponents(resultId: operands[0]));
8920	typeId = builder.makeStructResultType(type0: typeId0, type1: frexpIntType);
8921	consumedOperands = 1;
8922	}
8923	break;
8924	case glslang::EOpLdexp:
8925	libCall = spv::GLSLstd450Ldexp;
8926	break;
8927
8928	case glslang::EOpReadInvocation:
8929	return createInvocationsOperation(op, typeId, operands, typeProxy);
8930
8931	case glslang::EOpSubgroupBroadcast:
8932	case glslang::EOpSubgroupBallotBitExtract:
8933	case glslang::EOpSubgroupShuffle:
8934	case glslang::EOpSubgroupShuffleXor:
8935	case glslang::EOpSubgroupShuffleUp:
8936	case glslang::EOpSubgroupShuffleDown:
8937	case glslang::EOpSubgroupRotate:
8938	case glslang::EOpSubgroupClusteredRotate:
8939	case glslang::EOpSubgroupClusteredAdd:
8940	case glslang::EOpSubgroupClusteredMul:
8941	case glslang::EOpSubgroupClusteredMin:
8942	case glslang::EOpSubgroupClusteredMax:
8943	case glslang::EOpSubgroupClusteredAnd:
8944	case glslang::EOpSubgroupClusteredOr:
8945	case glslang::EOpSubgroupClusteredXor:
8946	case glslang::EOpSubgroupQuadBroadcast:
8947	case glslang::EOpSubgroupPartitionedAdd:
8948	case glslang::EOpSubgroupPartitionedMul:
8949	case glslang::EOpSubgroupPartitionedMin:
8950	case glslang::EOpSubgroupPartitionedMax:
8951	case glslang::EOpSubgroupPartitionedAnd:
8952	case glslang::EOpSubgroupPartitionedOr:
8953	case glslang::EOpSubgroupPartitionedXor:
8954	case glslang::EOpSubgroupPartitionedInclusiveAdd:
8955	case glslang::EOpSubgroupPartitionedInclusiveMul:
8956	case glslang::EOpSubgroupPartitionedInclusiveMin:
8957	case glslang::EOpSubgroupPartitionedInclusiveMax:
8958	case glslang::EOpSubgroupPartitionedInclusiveAnd:
8959	case glslang::EOpSubgroupPartitionedInclusiveOr:
8960	case glslang::EOpSubgroupPartitionedInclusiveXor:
8961	case glslang::EOpSubgroupPartitionedExclusiveAdd:
8962	case glslang::EOpSubgroupPartitionedExclusiveMul:
8963	case glslang::EOpSubgroupPartitionedExclusiveMin:
8964	case glslang::EOpSubgroupPartitionedExclusiveMax:
8965	case glslang::EOpSubgroupPartitionedExclusiveAnd:
8966	case glslang::EOpSubgroupPartitionedExclusiveOr:
8967	case glslang::EOpSubgroupPartitionedExclusiveXor:
8968	return createSubgroupOperation(op, typeId, operands, typeProxy);
8969
8970	case glslang::EOpSwizzleInvocations:
8971	extBuiltins = getExtBuiltins(name: spv::E_SPV_AMD_shader_ballot);
8972	libCall = spv::SwizzleInvocationsAMD;
8973	break;
8974	case glslang::EOpSwizzleInvocationsMasked:
8975	extBuiltins = getExtBuiltins(name: spv::E_SPV_AMD_shader_ballot);
8976	libCall = spv::SwizzleInvocationsMaskedAMD;
8977	break;
8978	case glslang::EOpWriteInvocation:
8979	extBuiltins = getExtBuiltins(name: spv::E_SPV_AMD_shader_ballot);
8980	libCall = spv::WriteInvocationAMD;
8981	break;
8982
8983	case glslang::EOpMin3:
8984	extBuiltins = getExtBuiltins(name: spv::E_SPV_AMD_shader_trinary_minmax);
8985	if (isFloat)
8986	libCall = spv::FMin3AMD;
8987	else {
8988	if (isUnsigned)
8989	libCall = spv::UMin3AMD;
8990	else
8991	libCall = spv::SMin3AMD;
8992	}
8993	break;
8994	case glslang::EOpMax3:
8995	extBuiltins = getExtBuiltins(name: spv::E_SPV_AMD_shader_trinary_minmax);
8996	if (isFloat)
8997	libCall = spv::FMax3AMD;
8998	else {
8999	if (isUnsigned)
9000	libCall = spv::UMax3AMD;
9001	else
9002	libCall = spv::SMax3AMD;
9003	}
9004	break;
9005	case glslang::EOpMid3:
9006	extBuiltins = getExtBuiltins(name: spv::E_SPV_AMD_shader_trinary_minmax);
9007	if (isFloat)
9008	libCall = spv::FMid3AMD;
9009	else {
9010	if (isUnsigned)
9011	libCall = spv::UMid3AMD;
9012	else
9013	libCall = spv::SMid3AMD;
9014	}
9015	break;
9016
9017	case glslang::EOpInterpolateAtVertex:
9018	if (typeProxy == glslang::EbtFloat16)
9019	builder.addExtension(ext: spv::E_SPV_AMD_gpu_shader_half_float);
9020	extBuiltins = getExtBuiltins(name: spv::E_SPV_AMD_shader_explicit_vertex_parameter);
9021	libCall = spv::InterpolateAtVertexAMD;
9022	break;
9023
9024	case glslang::EOpReportIntersection:
9025	typeId = builder.makeBoolType();
9026	opCode = spv::OpReportIntersectionKHR;
9027	break;
9028	case glslang::EOpTraceNV:
9029	builder.createNoResultOp(spv::OpTraceNV, operands);
9030	return 0;
9031	case glslang::EOpTraceRayMotionNV:
9032	builder.addExtension(ext: spv::E_SPV_NV_ray_tracing_motion_blur);
9033	builder.addCapability(cap: spv::CapabilityRayTracingMotionBlurNV);
9034	builder.createNoResultOp(spv::OpTraceRayMotionNV, operands);
9035	return 0;
9036	case glslang::EOpTraceKHR:
9037	builder.createNoResultOp(spv::OpTraceRayKHR, operands);
9038	return 0;
9039	case glslang::EOpExecuteCallableNV:
9040	builder.createNoResultOp(spv::OpExecuteCallableNV, operands);
9041	return 0;
9042	case glslang::EOpExecuteCallableKHR:
9043	builder.createNoResultOp(spv::OpExecuteCallableKHR, operands);
9044	return 0;
9045
9046	case glslang::EOpRayQueryInitialize:
9047	builder.createNoResultOp(spv::OpRayQueryInitializeKHR, operands);
9048	return 0;
9049	case glslang::EOpRayQueryTerminate:
9050	builder.createNoResultOp(spv::OpRayQueryTerminateKHR, operands);
9051	return 0;
9052	case glslang::EOpRayQueryGenerateIntersection:
9053	builder.createNoResultOp(spv::OpRayQueryGenerateIntersectionKHR, operands);
9054	return 0;
9055	case glslang::EOpRayQueryConfirmIntersection:
9056	builder.createNoResultOp(spv::OpRayQueryConfirmIntersectionKHR, operands);
9057	return 0;
9058	case glslang::EOpRayQueryProceed:
9059	typeId = builder.makeBoolType();
9060	opCode = spv::OpRayQueryProceedKHR;
9061	break;
9062	case glslang::EOpRayQueryGetIntersectionType:
9063	typeId = builder.makeUintType(width: 32);
9064	opCode = spv::OpRayQueryGetIntersectionTypeKHR;
9065	break;
9066	case glslang::EOpRayQueryGetRayTMin:
9067	typeId = builder.makeFloatType(width: 32);
9068	opCode = spv::OpRayQueryGetRayTMinKHR;
9069	break;
9070	case glslang::EOpRayQueryGetRayFlags:
9071	typeId = builder.makeIntType(width: 32);
9072	opCode = spv::OpRayQueryGetRayFlagsKHR;
9073	break;
9074	case glslang::EOpRayQueryGetIntersectionT:
9075	typeId = builder.makeFloatType(width: 32);
9076	opCode = spv::OpRayQueryGetIntersectionTKHR;
9077	break;
9078	case glslang::EOpRayQueryGetIntersectionInstanceCustomIndex:
9079	typeId = builder.makeIntType(width: 32);
9080	opCode = spv::OpRayQueryGetIntersectionInstanceCustomIndexKHR;
9081	break;
9082	case glslang::EOpRayQueryGetIntersectionInstanceId:
9083	typeId = builder.makeIntType(width: 32);
9084	opCode = spv::OpRayQueryGetIntersectionInstanceIdKHR;
9085	break;
9086	case glslang::EOpRayQueryGetIntersectionInstanceShaderBindingTableRecordOffset:
9087	typeId = builder.makeUintType(width: 32);
9088	opCode = spv::OpRayQueryGetIntersectionInstanceShaderBindingTableRecordOffsetKHR;
9089	break;
9090	case glslang::EOpRayQueryGetIntersectionGeometryIndex:
9091	typeId = builder.makeIntType(width: 32);
9092	opCode = spv::OpRayQueryGetIntersectionGeometryIndexKHR;
9093	break;
9094	case glslang::EOpRayQueryGetIntersectionPrimitiveIndex:
9095	typeId = builder.makeIntType(width: 32);
9096	opCode = spv::OpRayQueryGetIntersectionPrimitiveIndexKHR;
9097	break;
9098	case glslang::EOpRayQueryGetIntersectionBarycentrics:
9099	typeId = builder.makeVectorType(component: builder.makeFloatType(width: 32), size: 2);
9100	opCode = spv::OpRayQueryGetIntersectionBarycentricsKHR;
9101	break;
9102	case glslang::EOpRayQueryGetIntersectionFrontFace:
9103	typeId = builder.makeBoolType();
9104	opCode = spv::OpRayQueryGetIntersectionFrontFaceKHR;
9105	break;
9106	case glslang::EOpRayQueryGetIntersectionCandidateAABBOpaque:
9107	typeId = builder.makeBoolType();
9108	opCode = spv::OpRayQueryGetIntersectionCandidateAABBOpaqueKHR;
9109	break;
9110	case glslang::EOpRayQueryGetIntersectionObjectRayDirection:
9111	typeId = builder.makeVectorType(component: builder.makeFloatType(width: 32), size: 3);
9112	opCode = spv::OpRayQueryGetIntersectionObjectRayDirectionKHR;
9113	break;
9114	case glslang::EOpRayQueryGetIntersectionObjectRayOrigin:
9115	typeId = builder.makeVectorType(component: builder.makeFloatType(width: 32), size: 3);
9116	opCode = spv::OpRayQueryGetIntersectionObjectRayOriginKHR;
9117	break;
9118	case glslang::EOpRayQueryGetWorldRayDirection:
9119	typeId = builder.makeVectorType(component: builder.makeFloatType(width: 32), size: 3);
9120	opCode = spv::OpRayQueryGetWorldRayDirectionKHR;
9121	break;
9122	case glslang::EOpRayQueryGetWorldRayOrigin:
9123	typeId = builder.makeVectorType(component: builder.makeFloatType(width: 32), size: 3);
9124	opCode = spv::OpRayQueryGetWorldRayOriginKHR;
9125	break;
9126	case glslang::EOpRayQueryGetIntersectionObjectToWorld:
9127	typeId = builder.makeMatrixType(component: builder.makeFloatType(width: 32), cols: 4, rows: 3);
9128	opCode = spv::OpRayQueryGetIntersectionObjectToWorldKHR;
9129	break;
9130	case glslang::EOpRayQueryGetIntersectionWorldToObject:
9131	typeId = builder.makeMatrixType(component: builder.makeFloatType(width: 32), cols: 4, rows: 3);
9132	opCode = spv::OpRayQueryGetIntersectionWorldToObjectKHR;
9133	break;
9134	case glslang::EOpWritePackedPrimitiveIndices4x8NV:
9135	builder.createNoResultOp(spv::OpWritePackedPrimitiveIndices4x8NV, operands);
9136	return 0;
9137	case glslang::EOpEmitMeshTasksEXT:
9138	if (taskPayloadID)
9139	operands.push_back(x: taskPayloadID);
9140	// As per SPV_EXT_mesh_shader make it a terminating instruction in the current block
9141	builder.makeStatementTerminator(opcode: spv::OpEmitMeshTasksEXT, operands, name: "post-OpEmitMeshTasksEXT");
9142	return 0;
9143	case glslang::EOpSetMeshOutputsEXT:
9144	builder.createNoResultOp(spv::OpSetMeshOutputsEXT, operands);
9145	return 0;
9146	case glslang::EOpCooperativeMatrixMulAddNV:
9147	opCode = spv::OpCooperativeMatrixMulAddNV;
9148	break;
9149	case glslang::EOpHitObjectTraceRayNV:
9150	builder.createNoResultOp(spv::OpHitObjectTraceRayNV, operands);
9151	return 0;
9152	case glslang::EOpHitObjectTraceRayMotionNV:
9153	builder.createNoResultOp(spv::OpHitObjectTraceRayMotionNV, operands);
9154	return 0;
9155	case glslang::EOpHitObjectRecordHitNV:
9156	builder.createNoResultOp(spv::OpHitObjectRecordHitNV, operands);
9157	return 0;
9158	case glslang::EOpHitObjectRecordHitMotionNV:
9159	builder.createNoResultOp(spv::OpHitObjectRecordHitMotionNV, operands);
9160	return 0;
9161	case glslang::EOpHitObjectRecordHitWithIndexNV:
9162	builder.createNoResultOp(spv::OpHitObjectRecordHitWithIndexNV, operands);
9163	return 0;
9164	case glslang::EOpHitObjectRecordHitWithIndexMotionNV:
9165	builder.createNoResultOp(spv::OpHitObjectRecordHitWithIndexMotionNV, operands);
9166	return 0;
9167	case glslang::EOpHitObjectRecordMissNV:
9168	builder.createNoResultOp(spv::OpHitObjectRecordMissNV, operands);
9169	return 0;
9170	case glslang::EOpHitObjectRecordMissMotionNV:
9171	builder.createNoResultOp(spv::OpHitObjectRecordMissMotionNV, operands);
9172	return 0;
9173	case glslang::EOpHitObjectExecuteShaderNV:
9174	builder.createNoResultOp(spv::OpHitObjectExecuteShaderNV, operands);
9175	return 0;
9176	case glslang::EOpHitObjectIsEmptyNV:
9177	typeId = builder.makeBoolType();
9178	opCode = spv::OpHitObjectIsEmptyNV;
9179	break;
9180	case glslang::EOpHitObjectIsMissNV:
9181	typeId = builder.makeBoolType();
9182	opCode = spv::OpHitObjectIsMissNV;
9183	break;
9184	case glslang::EOpHitObjectIsHitNV:
9185	typeId = builder.makeBoolType();
9186	opCode = spv::OpHitObjectIsHitNV;
9187	break;
9188	case glslang::EOpHitObjectGetRayTMinNV:
9189	typeId = builder.makeFloatType(width: 32);
9190	opCode = spv::OpHitObjectGetRayTMinNV;
9191	break;
9192	case glslang::EOpHitObjectGetRayTMaxNV:
9193	typeId = builder.makeFloatType(width: 32);
9194	opCode = spv::OpHitObjectGetRayTMaxNV;
9195	break;
9196	case glslang::EOpHitObjectGetObjectRayOriginNV:
9197	typeId = builder.makeVectorType(component: builder.makeFloatType(width: 32), size: 3);
9198	opCode = spv::OpHitObjectGetObjectRayOriginNV;
9199	break;
9200	case glslang::EOpHitObjectGetObjectRayDirectionNV:
9201	typeId = builder.makeVectorType(component: builder.makeFloatType(width: 32), size: 3);
9202	opCode = spv::OpHitObjectGetObjectRayDirectionNV;
9203	break;
9204	case glslang::EOpHitObjectGetWorldRayOriginNV:
9205	typeId = builder.makeVectorType(component: builder.makeFloatType(width: 32), size: 3);
9206	opCode = spv::OpHitObjectGetWorldRayOriginNV;
9207	break;
9208	case glslang::EOpHitObjectGetWorldRayDirectionNV:
9209	typeId = builder.makeVectorType(component: builder.makeFloatType(width: 32), size: 3);
9210	opCode = spv::OpHitObjectGetWorldRayDirectionNV;
9211	break;
9212	case glslang::EOpHitObjectGetWorldToObjectNV:
9213	typeId = builder.makeMatrixType(component: builder.makeFloatType(width: 32), cols: 4, rows: 3);
9214	opCode = spv::OpHitObjectGetWorldToObjectNV;
9215	break;
9216	case glslang::EOpHitObjectGetObjectToWorldNV:
9217	typeId = builder.makeMatrixType(component: builder.makeFloatType(width: 32), cols: 4, rows: 3);
9218	opCode = spv::OpHitObjectGetObjectToWorldNV;
9219	break;
9220	case glslang::EOpHitObjectGetInstanceCustomIndexNV:
9221	typeId = builder.makeIntegerType(width: 32, hasSign: 1);
9222	opCode = spv::OpHitObjectGetInstanceCustomIndexNV;
9223	break;
9224	case glslang::EOpHitObjectGetInstanceIdNV:
9225	typeId = builder.makeIntegerType(width: 32, hasSign: 1);
9226	opCode = spv::OpHitObjectGetInstanceIdNV;
9227	break;
9228	case glslang::EOpHitObjectGetGeometryIndexNV:
9229	typeId = builder.makeIntegerType(width: 32, hasSign: 1);
9230	opCode = spv::OpHitObjectGetGeometryIndexNV;
9231	break;
9232	case glslang::EOpHitObjectGetPrimitiveIndexNV:
9233	typeId = builder.makeIntegerType(width: 32, hasSign: 1);
9234	opCode = spv::OpHitObjectGetPrimitiveIndexNV;
9235	break;
9236	case glslang::EOpHitObjectGetHitKindNV:
9237	typeId = builder.makeIntegerType(width: 32, hasSign: 0);
9238	opCode = spv::OpHitObjectGetHitKindNV;
9239	break;
9240	case glslang::EOpHitObjectGetCurrentTimeNV:
9241	typeId = builder.makeFloatType(width: 32);
9242	opCode = spv::OpHitObjectGetCurrentTimeNV;
9243	break;
9244	case glslang::EOpHitObjectGetShaderBindingTableRecordIndexNV:
9245	typeId = builder.makeIntegerType(width: 32, hasSign: 0);
9246	opCode = spv::OpHitObjectGetShaderBindingTableRecordIndexNV;
9247	return 0;
9248	case glslang::EOpHitObjectGetAttributesNV:
9249	builder.createNoResultOp(spv::OpHitObjectGetAttributesNV, operands);
9250	return 0;
9251	case glslang::EOpHitObjectGetShaderRecordBufferHandleNV:
9252	typeId = builder.makeVectorType(component: builder.makeUintType(width: 32), size: 2);
9253	opCode = spv::OpHitObjectGetShaderRecordBufferHandleNV;
9254	break;
9255	case glslang::EOpReorderThreadNV: {
9256	if (operands.size() == 2) {
9257	builder.createNoResultOp(spv::OpReorderThreadWithHintNV, operands);
9258	} else {
9259	builder.createNoResultOp(spv::OpReorderThreadWithHitObjectNV, operands);
9260	}
9261	return 0;
9262
9263	}
9264
9265	case glslang::EOpImageSampleWeightedQCOM:
9266	typeId = builder.makeVectorType(component: builder.makeFloatType(width: 32), size: 4);
9267	opCode = spv::OpImageSampleWeightedQCOM;
9268	addImageProcessingQCOMDecoration(id: operands[2], decor: spv::DecorationWeightTextureQCOM);
9269	break;
9270	case glslang::EOpImageBoxFilterQCOM:
9271	typeId = builder.makeVectorType(component: builder.makeFloatType(width: 32), size: 4);
9272	opCode = spv::OpImageBoxFilterQCOM;
9273	break;
9274	case glslang::EOpImageBlockMatchSADQCOM:
9275	typeId = builder.makeVectorType(component: builder.makeFloatType(width: 32), size: 4);
9276	opCode = spv::OpImageBlockMatchSADQCOM;
9277	addImageProcessingQCOMDecoration(id: operands[0], decor: spv::DecorationBlockMatchTextureQCOM);
9278	addImageProcessingQCOMDecoration(id: operands[2], decor: spv::DecorationBlockMatchTextureQCOM);
9279	break;
9280	case glslang::EOpImageBlockMatchSSDQCOM:
9281	typeId = builder.makeVectorType(component: builder.makeFloatType(width: 32), size: 4);
9282	opCode = spv::OpImageBlockMatchSSDQCOM;
9283	addImageProcessingQCOMDecoration(id: operands[0], decor: spv::DecorationBlockMatchTextureQCOM);
9284	addImageProcessingQCOMDecoration(id: operands[2], decor: spv::DecorationBlockMatchTextureQCOM);
9285	break;
9286
9287	case glslang::EOpFetchMicroTriangleVertexBarycentricNV:
9288	typeId = builder.makeVectorType(component: builder.makeFloatType(width: 32), size: 2);
9289	opCode = spv::OpFetchMicroTriangleVertexBarycentricNV;
9290	break;
9291
9292	case glslang::EOpFetchMicroTriangleVertexPositionNV:
9293	typeId = builder.makeVectorType(component: builder.makeFloatType(width: 32), size: 3);
9294	opCode = spv::OpFetchMicroTriangleVertexPositionNV;
9295	break;
9296
9297	case glslang::EOpImageBlockMatchWindowSSDQCOM:
9298	typeId = builder.makeVectorType(component: builder.makeFloatType(width: 32), size: 4);
9299	opCode = spv::OpImageBlockMatchWindowSSDQCOM;
9300	addImageProcessing2QCOMDecoration(id: operands[0], isForGather: false);
9301	addImageProcessing2QCOMDecoration(id: operands[2], isForGather: false);
9302	break;
9303	case glslang::EOpImageBlockMatchWindowSADQCOM:
9304	typeId = builder.makeVectorType(component: builder.makeFloatType(width: 32), size: 4);
9305	opCode = spv::OpImageBlockMatchWindowSADQCOM;
9306	addImageProcessing2QCOMDecoration(id: operands[0], isForGather: false);
9307	addImageProcessing2QCOMDecoration(id: operands[2], isForGather: false);
9308	break;
9309	case glslang::EOpImageBlockMatchGatherSSDQCOM:
9310	typeId = builder.makeVectorType(component: builder.makeFloatType(width: 32), size: 4);
9311	opCode = spv::OpImageBlockMatchGatherSSDQCOM;
9312	addImageProcessing2QCOMDecoration(id: operands[0], isForGather: true);
9313	addImageProcessing2QCOMDecoration(id: operands[2], isForGather: true);
9314	break;
9315	case glslang::EOpImageBlockMatchGatherSADQCOM:
9316	typeId = builder.makeVectorType(component: builder.makeFloatType(width: 32), size: 4);
9317	opCode = spv::OpImageBlockMatchGatherSADQCOM;
9318	addImageProcessing2QCOMDecoration(id: operands[0], isForGather: true);
9319	addImageProcessing2QCOMDecoration(id: operands[2], isForGather: true);
9320	break;
9321	case glslang::EOpCreateTensorLayoutNV:
9322	return builder.createOp(spv::OpCreateTensorLayoutNV, typeId, operands: std::vector<spv::Id>{});
9323	case glslang::EOpCreateTensorViewNV:
9324	return builder.createOp(spv::OpCreateTensorViewNV, typeId, operands: std::vector<spv::Id>{});
9325	case glslang::EOpTensorLayoutSetBlockSizeNV:
9326	opCode = spv::OpTensorLayoutSetBlockSizeNV;
9327	break;
9328	case glslang::EOpTensorLayoutSetDimensionNV:
9329	opCode = spv::OpTensorLayoutSetDimensionNV;
9330	break;
9331	case glslang::EOpTensorLayoutSetStrideNV:
9332	opCode = spv::OpTensorLayoutSetStrideNV;
9333	break;
9334	case glslang::EOpTensorLayoutSliceNV:
9335	opCode = spv::OpTensorLayoutSliceNV;
9336	break;
9337	case glslang::EOpTensorLayoutSetClampValueNV:
9338	opCode = spv::OpTensorLayoutSetClampValueNV;
9339	break;
9340	case glslang::EOpTensorViewSetDimensionNV:
9341	opCode = spv::OpTensorViewSetDimensionNV;
9342	break;
9343	case glslang::EOpTensorViewSetStrideNV:
9344	opCode = spv::OpTensorViewSetStrideNV;
9345	break;
9346	case glslang::EOpTensorViewSetClipNV:
9347	opCode = spv::OpTensorViewSetClipNV;
9348	break;
9349	default:
9350	return 0;
9351	}
9352
9353	spv::Id id = 0;
9354	if (libCall >= 0) {
9355	// Use an extended instruction from the standard library.
9356	// Construct the call arguments, without modifying the original operands vector.
9357	// We might need the remaining arguments, e.g. in the EOpFrexp case.
9358	std::vector<spv::Id> callArguments(consumedOperands);
9359	for (size_t i = 0; i < consumedOperands; ++i)
9360	callArguments[i] = operands[i];
9361	id = builder.createBuiltinCall(resultType: typeId, builtins: extBuiltins >= 0 ? extBuiltins : stdBuiltins, entryPoint: libCall, args: callArguments);
9362	} else if (opCode == spv::OpDot && !isFloat) {
9363	// int dot(int, int)
9364	// NOTE: never called for scalar/vector1, this is turned into simple mul before this can be reached
9365	const int componentCount = builder.getNumComponents(resultId: operands[0]);
9366	spv::Id mulOp = builder.createBinOp(spv::OpIMul, typeId: builder.getTypeId(resultId: operands[0]), operand1: operands[0], operand2: operands[1]);
9367	builder.setPrecision(id: mulOp, precision);
9368	id = builder.createCompositeExtract(composite: mulOp, typeId, index: 0);
9369	for (int i = 1; i < componentCount; ++i) {
9370	builder.setPrecision(id, precision);
9371	id = builder.createBinOp(spv::OpIAdd, typeId, operand1: id, operand2: builder.createCompositeExtract(composite: mulOp, typeId, index: i));
9372	}
9373	} else {
9374	switch (consumedOperands) {
9375	case 0:
9376	// should all be handled by visitAggregate and createNoArgOperation
9377	assert(0);
9378	return 0;
9379	case 1:
9380	// should all be handled by createUnaryOperation
9381	assert(0);
9382	return 0;
9383	case 2:
9384	id = builder.createBinOp(opCode, typeId, operand1: operands[0], operand2: operands[1]);
9385	break;
9386	default:
9387	// anything 3 or over doesn't have l-value operands, so all should be consumed
9388	assert(consumedOperands == operands.size());
9389	id = builder.createOp(opCode, typeId, operands);
9390	break;
9391	}
9392	}
9393
9394	// Decode the return types that were structures
9395	switch (op) {
9396	case glslang::EOpAddCarry:
9397	case glslang::EOpSubBorrow:
9398	builder.createStore(rValue: builder.createCompositeExtract(composite: id, typeId: typeId0, index: 1), lValue: operands[2]);
9399	id = builder.createCompositeExtract(composite: id, typeId: typeId0, index: 0);
9400	break;
9401	case glslang::EOpUMulExtended:
9402	case glslang::EOpIMulExtended:
9403	builder.createStore(rValue: builder.createCompositeExtract(composite: id, typeId: typeId0, index: 0), lValue: operands[3]);
9404	builder.createStore(rValue: builder.createCompositeExtract(composite: id, typeId: typeId0, index: 1), lValue: operands[2]);
9405	break;
9406	case glslang::EOpModf:
9407	{
9408	assert(operands.size() == 2);
9409	builder.createStore(rValue: builder.createCompositeExtract(composite: id, typeId: typeId0, index: 1), lValue: operands[1]);
9410	id = builder.createCompositeExtract(composite: id, typeId: typeId0, index: 0);
9411	}
9412	break;
9413	case glslang::EOpFrexp:
9414	{
9415	assert(operands.size() == 2);
9416	if (builder.isFloatType(typeId: builder.getScalarTypeId(typeId: typeId1))) {
9417	// "exp" is floating-point type (from HLSL intrinsic)
9418	spv::Id member1 = builder.createCompositeExtract(composite: id, typeId: frexpIntType, index: 1);
9419	member1 = builder.createUnaryOp(spv::OpConvertSToF, typeId: typeId1, operand: member1);
9420	builder.createStore(rValue: member1, lValue: operands[1]);
9421	} else
9422	// "exp" is integer type (from GLSL built-in function)
9423	builder.createStore(rValue: builder.createCompositeExtract(composite: id, typeId: frexpIntType, index: 1), lValue: operands[1]);
9424	id = builder.createCompositeExtract(composite: id, typeId: typeId0, index: 0);
9425	}
9426	break;
9427	default:
9428	break;
9429	}
9430
9431	return builder.setPrecision(id, precision);
9432	}
9433
9434	// Intrinsics with no arguments (or no return value, and no precision).
9435	spv::Id TGlslangToSpvTraverser::createNoArgOperation(glslang::TOperator op, spv::Decoration precision, spv::Id typeId)
9436	{
9437	// GLSL memory barriers use queuefamily scope in new model, device scope in old model
9438	spv::Scope memoryBarrierScope = glslangIntermediate->usingVulkanMemoryModel() ?
9439	spv::ScopeQueueFamilyKHR : spv::ScopeDevice;
9440
9441	switch (op) {
9442	case glslang::EOpBarrier:
9443	if (glslangIntermediate->getStage() == EShLangTessControl) {
9444	if (glslangIntermediate->usingVulkanMemoryModel()) {
9445	builder.createControlBarrier(execution: spv::ScopeWorkgroup, memory: spv::ScopeWorkgroup,
9446	spv::MemorySemanticsOutputMemoryKHRMask |
9447	spv::MemorySemanticsAcquireReleaseMask);
9448	builder.addCapability(cap: spv::CapabilityVulkanMemoryModelKHR);
9449	} else {
9450	builder.createControlBarrier(execution: spv::ScopeWorkgroup, memory: spv::ScopeInvocation, spv::MemorySemanticsMaskNone);
9451	}
9452	} else {
9453	builder.createControlBarrier(execution: spv::ScopeWorkgroup, memory: spv::ScopeWorkgroup,
9454	spv::MemorySemanticsWorkgroupMemoryMask |
9455	spv::MemorySemanticsAcquireReleaseMask);
9456	}
9457	return 0;
9458	case glslang::EOpMemoryBarrier:
9459	builder.createMemoryBarrier(executionScope: memoryBarrierScope, memorySemantics: spv::MemorySemanticsAllMemory |
9460	spv::MemorySemanticsAcquireReleaseMask);
9461	return 0;
9462	case glslang::EOpMemoryBarrierBuffer:
9463	builder.createMemoryBarrier(executionScope: memoryBarrierScope, memorySemantics: spv::MemorySemanticsUniformMemoryMask |
9464	spv::MemorySemanticsAcquireReleaseMask);
9465	return 0;
9466	case glslang::EOpMemoryBarrierShared:
9467	builder.createMemoryBarrier(executionScope: memoryBarrierScope, memorySemantics: spv::MemorySemanticsWorkgroupMemoryMask |
9468	spv::MemorySemanticsAcquireReleaseMask);
9469	return 0;
9470	case glslang::EOpGroupMemoryBarrier:
9471	builder.createMemoryBarrier(executionScope: spv::ScopeWorkgroup, memorySemantics: spv::MemorySemanticsAllMemory |
9472	spv::MemorySemanticsAcquireReleaseMask);
9473	return 0;
9474	case glslang::EOpMemoryBarrierAtomicCounter:
9475	builder.createMemoryBarrier(executionScope: memoryBarrierScope, memorySemantics: spv::MemorySemanticsAtomicCounterMemoryMask |
9476	spv::MemorySemanticsAcquireReleaseMask);
9477	return 0;
9478	case glslang::EOpMemoryBarrierImage:
9479	builder.createMemoryBarrier(executionScope: memoryBarrierScope, memorySemantics: spv::MemorySemanticsImageMemoryMask |
9480	spv::MemorySemanticsAcquireReleaseMask);
9481	return 0;
9482	case glslang::EOpAllMemoryBarrierWithGroupSync:
9483	builder.createControlBarrier(execution: spv::ScopeWorkgroup, memory: spv::ScopeDevice,
9484	spv::MemorySemanticsAllMemory |
9485	spv::MemorySemanticsAcquireReleaseMask);
9486	return 0;
9487	case glslang::EOpDeviceMemoryBarrier:
9488	builder.createMemoryBarrier(executionScope: spv::ScopeDevice, memorySemantics: spv::MemorySemanticsUniformMemoryMask |
9489	spv::MemorySemanticsImageMemoryMask |
9490	spv::MemorySemanticsAcquireReleaseMask);
9491	return 0;
9492	case glslang::EOpDeviceMemoryBarrierWithGroupSync:
9493	builder.createControlBarrier(execution: spv::ScopeWorkgroup, memory: spv::ScopeDevice, spv::MemorySemanticsUniformMemoryMask |
9494	spv::MemorySemanticsImageMemoryMask |
9495	spv::MemorySemanticsAcquireReleaseMask);
9496	return 0;
9497	case glslang::EOpWorkgroupMemoryBarrier:
9498	builder.createMemoryBarrier(executionScope: spv::ScopeWorkgroup, memorySemantics: spv::MemorySemanticsWorkgroupMemoryMask |
9499	spv::MemorySemanticsAcquireReleaseMask);
9500	return 0;
9501	case glslang::EOpWorkgroupMemoryBarrierWithGroupSync:
9502	builder.createControlBarrier(execution: spv::ScopeWorkgroup, memory: spv::ScopeWorkgroup,
9503	spv::MemorySemanticsWorkgroupMemoryMask |
9504	spv::MemorySemanticsAcquireReleaseMask);
9505	return 0;
9506	case glslang::EOpSubgroupBarrier:
9507	builder.createControlBarrier(execution: spv::ScopeSubgroup, memory: spv::ScopeSubgroup, spv::MemorySemanticsAllMemory |
9508	spv::MemorySemanticsAcquireReleaseMask);
9509	return spv::NoResult;
9510	case glslang::EOpSubgroupMemoryBarrier:
9511	builder.createMemoryBarrier(executionScope: spv::ScopeSubgroup, memorySemantics: spv::MemorySemanticsAllMemory |
9512	spv::MemorySemanticsAcquireReleaseMask);
9513	return spv::NoResult;
9514	case glslang::EOpSubgroupMemoryBarrierBuffer:
9515	builder.createMemoryBarrier(executionScope: spv::ScopeSubgroup, memorySemantics: spv::MemorySemanticsUniformMemoryMask |
9516	spv::MemorySemanticsAcquireReleaseMask);
9517	return spv::NoResult;
9518	case glslang::EOpSubgroupMemoryBarrierImage:
9519	builder.createMemoryBarrier(executionScope: spv::ScopeSubgroup, memorySemantics: spv::MemorySemanticsImageMemoryMask |
9520	spv::MemorySemanticsAcquireReleaseMask);
9521	return spv::NoResult;
9522	case glslang::EOpSubgroupMemoryBarrierShared:
9523	builder.createMemoryBarrier(executionScope: spv::ScopeSubgroup, memorySemantics: spv::MemorySemanticsWorkgroupMemoryMask |
9524	spv::MemorySemanticsAcquireReleaseMask);
9525	return spv::NoResult;
9526
9527	case glslang::EOpEmitVertex:
9528	builder.createNoResultOp(spv::OpEmitVertex);
9529	return 0;
9530	case glslang::EOpEndPrimitive:
9531	builder.createNoResultOp(spv::OpEndPrimitive);
9532	return 0;
9533
9534	case glslang::EOpSubgroupElect: {
9535	std::vector<spv::Id> operands;
9536	return createSubgroupOperation(op, typeId, operands, typeProxy: glslang::EbtVoid);
9537	}
9538	case glslang::EOpTime:
9539	{
9540	std::vector<spv::Id> args; // Dummy arguments
9541	spv::Id id = builder.createBuiltinCall(resultType: typeId, builtins: getExtBuiltins(name: spv::E_SPV_AMD_gcn_shader), entryPoint: spv::TimeAMD, args);
9542	return builder.setPrecision(id, precision);
9543	}
9544	case glslang::EOpIgnoreIntersectionNV:
9545	builder.createNoResultOp(spv::OpIgnoreIntersectionNV);
9546	return 0;
9547	case glslang::EOpTerminateRayNV:
9548	builder.createNoResultOp(spv::OpTerminateRayNV);
9549	return 0;
9550	case glslang::EOpRayQueryInitialize:
9551	builder.createNoResultOp(spv::OpRayQueryInitializeKHR);
9552	return 0;
9553	case glslang::EOpRayQueryTerminate:
9554	builder.createNoResultOp(spv::OpRayQueryTerminateKHR);
9555	return 0;
9556	case glslang::EOpRayQueryGenerateIntersection:
9557	builder.createNoResultOp(spv::OpRayQueryGenerateIntersectionKHR);
9558	return 0;
9559	case glslang::EOpRayQueryConfirmIntersection:
9560	builder.createNoResultOp(spv::OpRayQueryConfirmIntersectionKHR);
9561	return 0;
9562	case glslang::EOpBeginInvocationInterlock:
9563	builder.createNoResultOp(spv::OpBeginInvocationInterlockEXT);
9564	return 0;
9565	case glslang::EOpEndInvocationInterlock:
9566	builder.createNoResultOp(spv::OpEndInvocationInterlockEXT);
9567	return 0;
9568
9569	case glslang::EOpIsHelperInvocation:
9570	{
9571	std::vector<spv::Id> args; // Dummy arguments
9572	builder.addExtension(ext: spv::E_SPV_EXT_demote_to_helper_invocation);
9573	builder.addCapability(cap: spv::CapabilityDemoteToHelperInvocationEXT);
9574	return builder.createOp(spv::OpIsHelperInvocationEXT, typeId, operands: args);
9575	}
9576
9577	case glslang::EOpReadClockSubgroupKHR: {
9578	std::vector<spv::Id> args;
9579	args.push_back(x: builder.makeUintConstant(u: spv::ScopeSubgroup));
9580	builder.addExtension(ext: spv::E_SPV_KHR_shader_clock);
9581	builder.addCapability(cap: spv::CapabilityShaderClockKHR);
9582	return builder.createOp(spv::OpReadClockKHR, typeId, operands: args);
9583	}
9584
9585	case glslang::EOpReadClockDeviceKHR: {
9586	std::vector<spv::Id> args;
9587	args.push_back(x: builder.makeUintConstant(u: spv::ScopeDevice));
9588	builder.addExtension(ext: spv::E_SPV_KHR_shader_clock);
9589	builder.addCapability(cap: spv::CapabilityShaderClockKHR);
9590	return builder.createOp(spv::OpReadClockKHR, typeId, operands: args);
9591	}
9592	case glslang::EOpStencilAttachmentReadEXT:
9593	case glslang::EOpDepthAttachmentReadEXT:
9594	{
9595	builder.addExtension(ext: spv::E_SPV_EXT_shader_tile_image);
9596
9597	spv::Decoration precision;
9598	spv::Op spv_op;
9599	if (op == glslang::EOpStencilAttachmentReadEXT)
9600	{
9601	precision = spv::DecorationRelaxedPrecision;
9602	spv_op = spv::OpStencilAttachmentReadEXT;
9603	builder.addCapability(cap: spv::CapabilityTileImageStencilReadAccessEXT);
9604	}
9605	else
9606	{
9607	precision = spv::NoPrecision;
9608	spv_op = spv::OpDepthAttachmentReadEXT;
9609	builder.addCapability(cap: spv::CapabilityTileImageDepthReadAccessEXT);
9610	}
9611
9612	std::vector<spv::Id> args; // Dummy args
9613	spv::Id result = builder.createOp(spv_op, typeId, operands: args);
9614	return builder.setPrecision(id: result, precision);
9615	}
9616	default:
9617	break;
9618	}
9619
9620	logger->missingFunctionality(f: "unknown operation with no arguments");
9621
9622	return 0;
9623	}
9624
9625	spv::Id TGlslangToSpvTraverser::getSymbolId(const glslang::TIntermSymbol* symbol)
9626	{
9627	auto iter = symbolValues.find(x: symbol->getId());
9628	spv::Id id;
9629	if (symbolValues.end() != iter) {
9630	id = iter->second;
9631	return id;
9632	}
9633
9634	// it was not found, create it
9635	spv::BuiltIn builtIn = TranslateBuiltInDecoration(builtIn: symbol->getQualifier().builtIn, memberDeclaration: false);
9636	auto forcedType = getForcedType(glslangBuiltIn: symbol->getQualifier().builtIn, glslangType: symbol->getType());
9637
9638	// There are pairs of symbols that map to the same SPIR-V built-in:
9639	// gl_ObjectToWorldEXT and gl_ObjectToWorld3x4EXT, and gl_WorldToObjectEXT
9640	// and gl_WorldToObject3x4EXT. SPIR-V forbids having two OpVariables
9641	// with the same BuiltIn in the same storage class, so we must re-use one.
9642	const bool mayNeedToReuseBuiltIn =
9643	builtIn == spv::BuiltInObjectToWorldKHR ||
9644	builtIn == spv::BuiltInWorldToObjectKHR;
9645
9646	if (mayNeedToReuseBuiltIn) {
9647	auto iter = builtInVariableIds.find(x: uint32_t(builtIn));
9648	if (builtInVariableIds.end() != iter) {
9649	id = iter->second;
9650	symbolValues[symbol->getId()] = id;
9651	if (forcedType.second != spv::NoType)
9652	forceType[id] = forcedType.second;
9653	return id;
9654	}
9655	}
9656
9657	if (symbol->getBasicType() == glslang::EbtFunction) {
9658	return 0;
9659	}
9660
9661	id = createSpvVariable(node: symbol, forcedType: forcedType.first);
9662
9663	if (mayNeedToReuseBuiltIn) {
9664	builtInVariableIds.insert(x: {uint32_t(builtIn), id});
9665	}
9666
9667	symbolValues[symbol->getId()] = id;
9668	if (forcedType.second != spv::NoType)
9669	forceType[id] = forcedType.second;
9670
9671	if (symbol->getBasicType() != glslang::EbtBlock) {
9672	builder.addDecoration(id, TranslatePrecisionDecoration(type: symbol->getType()));
9673	builder.addDecoration(id, TranslateInterpolationDecoration(qualifier: symbol->getType().getQualifier()));
9674	builder.addDecoration(id, TranslateAuxiliaryStorageDecoration(qualifier: symbol->getType().getQualifier()));
9675	addMeshNVDecoration(id, /*member*/ -1, qualifier: symbol->getType().getQualifier());
9676	if (symbol->getQualifier().hasComponent())
9677	builder.addDecoration(id, spv::DecorationComponent, num: symbol->getQualifier().layoutComponent);
9678	if (symbol->getQualifier().hasIndex())
9679	builder.addDecoration(id, spv::DecorationIndex, num: symbol->getQualifier().layoutIndex);
9680	if (symbol->getType().getQualifier().hasSpecConstantId())
9681	builder.addDecoration(id, spv::DecorationSpecId, num: symbol->getType().getQualifier().layoutSpecConstantId);
9682	// atomic counters use this:
9683	if (symbol->getQualifier().hasOffset())
9684	builder.addDecoration(id, spv::DecorationOffset, num: symbol->getQualifier().layoutOffset);
9685	}
9686
9687	if (symbol->getQualifier().hasLocation()) {
9688	if (!(glslangIntermediate->isRayTracingStage() &&
9689	(glslangIntermediate->IsRequestedExtension(extension: glslang::E_GL_EXT_ray_tracing) ||
9690	glslangIntermediate->IsRequestedExtension(extension: glslang::E_GL_NV_shader_invocation_reorder))
9691	&& (builder.getStorageClass(resultId: id) == spv::StorageClassRayPayloadKHR ||
9692	builder.getStorageClass(resultId: id) == spv::StorageClassIncomingRayPayloadKHR ||
9693	builder.getStorageClass(resultId: id) == spv::StorageClassCallableDataKHR ||
9694	builder.getStorageClass(resultId: id) == spv::StorageClassIncomingCallableDataKHR ||
9695	builder.getStorageClass(resultId: id) == spv::StorageClassHitObjectAttributeNV))) {
9696	// Location values are used to link TraceRayKHR/ExecuteCallableKHR/HitObjectGetAttributesNV
9697	// to corresponding variables but are not valid in SPIRV since they are supported only
9698	// for Input/Output Storage classes.
9699	builder.addDecoration(id, spv::DecorationLocation, num: symbol->getQualifier().layoutLocation);
9700	}
9701	}
9702
9703	builder.addDecoration(id, TranslateInvariantDecoration(qualifier: symbol->getType().getQualifier()));
9704	if (symbol->getQualifier().hasStream() && glslangIntermediate->isMultiStream()) {
9705	builder.addCapability(cap: spv::CapabilityGeometryStreams);
9706	builder.addDecoration(id, spv::DecorationStream, num: symbol->getQualifier().layoutStream);
9707	}
9708	if (symbol->getQualifier().hasSet())
9709	builder.addDecoration(id, spv::DecorationDescriptorSet, num: symbol->getQualifier().layoutSet);
9710	else if (IsDescriptorResource(type: symbol->getType())) {
9711	// default to 0
9712	builder.addDecoration(id, spv::DecorationDescriptorSet, num: 0);
9713	}
9714	if (symbol->getQualifier().hasBinding())
9715	builder.addDecoration(id, spv::DecorationBinding, num: symbol->getQualifier().layoutBinding);
9716	else if (IsDescriptorResource(type: symbol->getType())) {
9717	// default to 0
9718	builder.addDecoration(id, spv::DecorationBinding, num: 0);
9719	}
9720	if (symbol->getQualifier().hasAttachment())
9721	builder.addDecoration(id, spv::DecorationInputAttachmentIndex, num: symbol->getQualifier().layoutAttachment);
9722	if (glslangIntermediate->getXfbMode()) {
9723	builder.addCapability(cap: spv::CapabilityTransformFeedback);
9724	if (symbol->getQualifier().hasXfbBuffer()) {
9725	builder.addDecoration(id, spv::DecorationXfbBuffer, num: symbol->getQualifier().layoutXfbBuffer);
9726	unsigned stride = glslangIntermediate->getXfbStride(buffer: symbol->getQualifier().layoutXfbBuffer);
9727	if (stride != glslang::TQualifier::layoutXfbStrideEnd)
9728	builder.addDecoration(id, spv::DecorationXfbStride, num: stride);
9729	}
9730	if (symbol->getQualifier().hasXfbOffset())
9731	builder.addDecoration(id, spv::DecorationOffset, num: symbol->getQualifier().layoutXfbOffset);
9732	}
9733
9734	// add built-in variable decoration
9735	if (builtIn != spv::BuiltInMax) {
9736	// WorkgroupSize deprecated in spirv1.6
9737	if (glslangIntermediate->getSpv().spv < glslang::EShTargetSpv_1_6 ||
9738	builtIn != spv::BuiltInWorkgroupSize)
9739	builder.addDecoration(id, spv::DecorationBuiltIn, num: (int)builtIn);
9740	}
9741
9742	// Add volatile decoration to HelperInvocation for spirv1.6 and beyond
9743	if (builtIn == spv::BuiltInHelperInvocation &&
9744	!glslangIntermediate->usingVulkanMemoryModel() &&
9745	glslangIntermediate->getSpv().spv >= glslang::EShTargetSpv_1_6) {
9746	builder.addDecoration(id, spv::DecorationVolatile);
9747	}
9748
9749	// Subgroup builtins which have input storage class are volatile for ray tracing stages.
9750	if (symbol->getType().isImage() || symbol->getQualifier().isPipeInput()) {
9751	std::vector<spv::Decoration> memory;
9752	TranslateMemoryDecoration(qualifier: symbol->getType().getQualifier(), memory,
9753	useVulkanMemoryModel: glslangIntermediate->usingVulkanMemoryModel());
9754	for (unsigned int i = 0; i < memory.size(); ++i)
9755	builder.addDecoration(id, memory[i]);
9756	}
9757
9758	if (builtIn == spv::BuiltInSampleMask) {
9759	spv::Decoration decoration;
9760	// GL_NV_sample_mask_override_coverage extension
9761	if (glslangIntermediate->getLayoutOverrideCoverage())
9762	decoration = (spv::Decoration)spv::DecorationOverrideCoverageNV;
9763	else
9764	decoration = (spv::Decoration)spv::DecorationMax;
9765	builder.addDecoration(id, decoration);
9766	if (decoration != spv::DecorationMax) {
9767	builder.addCapability(cap: spv::CapabilitySampleMaskOverrideCoverageNV);
9768	builder.addExtension(ext: spv::E_SPV_NV_sample_mask_override_coverage);
9769	}
9770	}
9771	else if (builtIn == spv::BuiltInLayer) {
9772	// SPV_NV_viewport_array2 extension
9773	if (symbol->getQualifier().layoutViewportRelative) {
9774	builder.addDecoration(id, (spv::Decoration)spv::DecorationViewportRelativeNV);
9775	builder.addCapability(cap: spv::CapabilityShaderViewportMaskNV);
9776	builder.addExtension(ext: spv::E_SPV_NV_viewport_array2);
9777	}
9778	if (symbol->getQualifier().layoutSecondaryViewportRelativeOffset != -2048) {
9779	builder.addDecoration(id, (spv::Decoration)spv::DecorationSecondaryViewportRelativeNV,
9780	num: symbol->getQualifier().layoutSecondaryViewportRelativeOffset);
9781	builder.addCapability(cap: spv::CapabilityShaderStereoViewNV);
9782	builder.addExtension(ext: spv::E_SPV_NV_stereo_view_rendering);
9783	}
9784	}
9785
9786	if (symbol->getQualifier().layoutPassthrough) {
9787	builder.addDecoration(id, spv::DecorationPassthroughNV);
9788	builder.addCapability(cap: spv::CapabilityGeometryShaderPassthroughNV);
9789	builder.addExtension(ext: spv::E_SPV_NV_geometry_shader_passthrough);
9790	}
9791	if (symbol->getQualifier().pervertexNV) {
9792	builder.addDecoration(id, spv::DecorationPerVertexNV);
9793	builder.addCapability(cap: spv::CapabilityFragmentBarycentricNV);
9794	builder.addExtension(ext: spv::E_SPV_NV_fragment_shader_barycentric);
9795	}
9796
9797	if (symbol->getQualifier().pervertexEXT) {
9798	builder.addDecoration(id, spv::DecorationPerVertexKHR);
9799	builder.addCapability(cap: spv::CapabilityFragmentBarycentricKHR);
9800	builder.addExtension(ext: spv::E_SPV_KHR_fragment_shader_barycentric);
9801	}
9802
9803	if (glslangIntermediate->getHlslFunctionality1() && symbol->getType().getQualifier().semanticName != nullptr) {
9804	builder.addExtension(ext: "SPV_GOOGLE_hlsl_functionality1");
9805	builder.addDecoration(id, (spv::Decoration)spv::DecorationHlslSemanticGOOGLE,
9806	symbol->getType().getQualifier().semanticName);
9807	}
9808
9809	if (symbol->isReference()) {
9810	builder.addDecoration(id, symbol->getType().getQualifier().restrict ?
9811	spv::DecorationRestrictPointerEXT : spv::DecorationAliasedPointerEXT);
9812	}
9813
9814	// Add SPIR-V decorations (GL_EXT_spirv_intrinsics)
9815	if (symbol->getType().getQualifier().hasSpirvDecorate())
9816	applySpirvDecorate(type: symbol->getType(), id, member: {});
9817
9818	return id;
9819	}
9820
9821	// add per-primitive, per-view. per-task decorations to a struct member (member >= 0) or an object
9822	void TGlslangToSpvTraverser::addMeshNVDecoration(spv::Id id, int member, const glslang::TQualifier& qualifier)
9823	{
9824	bool isMeshShaderExt = (glslangIntermediate->getRequestedExtensions().find(x: glslang::E_GL_EXT_mesh_shader) !=
9825	glslangIntermediate->getRequestedExtensions().end());
9826
9827	if (member >= 0) {
9828	if (qualifier.perPrimitiveNV) {
9829	// Need to add capability/extension for fragment shader.
9830	// Mesh shader already adds this by default.
9831	if (glslangIntermediate->getStage() == EShLangFragment) {
9832	if(isMeshShaderExt) {
9833	builder.addCapability(cap: spv::CapabilityMeshShadingEXT);
9834	builder.addExtension(ext: spv::E_SPV_EXT_mesh_shader);
9835	} else {
9836	builder.addCapability(cap: spv::CapabilityMeshShadingNV);
9837	builder.addExtension(ext: spv::E_SPV_NV_mesh_shader);
9838	}
9839	}
9840	builder.addMemberDecoration(id, member: (unsigned)member, spv::DecorationPerPrimitiveNV);
9841	}
9842	if (qualifier.perViewNV)
9843	builder.addMemberDecoration(id, member: (unsigned)member, spv::DecorationPerViewNV);
9844	if (qualifier.perTaskNV)
9845	builder.addMemberDecoration(id, member: (unsigned)member, spv::DecorationPerTaskNV);
9846	} else {
9847	if (qualifier.perPrimitiveNV) {
9848	// Need to add capability/extension for fragment shader.
9849	// Mesh shader already adds this by default.
9850	if (glslangIntermediate->getStage() == EShLangFragment) {
9851	if(isMeshShaderExt) {
9852	builder.addCapability(cap: spv::CapabilityMeshShadingEXT);
9853	builder.addExtension(ext: spv::E_SPV_EXT_mesh_shader);
9854	} else {
9855	builder.addCapability(cap: spv::CapabilityMeshShadingNV);
9856	builder.addExtension(ext: spv::E_SPV_NV_mesh_shader);
9857	}
9858	}
9859	builder.addDecoration(id, spv::DecorationPerPrimitiveNV);
9860	}
9861	if (qualifier.perViewNV)
9862	builder.addDecoration(id, spv::DecorationPerViewNV);
9863	if (qualifier.perTaskNV)
9864	builder.addDecoration(id, spv::DecorationPerTaskNV);
9865	}
9866	}
9867
9868	bool TGlslangToSpvTraverser::hasQCOMImageProceessingDecoration(spv::Id id, spv::Decoration decor)
9869	{
9870	std::vector<spv::Decoration> &decoVec = idToQCOMDecorations[id];
9871	for ( auto d : decoVec ) {
9872	if ( d == decor )
9873	return true;
9874	}
9875	return false;
9876	}
9877
9878	void TGlslangToSpvTraverser::addImageProcessingQCOMDecoration(spv::Id id, spv::Decoration decor)
9879	{
9880	spv::Op opc = builder.getOpCode(id);
9881	if (opc == spv::OpSampledImage) {
9882	id = builder.getIdOperand(resultId: id, idx: 0);
9883	opc = builder.getOpCode(id);
9884	}
9885
9886	if (opc == spv::OpLoad) {
9887	spv::Id texid = builder.getIdOperand(resultId: id, idx: 0);
9888	if (!hasQCOMImageProceessingDecoration(id: texid, decor)) {//
9889	builder.addDecoration(texid, decor);
9890	idToQCOMDecorations[texid].push_back(x: decor);
9891	}
9892	}
9893	}
9894
9895	void TGlslangToSpvTraverser::addImageProcessing2QCOMDecoration(spv::Id id, bool isForGather)
9896	{
9897	if (isForGather) {
9898	return addImageProcessingQCOMDecoration(id, decor: spv::DecorationBlockMatchTextureQCOM);
9899	}
9900
9901	auto addDecor =
9902	[this](spv::Id id, spv::Decoration decor) {
9903	spv::Id tsopc = this->builder.getOpCode(id);
9904	if (tsopc == spv::OpLoad) {
9905	spv::Id tsid = this->builder.getIdOperand(resultId: id, idx: 0);
9906	if (this->glslangIntermediate->getSpv().spv >= glslang::EShTargetSpv_1_4) {
9907	assert(iOSet.count(tsid) > 0);
9908	}
9909	if (!hasQCOMImageProceessingDecoration(id: tsid, decor)) {
9910	this->builder.addDecoration(tsid, decor);
9911	idToQCOMDecorations[tsid].push_back(x: decor);
9912	}
9913	}
9914	};
9915
9916	spv::Id opc = builder.getOpCode(id);
9917	bool isInterfaceObject = (opc != spv::OpSampledImage);
9918
9919	if (!isInterfaceObject) {
9920	addDecor(builder.getIdOperand(resultId: id, idx: 0), spv::DecorationBlockMatchTextureQCOM);
9921	addDecor(builder.getIdOperand(resultId: id, idx: 1), spv::DecorationBlockMatchSamplerQCOM);
9922	} else {
9923	addDecor(id, spv::DecorationBlockMatchTextureQCOM);
9924	addDecor(id, spv::DecorationBlockMatchSamplerQCOM);
9925	}
9926	}
9927
9928	// Make a full tree of instructions to build a SPIR-V specialization constant,
9929	// or regular constant if possible.
9930	//
9931	// TBD: this is not yet done, nor verified to be the best design, it does do the leaf symbols though
9932	//
9933	// Recursively walk the nodes. The nodes form a tree whose leaves are
9934	// regular constants, which themselves are trees that createSpvConstant()
9935	// recursively walks. So, this function walks the "top" of the tree:
9936	// - emit specialization constant-building instructions for specConstant
9937	// - when running into a non-spec-constant, switch to createSpvConstant()
9938	spv::Id TGlslangToSpvTraverser::createSpvConstant(const glslang::TIntermTyped& node)
9939	{
9940	assert(node.getQualifier().isConstant());
9941
9942	// Handle front-end constants first (non-specialization constants).
9943	if (! node.getQualifier().specConstant) {
9944	// hand off to the non-spec-constant path
9945	assert(node.getAsConstantUnion() != nullptr || node.getAsSymbolNode() != nullptr);
9946	int nextConst = 0;
9947	return createSpvConstantFromConstUnionArray(type: node.getType(), node.getAsConstantUnion() ?
9948	node.getAsConstantUnion()->getConstArray() : node.getAsSymbolNode()->getConstArray(),
9949	nextConst, specConstant: false);
9950	}
9951
9952	// We now know we have a specialization constant to build
9953
9954	// Extra capabilities may be needed.
9955	if (node.getType().contains8BitInt())
9956	builder.addCapability(cap: spv::CapabilityInt8);
9957	if (node.getType().contains16BitFloat())
9958	builder.addCapability(cap: spv::CapabilityFloat16);
9959	if (node.getType().contains16BitInt())
9960	builder.addCapability(cap: spv::CapabilityInt16);
9961	if (node.getType().contains64BitInt())
9962	builder.addCapability(cap: spv::CapabilityInt64);
9963	if (node.getType().containsDouble())
9964	builder.addCapability(cap: spv::CapabilityFloat64);
9965
9966	// gl_WorkGroupSize is a special case until the front-end handles hierarchical specialization constants,
9967	// even then, it's specialization ids are handled by special case syntax in GLSL: layout(local_size_x = ...
9968	if (node.getType().getQualifier().builtIn == glslang::EbvWorkGroupSize) {
9969	std::vector<spv::Id> dimConstId;
9970	for (int dim = 0; dim < 3; ++dim) {
9971	bool specConst = (glslangIntermediate->getLocalSizeSpecId(dim) != glslang::TQualifier::layoutNotSet);
9972	dimConstId.push_back(x: builder.makeUintConstant(u: glslangIntermediate->getLocalSize(dim), specConstant: specConst));
9973	if (specConst) {
9974	builder.addDecoration(dimConstId.back(), spv::DecorationSpecId,
9975	num: glslangIntermediate->getLocalSizeSpecId(dim));
9976	}
9977	}
9978	return builder.makeCompositeConstant(type: builder.makeVectorType(component: builder.makeUintType(width: 32), size: 3), comps: dimConstId, specConst: true);
9979	}
9980
9981	// An AST node labelled as specialization constant should be a symbol node.
9982	// Its initializer should either be a sub tree with constant nodes, or a constant union array.
9983	if (auto* sn = node.getAsSymbolNode()) {
9984	spv::Id result;
9985	if (auto* sub_tree = sn->getConstSubtree()) {
9986	// Traverse the constant constructor sub tree like generating normal run-time instructions.
9987	// During the AST traversal, if the node is marked as 'specConstant', SpecConstantOpModeGuard
9988	// will set the builder into spec constant op instruction generating mode.
9989	sub_tree->traverse(this);
9990	result = accessChainLoad(type: sub_tree->getType());
9991	} else if (auto* const_union_array = &sn->getConstArray()) {
9992	int nextConst = 0;
9993	result = createSpvConstantFromConstUnionArray(type: sn->getType(), *const_union_array, nextConst, specConstant: true);
9994	} else {
9995	logger->missingFunctionality(f: "Invalid initializer for spec onstant.");
9996	return spv::NoResult;
9997	}
9998	builder.addName(result, name: sn->getName().c_str());
9999	return result;
10000	}
10001
10002	// Neither a front-end constant node, nor a specialization constant node with constant union array or
10003	// constant sub tree as initializer.
10004	logger->missingFunctionality(f: "Neither a front-end constant nor a spec constant.");
10005	return spv::NoResult;
10006	}
10007
10008	// Use 'consts' as the flattened glslang source of scalar constants to recursively
10009	// build the aggregate SPIR-V constant.
10010	//
10011	// If there are not enough elements present in 'consts', 0 will be substituted;
10012	// an empty 'consts' can be used to create a fully zeroed SPIR-V constant.
10013	//
10014	spv::Id TGlslangToSpvTraverser::createSpvConstantFromConstUnionArray(const glslang::TType& glslangType,
10015	const glslang::TConstUnionArray& consts, int& nextConst, bool specConstant)
10016	{
10017	// vector of constants for SPIR-V
10018	std::vector<spv::Id> spvConsts;
10019
10020	// Type is used for struct and array constants
10021	spv::Id typeId = convertGlslangToSpvType(type: glslangType);
10022
10023	if (glslangType.isArray()) {
10024	glslang::TType elementType(glslangType, 0);
10025	for (int i = 0; i < glslangType.getOuterArraySize(); ++i)
10026	spvConsts.push_back(x: createSpvConstantFromConstUnionArray(glslangType: elementType, consts, nextConst, specConstant: false));
10027	} else if (glslangType.isMatrix()) {
10028	glslang::TType vectorType(glslangType, 0);
10029	for (int col = 0; col < glslangType.getMatrixCols(); ++col)
10030	spvConsts.push_back(x: createSpvConstantFromConstUnionArray(glslangType: vectorType, consts, nextConst, specConstant: false));
10031	} else if (glslangType.isCoopMat()) {
10032	glslang::TType componentType(glslangType.getBasicType());
10033	spvConsts.push_back(x: createSpvConstantFromConstUnionArray(glslangType: componentType, consts, nextConst, specConstant: false));
10034	} else if (glslangType.isStruct()) {
10035	glslang::TVector<glslang::TTypeLoc>::const_iterator iter;
10036	for (iter = glslangType.getStruct()->begin(); iter != glslangType.getStruct()->end(); ++iter)
10037	spvConsts.push_back(x: createSpvConstantFromConstUnionArray(glslangType: *iter->type, consts, nextConst, specConstant: false));
10038	} else if (glslangType.getVectorSize() > 1) {
10039	for (unsigned int i = 0; i < (unsigned int)glslangType.getVectorSize(); ++i) {
10040	bool zero = nextConst >= consts.size();
10041	switch (glslangType.getBasicType()) {
10042	case glslang::EbtInt:
10043	spvConsts.push_back(x: builder.makeIntConstant(i: zero ? 0 : consts[nextConst].getIConst()));
10044	break;
10045	case glslang::EbtUint:
10046	spvConsts.push_back(x: builder.makeUintConstant(u: zero ? 0 : consts[nextConst].getUConst()));
10047	break;
10048	case glslang::EbtFloat:
10049	spvConsts.push_back(x: builder.makeFloatConstant(f: zero ? 0.0F : (float)consts[nextConst].getDConst()));
10050	break;
10051	case glslang::EbtBool:
10052	spvConsts.push_back(x: builder.makeBoolConstant(b: zero ? false : consts[nextConst].getBConst()));
10053	break;
10054	case glslang::EbtInt8:
10055	builder.addCapability(cap: spv::CapabilityInt8);
10056	spvConsts.push_back(x: builder.makeInt8Constant(i: zero ? 0 : consts[nextConst].getI8Const()));
10057	break;
10058	case glslang::EbtUint8:
10059	builder.addCapability(cap: spv::CapabilityInt8);
10060	spvConsts.push_back(x: builder.makeUint8Constant(u: zero ? 0 : consts[nextConst].getU8Const()));
10061	break;
10062	case glslang::EbtInt16:
10063	builder.addCapability(cap: spv::CapabilityInt16);
10064	spvConsts.push_back(x: builder.makeInt16Constant(i: zero ? 0 : consts[nextConst].getI16Const()));
10065	break;
10066	case glslang::EbtUint16:
10067	builder.addCapability(cap: spv::CapabilityInt16);
10068	spvConsts.push_back(x: builder.makeUint16Constant(u: zero ? 0 : consts[nextConst].getU16Const()));
10069	break;
10070	case glslang::EbtInt64:
10071	spvConsts.push_back(x: builder.makeInt64Constant(i: zero ? 0 : consts[nextConst].getI64Const()));
10072	break;
10073	case glslang::EbtUint64:
10074	spvConsts.push_back(x: builder.makeUint64Constant(u: zero ? 0 : consts[nextConst].getU64Const()));
10075	break;
10076	case glslang::EbtDouble:
10077	spvConsts.push_back(x: builder.makeDoubleConstant(d: zero ? 0.0 : consts[nextConst].getDConst()));
10078	break;
10079	case glslang::EbtFloat16:
10080	builder.addCapability(cap: spv::CapabilityFloat16);
10081	spvConsts.push_back(x: builder.makeFloat16Constant(f16: zero ? 0.0F : (float)consts[nextConst].getDConst()));
10082	break;
10083	default:
10084	assert(0);
10085	break;
10086	}
10087	++nextConst;
10088	}
10089	} else {
10090	// we have a non-aggregate (scalar) constant
10091	bool zero = nextConst >= consts.size();
10092	spv::Id scalar = 0;
10093	switch (glslangType.getBasicType()) {
10094	case glslang::EbtInt:
10095	scalar = builder.makeIntConstant(i: zero ? 0 : consts[nextConst].getIConst(), specConstant);
10096	break;
10097	case glslang::EbtUint:
10098	scalar = builder.makeUintConstant(u: zero ? 0 : consts[nextConst].getUConst(), specConstant);
10099	break;
10100	case glslang::EbtFloat:
10101	scalar = builder.makeFloatConstant(f: zero ? 0.0F : (float)consts[nextConst].getDConst(), specConstant);
10102	break;
10103	case glslang::EbtBool:
10104	scalar = builder.makeBoolConstant(b: zero ? false : consts[nextConst].getBConst(), specConstant);
10105	break;
10106	case glslang::EbtInt8:
10107	builder.addCapability(cap: spv::CapabilityInt8);
10108	scalar = builder.makeInt8Constant(i: zero ? 0 : consts[nextConst].getI8Const(), specConstant);
10109	break;
10110	case glslang::EbtUint8:
10111	builder.addCapability(cap: spv::CapabilityInt8);
10112	scalar = builder.makeUint8Constant(u: zero ? 0 : consts[nextConst].getU8Const(), specConstant);
10113	break;
10114	case glslang::EbtInt16:
10115	builder.addCapability(cap: spv::CapabilityInt16);
10116	scalar = builder.makeInt16Constant(i: zero ? 0 : consts[nextConst].getI16Const(), specConstant);
10117	break;
10118	case glslang::EbtUint16:
10119	builder.addCapability(cap: spv::CapabilityInt16);
10120	scalar = builder.makeUint16Constant(u: zero ? 0 : consts[nextConst].getU16Const(), specConstant);
10121	break;
10122	case glslang::EbtInt64:
10123	scalar = builder.makeInt64Constant(i: zero ? 0 : consts[nextConst].getI64Const(), specConstant);
10124	break;
10125	case glslang::EbtUint64:
10126	scalar = builder.makeUint64Constant(u: zero ? 0 : consts[nextConst].getU64Const(), specConstant);
10127	break;
10128	case glslang::EbtDouble:
10129	scalar = builder.makeDoubleConstant(d: zero ? 0.0 : consts[nextConst].getDConst(), specConstant);
10130	break;
10131	case glslang::EbtFloat16:
10132	builder.addCapability(cap: spv::CapabilityFloat16);
10133	scalar = builder.makeFloat16Constant(f16: zero ? 0.0F : (float)consts[nextConst].getDConst(), specConstant);
10134	break;
10135	case glslang::EbtReference:
10136	scalar = builder.makeUint64Constant(u: zero ? 0 : consts[nextConst].getU64Const(), specConstant);
10137	scalar = builder.createUnaryOp(spv::OpBitcast, typeId, operand: scalar);
10138	break;
10139	case glslang::EbtString:
10140	scalar = builder.getStringId(str: consts[nextConst].getSConst()->c_str());
10141	break;
10142	default:
10143	assert(0);
10144	break;
10145	}
10146	++nextConst;
10147	return scalar;
10148	}
10149
10150	return builder.makeCompositeConstant(type: typeId, comps: spvConsts);
10151	}
10152
10153	// Return true if the node is a constant or symbol whose reading has no
10154	// non-trivial observable cost or effect.
10155	bool TGlslangToSpvTraverser::isTrivialLeaf(const glslang::TIntermTyped* node)
10156	{
10157	// don't know what this is
10158	if (node == nullptr)
10159	return false;
10160
10161	// a constant is safe
10162	if (node->getAsConstantUnion() != nullptr)
10163	return true;
10164
10165	// not a symbol means non-trivial
10166	if (node->getAsSymbolNode() == nullptr)
10167	return false;
10168
10169	// a symbol, depends on what's being read
10170	switch (node->getType().getQualifier().storage) {
10171	case glslang::EvqTemporary:
10172	case glslang::EvqGlobal:
10173	case glslang::EvqIn:
10174	case glslang::EvqInOut:
10175	case glslang::EvqConst:
10176	case glslang::EvqConstReadOnly:
10177	case glslang::EvqUniform:
10178	return true;
10179	default:
10180	return false;
10181	}
10182	}
10183
10184	// A node is trivial if it is a single operation with no side effects.
10185	// HLSL (and/or vectors) are always trivial, as it does not short circuit.
10186	// Otherwise, error on the side of saying non-trivial.
10187	// Return true if trivial.
10188	bool TGlslangToSpvTraverser::isTrivial(const glslang::TIntermTyped* node)
10189	{
10190	if (node == nullptr)
10191	return false;
10192
10193	// count non scalars as trivial, as well as anything coming from HLSL
10194	if (! node->getType().isScalarOrVec1() || glslangIntermediate->getSource() == glslang::EShSourceHlsl)
10195	return true;
10196
10197	// symbols and constants are trivial
10198	if (isTrivialLeaf(node))
10199	return true;
10200
10201	// otherwise, it needs to be a simple operation or one or two leaf nodes
10202
10203	// not a simple operation
10204	const glslang::TIntermBinary* binaryNode = node->getAsBinaryNode();
10205	const glslang::TIntermUnary* unaryNode = node->getAsUnaryNode();
10206	if (binaryNode == nullptr && unaryNode == nullptr)
10207	return false;
10208
10209	// not on leaf nodes
10210	if (binaryNode && (! isTrivialLeaf(node: binaryNode->getLeft()) || ! isTrivialLeaf(node: binaryNode->getRight())))
10211	return false;
10212
10213	if (unaryNode && ! isTrivialLeaf(node: unaryNode->getOperand())) {
10214	return false;
10215	}
10216
10217	if (IsOpNumericConv(op: node->getAsOperator()->getOp()) &&
10218	node->getType().getBasicType() == glslang::EbtBool) {
10219	return true;
10220	}
10221
10222	switch (node->getAsOperator()->getOp()) {
10223	case glslang::EOpLogicalNot:
10224	case glslang::EOpEqual:
10225	case glslang::EOpNotEqual:
10226	case glslang::EOpLessThan:
10227	case glslang::EOpGreaterThan:
10228	case glslang::EOpLessThanEqual:
10229	case glslang::EOpGreaterThanEqual:
10230	case glslang::EOpIndexDirect:
10231	case glslang::EOpIndexDirectStruct:
10232	case glslang::EOpLogicalXor:
10233	case glslang::EOpAny:
10234	case glslang::EOpAll:
10235	return true;
10236	default:
10237	return false;
10238	}
10239	}
10240
10241	// Emit short-circuiting code, where 'right' is never evaluated unless
10242	// the left side is true (for &&) or false (for ||).
10243	spv::Id TGlslangToSpvTraverser::createShortCircuit(glslang::TOperator op, glslang::TIntermTyped& left,
10244	glslang::TIntermTyped& right)
10245	{
10246	spv::Id boolTypeId = builder.makeBoolType();
10247
10248	// emit left operand
10249	builder.clearAccessChain();
10250	left.traverse(this);
10251	spv::Id leftId = accessChainLoad(type: left.getType());
10252
10253	// Operands to accumulate OpPhi operands
10254	std::vector<spv::Id> phiOperands;
10255	phiOperands.reserve(n: 4);
10256	// accumulate left operand's phi information
10257	phiOperands.push_back(x: leftId);
10258	phiOperands.push_back(x: builder.getBuildPoint()->getId());
10259
10260	// Make the two kinds of operation symmetric with a "!"
10261	// || => emit "if (! left) result = right"
10262	// && => emit "if ( left) result = right"
10263	//
10264	// TODO: this runtime "not" for || could be avoided by adding functionality
10265	// to 'builder' to have an "else" without an "then"
10266	if (op == glslang::EOpLogicalOr)
10267	leftId = builder.createUnaryOp(spv::OpLogicalNot, typeId: boolTypeId, operand: leftId);
10268
10269	// make an "if" based on the left value
10270	spv::Builder::If ifBuilder(leftId, spv::SelectionControlMaskNone, builder);
10271
10272	// emit right operand as the "then" part of the "if"
10273	builder.clearAccessChain();
10274	right.traverse(this);
10275	spv::Id rightId = accessChainLoad(type: right.getType());
10276
10277	// accumulate left operand's phi information
10278	phiOperands.push_back(x: rightId);
10279	phiOperands.push_back(x: builder.getBuildPoint()->getId());
10280
10281	// finish the "if"
10282	ifBuilder.makeEndIf();
10283
10284	// phi together the two results
10285	return builder.createOp(spv::OpPhi, typeId: boolTypeId, operands: phiOperands);
10286	}
10287
10288	// Return type Id of the imported set of extended instructions corresponds to the name.
10289	// Import this set if it has not been imported yet.
10290	spv::Id TGlslangToSpvTraverser::getExtBuiltins(const char* name)
10291	{
10292	if (extBuiltinMap.find(x: name) != extBuiltinMap.end())
10293	return extBuiltinMap[name];
10294	else {
10295	spv::Id extBuiltins = builder.import(name);
10296	extBuiltinMap[name] = extBuiltins;
10297	return extBuiltins;
10298	}
10299	}
10300
10301	} // end anonymous namespace
10302
10303	namespace QtShaderTools {
10304	namespace glslang {
10305
10306	void GetSpirvVersion(std::string& version)
10307	{
10308	const int bufSize = 100;
10309	char buf[bufSize];
10310	snprintf(s: buf, maxlen: bufSize, format: "0x%08x, Revision %d", spv::Version, spv::Revision);
10311	version = buf;
10312	}
10313
10314	// For low-order part of the generator's magic number. Bump up
10315	// when there is a change in the style (e.g., if SSA form changes,
10316	// or a different instruction sequence to do something gets used).
10317	int GetSpirvGeneratorVersion()
10318	{
10319	// return 1; // start
10320	// return 2; // EOpAtomicCounterDecrement gets a post decrement, to map between GLSL -> SPIR-V
10321	// return 3; // change/correct barrier-instruction operands, to match memory model group decisions
10322	// return 4; // some deeper access chains: for dynamic vector component, and local Boolean component
10323	// return 5; // make OpArrayLength result type be an int with signedness of 0
10324	// return 6; // revert version 5 change, which makes a different (new) kind of incorrect code,
10325	// versions 4 and 6 each generate OpArrayLength as it has long been done
10326	// return 7; // GLSL volatile keyword maps to both SPIR-V decorations Volatile and Coherent
10327	// return 8; // switch to new dead block eliminator; use OpUnreachable
10328	// return 9; // don't include opaque function parameters in OpEntryPoint global's operand list
10329	// return 10; // Generate OpFUnordNotEqual for != comparisons
10330	return 11; // Make OpEmitMeshTasksEXT a terminal instruction
10331	}
10332
10333	// Write SPIR-V out to a binary file
10334	bool OutputSpvBin(const std::vector<unsigned int>& spirv, const char* baseName)
10335	{
10336	std::ofstream out;
10337	out.open(s: baseName, mode: std::ios::binary | std::ios::out);
10338	if (out.fail()) {
10339	printf(format: "ERROR: Failed to open file: %s\n", baseName);
10340	return false;
10341	}
10342	for (int i = 0; i < (int)spirv.size(); ++i) {
10343	unsigned int word = spirv[i];
10344	out.write(s: (const char*)&word, n: 4);
10345	}
10346	out.close();
10347	return true;
10348	}
10349
10350	// Write SPIR-V out to a text file with 32-bit hexadecimal words
10351	bool OutputSpvHex(const std::vector<unsigned int>& spirv, const char* baseName, const char* varName)
10352	{
10353	std::ofstream out;
10354	out.open(s: baseName, mode: std::ios::binary | std::ios::out);
10355	if (out.fail()) {
10356	printf(format: "ERROR: Failed to open file: %s\n", baseName);
10357	return false;
10358	}
10359	out << "\t// " <<
10360	GetSpirvGeneratorVersion() <<
10361	GLSLANG_VERSION_MAJOR << "." << GLSLANG_VERSION_MINOR << "." << GLSLANG_VERSION_PATCH <<
10362	GLSLANG_VERSION_FLAVOR << std::endl;
10363	if (varName != nullptr) {
10364	out << "\t #pragma once" << std::endl;
10365	out << "const uint32_t " << varName << "[] = {" << std::endl;
10366	}
10367	const int WORDS_PER_LINE = 8;
10368	for (int i = 0; i < (int)spirv.size(); i += WORDS_PER_LINE) {
10369	out << "\t";
10370	for (int j = 0; j < WORDS_PER_LINE && i + j < (int)spirv.size(); ++j) {
10371	const unsigned int word = spirv[i + j];
10372	out << "0x" << std::hex << std::setw(8) << std::setfill('0') << word;
10373	if (i + j + 1 < (int)spirv.size()) {
10374	out << ",";
10375	}
10376	}
10377	out << std::endl;
10378	}
10379	if (varName != nullptr) {
10380	out << "};";
10381	out << std::endl;
10382	}
10383	out.close();
10384	return true;
10385	}
10386
10387	//
10388	// Set up the glslang traversal
10389	//
10390	void GlslangToSpv(const TIntermediate& intermediate, std::vector<unsigned int>& spirv, SpvOptions* options)
10391	{
10392	spv::SpvBuildLogger logger;
10393	GlslangToSpv(intermediate, spirv, logger: &logger, options);
10394	}
10395
10396	void GlslangToSpv(const TIntermediate& intermediate, std::vector<unsigned int>& spirv,
10397	spv::SpvBuildLogger* logger, SpvOptions* options)
10398	{
10399	TIntermNode* root = intermediate.getTreeRoot();
10400
10401	if (root == nullptr)
10402	return;
10403
10404	SpvOptions defaultOptions;
10405	if (options == nullptr)
10406	options = &defaultOptions;
10407
10408	GetThreadPoolAllocator().push();
10409
10410	TGlslangToSpvTraverser it(intermediate.getSpv().spv, &intermediate, logger, *options);
10411	root->traverse(&it);
10412	it.finishSpv(compileOnly: options->compileOnly);
10413	it.dumpSpv(out&: spirv);
10414
10415	#if ENABLE_OPT
10416	// If from HLSL, run spirv-opt to "legalize" the SPIR-V for Vulkan
10417	// eg. forward and remove memory writes of opaque types.
10418	bool prelegalization = intermediate.getSource() == EShSourceHlsl;
10419	if ((prelegalization || options->optimizeSize) && !options->disableOptimizer) {
10420	SpirvToolsTransform(intermediate, spirv, logger, options);
10421	prelegalization = false;
10422	}
10423	else if (options->stripDebugInfo) {
10424	// Strip debug info even if optimization is disabled.
10425	SpirvToolsStripDebugInfo(intermediate, spirv, logger);
10426	}
10427
10428	if (options->validate)
10429	SpirvToolsValidate(intermediate, spirv, logger, prelegalization);
10430
10431	if (options->disassemble)
10432	SpirvToolsDisassemble(std::cout, spirv);
10433
10434	#endif
10435
10436	GetThreadPoolAllocator().pop();
10437	}
10438
10439	} // end namespace glslang
10440	} // namespace QtShaderTools
10441