1	// Copyright 2009-2021 Intel Corporation
2	// SPDX-License-Identifier: Apache-2.0
3
4	#pragma once
5
6	#include "default.h"
7	#include "instance_stack.h"
8
9	// FIXME: if ray gets seperated into ray* and hit, uload4 needs to be adjusted
10
11	namespace embree
12	{
13	static const size_t MAX_INTERNAL_STREAM_SIZE = 32;
14
15	/* Ray structure for K rays */
16	template<int K>
17	struct RayK
18	{
19	/* Default construction does nothing */
20	__forceinline RayK() {}
21
22	/* Constructs a ray from origin, direction, and ray segment. Near
23	* has to be smaller than far */
24	__forceinline RayK(const Vec3vf<K>& org, const Vec3vf<K>& dir,
25	const vfloat<K>& tnear = zero, const vfloat<K>& tfar = inf,
26	const vfloat<K>& time = zero, const vint<K>& mask = -1, const vint<K>& id = 0, const vint<K>& flags = 0)
27	: org(org), dir(dir), _tnear(tnear), tfar(tfar), _time(time), mask(mask), id(id), flags(flags) {}
28
29	/* Returns the size of the ray */
30	static __forceinline size_t size() { return K; }
31
32	/* Calculates if this is a valid ray that does not cause issues during traversal */
33	__forceinline vbool<K> valid() const
34	{
35	const vbool<K> vx = (abs(org.x) <= vfloat<K>(FLT_LARGE)) & (abs(dir.x) <= vfloat<K>(FLT_LARGE));
36	const vbool<K> vy = (abs(org.y) <= vfloat<K>(FLT_LARGE)) & (abs(dir.y) <= vfloat<K>(FLT_LARGE));
37	const vbool<K> vz = (abs(org.z) <= vfloat<K>(FLT_LARGE)) & (abs(dir.z) <= vfloat<K>(FLT_LARGE));
38	const vbool<K> vn = abs(tnear()) <= vfloat<K>(inf);
39	const vbool<K> vf = abs(tfar) <= vfloat<K>(inf);
40	return vx & vy & vz & vn & vf;
41	}
42
43	__forceinline void get(RayK<1>* ray) const;
44	__forceinline void get(size_t i, RayK<1>& ray) const;
45	__forceinline void set(const RayK<1>* ray);
46	__forceinline void set(size_t i, const RayK<1>& ray);
47
48	__forceinline void copy(size_t dest, size_t source);
49
50	__forceinline vint<K> octant() const
51	{
52	return select(dir.x < 0.0f, vint<K>(1), vint<K>(zero)) |
53	select(dir.y < 0.0f, vint<K>(2), vint<K>(zero)) |
54	select(dir.z < 0.0f, vint<K>(4), vint<K>(zero));
55	}
56
57	/* Ray data */
58	Vec3vf<K> org; // ray origin
59	vfloat<K> _tnear; // start of ray segment
60	Vec3vf<K> dir; // ray direction
61	vfloat<K> _time; // time of this ray for motion blur
62	vfloat<K> tfar; // end of ray segment
63	vint<K> mask; // used to mask out objects during traversal
64	vint<K> id;
65	vint<K> flags;
66
67	__forceinline vfloat<K>& tnear() { return _tnear; }
68	__forceinline vfloat<K>& time() { return _time; }
69	__forceinline const vfloat<K>& tnear() const { return _tnear; }
70	__forceinline const vfloat<K>& time() const { return _time; }
71	};
72
73	/* Ray+hit structure for K rays */
74	template<int K>
75	struct RayHitK : RayK<K>
76	{
77	using RayK<K>::org;
78	using RayK<K>::_tnear;
79	using RayK<K>::dir;
80	using RayK<K>::_time;
81	using RayK<K>::tfar;
82	using RayK<K>::mask;
83	using RayK<K>::id;
84	using RayK<K>::flags;
85
86	using RayK<K>::tnear;
87	using RayK<K>::time;
88
89	/* Default construction does nothing */
90	__forceinline RayHitK() {}
91
92	/* Constructs a ray from origin, direction, and ray segment. Near
93	* has to be smaller than far */
94	__forceinline RayHitK(const Vec3vf<K>& org, const Vec3vf<K>& dir,
95	const vfloat<K>& tnear = zero, const vfloat<K>& tfar = inf,
96	const vfloat<K>& time = zero, const vint<K>& mask = -1, const vint<K>& id = 0, const vint<K>& flags = 0)
97	: RayK<K>(org, dir, tnear, tfar, time, mask, id, flags),
98	geomID(RTC_INVALID_GEOMETRY_ID)
99	{
100	for (unsigned l = 0; l < RTC_MAX_INSTANCE_LEVEL_COUNT; ++l)
101	instID[l] = RTC_INVALID_GEOMETRY_ID;
102	}
103
104	__forceinline RayHitK(const RayK<K>& ray)
105	: RayK<K>(ray),
106	geomID(RTC_INVALID_GEOMETRY_ID)
107	{
108	for (unsigned l = 0; l < RTC_MAX_INSTANCE_LEVEL_COUNT; ++l)
109	instID[l] = RTC_INVALID_GEOMETRY_ID;
110	}
111
112	__forceinline RayHitK<K>& operator =(const RayK<K>& ray)
113	{
114	org = ray.org;
115	_tnear = ray._tnear;
116	dir = ray.dir;
117	_time = ray._time;
118	tfar = ray.tfar;
119	mask = ray.mask;
120	id = ray.id;
121	flags = ray.flags;
122
123	geomID = RTC_INVALID_GEOMETRY_ID;
124	for (unsigned l = 0; l < RTC_MAX_INSTANCE_LEVEL_COUNT; ++l)
125	instID[l] = RTC_INVALID_GEOMETRY_ID;
126
127	return *this;
128	}
129
130	/* Calculates if the hit is valid */
131	__forceinline void verifyHit(const vbool<K>& valid0) const
132	{
133	vbool<K> valid = valid0 & geomID != vuint<K>(RTC_INVALID_GEOMETRY_ID);
134	const vbool<K> vt = (abs(tfar) <= vfloat<K>(FLT_LARGE)) | (tfar == vfloat<K>(neg_inf));
135	const vbool<K> vu = (abs(u) <= vfloat<K>(FLT_LARGE));
136	const vbool<K> vv = (abs(u) <= vfloat<K>(FLT_LARGE));
137	const vbool<K> vnx = abs(Ng.x) <= vfloat<K>(FLT_LARGE);
138	const vbool<K> vny = abs(Ng.y) <= vfloat<K>(FLT_LARGE);
139	const vbool<K> vnz = abs(Ng.z) <= vfloat<K>(FLT_LARGE);
140	if (any(valid & !vt)) throw_RTCError(RTC_ERROR_UNKNOWN,"invalid t");
141	if (any(valid & !vu)) throw_RTCError(RTC_ERROR_UNKNOWN,"invalid u");
142	if (any(valid & !vv)) throw_RTCError(RTC_ERROR_UNKNOWN,"invalid v");
143	if (any(valid & !vnx)) throw_RTCError(RTC_ERROR_UNKNOWN,"invalid Ng.x");
144	if (any(valid & !vny)) throw_RTCError(RTC_ERROR_UNKNOWN,"invalid Ng.y");
145	if (any(valid & !vnz)) throw_RTCError(RTC_ERROR_UNKNOWN,"invalid Ng.z");
146	}
147
148	__forceinline void get(RayHitK<1>* ray) const;
149	__forceinline void get(size_t i, RayHitK<1>& ray) const;
150	__forceinline void set(const RayHitK<1>* ray);
151	__forceinline void set(size_t i, const RayHitK<1>& ray);
152
153	__forceinline void copy(size_t dest, size_t source);
154
155	/* Hit data */
156	Vec3vf<K> Ng; // geometry normal
157	vfloat<K> u; // barycentric u coordinate of hit
158	vfloat<K> v; // barycentric v coordinate of hit
159	vuint<K> primID; // primitive ID
160	vuint<K> geomID; // geometry ID
161	vuint<K> instID[RTC_MAX_INSTANCE_LEVEL_COUNT]; // instance ID
162	};
163
164	/* Specialization for a single ray */
165	template<>
166	struct RayK<1>
167	{
168	/* Default construction does nothing */
169	__forceinline RayK() {}
170
171	/* Constructs a ray from origin, direction, and ray segment. Near
172	* has to be smaller than far */
173	__forceinline RayK(const Vec3fa& org, const Vec3fa& dir, float tnear = zero, float tfar = inf, float time = zero, int mask = -1, int id = 0, int flags = 0)
174	: org(org,tnear), dir(dir,time), tfar(tfar), mask(mask), id(id), flags(flags) {}
175
176	/* Calculates if this is a valid ray that does not cause issues during traversal */
177	__forceinline bool valid() const {
178	return all(b: le_mask(a: abs(a: Vec3fa(org)), b: Vec3fa(FLT_LARGE)) & le_mask(a: abs(a: Vec3fa(dir)), b: Vec3fa(FLT_LARGE))) && abs(x: tnear()) <= float(inf) && abs(x: tfar) <= float(inf);
179	}
180
181	/* Ray data */
182	Vec3ff org; // 3 floats for ray origin, 1 float for tnear
183	//float tnear; // start of ray segment
184	Vec3ff dir; // 3 floats for ray direction, 1 float for time
185	// float time;
186	float tfar; // end of ray segment
187	int mask; // used to mask out objects during traversal
188	int id; // ray ID
189	int flags; // ray flags
190
191	__forceinline float& tnear() { return org.w; };
192	__forceinline const float& tnear() const { return org.w; };
193
194	__forceinline float& time() { return dir.w; };
195	__forceinline const float& time() const { return dir.w; };
196
197	};
198
199	template<>
200	struct RayHitK<1> : RayK<1>
201	{
202	/* Default construction does nothing */
203	__forceinline RayHitK() {}
204
205	/* Constructs a ray from origin, direction, and ray segment. Near
206	* has to be smaller than far */
207	__forceinline RayHitK(const Vec3fa& org, const Vec3fa& dir, float tnear = zero, float tfar = inf, float time = zero, int mask = -1, int id = 0, int flags = 0)
208	: RayK<1>(org, dir, tnear, tfar, time, mask, id, flags),
209	geomID(RTC_INVALID_GEOMETRY_ID) {}
210
211	__forceinline RayHitK(const RayK<1>& ray)
212	: RayK<1>(ray),
213	geomID(RTC_INVALID_GEOMETRY_ID) {}
214
215	__forceinline RayHitK<1>& operator =(const RayK<1>& ray)
216	{
217	org = ray.org;
218	dir = ray.dir;
219	tfar = ray.tfar;
220	mask = ray.mask;
221	id = ray.id;
222	flags = ray.flags;
223
224	geomID = RTC_INVALID_GEOMETRY_ID;
225
226	return *this;
227	}
228
229	/* Calculates if the hit is valid */
230	__forceinline void verifyHit() const
231	{
232	if (geomID == RTC_INVALID_GEOMETRY_ID) return;
233	const bool vt = (abs(x: tfar) <= FLT_LARGE) || (tfar == float(neg_inf));
234	const bool vu = (abs(x: u) <= FLT_LARGE);
235	const bool vv = (abs(x: u) <= FLT_LARGE);
236	const bool vnx = abs(x: Ng.x) <= FLT_LARGE;
237	const bool vny = abs(x: Ng.y) <= FLT_LARGE;
238	const bool vnz = abs(x: Ng.z) <= FLT_LARGE;
239	if (!vt) throw_RTCError(RTC_ERROR_UNKNOWN, "invalid t");
240	if (!vu) throw_RTCError(RTC_ERROR_UNKNOWN, "invalid u");
241	if (!vv) throw_RTCError(RTC_ERROR_UNKNOWN, "invalid v");
242	if (!vnx) throw_RTCError(RTC_ERROR_UNKNOWN, "invalid Ng.x");
243	if (!vny) throw_RTCError(RTC_ERROR_UNKNOWN, "invalid Ng.y");
244	if (!vnz) throw_RTCError(RTC_ERROR_UNKNOWN, "invalid Ng.z");
245	}
246
247	/* Hit data */
248	Vec3f Ng; // not normalized geometry normal
249	float u; // barycentric u coordinate of hit
250	float v; // barycentric v coordinate of hit
251	unsigned int primID; // primitive ID
252	unsigned int geomID; // geometry ID
253	unsigned int instID[RTC_MAX_INSTANCE_LEVEL_COUNT]; // instance ID
254	};
255
256	/* Converts ray packet to single rays */
257	template<int K>
258	__forceinline void RayK<K>::get(RayK<1>* ray) const
259	{
260	for (size_t i = 0; i < K; i++) // FIXME: use SIMD transpose
261	{
262	ray[i].org.x = org.x[i]; ray[i].org.y = org.y[i]; ray[i].org.z = org.z[i]; ray[i].tnear() = tnear()[i];
263	ray[i].dir.x = dir.x[i]; ray[i].dir.y = dir.y[i]; ray[i].dir.z = dir.z[i]; ray[i].time() = time()[i];
264	ray[i].tfar = tfar[i]; ray[i].mask = mask[i]; ray[i].id = id[i]; ray[i].flags = flags[i];
265	}
266	}
267
268	template<int K>
269	__forceinline void RayHitK<K>::get(RayHitK<1>* ray) const
270	{
271	// FIXME: use SIMD transpose
272	for (size_t i = 0; i < K; i++)
273	get(i, ray[i]);
274	}
275
276	/* Extracts a single ray out of a ray packet*/
277	template<int K>
278	__forceinline void RayK<K>::get(size_t i, RayK<1>& ray) const
279	{
280	ray.org.x = org.x[i]; ray.org.y = org.y[i]; ray.org.z = org.z[i]; ray.tnear() = tnear()[i];
281	ray.dir.x = dir.x[i]; ray.dir.y = dir.y[i]; ray.dir.z = dir.z[i]; ray.time() = time()[i];
282	ray.tfar = tfar[i]; ray.mask = mask[i]; ray.id = id[i]; ray.flags = flags[i];
283	}
284
285	template<int K>
286	__forceinline void RayHitK<K>::get(size_t i, RayHitK<1>& ray) const
287	{
288	ray.org.x = org.x[i]; ray.org.y = org.y[i]; ray.org.z = org.z[i]; ray.tnear() = tnear()[i];
289	ray.dir.x = dir.x[i]; ray.dir.y = dir.y[i]; ray.dir.z = dir.z[i]; ray.tfar = tfar[i]; ray.time() = time()[i];
290	ray.mask = mask[i]; ray.id = id[i]; ray.flags = flags[i];
291	ray.Ng.x = Ng.x[i]; ray.Ng.y = Ng.y[i]; ray.Ng.z = Ng.z[i];
292	ray.u = u[i]; ray.v = v[i];
293	ray.primID = primID[i]; ray.geomID = geomID[i];
294
295	instance_id_stack::copy_VU<K>(instID, ray.instID, i);
296	}
297
298	/* Converts single rays to ray packet */
299	template<int K>
300	__forceinline void RayK<K>::set(const RayK<1>* ray)
301	{
302	// FIXME: use SIMD transpose
303	for (size_t i = 0; i < K; i++)
304	set(i, ray[i]);
305	}
306
307	template<int K>
308	__forceinline void RayHitK<K>::set(const RayHitK<1>* ray)
309	{
310	// FIXME: use SIMD transpose
311	for (size_t i = 0; i < K; i++)
312	set(i, ray[i]);
313	}
314
315	/* inserts a single ray into a ray packet element */
316	template<int K>
317	__forceinline void RayK<K>::set(size_t i, const RayK<1>& ray)
318	{
319	org.x[i] = ray.org.x; org.y[i] = ray.org.y; org.z[i] = ray.org.z; tnear()[i] = ray.tnear();
320	dir.x[i] = ray.dir.x; dir.y[i] = ray.dir.y; dir.z[i] = ray.dir.z; time()[i] = ray.time();
321	tfar[i] = ray.tfar; mask[i] = ray.mask; id[i] = ray.id; flags[i] = ray.flags;
322	}
323
324	template<int K>
325	__forceinline void RayHitK<K>::set(size_t i, const RayHitK<1>& ray)
326	{
327	org.x[i] = ray.org.x; org.y[i] = ray.org.y; org.z[i] = ray.org.z; tnear()[i] = ray.tnear();
328	dir.x[i] = ray.dir.x; dir.y[i] = ray.dir.y; dir.z[i] = ray.dir.z; time()[i] = ray.time();
329	tfar[i] = ray.tfar; mask[i] = ray.mask; id[i] = ray.id; flags[i] = ray.flags;
330	Ng.x[i] = ray.Ng.x; Ng.y[i] = ray.Ng.y; Ng.z[i] = ray.Ng.z;
331	u[i] = ray.u; v[i] = ray.v;
332	primID[i] = ray.primID; geomID[i] = ray.geomID;
333
334	instance_id_stack::copy_UV<K>(ray.instID, instID, i);
335	}
336
337	/* copies a ray packet element into another element*/
338	template<int K>
339	__forceinline void RayK<K>::copy(size_t dest, size_t source)
340	{
341	org.x[dest] = org.x[source]; org.y[dest] = org.y[source]; org.z[dest] = org.z[source]; tnear()[dest] = tnear()[source];
342	dir.x[dest] = dir.x[source]; dir.y[dest] = dir.y[source]; dir.z[dest] = dir.z[source]; time()[dest] = time()[source];
343	tfar [dest] = tfar[source]; mask[dest] = mask[source]; id[dest] = id[source]; flags[dest] = flags[source];
344	}
345
346	template<int K>
347	__forceinline void RayHitK<K>::copy(size_t dest, size_t source)
348	{
349	org.x[dest] = org.x[source]; org.y[dest] = org.y[source]; org.z[dest] = org.z[source]; tnear()[dest] = tnear()[source];
350	dir.x[dest] = dir.x[source]; dir.y[dest] = dir.y[source]; dir.z[dest] = dir.z[source]; time()[dest] = time()[source];
351	tfar [dest] = tfar[source]; mask[dest] = mask[source]; id[dest] = id[source]; flags[dest] = flags[source];
352	Ng.x[dest] = Ng.x[source]; Ng.y[dest] = Ng.y[source]; Ng.z[dest] = Ng.z[source];
353	u[dest] = u[source]; v[dest] = v[source];
354	primID[dest] = primID[source]; geomID[dest] = geomID[source];
355
356	instance_id_stack::copy_VV<K>(instID, instID, source, dest);
357	}
358
359	/* Shortcuts */
360	typedef RayK<1> Ray;
361	typedef RayK<4> Ray4;
362	typedef RayK<8> Ray8;
363	typedef RayK<16> Ray16;
364	struct RayN;
365
366	typedef RayHitK<1> RayHit;
367	typedef RayHitK<4> RayHit4;
368	typedef RayHitK<8> RayHit8;
369	typedef RayHitK<16> RayHit16;
370	struct RayHitN;
371
372	template<int K, bool intersect>
373	struct RayTypeHelper;
374
375	template<int K>
376	struct RayTypeHelper<K, true>
377	{
378	typedef RayHitK<K> Ty;
379	};
380
381	template<int K>
382	struct RayTypeHelper<K, false>
383	{
384	typedef RayK<K> Ty;
385	};
386
387	template<bool intersect>
388	using RayType = typename RayTypeHelper<1, intersect>::Ty;
389
390	template<int K, bool intersect>
391	using RayTypeK = typename RayTypeHelper<K, intersect>::Ty;
392
393	/* Outputs ray to stream */
394	template<int K>
395	__forceinline embree_ostream operator <<(embree_ostream cout, const RayK<K>& ray)
396	{
397	return cout << "{ " << embree_endl
398	<< " org = " << ray.org << embree_endl
399	<< " dir = " << ray.dir << embree_endl
400	<< " near = " << ray.tnear() << embree_endl
401	<< " far = " << ray.tfar << embree_endl
402	<< " time = " << ray.time() << embree_endl
403	<< " mask = " << ray.mask << embree_endl
404	<< " id = " << ray.id << embree_endl
405	<< " flags = " << ray.flags << embree_endl
406	<< "}";
407	}
408
409	template<int K>
410	__forceinline embree_ostream operator <<(embree_ostream cout, const RayHitK<K>& ray)
411	{
412	cout << "{ " << embree_endl
413	<< " org = " << ray.org << embree_endl
414	<< " dir = " << ray.dir << embree_endl
415	<< " near = " << ray.tnear() << embree_endl
416	<< " far = " << ray.tfar << embree_endl
417	<< " time = " << ray.time() << embree_endl
418	<< " mask = " << ray.mask << embree_endl
419	<< " id = " << ray.id << embree_endl
420	<< " flags = " << ray.flags << embree_endl
421	<< " Ng = " << ray.Ng
422	<< " u = " << ray.u << embree_endl
423	<< " v = " << ray.v << embree_endl
424	<< " primID = " << ray.primID << embree_endl
425	<< " geomID = " << ray.geomID << embree_endl
426	<< " instID =";
427	for (unsigned l = 0; l < RTC_MAX_INSTANCE_LEVEL_COUNT; ++l)
428	{
429	cout << " " << ray.instID[l];
430	}
431	cout << embree_endl;
432	return cout << "}";
433	}
434
435	struct RayStreamSOA
436	{
437	__forceinline RayStreamSOA(void* rays, size_t N)
438	: ptr((char*)rays), N(N) {}
439
440	/* ray data access functions */
441	__forceinline float* org_x(size_t offset = 0) { return (float*)&ptr[0*4*N+offset]; } // x coordinate of ray origin
442	__forceinline float* org_y(size_t offset = 0) { return (float*)&ptr[1*4*N+offset]; } // y coordinate of ray origin
443	__forceinline float* org_z(size_t offset = 0) { return (float*)&ptr[2*4*N+offset]; }; // z coordinate of ray origin
444	__forceinline float* tnear(size_t offset = 0) { return (float*)&ptr[3*4*N+offset]; }; // start of ray segment
445
446	__forceinline float* dir_x(size_t offset = 0) { return (float*)&ptr[4*4*N+offset]; }; // x coordinate of ray direction
447	__forceinline float* dir_y(size_t offset = 0) { return (float*)&ptr[5*4*N+offset]; }; // y coordinate of ray direction
448	__forceinline float* dir_z(size_t offset = 0) { return (float*)&ptr[6*4*N+offset]; }; // z coordinate of ray direction
449	__forceinline float* time (size_t offset = 0) { return (float*)&ptr[7*4*N+offset]; }; // time of this ray for motion blur
450
451	__forceinline float* tfar (size_t offset = 0) { return (float*)&ptr[8*4*N+offset]; }; // end of ray segment (set to hit distance)
452	__forceinline int* mask (size_t offset = 0) { return (int*)&ptr[9*4*N+offset]; }; // used to mask out objects during traversal (optional)
453	__forceinline int* id (size_t offset = 0) { return (int*)&ptr[10*4*N+offset]; }; // id
454	__forceinline int* flags(size_t offset = 0) { return (int*)&ptr[11*4*N+offset]; }; // flags
455
456	/* hit data access functions */
457	__forceinline float* Ng_x(size_t offset = 0) { return (float*)&ptr[12*4*N+offset]; }; // x coordinate of geometry normal
458	__forceinline float* Ng_y(size_t offset = 0) { return (float*)&ptr[13*4*N+offset]; }; // y coordinate of geometry normal
459	__forceinline float* Ng_z(size_t offset = 0) { return (float*)&ptr[14*4*N+offset]; }; // z coordinate of geometry normal
460
461	__forceinline float* u(size_t offset = 0) { return (float*)&ptr[15*4*N+offset]; }; // barycentric u coordinate of hit
462	__forceinline float* v(size_t offset = 0) { return (float*)&ptr[16*4*N+offset]; }; // barycentric v coordinate of hit
463
464	__forceinline unsigned int* primID(size_t offset = 0) { return (unsigned int*)&ptr[17*4*N+offset]; }; // primitive ID
465	__forceinline unsigned int* geomID(size_t offset = 0) { return (unsigned int*)&ptr[18*4*N+offset]; }; // geometry ID
466	__forceinline unsigned int* instID(size_t level, size_t offset = 0) { return (unsigned int*)&ptr[19*4*N+level*4*N+offset]; }; // instance ID
467
468	__forceinline Ray getRayByOffset(size_t offset)
469	{
470	Ray ray;
471	ray.org.x = org_x(offset)[0];
472	ray.org.y = org_y(offset)[0];
473	ray.org.z = org_z(offset)[0];
474	ray.tnear() = tnear(offset)[0];
475	ray.dir.x = dir_x(offset)[0];
476	ray.dir.y = dir_y(offset)[0];
477	ray.dir.z = dir_z(offset)[0];
478	ray.time() = time(offset)[0];
479	ray.tfar = tfar(offset)[0];
480	ray.mask = mask(offset)[0];
481	ray.id = id(offset)[0];
482	ray.flags = flags(offset)[0];
483	return ray;
484	}
485
486	template<int K>
487	__forceinline RayK<K> getRayByOffset(size_t offset)
488	{
489	RayK<K> ray;
490	ray.org.x = vfloat<K>::loadu(org_x(offset));
491	ray.org.y = vfloat<K>::loadu(org_y(offset));
492	ray.org.z = vfloat<K>::loadu(org_z(offset));
493	ray.tnear = vfloat<K>::loadu(tnear(offset));
494	ray.dir.x = vfloat<K>::loadu(dir_x(offset));
495	ray.dir.y = vfloat<K>::loadu(dir_y(offset));
496	ray.dir.z = vfloat<K>::loadu(dir_z(offset));
497	ray.time = vfloat<K>::loadu(time(offset));
498	ray.tfar = vfloat<K>::loadu(tfar(offset));
499	ray.mask = vint<K>::loadu(mask(offset));
500	ray.id = vint<K>::loadu(id(offset));
501	ray.flags = vint<K>::loadu(flags(offset));
502	return ray;
503	}
504
505	template<int K>
506	__forceinline RayK<K> getRayByOffset(const vbool<K>& valid, size_t offset)
507	{
508	RayK<K> ray;
509	ray.org.x = vfloat<K>::loadu(valid, org_x(offset));
510	ray.org.y = vfloat<K>::loadu(valid, org_y(offset));
511	ray.org.z = vfloat<K>::loadu(valid, org_z(offset));
512	ray.tnear() = vfloat<K>::loadu(valid, tnear(offset));
513	ray.dir.x = vfloat<K>::loadu(valid, dir_x(offset));
514	ray.dir.y = vfloat<K>::loadu(valid, dir_y(offset));
515	ray.dir.z = vfloat<K>::loadu(valid, dir_z(offset));
516	ray.time() = vfloat<K>::loadu(valid, time(offset));
517	ray.tfar = vfloat<K>::loadu(valid, tfar(offset));
518
519	#if !defined(__AVX__)
520	/* SSE: some ray members must be loaded with scalar instructions to ensure that we don't cause memory faults,
521	because the SSE masked loads always access the entire vector */
522	if (unlikely(!all(valid)))
523	{
524	ray.mask = zero;
525	ray.id = zero;
526	ray.flags = zero;
527
528	for (size_t k = 0; k < K; k++)
529	{
530	if (likely(valid[k]))
531	{
532	ray.mask[k] = mask(offset)[k];
533	ray.id[k] = id(offset)[k];
534	ray.flags[k] = flags(offset)[k];
535	}
536	}
537	}
538	else
539	#endif
540	{
541	ray.mask = vint<K>::loadu(valid, mask(offset));
542	ray.id = vint<K>::loadu(valid, id(offset));
543	ray.flags = vint<K>::loadu(valid, flags(offset));
544	}
545
546	return ray;
547	}
548
549	template<int K>
550	__forceinline void setHitByOffset(const vbool<K>& valid_i, size_t offset, const RayHitK<K>& ray)
551	{
552	/*
553	* valid_i: stores which of the input rays exist (do not access nonexistent rays!)
554	* valid: stores which of the rays actually hit something.
555	*/
556	vbool<K> valid = valid_i;
557	valid &= (ray.geomID != RTC_INVALID_GEOMETRY_ID);
558
559	if (likely(any(valid)))
560	{
561	vfloat<K>::storeu(valid, tfar(offset), ray.tfar);
562	vfloat<K>::storeu(valid, Ng_x(offset), ray.Ng.x);
563	vfloat<K>::storeu(valid, Ng_y(offset), ray.Ng.y);
564	vfloat<K>::storeu(valid, Ng_z(offset), ray.Ng.z);
565	vfloat<K>::storeu(valid, u(offset), ray.u);
566	vfloat<K>::storeu(valid, v(offset), ray.v);
567
568	#if !defined(__AVX__)
569	/* SSE: some ray members must be stored with scalar instructions to ensure that we don't cause memory faults,
570	because the SSE masked stores always access the entire vector */
571	if (unlikely(!all(valid_i)))
572	{
573	for (size_t k = 0; k < K; k++)
574	{
575	if (likely(valid[k]))
576	{
577	primID(offset)[k] = ray.primID[k];
578	geomID(offset)[k] = ray.geomID[k];
579
580	instID(level: 0, offset)[k] = ray.instID[0][k];
581	#if (RTC_MAX_INSTANCE_LEVEL_COUNT > 1)
582	for (unsigned l = 1; l < RTC_MAX_INSTANCE_LEVEL_COUNT && ray.instID[l-1][k] != RTC_INVALID_GEOMETRY_ID; ++l)
583	instID(l, offset)[k] = ray.instID[l][k];
584	#endif
585	}
586	}
587	}
588	else
589	#endif
590	{
591	vuint<K>::storeu(valid, primID(offset), ray.primID);
592	vuint<K>::storeu(valid, geomID(offset), ray.geomID);
593
594	vuint<K>::storeu(valid, instID(level: 0, offset), ray.instID[0]);
595	#if (RTC_MAX_INSTANCE_LEVEL_COUNT > 1)
596	for (unsigned l = 1; l < RTC_MAX_INSTANCE_LEVEL_COUNT && any(valid & (ray.instID[l-1] != RTC_INVALID_GEOMETRY_ID)); ++l)
597	vuint<K>::storeu(valid, instID(l, offset), ray.instID[l]);
598	#endif
599	}
600	}
601	}
602
603	template<int K>
604	__forceinline void setHitByOffset(const vbool<K>& valid_i, size_t offset, const RayK<K>& ray)
605	{
606	vbool<K> valid = valid_i;
607	valid &= (ray.tfar < 0.0f);
608
609	if (likely(any(valid)))
610	vfloat<K>::storeu(valid, tfar(offset), ray.tfar);
611	}
612
613	__forceinline size_t getOctantByOffset(size_t offset)
614	{
615	const float dx = dir_x(offset)[0];
616	const float dy = dir_y(offset)[0];
617	const float dz = dir_z(offset)[0];
618	const size_t octantID = (dx < 0.0f ? 1 : 0) + (dy < 0.0f ? 2 : 0) + (dz < 0.0f ? 4 : 0);
619	return octantID;
620	}
621
622	__forceinline bool isValidByOffset(size_t offset)
623	{
624	const float nnear = tnear(offset)[0];
625	const float ffar = tfar(offset)[0];
626	return nnear <= ffar;
627	}
628
629	template<int K>
630	__forceinline RayK<K> getRayByOffset(const vbool<K>& valid, const vint<K>& offset)
631	{
632	RayK<K> ray;
633
634	#if defined(__AVX2__)
635	ray.org.x = vfloat<K>::template gather<1>(valid, org_x(), offset);
636	ray.org.y = vfloat<K>::template gather<1>(valid, org_y(), offset);
637	ray.org.z = vfloat<K>::template gather<1>(valid, org_z(), offset);
638	ray.tnear() = vfloat<K>::template gather<1>(valid, tnear(), offset);
639	ray.dir.x = vfloat<K>::template gather<1>(valid, dir_x(), offset);
640	ray.dir.y = vfloat<K>::template gather<1>(valid, dir_y(), offset);
641	ray.dir.z = vfloat<K>::template gather<1>(valid, dir_z(), offset);
642	ray.time() = vfloat<K>::template gather<1>(valid, time(), offset);
643	ray.tfar = vfloat<K>::template gather<1>(valid, tfar(), offset);
644	ray.mask = vint<K>::template gather<1>(valid, mask(), offset);
645	ray.id = vint<K>::template gather<1>(valid, id(), offset);
646	ray.flags = vint<K>::template gather<1>(valid, flags(), offset);
647	#else
648	ray.org = zero;
649	ray.tnear() = zero;
650	ray.dir = zero;
651	ray.time() = zero;
652	ray.tfar = zero;
653	ray.mask = zero;
654	ray.id = zero;
655	ray.flags = zero;
656
657	for (size_t k = 0; k < K; k++)
658	{
659	if (likely(valid[k]))
660	{
661	const size_t ofs = offset[k];
662
663	ray.org.x[k] = *org_x(offset: ofs);
664	ray.org.y[k] = *org_y(offset: ofs);
665	ray.org.z[k] = *org_z(offset: ofs);
666	ray.tnear()[k] = *tnear(offset: ofs);
667	ray.dir.x[k] = *dir_x(offset: ofs);
668	ray.dir.y[k] = *dir_y(offset: ofs);
669	ray.dir.z[k] = *dir_z(offset: ofs);
670	ray.time()[k] = *time(offset: ofs);
671	ray.tfar[k] = *tfar(offset: ofs);
672	ray.mask[k] = *mask(offset: ofs);
673	ray.id[k] = *id(offset: ofs);
674	ray.flags[k] = *flags(offset: ofs);
675	}
676	}
677	#endif
678
679	return ray;
680	}
681
682	template<int K>
683	__forceinline void setHitByOffset(const vbool<K>& valid_i, const vint<K>& offset, const RayHitK<K>& ray)
684	{
685	vbool<K> valid = valid_i;
686	valid &= (ray.geomID != RTC_INVALID_GEOMETRY_ID);
687
688	if (likely(any(valid)))
689	{
690	#if defined(__AVX512F__)
691	vfloat<K>::template scatter<1>(valid, tfar(), offset, ray.tfar);
692	vfloat<K>::template scatter<1>(valid, Ng_x(), offset, ray.Ng.x);
693	vfloat<K>::template scatter<1>(valid, Ng_y(), offset, ray.Ng.y);
694	vfloat<K>::template scatter<1>(valid, Ng_z(), offset, ray.Ng.z);
695	vfloat<K>::template scatter<1>(valid, u(), offset, ray.u);
696	vfloat<K>::template scatter<1>(valid, v(), offset, ray.v);
697	vuint<K>::template scatter<1>(valid, primID(), offset, ray.primID);
698	vuint<K>::template scatter<1>(valid, geomID(), offset, ray.geomID);
699
700	vuint<K>::template scatter<1>(valid, instID(0), offset, ray.instID[0]);
701	#if (RTC_MAX_INSTANCE_LEVEL_COUNT > 1)
702	for (unsigned l = 1; l < RTC_MAX_INSTANCE_LEVEL_COUNT && any(valid & (ray.instID[l-1] != RTC_INVALID_GEOMETRY_ID)); ++l)
703	vuint<K>::template scatter<1>(valid, instID(l), offset, ray.instID[l]);
704	#endif
705	#else
706	size_t valid_bits = movemask(valid);
707	while (valid_bits != 0)
708	{
709	const size_t k = bscf(v&: valid_bits);
710	const size_t ofs = offset[k];
711
712	*tfar(offset: ofs) = ray.tfar[k];
713
714	*Ng_x(offset: ofs) = ray.Ng.x[k];
715	*Ng_y(offset: ofs) = ray.Ng.y[k];
716	*Ng_z(offset: ofs) = ray.Ng.z[k];
717	*u(offset: ofs) = ray.u[k];
718	*v(offset: ofs) = ray.v[k];
719	*primID(offset: ofs) = ray.primID[k];
720	*geomID(offset: ofs) = ray.geomID[k];
721
722	*instID(level: 0, offset: ofs) = ray.instID[0][k];
723	#if (RTC_MAX_INSTANCE_LEVEL_COUNT > 1)
724	for (unsigned l = 1; l < RTC_MAX_INSTANCE_LEVEL_COUNT && ray.instID[l-1][k] != RTC_INVALID_GEOMETRY_ID; ++l)
725	*instID(l, ofs) = ray.instID[l][k];
726	#endif
727	}
728	#endif
729	}
730	}
731
732	template<int K>
733	__forceinline void setHitByOffset(const vbool<K>& valid_i, const vint<K>& offset, const RayK<K>& ray)
734	{
735	vbool<K> valid = valid_i;
736	valid &= (ray.tfar < 0.0f);
737
738	if (likely(any(valid)))
739	{
740	#if defined(__AVX512F__)
741	vfloat<K>::template scatter<1>(valid, tfar(), offset, ray.tfar);
742	#else
743	size_t valid_bits = movemask(valid);
744	while (valid_bits != 0)
745	{
746	const size_t k = bscf(v&: valid_bits);
747	const size_t ofs = offset[k];
748
749	*tfar(offset: ofs) = ray.tfar[k];
750	}
751	#endif
752	}
753	}
754
755	char* __restrict__ ptr;
756	size_t N;
757	};
758
759	template<size_t MAX_K>
760	struct StackRayStreamSOA : public RayStreamSOA
761	{
762	__forceinline StackRayStreamSOA(size_t K)
763	: RayStreamSOA(data, K) { assert(K <= MAX_K); }
764
765	char data[MAX_K / 4 * sizeof(RayHit4)];
766	};
767
768
769	struct RayStreamSOP
770	{
771	template<class T>
772	__forceinline void init(T& t)
773	{
774	org_x = (float*)&t.org.x;
775	org_y = (float*)&t.org.y;
776	org_z = (float*)&t.org.z;
777	tnear = (float*)&t.tnear;
778	dir_x = (float*)&t.dir.x;
779	dir_y = (float*)&t.dir.y;
780	dir_z = (float*)&t.dir.z;
781	time = (float*)&t.time;
782	tfar = (float*)&t.tfar;
783	mask = (unsigned int*)&t.mask;
784	id = (unsigned int*)&t.id;
785	flags = (unsigned int*)&t.flags;
786
787	Ng_x = (float*)&t.Ng.x;
788	Ng_y = (float*)&t.Ng.y;
789	Ng_z = (float*)&t.Ng.z;
790	u = (float*)&t.u;
791	v = (float*)&t.v;
792	primID = (unsigned int*)&t.primID;
793	geomID = (unsigned int*)&t.geomID;
794
795	for (unsigned l = 0; l < RTC_MAX_INSTANCE_LEVEL_COUNT; ++l)
796	instID[l] = (unsigned int*)&t.instID[l];
797	}
798
799	__forceinline Ray getRayByOffset(size_t offset)
800	{
801	Ray ray;
802	ray.org.x = *(float* __restrict__)((char*)org_x + offset);
803	ray.org.y = *(float* __restrict__)((char*)org_y + offset);
804	ray.org.z = *(float* __restrict__)((char*)org_z + offset);
805	ray.dir.x = *(float* __restrict__)((char*)dir_x + offset);
806	ray.dir.y = *(float* __restrict__)((char*)dir_y + offset);
807	ray.dir.z = *(float* __restrict__)((char*)dir_z + offset);
808	ray.tfar = *(float* __restrict__)((char*)tfar + offset);
809	ray.tnear() = tnear ? *(float* __restrict__)((char*)tnear + offset) : 0.0f;
810	ray.time() = time ? *(float* __restrict__)((char*)time + offset) : 0.0f;
811	ray.mask = mask ? *(unsigned int* __restrict__)((char*)mask + offset) : -1;
812	ray.id = id ? *(unsigned int* __restrict__)((char*)id + offset) : -1;
813	ray.flags = flags ? *(unsigned int* __restrict__)((char*)flags + offset) : -1;
814	return ray;
815	}
816
817	template<int K>
818	__forceinline RayK<K> getRayByOffset(const vbool<K>& valid, size_t offset)
819	{
820	RayK<K> ray;
821	ray.org.x = vfloat<K>::loadu(valid, (float* __restrict__)((char*)org_x + offset));
822	ray.org.y = vfloat<K>::loadu(valid, (float* __restrict__)((char*)org_y + offset));
823	ray.org.z = vfloat<K>::loadu(valid, (float* __restrict__)((char*)org_z + offset));
824	ray.dir.x = vfloat<K>::loadu(valid, (float* __restrict__)((char*)dir_x + offset));
825	ray.dir.y = vfloat<K>::loadu(valid, (float* __restrict__)((char*)dir_y + offset));
826	ray.dir.z = vfloat<K>::loadu(valid, (float* __restrict__)((char*)dir_z + offset));
827	ray.tfar = vfloat<K>::loadu(valid, (float* __restrict__)((char*)tfar + offset));
828	ray.tnear() = tnear ? vfloat<K>::loadu(valid, (float* __restrict__)((char*)tnear + offset)) : 0.0f;
829	ray.time() = time ? vfloat<K>::loadu(valid, (float* __restrict__)((char*)time + offset)) : 0.0f;
830	ray.mask = mask ? vint<K>::loadu(valid, (const void* __restrict__)((char*)mask + offset)) : -1;
831	ray.id = id ? vint<K>::loadu(valid, (const void* __restrict__)((char*)id + offset)) : -1;
832	ray.flags = flags ? vint<K>::loadu(valid, (const void* __restrict__)((char*)flags + offset)) : -1;
833	return ray;
834	}
835
836	template<int K>
837	__forceinline Vec3vf<K> getDirByOffset(const vbool<K>& valid, size_t offset)
838	{
839	Vec3vf<K> dir;
840	dir.x = vfloat<K>::loadu(valid, (float* __restrict__)((char*)dir_x + offset));
841	dir.y = vfloat<K>::loadu(valid, (float* __restrict__)((char*)dir_y + offset));
842	dir.z = vfloat<K>::loadu(valid, (float* __restrict__)((char*)dir_z + offset));
843	return dir;
844	}
845
846	__forceinline void setHitByOffset(size_t offset, const RayHit& ray)
847	{
848	if (ray.geomID != RTC_INVALID_GEOMETRY_ID)
849	{
850	*(float* __restrict__)((char*)tfar + offset) = ray.tfar;
851
852	if (likely(Ng_x)) *(float* __restrict__)((char*)Ng_x + offset) = ray.Ng.x;
853	if (likely(Ng_y)) *(float* __restrict__)((char*)Ng_y + offset) = ray.Ng.y;
854	if (likely(Ng_z)) *(float* __restrict__)((char*)Ng_z + offset) = ray.Ng.z;
855	*(float* __restrict__)((char*)u + offset) = ray.u;
856	*(float* __restrict__)((char*)v + offset) = ray.v;
857	*(unsigned int* __restrict__)((char*)geomID + offset) = ray.geomID;
858	*(unsigned int* __restrict__)((char*)primID + offset) = ray.primID;
859
860	if (likely(instID[0])) {
861	*(unsigned int* __restrict__)((char*)instID[0] + offset) = ray.instID[0];
862	#if (RTC_MAX_INSTANCE_LEVEL_COUNT > 1)
863	for (unsigned l = 1; l < RTC_MAX_INSTANCE_LEVEL_COUNT && ray.instID[l-1] != RTC_INVALID_GEOMETRY_ID; ++l)
864	*(unsigned int* __restrict__)((char*)instID[l] + offset) = ray.instID[l];
865	#endif
866	}
867	}
868	}
869
870	__forceinline void setHitByOffset(size_t offset, const Ray& ray)
871	{
872	*(float* __restrict__)((char*)tfar + offset) = ray.tfar;
873	}
874
875	template<int K>
876	__forceinline void setHitByOffset(const vbool<K>& valid_i, size_t offset, const RayHitK<K>& ray)
877	{
878	vbool<K> valid = valid_i;
879	valid &= (ray.geomID != RTC_INVALID_GEOMETRY_ID);
880
881	if (likely(any(valid)))
882	{
883	vfloat<K>::storeu(valid, (float* __restrict__)((char*)tfar + offset), ray.tfar);
884
885	if (likely(Ng_x)) vfloat<K>::storeu(valid, (float* __restrict__)((char*)Ng_x + offset), ray.Ng.x);
886	if (likely(Ng_y)) vfloat<K>::storeu(valid, (float* __restrict__)((char*)Ng_y + offset), ray.Ng.y);
887	if (likely(Ng_z)) vfloat<K>::storeu(valid, (float* __restrict__)((char*)Ng_z + offset), ray.Ng.z);
888	vfloat<K>::storeu(valid, (float* __restrict__)((char*)u + offset), ray.u);
889	vfloat<K>::storeu(valid, (float* __restrict__)((char*)v + offset), ray.v);
890	vuint<K>::storeu(valid, (unsigned int* __restrict__)((char*)primID + offset), ray.primID);
891	vuint<K>::storeu(valid, (unsigned int* __restrict__)((char*)geomID + offset), ray.geomID);
892
893	if (likely(instID[0])) {
894	vuint<K>::storeu(valid, (unsigned int* __restrict__)((char*)instID[0] + offset), ray.instID[0]);
895	#if (RTC_MAX_INSTANCE_LEVEL_COUNT > 1)
896	for (unsigned l = 1; l < RTC_MAX_INSTANCE_LEVEL_COUNT && any(valid & (ray.instID[l-1] != RTC_INVALID_GEOMETRY_ID)); ++l)
897	vuint<K>::storeu(valid, (unsigned int* __restrict__)((char*)instID[l] + offset), ray.instID[l]);
898	#endif
899	}
900	}
901	}
902
903	template<int K>
904	__forceinline void setHitByOffset(const vbool<K>& valid_i, size_t offset, const RayK<K>& ray)
905	{
906	vbool<K> valid = valid_i;
907	valid &= (ray.tfar < 0.0f);
908
909	if (likely(any(valid)))
910	vfloat<K>::storeu(valid, (float* __restrict__)((char*)tfar + offset), ray.tfar);
911	}
912
913	__forceinline size_t getOctantByOffset(size_t offset)
914	{
915	const float dx = *(float* __restrict__)((char*)dir_x + offset);
916	const float dy = *(float* __restrict__)((char*)dir_y + offset);
917	const float dz = *(float* __restrict__)((char*)dir_z + offset);
918	const size_t octantID = (dx < 0.0f ? 1 : 0) + (dy < 0.0f ? 2 : 0) + (dz < 0.0f ? 4 : 0);
919	return octantID;
920	}
921
922	__forceinline bool isValidByOffset(size_t offset)
923	{
924	const float nnear = tnear ? *(float* __restrict__)((char*)tnear + offset) : 0.0f;
925	const float ffar = *(float* __restrict__)((char*)tfar + offset);
926	return nnear <= ffar;
927	}
928
929	template<int K>
930	__forceinline vbool<K> isValidByOffset(const vbool<K>& valid, size_t offset)
931	{
932	const vfloat<K> nnear = tnear ? vfloat<K>::loadu(valid, (float* __restrict__)((char*)tnear + offset)) : 0.0f;
933	const vfloat<K> ffar = vfloat<K>::loadu(valid, (float* __restrict__)((char*)tfar + offset));
934	return nnear <= ffar;
935	}
936
937	template<int K>
938	__forceinline RayK<K> getRayByOffset(const vbool<K>& valid, const vint<K>& offset)
939	{
940	RayK<K> ray;
941
942	#if defined(__AVX2__)
943	ray.org.x = vfloat<K>::template gather<1>(valid, org_x, offset);
944	ray.org.y = vfloat<K>::template gather<1>(valid, org_y, offset);
945	ray.org.z = vfloat<K>::template gather<1>(valid, org_z, offset);
946	ray.dir.x = vfloat<K>::template gather<1>(valid, dir_x, offset);
947	ray.dir.y = vfloat<K>::template gather<1>(valid, dir_y, offset);
948	ray.dir.z = vfloat<K>::template gather<1>(valid, dir_z, offset);
949	ray.tfar = vfloat<K>::template gather<1>(valid, tfar, offset);
950	ray.tnear() = tnear ? vfloat<K>::template gather<1>(valid, tnear, offset) : vfloat<K>(zero);
951	ray.time() = time ? vfloat<K>::template gather<1>(valid, time, offset) : vfloat<K>(zero);
952	ray.mask = mask ? vint<K>::template gather<1>(valid, (int*)mask, offset) : vint<K>(-1);
953	ray.id = id ? vint<K>::template gather<1>(valid, (int*)id, offset) : vint<K>(-1);
954	ray.flags = flags ? vint<K>::template gather<1>(valid, (int*)flags, offset) : vint<K>(-1);
955	#else
956	ray.org = zero;
957	ray.tnear() = zero;
958	ray.dir = zero;
959	ray.tfar = zero;
960	ray.time() = zero;
961	ray.mask = zero;
962	ray.id = zero;
963	ray.flags = zero;
964
965	for (size_t k = 0; k < K; k++)
966	{
967	if (likely(valid[k]))
968	{
969	const size_t ofs = offset[k];
970
971	ray.org.x[k] = *(float* __restrict__)((char*)org_x + ofs);
972	ray.org.y[k] = *(float* __restrict__)((char*)org_y + ofs);
973	ray.org.z[k] = *(float* __restrict__)((char*)org_z + ofs);
974	ray.dir.x[k] = *(float* __restrict__)((char*)dir_x + ofs);
975	ray.dir.y[k] = *(float* __restrict__)((char*)dir_y + ofs);
976	ray.dir.z[k] = *(float* __restrict__)((char*)dir_z + ofs);
977	ray.tfar[k] = *(float* __restrict__)((char*)tfar + ofs);
978	ray.tnear()[k] = tnear ? *(float* __restrict__)((char*)tnear + ofs) : 0.0f;
979	ray.time()[k] = time ? *(float* __restrict__)((char*)time + ofs) : 0.0f;
980	ray.mask[k] = mask ? *(int* __restrict__)((char*)mask + ofs) : -1;
981	ray.id[k] = id ? *(int* __restrict__)((char*)id + ofs) : -1;
982	ray.flags[k] = flags ? *(int* __restrict__)((char*)flags + ofs) : -1;
983	}
984	}
985	#endif
986
987	return ray;
988	}
989
990	template<int K>
991	__forceinline void setHitByOffset(const vbool<K>& valid_i, const vint<K>& offset, const RayHitK<K>& ray)
992	{
993	vbool<K> valid = valid_i;
994	valid &= (ray.geomID != RTC_INVALID_GEOMETRY_ID);
995
996	if (likely(any(valid)))
997	{
998	#if defined(__AVX512F__)
999	vfloat<K>::template scatter<1>(valid, tfar, offset, ray.tfar);
1000
1001	if (likely(Ng_x)) vfloat<K>::template scatter<1>(valid, Ng_x, offset, ray.Ng.x);
1002	if (likely(Ng_y)) vfloat<K>::template scatter<1>(valid, Ng_y, offset, ray.Ng.y);
1003	if (likely(Ng_z)) vfloat<K>::template scatter<1>(valid, Ng_z, offset, ray.Ng.z);
1004	vfloat<K>::template scatter<1>(valid, u, offset, ray.u);
1005	vfloat<K>::template scatter<1>(valid, v, offset, ray.v);
1006	vuint<K>::template scatter<1>(valid, (unsigned int*)geomID, offset, ray.geomID);
1007	vuint<K>::template scatter<1>(valid, (unsigned int*)primID, offset, ray.primID);
1008
1009	if (likely(instID[0])) {
1010	vuint<K>::template scatter<1>(valid, (unsigned int*)instID[0], offset, ray.instID[0]);
1011	#if (RTC_MAX_INSTANCE_LEVEL_COUNT > 1)
1012	for (unsigned l = 1; l < RTC_MAX_INSTANCE_LEVEL_COUNT && any(valid & (ray.instID[l-1] != RTC_INVALID_GEOMETRY_ID)); ++l)
1013	vuint<K>::template scatter<1>(valid, (unsigned int*)instID[l], offset, ray.instID[l]);
1014	#endif
1015	}
1016	#else
1017	size_t valid_bits = movemask(valid);
1018	while (valid_bits != 0)
1019	{
1020	const size_t k = bscf(v&: valid_bits);
1021	const size_t ofs = offset[k];
1022
1023	*(float* __restrict__)((char*)tfar + ofs) = ray.tfar[k];
1024
1025	if (likely(Ng_x)) *(float* __restrict__)((char*)Ng_x + ofs) = ray.Ng.x[k];
1026	if (likely(Ng_y)) *(float* __restrict__)((char*)Ng_y + ofs) = ray.Ng.y[k];
1027	if (likely(Ng_z)) *(float* __restrict__)((char*)Ng_z + ofs) = ray.Ng.z[k];
1028	*(float* __restrict__)((char*)u + ofs) = ray.u[k];
1029	*(float* __restrict__)((char*)v + ofs) = ray.v[k];
1030	*(unsigned int* __restrict__)((char*)primID + ofs) = ray.primID[k];
1031	*(unsigned int* __restrict__)((char*)geomID + ofs) = ray.geomID[k];
1032
1033	if (likely(instID[0])) {
1034	*(unsigned int* __restrict__)((char*)instID[0] + ofs) = ray.instID[0][k];
1035	#if (RTC_MAX_INSTANCE_LEVEL_COUNT > 1)
1036	for (unsigned l = 1; l < RTC_MAX_INSTANCE_LEVEL_COUNT && ray.instID[l-1][k] != RTC_INVALID_GEOMETRY_ID; ++l)
1037	*(unsigned int* __restrict__)((char*)instID[l] + ofs) = ray.instID[l][k];
1038	#endif
1039	}
1040	}
1041	#endif
1042	}
1043	}
1044
1045	template<int K>
1046	__forceinline void setHitByOffset(const vbool<K>& valid_i, const vint<K>& offset, const RayK<K>& ray)
1047	{
1048	vbool<K> valid = valid_i;
1049	valid &= (ray.tfar < 0.0f);
1050
1051	if (likely(any(valid)))
1052	{
1053	#if defined(__AVX512F__)
1054	vfloat<K>::template scatter<1>(valid, tfar, offset, ray.tfar);
1055	#else
1056	size_t valid_bits = movemask(valid);
1057	while (valid_bits != 0)
1058	{
1059	const size_t k = bscf(v&: valid_bits);
1060	const size_t ofs = offset[k];
1061
1062	*(float* __restrict__)((char*)tfar + ofs) = ray.tfar[k];
1063	}
1064	#endif
1065	}
1066	}
1067
1068	/* ray data */
1069	float* __restrict__ org_x; // x coordinate of ray origin
1070	float* __restrict__ org_y; // y coordinate of ray origin
1071	float* __restrict__ org_z; // z coordinate of ray origin
1072	float* __restrict__ tnear; // start of ray segment (optional)
1073
1074	float* __restrict__ dir_x; // x coordinate of ray direction
1075	float* __restrict__ dir_y; // y coordinate of ray direction
1076	float* __restrict__ dir_z; // z coordinate of ray direction
1077	float* __restrict__ time; // time of this ray for motion blur (optional)
1078
1079	float* __restrict__ tfar; // end of ray segment (set to hit distance)
1080	unsigned int* __restrict__ mask; // used to mask out objects during traversal (optional)
1081	unsigned int* __restrict__ id; // ray ID
1082	unsigned int* __restrict__ flags; // ray flags
1083
1084	/* hit data */
1085	float* __restrict__ Ng_x; // x coordinate of geometry normal (optional)
1086	float* __restrict__ Ng_y; // y coordinate of geometry normal (optional)
1087	float* __restrict__ Ng_z; // z coordinate of geometry normal (optional)
1088
1089	float* __restrict__ u; // barycentric u coordinate of hit
1090	float* __restrict__ v; // barycentric v coordinate of hit
1091
1092	unsigned int* __restrict__ primID; // primitive ID
1093	unsigned int* __restrict__ geomID; // geometry ID
1094	unsigned int* __restrict__ instID[RTC_MAX_INSTANCE_LEVEL_COUNT]; // instance ID (optional)
1095	};
1096
1097
1098	struct RayStreamAOS
1099	{
1100	__forceinline RayStreamAOS(void* rays)
1101	: ptr((Ray*)rays) {}
1102
1103	__forceinline Ray& getRayByOffset(size_t offset)
1104	{
1105	return *(Ray*)((char*)ptr + offset);
1106	}
1107
1108	template<int K>
1109	__forceinline RayK<K> getRayByOffset(const vint<K>& offset);
1110
1111	template<int K>
1112	__forceinline RayK<K> getRayByOffset(const vbool<K>& valid, const vint<K>& offset)
1113	{
1114	const vint<K> valid_offset = select(valid, offset, vintx(zero));
1115	return getRayByOffset<K>(valid_offset);
1116	}
1117
1118	template<int K>
1119	__forceinline void setHitByOffset(const vbool<K>& valid_i, const vint<K>& offset, const RayHitK<K>& ray)
1120	{
1121	vbool<K> valid = valid_i;
1122	valid &= (ray.geomID != RTC_INVALID_GEOMETRY_ID);
1123
1124	if (likely(any(valid)))
1125	{
1126	#if defined(__AVX512F__)
1127	vfloat<K>::template scatter<1>(valid, &ptr->tfar, offset, ray.tfar);
1128	vfloat<K>::template scatter<1>(valid, &((RayHit*)ptr)->Ng.x, offset, ray.Ng.x);
1129	vfloat<K>::template scatter<1>(valid, &((RayHit*)ptr)->Ng.y, offset, ray.Ng.y);
1130	vfloat<K>::template scatter<1>(valid, &((RayHit*)ptr)->Ng.z, offset, ray.Ng.z);
1131	vfloat<K>::template scatter<1>(valid, &((RayHit*)ptr)->u, offset, ray.u);
1132	vfloat<K>::template scatter<1>(valid, &((RayHit*)ptr)->v, offset, ray.v);
1133	vuint<K>::template scatter<1>(valid, (unsigned int*)&((RayHit*)ptr)->primID, offset, ray.primID);
1134	vuint<K>::template scatter<1>(valid, (unsigned int*)&((RayHit*)ptr)->geomID, offset, ray.geomID);
1135
1136	vuint<K>::template scatter<1>(valid, (unsigned int*)&((RayHit*)ptr)->instID[0], offset, ray.instID[0]);
1137	#if (RTC_MAX_INSTANCE_LEVEL_COUNT > 1)
1138	for (unsigned l = 1; l < RTC_MAX_INSTANCE_LEVEL_COUNT && any(valid & (ray.instID[l-1] != RTC_INVALID_GEOMETRY_ID)); ++l)
1139	vuint<K>::template scatter<1>(valid, (unsigned int*)&((RayHit*)ptr)->instID[l], offset, ray.instID[l]);
1140	#endif
1141	#else
1142	size_t valid_bits = movemask(valid);
1143	while (valid_bits != 0)
1144	{
1145	const size_t k = bscf(v&: valid_bits);
1146	RayHit* __restrict__ ray_k = (RayHit*)((char*)ptr + offset[k]);
1147	ray_k->tfar = ray.tfar[k];
1148	ray_k->Ng.x = ray.Ng.x[k];
1149	ray_k->Ng.y = ray.Ng.y[k];
1150	ray_k->Ng.z = ray.Ng.z[k];
1151	ray_k->u = ray.u[k];
1152	ray_k->v = ray.v[k];
1153	ray_k->primID = ray.primID[k];
1154	ray_k->geomID = ray.geomID[k];
1155
1156	instance_id_stack::copy_VU<K>(ray.instID, ray_k->instID, k);
1157	}
1158	#endif
1159	}
1160	}
1161
1162	template<int K>
1163	__forceinline void setHitByOffset(const vbool<K>& valid_i, const vint<K>& offset, const RayK<K>& ray)
1164	{
1165	vbool<K> valid = valid_i;
1166	valid &= (ray.tfar < 0.0f);
1167
1168	if (likely(any(valid)))
1169	{
1170	#if defined(__AVX512F__)
1171	vfloat<K>::template scatter<1>(valid, &ptr->tfar, offset, ray.tfar);
1172	#else
1173	size_t valid_bits = movemask(valid);
1174	while (valid_bits != 0)
1175	{
1176	const size_t k = bscf(v&: valid_bits);
1177	Ray* __restrict__ ray_k = (Ray*)((char*)ptr + offset[k]);
1178	ray_k->tfar = ray.tfar[k];
1179	}
1180	#endif
1181	}
1182	}
1183
1184	Ray* __restrict__ ptr;
1185	};
1186
1187	template<>
1188	__forceinline Ray4 RayStreamAOS::getRayByOffset<4>(const vint4& offset)
1189	{
1190	Ray4 ray;
1191
1192	/* load and transpose: org.x, org.y, org.z, tnear */
1193	const vfloat4 a0 = vfloat4::loadu(a: &((Ray*)((char*)ptr + offset[0]))->org);
1194	const vfloat4 a1 = vfloat4::loadu(a: &((Ray*)((char*)ptr + offset[1]))->org);
1195	const vfloat4 a2 = vfloat4::loadu(a: &((Ray*)((char*)ptr + offset[2]))->org);
1196	const vfloat4 a3 = vfloat4::loadu(a: &((Ray*)((char*)ptr + offset[3]))->org);
1197
1198	transpose(r0: a0,r1: a1,r2: a2,r3: a3, c0&: ray.org.x, c1&: ray.org.y, c2&: ray.org.z, c3&: ray.tnear());
1199
1200	/* load and transpose: dir.x, dir.y, dir.z, time */
1201	const vfloat4 b0 = vfloat4::loadu(a: &((Ray*)((char*)ptr + offset[0]))->dir);
1202	const vfloat4 b1 = vfloat4::loadu(a: &((Ray*)((char*)ptr + offset[1]))->dir);
1203	const vfloat4 b2 = vfloat4::loadu(a: &((Ray*)((char*)ptr + offset[2]))->dir);
1204	const vfloat4 b3 = vfloat4::loadu(a: &((Ray*)((char*)ptr + offset[3]))->dir);
1205
1206	transpose(r0: b0,r1: b1,r2: b2,r3: b3, c0&: ray.dir.x, c1&: ray.dir.y, c2&: ray.dir.z, c3&: ray.time());
1207
1208	/* load and transpose: tfar, mask, id, flags */
1209	const vfloat4 c0 = vfloat4::loadu(a: &((Ray*)((char*)ptr + offset[0]))->tfar);
1210	const vfloat4 c1 = vfloat4::loadu(a: &((Ray*)((char*)ptr + offset[1]))->tfar);
1211	const vfloat4 c2 = vfloat4::loadu(a: &((Ray*)((char*)ptr + offset[2]))->tfar);
1212	const vfloat4 c3 = vfloat4::loadu(a: &((Ray*)((char*)ptr + offset[3]))->tfar);
1213
1214	vfloat4 maskf, idf, flagsf;
1215	transpose(r0: c0,r1: c1,r2: c2,r3: c3, c0&: ray.tfar, c1&: maskf, c2&: idf, c3&: flagsf);
1216	ray.mask = asInt(a: maskf);
1217	ray.id = asInt(a: idf);
1218	ray.flags = asInt(a: flagsf);
1219
1220	return ray;
1221	}
1222
1223	#if defined(__AVX__)
1224	template<>
1225	__forceinline Ray8 RayStreamAOS::getRayByOffset<8>(const vint8& offset)
1226	{
1227	Ray8 ray;
1228
1229	/* load and transpose: org.x, org.y, org.z, tnear, dir.x, dir.y, dir.z, time */
1230	const vfloat8 ab0 = vfloat8::loadu(&((Ray*)((char*)ptr + offset[0]))->org);
1231	const vfloat8 ab1 = vfloat8::loadu(&((Ray*)((char*)ptr + offset[1]))->org);
1232	const vfloat8 ab2 = vfloat8::loadu(&((Ray*)((char*)ptr + offset[2]))->org);
1233	const vfloat8 ab3 = vfloat8::loadu(&((Ray*)((char*)ptr + offset[3]))->org);
1234	const vfloat8 ab4 = vfloat8::loadu(&((Ray*)((char*)ptr + offset[4]))->org);
1235	const vfloat8 ab5 = vfloat8::loadu(&((Ray*)((char*)ptr + offset[5]))->org);
1236	const vfloat8 ab6 = vfloat8::loadu(&((Ray*)((char*)ptr + offset[6]))->org);
1237	const vfloat8 ab7 = vfloat8::loadu(&((Ray*)((char*)ptr + offset[7]))->org);
1238
1239	transpose(ab0,ab1,ab2,ab3,ab4,ab5,ab6,ab7, ray.org.x, ray.org.y, ray.org.z, ray.tnear(), ray.dir.x, ray.dir.y, ray.dir.z, ray.time());
1240
1241	/* load and transpose: tfar, mask, id, flags */
1242	const vfloat4 c0 = vfloat4::loadu(&((Ray*)((char*)ptr + offset[0]))->tfar);
1243	const vfloat4 c1 = vfloat4::loadu(&((Ray*)((char*)ptr + offset[1]))->tfar);
1244	const vfloat4 c2 = vfloat4::loadu(&((Ray*)((char*)ptr + offset[2]))->tfar);
1245	const vfloat4 c3 = vfloat4::loadu(&((Ray*)((char*)ptr + offset[3]))->tfar);
1246	const vfloat4 c4 = vfloat4::loadu(&((Ray*)((char*)ptr + offset[4]))->tfar);
1247	const vfloat4 c5 = vfloat4::loadu(&((Ray*)((char*)ptr + offset[5]))->tfar);
1248	const vfloat4 c6 = vfloat4::loadu(&((Ray*)((char*)ptr + offset[6]))->tfar);
1249	const vfloat4 c7 = vfloat4::loadu(&((Ray*)((char*)ptr + offset[7]))->tfar);
1250
1251	vfloat8 maskf, idf, flagsf;
1252	transpose(c0,c1,c2,c3,c4,c5,c6,c7, ray.tfar, maskf, idf, flagsf);
1253	ray.mask = asInt(maskf);
1254	ray.id = asInt(idf);
1255	ray.flags = asInt(flagsf);
1256
1257	return ray;
1258	}
1259	#endif
1260
1261	#if defined(__AVX512F__)
1262	template<>
1263	__forceinline Ray16 RayStreamAOS::getRayByOffset<16>(const vint16& offset)
1264	{
1265	Ray16 ray;
1266
1267	/* load and transpose: org.x, org.y, org.z, tnear, dir.x, dir.y, dir.z, time */
1268	const vfloat8 ab0 = vfloat8::loadu(&((Ray*)((char*)ptr + offset[ 0]))->org);
1269	const vfloat8 ab1 = vfloat8::loadu(&((Ray*)((char*)ptr + offset[ 1]))->org);
1270	const vfloat8 ab2 = vfloat8::loadu(&((Ray*)((char*)ptr + offset[ 2]))->org);
1271	const vfloat8 ab3 = vfloat8::loadu(&((Ray*)((char*)ptr + offset[ 3]))->org);
1272	const vfloat8 ab4 = vfloat8::loadu(&((Ray*)((char*)ptr + offset[ 4]))->org);
1273	const vfloat8 ab5 = vfloat8::loadu(&((Ray*)((char*)ptr + offset[ 5]))->org);
1274	const vfloat8 ab6 = vfloat8::loadu(&((Ray*)((char*)ptr + offset[ 6]))->org);
1275	const vfloat8 ab7 = vfloat8::loadu(&((Ray*)((char*)ptr + offset[ 7]))->org);
1276	const vfloat8 ab8 = vfloat8::loadu(&((Ray*)((char*)ptr + offset[ 8]))->org);
1277	const vfloat8 ab9 = vfloat8::loadu(&((Ray*)((char*)ptr + offset[ 9]))->org);
1278	const vfloat8 ab10 = vfloat8::loadu(&((Ray*)((char*)ptr + offset[10]))->org);
1279	const vfloat8 ab11 = vfloat8::loadu(&((Ray*)((char*)ptr + offset[11]))->org);
1280	const vfloat8 ab12 = vfloat8::loadu(&((Ray*)((char*)ptr + offset[12]))->org);
1281	const vfloat8 ab13 = vfloat8::loadu(&((Ray*)((char*)ptr + offset[13]))->org);
1282	const vfloat8 ab14 = vfloat8::loadu(&((Ray*)((char*)ptr + offset[14]))->org);
1283	const vfloat8 ab15 = vfloat8::loadu(&((Ray*)((char*)ptr + offset[15]))->org);
1284
1285	transpose(ab0,ab1,ab2,ab3,ab4,ab5,ab6,ab7,ab8,ab9,ab10,ab11,ab12,ab13,ab14,ab15,
1286	ray.org.x, ray.org.y, ray.org.z, ray.tnear(), ray.dir.x, ray.dir.y, ray.dir.z, ray.time());
1287
1288	/* load and transpose: tfar, mask, id, flags */
1289	const vfloat4 c0 = vfloat4::loadu(&((Ray*)((char*)ptr + offset[ 0]))->tfar);
1290	const vfloat4 c1 = vfloat4::loadu(&((Ray*)((char*)ptr + offset[ 1]))->tfar);
1291	const vfloat4 c2 = vfloat4::loadu(&((Ray*)((char*)ptr + offset[ 2]))->tfar);
1292	const vfloat4 c3 = vfloat4::loadu(&((Ray*)((char*)ptr + offset[ 3]))->tfar);
1293	const vfloat4 c4 = vfloat4::loadu(&((Ray*)((char*)ptr + offset[ 4]))->tfar);
1294	const vfloat4 c5 = vfloat4::loadu(&((Ray*)((char*)ptr + offset[ 5]))->tfar);
1295	const vfloat4 c6 = vfloat4::loadu(&((Ray*)((char*)ptr + offset[ 6]))->tfar);
1296	const vfloat4 c7 = vfloat4::loadu(&((Ray*)((char*)ptr + offset[ 7]))->tfar);
1297	const vfloat4 c8 = vfloat4::loadu(&((Ray*)((char*)ptr + offset[ 8]))->tfar);
1298	const vfloat4 c9 = vfloat4::loadu(&((Ray*)((char*)ptr + offset[ 9]))->tfar);
1299	const vfloat4 c10 = vfloat4::loadu(&((Ray*)((char*)ptr + offset[10]))->tfar);
1300	const vfloat4 c11 = vfloat4::loadu(&((Ray*)((char*)ptr + offset[11]))->tfar);
1301	const vfloat4 c12 = vfloat4::loadu(&((Ray*)((char*)ptr + offset[12]))->tfar);
1302	const vfloat4 c13 = vfloat4::loadu(&((Ray*)((char*)ptr + offset[13]))->tfar);
1303	const vfloat4 c14 = vfloat4::loadu(&((Ray*)((char*)ptr + offset[14]))->tfar);
1304	const vfloat4 c15 = vfloat4::loadu(&((Ray*)((char*)ptr + offset[15]))->tfar);
1305
1306	vfloat16 maskf, idf, flagsf;
1307	transpose(c0,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,
1308	ray.tfar, maskf, idf, flagsf);
1309	ray.mask = asInt(maskf);
1310	ray.id = asInt(idf);
1311	ray.flags = asInt(flagsf);
1312
1313	return ray;
1314	}
1315	#endif
1316
1317
1318	struct RayStreamAOP
1319	{
1320	__forceinline RayStreamAOP(void* rays)
1321	: ptr((Ray**)rays) {}
1322
1323	__forceinline Ray& getRayByIndex(size_t index)
1324	{
1325	return *ptr[index];
1326	}
1327
1328	template<int K>
1329	__forceinline RayK<K> getRayByIndex(const vint<K>& index);
1330
1331	template<int K>
1332	__forceinline RayK<K> getRayByIndex(const vbool<K>& valid, const vint<K>& index)
1333	{
1334	const vint<K> valid_index = select(valid, index, vintx(zero));
1335	return getRayByIndex<K>(valid_index);
1336	}
1337
1338	template<int K>
1339	__forceinline void setHitByIndex(const vbool<K>& valid_i, const vint<K>& index, const RayHitK<K>& ray)
1340	{
1341	vbool<K> valid = valid_i;
1342	valid &= (ray.geomID != RTC_INVALID_GEOMETRY_ID);
1343
1344	if (likely(any(valid)))
1345	{
1346	size_t valid_bits = movemask(valid);
1347	while (valid_bits != 0)
1348	{
1349	const size_t k = bscf(v&: valid_bits);
1350	RayHit* __restrict__ ray_k = (RayHit*)ptr[index[k]];
1351
1352	ray_k->tfar = ray.tfar[k];
1353	ray_k->Ng.x = ray.Ng.x[k];
1354	ray_k->Ng.y = ray.Ng.y[k];
1355	ray_k->Ng.z = ray.Ng.z[k];
1356	ray_k->u = ray.u[k];
1357	ray_k->v = ray.v[k];
1358	ray_k->primID = ray.primID[k];
1359	ray_k->geomID = ray.geomID[k];
1360	instance_id_stack::copy_VU<K>(ray.instID, ray_k->instID, k);
1361	}
1362	}
1363	}
1364
1365	template<int K>
1366	__forceinline void setHitByIndex(const vbool<K>& valid_i, const vint<K>& index, const RayK<K>& ray)
1367	{
1368	vbool<K> valid = valid_i;
1369	valid &= (ray.tfar < 0.0f);
1370
1371	if (likely(any(valid)))
1372	{
1373	size_t valid_bits = movemask(valid);
1374	while (valid_bits != 0)
1375	{
1376	const size_t k = bscf(v&: valid_bits);
1377	Ray* __restrict__ ray_k = ptr[index[k]];
1378
1379	ray_k->tfar = ray.tfar[k];
1380	}
1381	}
1382	}
1383
1384	Ray** __restrict__ ptr;
1385	};
1386
1387	template<>
1388	__forceinline Ray4 RayStreamAOP::getRayByIndex<4>(const vint4& index)
1389	{
1390	Ray4 ray;
1391
1392	/* load and transpose: org.x, org.y, org.z, tnear */
1393	const vfloat4 a0 = vfloat4::loadu(a: &ptr[index[0]]->org);
1394	const vfloat4 a1 = vfloat4::loadu(a: &ptr[index[1]]->org);
1395	const vfloat4 a2 = vfloat4::loadu(a: &ptr[index[2]]->org);
1396	const vfloat4 a3 = vfloat4::loadu(a: &ptr[index[3]]->org);
1397
1398	transpose(r0: a0,r1: a1,r2: a2,r3: a3, c0&: ray.org.x, c1&: ray.org.y, c2&: ray.org.z, c3&: ray.tnear());
1399
1400	/* load and transpose: dir.x, dir.y, dir.z, time */
1401	const vfloat4 b0 = vfloat4::loadu(a: &ptr[index[0]]->dir);
1402	const vfloat4 b1 = vfloat4::loadu(a: &ptr[index[1]]->dir);
1403	const vfloat4 b2 = vfloat4::loadu(a: &ptr[index[2]]->dir);
1404	const vfloat4 b3 = vfloat4::loadu(a: &ptr[index[3]]->dir);
1405
1406	transpose(r0: b0,r1: b1,r2: b2,r3: b3, c0&: ray.dir.x, c1&: ray.dir.y, c2&: ray.dir.z, c3&: ray.time());
1407
1408	/* load and transpose: tfar, mask, id, flags */
1409	const vfloat4 c0 = vfloat4::loadu(a: &ptr[index[0]]->tfar);
1410	const vfloat4 c1 = vfloat4::loadu(a: &ptr[index[1]]->tfar);
1411	const vfloat4 c2 = vfloat4::loadu(a: &ptr[index[2]]->tfar);
1412	const vfloat4 c3 = vfloat4::loadu(a: &ptr[index[3]]->tfar);
1413
1414	vfloat4 maskf, idf, flagsf;
1415	transpose(r0: c0,r1: c1,r2: c2,r3: c3, c0&: ray.tfar, c1&: maskf, c2&: idf, c3&: flagsf);
1416	ray.mask = asInt(a: maskf);
1417	ray.id = asInt(a: idf);
1418	ray.flags = asInt(a: flagsf);
1419
1420	return ray;
1421	}
1422
1423	#if defined(__AVX__)
1424	template<>
1425	__forceinline Ray8 RayStreamAOP::getRayByIndex<8>(const vint8& index)
1426	{
1427	Ray8 ray;
1428
1429	/* load and transpose: org.x, org.y, org.z, tnear, dir.x, dir.y, dir.z, time */
1430	const vfloat8 ab0 = vfloat8::loadu(&ptr[index[0]]->org);
1431	const vfloat8 ab1 = vfloat8::loadu(&ptr[index[1]]->org);
1432	const vfloat8 ab2 = vfloat8::loadu(&ptr[index[2]]->org);
1433	const vfloat8 ab3 = vfloat8::loadu(&ptr[index[3]]->org);
1434	const vfloat8 ab4 = vfloat8::loadu(&ptr[index[4]]->org);
1435	const vfloat8 ab5 = vfloat8::loadu(&ptr[index[5]]->org);
1436	const vfloat8 ab6 = vfloat8::loadu(&ptr[index[6]]->org);
1437	const vfloat8 ab7 = vfloat8::loadu(&ptr[index[7]]->org);
1438
1439	transpose(ab0,ab1,ab2,ab3,ab4,ab5,ab6,ab7, ray.org.x, ray.org.y, ray.org.z, ray.tnear(), ray.dir.x, ray.dir.y, ray.dir.z, ray.time());
1440
1441	/* load and transpose: tfar, mask, id, flags */
1442	const vfloat4 c0 = vfloat4::loadu(&ptr[index[0]]->tfar);
1443	const vfloat4 c1 = vfloat4::loadu(&ptr[index[1]]->tfar);
1444	const vfloat4 c2 = vfloat4::loadu(&ptr[index[2]]->tfar);
1445	const vfloat4 c3 = vfloat4::loadu(&ptr[index[3]]->tfar);
1446	const vfloat4 c4 = vfloat4::loadu(&ptr[index[4]]->tfar);
1447	const vfloat4 c5 = vfloat4::loadu(&ptr[index[5]]->tfar);
1448	const vfloat4 c6 = vfloat4::loadu(&ptr[index[6]]->tfar);
1449	const vfloat4 c7 = vfloat4::loadu(&ptr[index[7]]->tfar);
1450
1451	vfloat8 maskf, idf, flagsf;
1452	transpose(c0,c1,c2,c3,c4,c5,c6,c7, ray.tfar, maskf, idf, flagsf);
1453	ray.mask = asInt(maskf);
1454	ray.id = asInt(idf);
1455	ray.flags = asInt(flagsf);
1456
1457	return ray;
1458	}
1459	#endif
1460
1461	#if defined(__AVX512F__)
1462	template<>
1463	__forceinline Ray16 RayStreamAOP::getRayByIndex<16>(const vint16& index)
1464	{
1465	Ray16 ray;
1466
1467	/* load and transpose: org.x, org.y, org.z, tnear, dir.x, dir.y, dir.z, time */
1468	const vfloat8 ab0 = vfloat8::loadu(&ptr[index[0]]->org);
1469	const vfloat8 ab1 = vfloat8::loadu(&ptr[index[1]]->org);
1470	const vfloat8 ab2 = vfloat8::loadu(&ptr[index[2]]->org);
1471	const vfloat8 ab3 = vfloat8::loadu(&ptr[index[3]]->org);
1472	const vfloat8 ab4 = vfloat8::loadu(&ptr[index[4]]->org);
1473	const vfloat8 ab5 = vfloat8::loadu(&ptr[index[5]]->org);
1474	const vfloat8 ab6 = vfloat8::loadu(&ptr[index[6]]->org);
1475	const vfloat8 ab7 = vfloat8::loadu(&ptr[index[7]]->org);
1476	const vfloat8 ab8 = vfloat8::loadu(&ptr[index[8]]->org);
1477	const vfloat8 ab9 = vfloat8::loadu(&ptr[index[9]]->org);
1478	const vfloat8 ab10 = vfloat8::loadu(&ptr[index[10]]->org);
1479	const vfloat8 ab11 = vfloat8::loadu(&ptr[index[11]]->org);
1480	const vfloat8 ab12 = vfloat8::loadu(&ptr[index[12]]->org);
1481	const vfloat8 ab13 = vfloat8::loadu(&ptr[index[13]]->org);
1482	const vfloat8 ab14 = vfloat8::loadu(&ptr[index[14]]->org);
1483	const vfloat8 ab15 = vfloat8::loadu(&ptr[index[15]]->org);
1484
1485	transpose(ab0,ab1,ab2,ab3,ab4,ab5,ab6,ab7,ab8,ab9,ab10,ab11,ab12,ab13,ab14,ab15,
1486	ray.org.x, ray.org.y, ray.org.z, ray.tnear(), ray.dir.x, ray.dir.y, ray.dir.z, ray.time());
1487
1488	/* load and transpose: tfar, mask, id, flags */
1489	const vfloat4 c0 = vfloat4::loadu(&ptr[index[0]]->tfar);
1490	const vfloat4 c1 = vfloat4::loadu(&ptr[index[1]]->tfar);
1491	const vfloat4 c2 = vfloat4::loadu(&ptr[index[2]]->tfar);
1492	const vfloat4 c3 = vfloat4::loadu(&ptr[index[3]]->tfar);
1493	const vfloat4 c4 = vfloat4::loadu(&ptr[index[4]]->tfar);
1494	const vfloat4 c5 = vfloat4::loadu(&ptr[index[5]]->tfar);
1495	const vfloat4 c6 = vfloat4::loadu(&ptr[index[6]]->tfar);
1496	const vfloat4 c7 = vfloat4::loadu(&ptr[index[7]]->tfar);
1497	const vfloat4 c8 = vfloat4::loadu(&ptr[index[8]]->tfar);
1498	const vfloat4 c9 = vfloat4::loadu(&ptr[index[9]]->tfar);
1499	const vfloat4 c10 = vfloat4::loadu(&ptr[index[10]]->tfar);
1500	const vfloat4 c11 = vfloat4::loadu(&ptr[index[11]]->tfar);
1501	const vfloat4 c12 = vfloat4::loadu(&ptr[index[12]]->tfar);
1502	const vfloat4 c13 = vfloat4::loadu(&ptr[index[13]]->tfar);
1503	const vfloat4 c14 = vfloat4::loadu(&ptr[index[14]]->tfar);
1504	const vfloat4 c15 = vfloat4::loadu(&ptr[index[15]]->tfar);
1505
1506	vfloat16 maskf, idf, flagsf;
1507	transpose(c0,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,
1508	ray.tfar, maskf, idf, flagsf);
1509
1510	ray.mask = asInt(maskf);
1511	ray.id = asInt(idf);
1512	ray.flags = asInt(flagsf);
1513
1514	return ray;
1515	}
1516	#endif
1517	}
1518