1//===-- lib/runtime/copy.cpp ------------------------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "copy.h"
10#include "stack.h"
11#include "flang-rt/runtime/descriptor.h"
12#include "flang-rt/runtime/terminator.h"
13#include "flang-rt/runtime/type-info.h"
14#include "flang/Runtime/allocatable.h"
15#include <cstring>
16
17namespace Fortran::runtime {
18namespace {
19using StaticDescTy = StaticDescriptor<maxRank, true, 0>;
20
21// A structure describing the data copy that needs to be done
22// from one descriptor to another. It is a helper structure
23// for CopyElement.
24struct CopyDescriptor {
25 // A constructor specifying all members explicitly.
26 // The toAt and fromAt specify subscript storages that might be
27 // external to CopyElement, and cannot be modified.
28 // The copy descriptor only establishes toAtPtr_ and fromAtPtr_
29 // pointers to point to these storages.
30 RT_API_ATTRS CopyDescriptor(const Descriptor &to, const SubscriptValue toAt[],
31 const Descriptor &from, const SubscriptValue fromAt[],
32 std::size_t elements, bool usesStaticDescriptors = false)
33 : to_(to), from_(from), elements_(elements),
34 usesStaticDescriptors_(usesStaticDescriptors) {
35 toAtPtr_ = toAt;
36 fromAtPtr_ = fromAt;
37 }
38 // The number of elements to copy is initialized from the to descriptor.
39 // The current element subscripts are initialized from the lower bounds
40 // of the to and from descriptors.
41 RT_API_ATTRS CopyDescriptor(const Descriptor &to, const Descriptor &from,
42 bool usesStaticDescriptors = false)
43 : to_(to), from_(from), elements_(to.Elements()),
44 usesStaticDescriptors_(usesStaticDescriptors) {
45 to.GetLowerBounds(toAt_);
46 from.GetLowerBounds(fromAt_);
47 }
48
49 // Increment the toAt_ and fromAt_ subscripts to the next
50 // element.
51 RT_API_ATTRS void IncrementSubscripts(Terminator &terminator) {
52 // This method must not be called for copy descriptors
53 // using external non-modifiable subscript storage.
54 RUNTIME_CHECK(terminator, toAt_ == toAtPtr_ && fromAt_ == fromAtPtr_);
55 to_.IncrementSubscripts(toAt_);
56 from_.IncrementSubscripts(fromAt_);
57 }
58
59 // Descriptor of the destination.
60 const Descriptor &to_;
61 // A subscript specifying the current element position to copy to.
62 SubscriptValue toAt_[maxRank];
63 // A pointer to the storage of the 'to' subscript.
64 // It may point to toAt_ or to an external non-modifiable
65 // subscript storage.
66 const SubscriptValue *toAtPtr_{toAt_};
67 // Descriptor of the source.
68 const Descriptor &from_;
69 // A subscript specifying the current element position to copy from.
70 SubscriptValue fromAt_[maxRank];
71 // A pointer to the storage of the 'from' subscript.
72 // It may point to fromAt_ or to an external non-modifiable
73 // subscript storage.
74 const SubscriptValue *fromAtPtr_{fromAt_};
75 // Number of elements left to copy.
76 std::size_t elements_;
77 // Must be true, if the to and from descriptors are allocated
78 // by the CopyElement runtime. The allocated memory belongs
79 // to a separate stack that needs to be popped in correspondence
80 // with popping such a CopyDescriptor node.
81 bool usesStaticDescriptors_;
82};
83
84// A pair of StaticDescTy elements.
85struct StaticDescriptorsPair {
86 StaticDescTy to;
87 StaticDescTy from;
88};
89} // namespace
90
91RT_OFFLOAD_API_GROUP_BEGIN
92
93RT_API_ATTRS void CopyElement(const Descriptor &to, const SubscriptValue toAt[],
94 const Descriptor &from, const SubscriptValue fromAt[],
95 Terminator &terminator) {
96 if (!to.Addendum()) {
97 // Avoid the overhead of creating the work stacks below
98 // for the simple non-derived type cases, because the overhead
99 // might be noticeable over the total amount of work that
100 // needs to be done for the copy.
101 char *toPtr{to.Element<char>(toAt)};
102 char *fromPtr{from.Element<char>(fromAt)};
103 RUNTIME_CHECK(terminator, to.ElementBytes() == from.ElementBytes());
104 std::memcpy(dest: toPtr, src: fromPtr, n: to.ElementBytes());
105 return;
106 }
107
108#if !defined(RT_DEVICE_COMPILATION)
109 constexpr unsigned copyStackReserve{16};
110 constexpr unsigned descriptorStackReserve{6};
111#else
112 // Always use dynamic allocation on the device to avoid
113 // big stack sizes. This may be tuned as needed.
114 constexpr unsigned copyStackReserve{0};
115 constexpr unsigned descriptorStackReserve{0};
116#endif
117 // Keep a stack of CopyDescriptor's to avoid recursive calls.
118 Stack<CopyDescriptor, copyStackReserve> copyStack{terminator};
119 // Keep a separate stack of StaticDescTy pairs. These descriptors
120 // may be used for representing copies of Component::Genre::Data
121 // components (since they do not have their descriptors allocated
122 // in memory).
123 Stack<StaticDescriptorsPair, descriptorStackReserve> descriptorsStack{
124 terminator};
125 copyStack.emplace(to, toAt, from, fromAt, /*elements=*/std::size_t{1});
126
127 while (!copyStack.empty()) {
128 CopyDescriptor &currentCopy{copyStack.top()};
129 std::size_t &elements{currentCopy.elements_};
130 if (elements == 0) {
131 // This copy has been exhausted.
132 if (currentCopy.usesStaticDescriptors_) {
133 // Pop the static descriptors, if they were used
134 // for the current copy.
135 descriptorsStack.pop();
136 }
137 copyStack.pop();
138 continue;
139 }
140 const Descriptor &curTo{currentCopy.to_};
141 const SubscriptValue *curToAt{currentCopy.toAtPtr_};
142 const Descriptor &curFrom{currentCopy.from_};
143 const SubscriptValue *curFromAt{currentCopy.fromAtPtr_};
144 char *toPtr{curTo.Element<char>(curToAt)};
145 char *fromPtr{curFrom.Element<char>(curFromAt)};
146 RUNTIME_CHECK(terminator, curTo.ElementBytes() == curFrom.ElementBytes());
147 // TODO: the memcpy can be optimized when both to and from are contiguous.
148 // Moreover, if we came here from an Component::Genre::Data component,
149 // all the per-element copies are redundant, because the parent
150 // has already been copied as a whole.
151 std::memcpy(dest: toPtr, src: fromPtr, n: curTo.ElementBytes());
152 --elements;
153 if (elements != 0) {
154 currentCopy.IncrementSubscripts(terminator);
155 }
156
157 // Deep copy allocatable and automatic components if any.
158 if (const auto *addendum{curTo.Addendum()}) {
159 if (const auto *derived{addendum->derivedType()};
160 derived && !derived->noDestructionNeeded()) {
161 RUNTIME_CHECK(terminator,
162 curFrom.Addendum() && derived == curFrom.Addendum()->derivedType());
163 const Descriptor &componentDesc{derived->component()};
164 const typeInfo::Component *component{
165 componentDesc.OffsetElement<typeInfo::Component>()};
166 std::size_t nComponents{componentDesc.Elements()};
167 for (std::size_t j{0}; j < nComponents; ++j, ++component) {
168 if (component->genre() == typeInfo::Component::Genre::Allocatable ||
169 component->genre() == typeInfo::Component::Genre::Automatic) {
170 Descriptor &toDesc{
171 *reinterpret_cast<Descriptor *>(toPtr + component->offset())};
172 if (toDesc.raw().base_addr != nullptr) {
173 toDesc.set_base_addr(nullptr);
174 RUNTIME_CHECK(terminator,
175 toDesc.Allocate(/*asyncObject=*/nullptr) == CFI_SUCCESS);
176 const Descriptor &fromDesc{*reinterpret_cast<const Descriptor *>(
177 fromPtr + component->offset())};
178 copyStack.emplace(toDesc, fromDesc);
179 }
180 } else if (component->genre() == typeInfo::Component::Genre::Data &&
181 component->derivedType() &&
182 !component->derivedType()->noDestructionNeeded()) {
183 SubscriptValue extents[maxRank];
184 const typeInfo::Value *bounds{component->bounds()};
185 std::size_t elements{1};
186 for (int dim{0}; dim < component->rank(); ++dim) {
187 typeInfo::TypeParameterValue lb{
188 bounds[2 * dim].GetValue(&curTo).value_or(0)};
189 typeInfo::TypeParameterValue ub{
190 bounds[2 * dim + 1].GetValue(&curTo).value_or(0)};
191 extents[dim] = ub >= lb ? ub - lb + 1 : 0;
192 elements *= extents[dim];
193 }
194 if (elements != 0) {
195 const typeInfo::DerivedType &compType{*component->derivedType()};
196 // Place a pair of static descriptors onto the descriptors stack.
197 descriptorsStack.emplace();
198 StaticDescriptorsPair &descs{descriptorsStack.top()};
199 Descriptor &toCompDesc{descs.to.descriptor()};
200 toCompDesc.Establish(compType, toPtr + component->offset(),
201 component->rank(), extents);
202 Descriptor &fromCompDesc{descs.from.descriptor()};
203 fromCompDesc.Establish(compType, fromPtr + component->offset(),
204 component->rank(), extents);
205 copyStack.emplace(toCompDesc, fromCompDesc,
206 /*usesStaticDescriptors=*/true);
207 }
208 }
209 }
210 }
211 }
212 }
213}
214RT_OFFLOAD_API_GROUP_END
215} // namespace Fortran::runtime
216

source code of flang-rt/lib/runtime/copy.cpp