1 | /* LTO IL compression streams. |
2 | |
3 | Copyright (C) 2009-2023 Free Software Foundation, Inc. |
4 | Contributed by Simon Baldwin <simonb@google.com> |
5 | |
6 | This file is part of GCC. |
7 | |
8 | GCC is free software; you can redistribute it and/or modify it |
9 | under the terms of the GNU General Public License as published by |
10 | the Free Software Foundation; either version 3, or (at your option) |
11 | any later version. |
12 | |
13 | GCC is distributed in the hope that it will be useful, but WITHOUT |
14 | ANY WARRANTY; without even the implied warranty of MERCHANTABILITY |
15 | or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public |
16 | License for more details. |
17 | |
18 | You should have received a copy of the GNU General Public License |
19 | along with GCC; see the file COPYING3. If not see |
20 | <http://www.gnu.org/licenses/>. */ |
21 | |
22 | #include "config.h" |
23 | #include "system.h" |
24 | #include "coretypes.h" |
25 | #include "backend.h" |
26 | #include "tree.h" |
27 | #include "gimple.h" |
28 | #include "cgraph.h" |
29 | #include "lto-streamer.h" |
30 | /* zlib.h includes other system headers. Those headers may test feature |
31 | test macros. config.h may define feature test macros. For this reason, |
32 | zlib.h needs to be included after, rather than before, config.h and |
33 | system.h. */ |
34 | #include <zlib.h> |
35 | #include "lto-compress.h" |
36 | #include "timevar.h" |
37 | |
38 | #ifdef HAVE_ZSTD_H |
39 | #include <zstd.h> |
40 | #endif |
41 | |
42 | /* Compression stream structure, holds the flush callback and opaque token, |
43 | the buffered data, and a note of whether compressing or uncompressing. */ |
44 | |
45 | struct lto_compression_stream |
46 | { |
47 | void (*callback) (const char *, unsigned, void *); |
48 | void *opaque; |
49 | char *buffer; |
50 | size_t bytes; |
51 | size_t allocation; |
52 | bool is_compression; |
53 | }; |
54 | |
55 | /* Overall compression constants for zlib. */ |
56 | |
57 | static const size_t Z_BUFFER_LENGTH = 4096; |
58 | static const size_t MIN_STREAM_ALLOCATION = 1024; |
59 | |
60 | /* For zlib, allocate SIZE count of ITEMS and return the address, OPAQUE |
61 | is unused. */ |
62 | |
63 | static void * |
64 | lto_zalloc (void *opaque, unsigned items, unsigned size) |
65 | { |
66 | gcc_assert (opaque == Z_NULL); |
67 | return xmalloc (items * size); |
68 | } |
69 | |
70 | /* For zlib, free memory at ADDRESS, OPAQUE is unused. */ |
71 | |
72 | static void |
73 | lto_zfree (void *opaque, void *address) |
74 | { |
75 | gcc_assert (opaque == Z_NULL); |
76 | free (ptr: address); |
77 | } |
78 | |
79 | /* Return a zlib compression level that zlib will not reject. Normalizes |
80 | the compression level from the command line flag, clamping non-default |
81 | values to the appropriate end of their valid range. */ |
82 | |
83 | static int |
84 | lto_normalized_zlib_level (void) |
85 | { |
86 | int level = flag_lto_compression_level; |
87 | |
88 | if (level != Z_DEFAULT_COMPRESSION) |
89 | { |
90 | if (level < Z_NO_COMPRESSION) |
91 | level = Z_NO_COMPRESSION; |
92 | else if (level > Z_BEST_COMPRESSION) |
93 | level = Z_BEST_COMPRESSION; |
94 | } |
95 | |
96 | return level; |
97 | } |
98 | |
99 | /* Free the buffer and memory associated with STREAM. */ |
100 | |
101 | static void |
102 | lto_destroy_compression_stream (struct lto_compression_stream *stream) |
103 | { |
104 | free (ptr: stream->buffer); |
105 | free (ptr: stream); |
106 | } |
107 | |
108 | #ifdef HAVE_ZSTD_H |
109 | /* Return a zstd compression level that zstd will not reject. Normalizes |
110 | the compression level from the command line flag, clamping non-default |
111 | values to the appropriate end of their valid range. */ |
112 | |
113 | static int |
114 | lto_normalized_zstd_level (void) |
115 | { |
116 | int level = flag_lto_compression_level; |
117 | |
118 | if (level < 0) |
119 | level = 0; |
120 | else if (level > ZSTD_maxCLevel ()) |
121 | level = ZSTD_maxCLevel (); |
122 | |
123 | return level; |
124 | } |
125 | |
126 | /* Compress STREAM using ZSTD algorithm. */ |
127 | |
128 | static void |
129 | lto_compression_zstd (struct lto_compression_stream *stream) |
130 | { |
131 | unsigned char *cursor = (unsigned char *) stream->buffer; |
132 | size_t size = stream->bytes; |
133 | |
134 | timevar_push (tv: TV_IPA_LTO_COMPRESS); |
135 | size_t const outbuf_length = ZSTD_compressBound (srcSize: size); |
136 | char *outbuf = (char *) xmalloc (outbuf_length); |
137 | |
138 | size_t const csize = ZSTD_compress (dst: outbuf, dstCapacity: outbuf_length, src: cursor, srcSize: size, |
139 | compressionLevel: lto_normalized_zstd_level ()); |
140 | |
141 | if (ZSTD_isError (code: csize)) |
142 | internal_error ("compressed stream: %s" , ZSTD_getErrorName (code: csize)); |
143 | |
144 | lto_stats.num_compressed_il_bytes += csize; |
145 | stream->callback (outbuf, csize, NULL); |
146 | |
147 | lto_destroy_compression_stream (stream); |
148 | free (ptr: outbuf); |
149 | timevar_pop (tv: TV_IPA_LTO_COMPRESS); |
150 | } |
151 | |
152 | /* Uncompress STREAM using ZSTD algorithm. */ |
153 | |
154 | static void |
155 | lto_uncompression_zstd (struct lto_compression_stream *stream) |
156 | { |
157 | unsigned char *cursor = (unsigned char *) stream->buffer; |
158 | size_t size = stream->bytes; |
159 | |
160 | timevar_push (tv: TV_IPA_LTO_DECOMPRESS); |
161 | unsigned long long const rsize = ZSTD_getFrameContentSize (src: cursor, srcSize: size); |
162 | if (rsize == ZSTD_CONTENTSIZE_ERROR) |
163 | internal_error ("original not compressed with zstd" ); |
164 | else if (rsize == ZSTD_CONTENTSIZE_UNKNOWN) |
165 | internal_error ("original size unknown" ); |
166 | |
167 | char *outbuf = (char *) xmalloc (rsize); |
168 | size_t const dsize = ZSTD_decompress (dst: outbuf, dstCapacity: rsize, src: cursor, compressedSize: size); |
169 | |
170 | if (ZSTD_isError (code: dsize)) |
171 | internal_error ("decompressed stream: %s" , ZSTD_getErrorName (code: dsize)); |
172 | |
173 | lto_stats.num_uncompressed_il_bytes += dsize; |
174 | stream->callback (outbuf, dsize, stream->opaque); |
175 | |
176 | lto_destroy_compression_stream (stream); |
177 | free (ptr: outbuf); |
178 | timevar_pop (tv: TV_IPA_LTO_DECOMPRESS); |
179 | } |
180 | |
181 | #endif |
182 | |
183 | /* Create a new compression stream, with CALLBACK flush function passed |
184 | OPAQUE token, IS_COMPRESSION indicates if compressing or uncompressing. */ |
185 | |
186 | static struct lto_compression_stream * |
187 | lto_new_compression_stream (void (*callback) (const char *, unsigned, void *), |
188 | void *opaque, bool is_compression) |
189 | { |
190 | struct lto_compression_stream *stream |
191 | = (struct lto_compression_stream *) xmalloc (sizeof (*stream)); |
192 | |
193 | memset (s: stream, c: 0, n: sizeof (*stream)); |
194 | stream->callback = callback; |
195 | stream->opaque = opaque; |
196 | stream->is_compression = is_compression; |
197 | |
198 | return stream; |
199 | } |
200 | |
201 | /* Append NUM_CHARS from address BASE to STREAM. */ |
202 | |
203 | static void |
204 | lto_append_to_compression_stream (struct lto_compression_stream *stream, |
205 | const char *base, size_t num_chars) |
206 | { |
207 | size_t required = stream->bytes + num_chars; |
208 | |
209 | if (stream->allocation < required) |
210 | { |
211 | if (stream->allocation == 0) |
212 | stream->allocation = MIN_STREAM_ALLOCATION; |
213 | while (stream->allocation < required) |
214 | stream->allocation *= 2; |
215 | |
216 | stream->buffer = (char *) xrealloc (stream->buffer, stream->allocation); |
217 | } |
218 | |
219 | memcpy (dest: stream->buffer + stream->bytes, src: base, n: num_chars); |
220 | stream->bytes += num_chars; |
221 | } |
222 | |
223 | /* Return a new compression stream, with CALLBACK flush function passed |
224 | OPAQUE token. */ |
225 | |
226 | struct lto_compression_stream * |
227 | lto_start_compression (void (*callback) (const char *, unsigned, void *), |
228 | void *opaque) |
229 | { |
230 | return lto_new_compression_stream (callback, opaque, is_compression: true); |
231 | } |
232 | |
233 | /* Append NUM_CHARS from address BASE to STREAM. */ |
234 | |
235 | void |
236 | lto_compress_block (struct lto_compression_stream *stream, |
237 | const char *base, size_t num_chars) |
238 | { |
239 | gcc_assert (stream->is_compression); |
240 | |
241 | lto_append_to_compression_stream (stream, base, num_chars); |
242 | lto_stats.num_output_il_bytes += num_chars; |
243 | } |
244 | |
245 | static void ATTRIBUTE_UNUSED |
246 | lto_compression_zlib (struct lto_compression_stream *stream) |
247 | { |
248 | unsigned char *cursor = (unsigned char *) stream->buffer; |
249 | size_t remaining = stream->bytes; |
250 | const size_t outbuf_length = Z_BUFFER_LENGTH; |
251 | unsigned char *outbuf = (unsigned char *) xmalloc (outbuf_length); |
252 | z_stream out_stream; |
253 | int status; |
254 | |
255 | gcc_assert (stream->is_compression); |
256 | |
257 | timevar_push (tv: TV_IPA_LTO_COMPRESS); |
258 | |
259 | out_stream.next_out = outbuf; |
260 | out_stream.avail_out = outbuf_length; |
261 | out_stream.next_in = cursor; |
262 | out_stream.avail_in = remaining; |
263 | out_stream.zalloc = lto_zalloc; |
264 | out_stream.zfree = lto_zfree; |
265 | out_stream.opaque = Z_NULL; |
266 | |
267 | status = deflateInit (&out_stream, lto_normalized_zlib_level ()); |
268 | if (status != Z_OK) |
269 | internal_error ("compressed stream: %s" , zError (status)); |
270 | |
271 | do |
272 | { |
273 | size_t in_bytes, out_bytes; |
274 | |
275 | status = deflate (strm: &out_stream, Z_FINISH); |
276 | if (status != Z_OK && status != Z_STREAM_END) |
277 | internal_error ("compressed stream: %s" , zError (status)); |
278 | |
279 | in_bytes = remaining - out_stream.avail_in; |
280 | out_bytes = outbuf_length - out_stream.avail_out; |
281 | |
282 | stream->callback ((const char *) outbuf, out_bytes, stream->opaque); |
283 | lto_stats.num_compressed_il_bytes += out_bytes; |
284 | |
285 | cursor += in_bytes; |
286 | remaining -= in_bytes; |
287 | |
288 | out_stream.next_out = outbuf; |
289 | out_stream.avail_out = outbuf_length; |
290 | out_stream.next_in = cursor; |
291 | out_stream.avail_in = remaining; |
292 | } |
293 | while (status != Z_STREAM_END); |
294 | |
295 | status = deflateEnd (strm: &out_stream); |
296 | if (status != Z_OK) |
297 | internal_error ("compressed stream: %s" , zError (status)); |
298 | |
299 | lto_destroy_compression_stream (stream); |
300 | free (ptr: outbuf); |
301 | timevar_pop (tv: TV_IPA_LTO_COMPRESS); |
302 | } |
303 | |
304 | void |
305 | lto_end_compression (struct lto_compression_stream *stream) |
306 | { |
307 | #ifdef HAVE_ZSTD_H |
308 | lto_compression_zstd (stream); |
309 | #else |
310 | lto_compression_zlib (stream); |
311 | #endif |
312 | } |
313 | |
314 | /* Return a new uncompression stream, with CALLBACK flush function passed |
315 | OPAQUE token. */ |
316 | |
317 | struct lto_compression_stream * |
318 | lto_start_uncompression (void (*callback) (const char *, unsigned, void *), |
319 | void *opaque) |
320 | { |
321 | return lto_new_compression_stream (callback, opaque, is_compression: false); |
322 | } |
323 | |
324 | /* Append NUM_CHARS from address BASE to STREAM. */ |
325 | |
326 | void |
327 | lto_uncompress_block (struct lto_compression_stream *stream, |
328 | const char *base, size_t num_chars) |
329 | { |
330 | gcc_assert (!stream->is_compression); |
331 | |
332 | lto_append_to_compression_stream (stream, base, num_chars); |
333 | lto_stats.num_input_il_bytes += num_chars; |
334 | } |
335 | |
336 | static void |
337 | lto_uncompression_zlib (struct lto_compression_stream *stream) |
338 | { |
339 | unsigned char *cursor = (unsigned char *) stream->buffer; |
340 | size_t remaining = stream->bytes; |
341 | const size_t outbuf_length = Z_BUFFER_LENGTH; |
342 | unsigned char *outbuf = (unsigned char *) xmalloc (outbuf_length); |
343 | |
344 | gcc_assert (!stream->is_compression); |
345 | timevar_push (tv: TV_IPA_LTO_DECOMPRESS); |
346 | |
347 | while (remaining > 0) |
348 | { |
349 | z_stream in_stream; |
350 | size_t out_bytes; |
351 | int status; |
352 | |
353 | in_stream.next_out = outbuf; |
354 | in_stream.avail_out = outbuf_length; |
355 | in_stream.next_in = cursor; |
356 | in_stream.avail_in = remaining; |
357 | in_stream.zalloc = lto_zalloc; |
358 | in_stream.zfree = lto_zfree; |
359 | in_stream.opaque = Z_NULL; |
360 | |
361 | status = inflateInit (&in_stream); |
362 | if (status != Z_OK) |
363 | internal_error ("compressed stream: %s" , zError (status)); |
364 | |
365 | do |
366 | { |
367 | size_t in_bytes; |
368 | |
369 | status = inflate (strm: &in_stream, Z_SYNC_FLUSH); |
370 | if (status != Z_OK && status != Z_STREAM_END) |
371 | internal_error ("compressed stream: %s" , zError (status)); |
372 | |
373 | in_bytes = remaining - in_stream.avail_in; |
374 | out_bytes = outbuf_length - in_stream.avail_out; |
375 | |
376 | stream->callback ((const char *) outbuf, out_bytes, stream->opaque); |
377 | lto_stats.num_uncompressed_il_bytes += out_bytes; |
378 | |
379 | cursor += in_bytes; |
380 | remaining -= in_bytes; |
381 | |
382 | in_stream.next_out = outbuf; |
383 | in_stream.avail_out = outbuf_length; |
384 | in_stream.next_in = cursor; |
385 | in_stream.avail_in = remaining; |
386 | } |
387 | while (!(status == Z_STREAM_END && out_bytes == 0)); |
388 | |
389 | status = inflateEnd (strm: &in_stream); |
390 | if (status != Z_OK) |
391 | internal_error ("compressed stream: %s" , zError (status)); |
392 | } |
393 | |
394 | lto_destroy_compression_stream (stream); |
395 | free (ptr: outbuf); |
396 | timevar_pop (tv: TV_IPA_LTO_DECOMPRESS); |
397 | } |
398 | |
399 | void |
400 | lto_end_uncompression (struct lto_compression_stream *stream, |
401 | lto_compression compression) |
402 | { |
403 | #ifdef HAVE_ZSTD_H |
404 | if (compression == ZSTD) |
405 | { |
406 | lto_uncompression_zstd (stream); |
407 | return; |
408 | } |
409 | #endif |
410 | if (compression == ZSTD) |
411 | internal_error ("compiler does not support ZSTD LTO compression" ); |
412 | |
413 | lto_uncompression_zlib (stream); |
414 | } |
415 | |