1 /*
2 * Copyright © 2018 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included
12 * in all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
15 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
21 */
22
23 /**
24 * @file iris_disk_cache.c
25 *
26 * Functions for interacting with the on-disk shader cache.
27 */
28
29 #include <stdio.h>
30 #include <stdint.h>
31 #include <assert.h>
32 #include <string.h>
33
34 #include "compiler/nir/nir.h"
35 #include "util/blob.h"
36 #include "util/build_id.h"
37 #include "util/disk_cache.h"
38 #include "util/mesa-sha1.h"
39 #include "intel/compiler/brw_compiler.h"
40 #include "intel/compiler/elk/elk_compiler.h"
41
42 #include "iris_context.h"
43
44 static bool debug = false;
45
46 /**
47 * Compute a disk cache key for the given uncompiled shader and NOS key.
48 */
49 static void
iris_disk_cache_compute_key(struct disk_cache * cache,const struct iris_uncompiled_shader * ish,const void * orig_prog_key,uint32_t prog_key_size,cache_key cache_key)50 iris_disk_cache_compute_key(struct disk_cache *cache,
51 const struct iris_uncompiled_shader *ish,
52 const void *orig_prog_key,
53 uint32_t prog_key_size,
54 cache_key cache_key)
55 {
56 /* Create a copy of the program key with program_string_id zeroed out.
57 * It's essentially random data which we don't want to include in our
58 * hashing and comparisons. We'll set a proper value on a cache hit.
59 */
60 union brw_any_prog_key prog_key;
61 memcpy(&prog_key, orig_prog_key, prog_key_size);
62 prog_key.base.program_string_id = 0;
63
64 uint8_t data[sizeof(prog_key) + sizeof(ish->nir_sha1)];
65 uint32_t data_size = prog_key_size + sizeof(ish->nir_sha1);
66
67 memcpy(data, ish->nir_sha1, sizeof(ish->nir_sha1));
68 memcpy(data + sizeof(ish->nir_sha1), &prog_key, prog_key_size);
69
70 disk_cache_compute_key(cache, data, data_size, cache_key);
71 }
72
73 /**
74 * Store the given compiled shader in the disk cache.
75 *
76 * This should only be called on newly compiled shaders. No checking is
77 * done to prevent repeated stores of the same shader.
78 */
79 void
iris_disk_cache_store(struct disk_cache * cache,const struct iris_uncompiled_shader * ish,const struct iris_compiled_shader * shader,const void * prog_key,uint32_t prog_key_size)80 iris_disk_cache_store(struct disk_cache *cache,
81 const struct iris_uncompiled_shader *ish,
82 const struct iris_compiled_shader *shader,
83 const void *prog_key,
84 uint32_t prog_key_size)
85 {
86 #ifdef ENABLE_SHADER_CACHE
87 if (!cache)
88 return;
89
90 gl_shader_stage stage = ish->nir->info.stage;
91 const struct brw_stage_prog_data *brw = shader->brw_prog_data;
92 const struct elk_stage_prog_data *elk = shader->elk_prog_data;
93 assert((brw == NULL) != (elk == NULL));
94
95 cache_key cache_key;
96 iris_disk_cache_compute_key(cache, ish, prog_key, prog_key_size, cache_key);
97
98 if (debug) {
99 char sha1[41];
100 _mesa_sha1_format(sha1, cache_key);
101 fprintf(stderr, "[mesa disk cache] storing %s\n", sha1);
102 }
103
104 struct blob blob;
105 blob_init(&blob);
106
107 /* We write the following data to the cache blob:
108 *
109 * 1. Prog data (must come first because it has the assembly size)
110 * - Zero out pointer values in prog data, so cache entries will be
111 * consistent.
112 * 2. Assembly code
113 * 3. Number of entries in the system value array
114 * 4. System value array
115 * 5. Size (in bytes) of kernel inputs
116 * 6. Shader relocations
117 * 7. Legacy param array (only used for compute workgroup ID)
118 * 8. Binding table
119 */
120 if (brw) {
121 size_t prog_data_s = brw_prog_data_size(stage);
122 union brw_any_prog_data serializable;
123 assert(prog_data_s <= sizeof(serializable));
124 memcpy(&serializable, shader->brw_prog_data, prog_data_s);
125 serializable.base.param = NULL;
126 serializable.base.relocs = NULL;
127 blob_write_bytes(&blob, &serializable, prog_data_s);
128 } else {
129 size_t prog_data_s = elk_prog_data_size(stage);
130 union elk_any_prog_data serializable;
131 assert(prog_data_s <= sizeof(serializable));
132 memcpy(&serializable, shader->elk_prog_data, prog_data_s);
133 serializable.base.param = NULL;
134 serializable.base.relocs = NULL;
135 blob_write_bytes(&blob, &serializable, prog_data_s);
136 }
137
138 blob_write_bytes(&blob, shader->map, shader->program_size);
139 blob_write_uint32(&blob, shader->num_system_values);
140 blob_write_bytes(&blob, shader->system_values,
141 shader->num_system_values * sizeof(uint32_t));
142 blob_write_uint32(&blob, shader->kernel_input_size);
143 if (brw) {
144 blob_write_bytes(&blob, brw->relocs,
145 brw->num_relocs * sizeof(struct brw_shader_reloc));
146 blob_write_bytes(&blob, brw->param,
147 brw->nr_params * sizeof(uint32_t));
148 } else {
149 blob_write_bytes(&blob, elk->relocs,
150 elk->num_relocs * sizeof(struct elk_shader_reloc));
151 blob_write_bytes(&blob, elk->param,
152 elk->nr_params * sizeof(uint32_t));
153 }
154 blob_write_bytes(&blob, &shader->bt, sizeof(shader->bt));
155
156 disk_cache_put(cache, cache_key, blob.data, blob.size, NULL);
157 blob_finish(&blob);
158 #endif
159 }
160
161 static const enum iris_program_cache_id cache_id_for_stage[] = {
162 [MESA_SHADER_VERTEX] = IRIS_CACHE_VS,
163 [MESA_SHADER_TESS_CTRL] = IRIS_CACHE_TCS,
164 [MESA_SHADER_TESS_EVAL] = IRIS_CACHE_TES,
165 [MESA_SHADER_GEOMETRY] = IRIS_CACHE_GS,
166 [MESA_SHADER_FRAGMENT] = IRIS_CACHE_FS,
167 [MESA_SHADER_COMPUTE] = IRIS_CACHE_CS,
168 };
169
170 /**
171 * Search for a compiled shader in the disk cache. If found, upload it
172 * to the in-memory program cache so we can use it.
173 */
174 bool
iris_disk_cache_retrieve(struct iris_screen * screen,struct u_upload_mgr * uploader,struct iris_uncompiled_shader * ish,struct iris_compiled_shader * shader,const void * prog_key,uint32_t key_size)175 iris_disk_cache_retrieve(struct iris_screen *screen,
176 struct u_upload_mgr *uploader,
177 struct iris_uncompiled_shader *ish,
178 struct iris_compiled_shader *shader,
179 const void *prog_key,
180 uint32_t key_size)
181 {
182 #ifdef ENABLE_SHADER_CACHE
183 struct disk_cache *cache = screen->disk_cache;
184 gl_shader_stage stage = ish->nir->info.stage;
185
186 if (!cache)
187 return false;
188
189 cache_key cache_key;
190 iris_disk_cache_compute_key(cache, ish, prog_key, key_size, cache_key);
191
192 if (debug) {
193 char sha1[41];
194 _mesa_sha1_format(sha1, cache_key);
195 fprintf(stderr, "[mesa disk cache] retrieving %s: ", sha1);
196 }
197
198 size_t size;
199 void *buffer = disk_cache_get(screen->disk_cache, cache_key, &size);
200
201 if (debug)
202 fprintf(stderr, "%s\n", buffer ? "found" : "missing");
203
204 if (!buffer)
205 return false;
206
207 const uint32_t prog_data_size = screen->brw ? brw_prog_data_size(stage)
208 : elk_prog_data_size(stage);
209
210 void *prog_data = ralloc_size(NULL, prog_data_size);
211 const void *assembly;
212 uint32_t num_system_values;
213 uint32_t kernel_input_size;
214 uint32_t *system_values = NULL;
215 uint32_t *so_decls = NULL;
216
217 struct brw_stage_prog_data *brw = screen->brw ? prog_data : NULL;
218 struct elk_stage_prog_data *elk = screen->elk ? prog_data : NULL;
219 assert((brw == NULL) != (elk == NULL));
220
221 struct blob_reader blob;
222 blob_reader_init(&blob, buffer, size);
223 blob_copy_bytes(&blob, prog_data, prog_data_size);
224 assembly = blob_read_bytes(&blob, brw ? brw->program_size : elk->program_size);
225 num_system_values = blob_read_uint32(&blob);
226 if (num_system_values) {
227 system_values =
228 ralloc_array(NULL, uint32_t, num_system_values);
229 blob_copy_bytes(&blob, system_values,
230 num_system_values * sizeof(uint32_t));
231 }
232
233 kernel_input_size = blob_read_uint32(&blob);
234
235 if (brw) {
236 brw->relocs = NULL;
237 if (brw->num_relocs) {
238 struct brw_shader_reloc *relocs =
239 ralloc_array(NULL, struct brw_shader_reloc, brw->num_relocs);
240 blob_copy_bytes(&blob, relocs,
241 brw->num_relocs * sizeof(struct brw_shader_reloc));
242 brw->relocs = relocs;
243 }
244
245 brw->param = NULL;
246 if (brw->nr_params) {
247 brw->param = ralloc_array(NULL, uint32_t, brw->nr_params);
248 blob_copy_bytes(&blob, brw->param, brw->nr_params * sizeof(uint32_t));
249 }
250 } else {
251 elk->relocs = NULL;
252 if (elk->num_relocs) {
253 struct elk_shader_reloc *relocs =
254 ralloc_array(NULL, struct elk_shader_reloc, elk->num_relocs);
255 blob_copy_bytes(&blob, relocs,
256 elk->num_relocs * sizeof(struct elk_shader_reloc));
257 elk->relocs = relocs;
258 }
259
260 elk->param = NULL;
261 if (elk->nr_params) {
262 elk->param = ralloc_array(NULL, uint32_t, elk->nr_params);
263 blob_copy_bytes(&blob, elk->param,
264 elk->nr_params * sizeof(uint32_t));
265 }
266 }
267
268 struct iris_binding_table bt;
269 blob_copy_bytes(&blob, &bt, sizeof(bt));
270
271 if (stage == MESA_SHADER_VERTEX ||
272 stage == MESA_SHADER_TESS_EVAL ||
273 stage == MESA_SHADER_GEOMETRY) {
274 struct intel_vue_map *vue_map =
275 screen->brw ? &brw_vue_prog_data(prog_data)->vue_map
276 : &elk_vue_prog_data(prog_data)->vue_map;
277 so_decls = screen->vtbl.create_so_decl_list(&ish->stream_output, vue_map);
278 }
279
280 /* System values and uniforms are stored in constant buffer 0, the
281 * user-facing UBOs are indexed by one. So if any constant buffer is
282 * needed, the constant buffer 0 will be needed, so account for it.
283 */
284 unsigned num_cbufs = ish->nir->info.num_ubos;
285
286 if (num_cbufs || ish->nir->num_uniforms)
287 num_cbufs++;
288
289 if (num_system_values || kernel_input_size)
290 num_cbufs++;
291
292 if (brw)
293 iris_apply_brw_prog_data(shader, brw);
294 else
295 iris_apply_elk_prog_data(shader, elk);
296
297 iris_finalize_program(shader, so_decls, system_values,
298 num_system_values, kernel_input_size, num_cbufs,
299 &bt);
300
301 assert(stage < ARRAY_SIZE(cache_id_for_stage));
302 enum iris_program_cache_id cache_id = cache_id_for_stage[stage];
303
304 /* Upload our newly read shader to the in-memory program cache. */
305 iris_upload_shader(screen, ish, shader, NULL, uploader,
306 cache_id, key_size, prog_key, assembly);
307
308 free(buffer);
309
310 return true;
311 #else
312 return false;
313 #endif
314 }
315
316 /**
317 * Initialize the on-disk shader cache.
318 */
319 void
iris_disk_cache_init(struct iris_screen * screen)320 iris_disk_cache_init(struct iris_screen *screen)
321 {
322 #ifdef ENABLE_SHADER_CACHE
323 if (INTEL_DEBUG(DEBUG_DISK_CACHE_DISABLE_MASK))
324 return;
325
326 /* array length = strlen("iris_") + sha + nul char */
327 char renderer[5 + 40 + 1] = {0};
328
329 if (screen->brw) {
330 char device_info_sha[41];
331 brw_device_sha1(device_info_sha, screen->devinfo);
332 memcpy(renderer, "iris_", 5);
333 memcpy(renderer + 5, device_info_sha, 40);
334 } else {
335 /* For Gfx8, just use PCI ID. */
336 ASSERTED int len = snprintf(renderer, sizeof(renderer),
337 "iris_%04x", screen->devinfo->pci_device_id);
338 assert(len < ARRAY_SIZE(renderer) - 1);
339 }
340
341 const struct build_id_note *note =
342 build_id_find_nhdr_for_addr(iris_disk_cache_init);
343 assert(note && build_id_length(note) == 20); /* sha1 */
344
345 const uint8_t *id_sha1 = build_id_data(note);
346 assert(id_sha1);
347
348 char timestamp[41];
349 _mesa_sha1_format(timestamp, id_sha1);
350
351 const uint64_t driver_flags = screen->brw ?
352 brw_get_compiler_config_value(screen->brw) :
353 elk_get_compiler_config_value(screen->elk);
354 screen->disk_cache = disk_cache_create(renderer, timestamp, driver_flags);
355 #endif
356 }
357