1 /*
2 * Copyright (c) 2017, Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included
12 * in all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
15 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 */
22 //!
23 //! \file cm_program.cpp
24 //! \brief Contains Class CmProgram definitions
25 //!
26
27 #include "cm_program.h"
28
29 #include "cm_device_rt.h"
30 #include "cm_mem.h"
31 #include "cm_hal.h"
32
33 #if USE_EXTENSION_CODE
34 #include "cm_hw_debugger.h"
35 #endif
36
37 #include <string>
38 #include <functional>
39
40 #define READ_FIELD_FROM_BUF( dst, type ) \
41 dst = *((type *) &buf[bytePos]); \
42 bytePos += sizeof(type);
43
44 #define CM_RT_JITTER_DEBUG_FLAG "-debug"
45 #define CM_RT_JITTER_NCSTATELESS_FLAG "-ncstateless"
46
47 #define CM_RT_JITTER_MAX_NUM_FLAGS 30
48 #define CM_RT_JITTER_NUM_RESERVED_FLAGS 3 // one for gtpin; two for hw stepping info.
49 #define CM_RT_JITTER_MAX_NUM_USER_FLAGS (CM_RT_JITTER_MAX_NUM_FLAGS - CM_RT_JITTER_NUM_RESERVED_FLAGS)
50
51 namespace CMRT_UMD
52 {
53 //*-----------------------------------------------------------------------------
54 //| Purpose: Create Cm Program
55 //| Arguments :
56 //| device [in] Pointer to Cm Device
57 //| commonISACode [in] Pointer to memory where common isa locates
58 //| size [in] Size of memory
59 //| pProgram [in] Reference to pointer to CmProgram
60 //| options [in] jitter or non-jitter
61 //| Returns: Result of the operation.
62 //*-----------------------------------------------------------------------------
Create(CmDeviceRT * device,void * cisaCode,const uint32_t cisaCodeSize,CmProgramRT * & pProgram,const char * options,const uint32_t programId)63 int32_t CmProgramRT::Create( CmDeviceRT* device, void* cisaCode, const uint32_t cisaCodeSize, CmProgramRT*& pProgram, const char* options, const uint32_t programId )
64 {
65 int32_t result = CM_SUCCESS;
66 pProgram = new (std::nothrow) CmProgramRT( device, programId );
67 if( pProgram )
68 {
69
70 pProgram->Acquire();
71 result = pProgram->Initialize( cisaCode, cisaCodeSize, options );
72 device->m_memObjectCount.programCount++;
73 if( result != CM_SUCCESS )
74 {
75 CmProgramRT::Destroy( pProgram);
76 }
77 }
78 else
79 {
80 CM_ASSERTMESSAGE("Error: Failed to create CmProgram due to out of system memory.");
81 result = CM_OUT_OF_HOST_MEMORY;
82 }
83 return result;
84 }
85
86 //*-----------------------------------------------------------------------------
87 //| Purpose: Destroy Cm Program
88 //| Returns: Result of the operation.
89 //*-----------------------------------------------------------------------------
Destroy(CmProgramRT * & program)90 int32_t CmProgramRT::Destroy( CmProgramRT* &program )
91 {
92 long refCount = program->SafeRelease( );
93 if( refCount == 0 )
94 {
95 program = nullptr;
96 }
97 return CM_SUCCESS;
98 }
99
100 //*-----------------------------------------------------------------------------
101 //| Purpose: Acquire: Increase refcount
102 //| Returns: Result of the operation.
103 //*-----------------------------------------------------------------------------
Acquire(void)104 int32_t CmProgramRT::Acquire( void )
105 {
106 m_refCount++;
107 return CM_SUCCESS;
108 }
109
110 //*-----------------------------------------------------------------------------
111 //| Purpose: SafeRelease:
112 //| Returns: Result of the operation.
113 //*-----------------------------------------------------------------------------
SafeRelease(void)114 int32_t CmProgramRT::SafeRelease( void )
115 {
116 --m_refCount;
117 if( m_refCount == 0 )
118 {
119 m_device->m_memObjectCount.programCount--;
120 delete this;
121 return 0;
122 }
123 return m_refCount;
124 }
125
126 //*-----------------------------------------------------------------------------
127 //| Purpose: Constructor of Cm Program
128 //| Returns: Result of the operation.
129 //*-----------------------------------------------------------------------------
CmProgramRT(CmDeviceRT * device,uint32_t programId)130 CmProgramRT::CmProgramRT( CmDeviceRT* device, uint32_t programId ):
131 m_device( device ),
132 m_programCodeSize( 0 ),
133 m_programCode(nullptr),
134 m_isaFile(nullptr),
135 m_options( nullptr ),
136 m_surfaceCount( 0 ),
137 m_kernelCount( 0 ),
138 m_kernelInfo( CM_INIT_KERNEL_PER_PROGRAM ),
139 m_isJitterEnabled(false),
140 m_isHwDebugEnabled(false),
141 m_refCount(0),
142 m_programIndex(programId),
143 m_kernelIndex(0),
144 m_fJITCompile(nullptr),
145 m_fFreeBlock(nullptr),
146 m_fJITVersion(nullptr),
147 m_fJITCompile_v2(nullptr),
148 m_cisaMagicNumber(0),
149 m_cisaMajorVersion(0),
150 m_cisaMinorVersion(0)
151 {
152 CmSafeMemSet(m_isaFileName,0,sizeof(char)*CM_MAX_ISA_FILE_NAME_SIZE_IN_BYTE);
153 }
154
155 //*-----------------------------------------------------------------------------
156 //| Purpose: Destructor of Cm Program
157 //| Returns: Result of the operation.
158 //*-----------------------------------------------------------------------------
~CmProgramRT(void)159 CmProgramRT::~CmProgramRT( void )
160 {
161 MosSafeDeleteArray( m_options );
162 MosSafeDeleteArray( m_programCode );
163 for( uint32_t i = 0; i < m_kernelCount; i ++ )
164 {
165 uint32_t refCount = this->ReleaseKernelInfo(i);
166 CM_ASSERT(refCount == 0);
167
168 }
169 m_kernelInfo.Delete();
170 CmSafeDelete(m_isaFile);
171 }
172
173 //*-----------------------------------------------------------------------------
174 //| Purpose: Initialize Cm Program
175 //| Returns: Result of the operation.
176 //*-----------------------------------------------------------------------------
Initialize(void * cisaCode,const uint32_t cisaCodeSize,const char * options)177 int32_t CmProgramRT::Initialize( void* cisaCode, const uint32_t cisaCodeSize, const char* options )
178 {
179 bool loadingGPUCopyKernel = false;
180 int32_t result = CM_SUCCESS;
181 int32_t hr = CM_FAILURE;
182
183 m_isJitterEnabled = true; //by default jitter is ON
184 //get first kernel starting slot index
185 m_kernelIndex = m_device->GetKernelSlot();
186
187 int numJitFlags = 0;
188 const char *jitFlags[CM_RT_JITTER_MAX_NUM_FLAGS];
189 CmSafeMemSet(jitFlags, 0, sizeof(char *) * CM_RT_JITTER_MAX_NUM_FLAGS);
190
191 char* flagStepInfo = nullptr;
192
193 if( options )
194 {
195 size_t length = strnlen( options, CM_MAX_OPTION_SIZE_IN_BYTE );
196 if(length >= CM_MAX_OPTION_SIZE_IN_BYTE)
197 {
198 CM_ASSERTMESSAGE("Error: option size is too long.");
199 return CM_INVALID_ARG_VALUE;
200 }
201 else
202 {
203 m_options = MOS_NewArray(char, (length + 1));
204 if( !m_options )
205 {
206 CM_ASSERTMESSAGE("Error: Out of system memory.");
207 return CM_OUT_OF_HOST_MEMORY;
208
209 }
210 CmSafeMemCopy( m_options, options, length);
211 m_options[ length ] = '\0';
212
213 if(strstr(options, "nojitter"))
214 m_isJitterEnabled = false;
215
216 if(!strcmp(m_options, "PredefinedGPUKernel"))
217 loadingGPUCopyKernel = true;
218 if( (m_isJitterEnabled == true) && (loadingGPUCopyKernel == false))
219 { // option: "nonjitter" and "PredefinedGPUCopyKernel" should not be passed to jitter
220 char *token = nullptr;
221 char *nextToken = nullptr;
222 char *ptr = m_options;
223
224 while( nullptr != (token = strtok_s(ptr," ",&nextToken)))
225 {
226 if(numJitFlags >= CM_RT_JITTER_MAX_NUM_USER_FLAGS)
227 {
228 CM_ASSERTMESSAGE("Error: Invalid jitter user flags number.");
229 MosSafeDeleteArray(m_options);
230 return CM_FAILURE;
231 }
232 if(!strcmp(token, CM_RT_JITTER_DEBUG_FLAG))
233 {
234 m_isHwDebugEnabled = true;
235 }
236 jitFlags[numJitFlags] = token;
237 numJitFlags++;
238 ptr = nextToken;
239 }
240 }
241 }
242 }
243
244 uint8_t *buf = (uint8_t*)cisaCode;
245 uint32_t bytePos = 0;
246
247 READ_FIELD_FROM_BUF(m_cisaMagicNumber, uint32_t);
248 READ_FIELD_FROM_BUF(m_cisaMajorVersion, uint8_t);
249 READ_FIELD_FROM_BUF(m_cisaMinorVersion, uint8_t);
250
251 bool useVisaApi = true;
252 vISA::Header *header = nullptr;
253
254 auto getVersionAsInt = [](int major, int minor) {return major * 100 + minor;};
255 if (getVersionAsInt(m_cisaMajorVersion, m_cisaMinorVersion) < getVersionAsInt(3, 2))
256 {
257 useVisaApi = false;
258 }
259 else
260 {
261 m_isaFile = new vISA::ISAfile((uint8_t*)cisaCode, cisaCodeSize);
262 if (!m_isaFile->readFile())
263 {
264 CM_ASSERTMESSAGE("Error: invalid VISA.");
265 MosSafeDeleteArray(m_options);
266 return CM_INVALID_COMMON_ISA;
267 }
268 header = m_isaFile->getHeader();
269 }
270
271 if (m_cisaMagicNumber != CISA_MAGIC_NUMBER)
272 {
273 CM_ASSERTMESSAGE("Error: Invalid CISA magic number.");
274 MosSafeDeleteArray(m_options);
275 return CM_INVALID_COMMON_ISA;
276 }
277
278 if(loadingGPUCopyKernel)//for predefined kernel(GPUCopy), forcely disable jitting
279 {
280 m_isJitterEnabled = false;
281 }
282
283 const char *platform = nullptr;
284
285 PCM_HAL_STATE cmHalState = \
286 ((PCM_CONTEXT_DATA)m_device->GetAccelData())->cmHalState;
287 CM_CHK_NULL_GOTOFINISH_CMERROR(cmHalState);
288
289 cmHalState->cmHalInterface->GetGenPlatformInfo(nullptr, nullptr, &platform);
290
291 if( m_isJitterEnabled )
292 {
293 #if (_DEBUG || _RELEASE_INTERNAL)
294 //reg control for svm IA/GT cache coherence
295 MOS_USER_FEATURE_VALUE_DATA userFeatureData;
296 MOS_ZeroMemory(&userFeatureData, sizeof(userFeatureData));
297 CM_HAL_STATE *hal_state = m_device->GetHalState();
298 MOS_UserFeature_ReadValue_ID(
299 nullptr, __MEDIA_USER_FEATURE_VALUE_MDF_FORCE_COHERENT_STATELESSBTI_ID,
300 &userFeatureData, hal_state->osInterface->pOsContext);
301 if (userFeatureData.i32Data == 1)
302 {
303 jitFlags[numJitFlags] = CM_RT_JITTER_NCSTATELESS_FLAG;
304 numJitFlags++;
305 }
306 #endif // (_DEBUG || _RELEASE_INTERNAL)
307
308 //Load jitter library and add function pointers to program
309 // Get hmodule from CmDevice_RT or CmDevice_Sim, which is casted from CmDevice
310 result = m_device->LoadJITDll();
311 if(result != CM_SUCCESS)
312 {
313 CM_ASSERTMESSAGE("Error: Load jitter library failure.");
314 MosSafeDeleteArray(m_options);
315 return result;
316 }
317
318 m_device->GetJITCompileFnt(m_fJITCompile);
319 m_device->GetJITCompileFntV2(m_fJITCompile_v2);
320 m_device->GetFreeBlockFnt(m_fFreeBlock);
321 m_device->GetJITVersionFnt(m_fJITVersion);
322
323 uint32_t jitMajor = 0;
324 uint32_t jitMinor = 0;
325 m_fJITVersion(jitMajor, jitMinor);
326 if((jitMajor < m_cisaMajorVersion) || (jitMajor == m_cisaMajorVersion && jitMinor < m_cisaMinorVersion))
327 return CM_JITDLL_OLDER_THAN_ISA;
328 #if USE_EXTENSION_CODE
329 if( m_device->CheckGTPinEnabled() && !loadingGPUCopyKernel)
330 {
331 hr = InitForGTPin(jitFlags, numJitFlags);
332 if (hr != CM_SUCCESS)
333 {
334 goto finish;
335 }
336 }
337 #endif
338 //Pass stepping info to JITTER..."
339 char *stepstr = nullptr;
340 m_device->GetGenStepInfo(stepstr);
341 if(stepstr != nullptr)
342 {
343 flagStepInfo = MOS_NewArray(char, CM_JIT_FLAG_SIZE);
344 if (flagStepInfo)
345 {
346 jitFlags[numJitFlags] = "-stepping";
347 numJitFlags++;
348
349 CmSafeMemSet(flagStepInfo, 0, CM_JIT_FLAG_SIZE);
350 MOS_SecureStringPrint(flagStepInfo, CM_JIT_FLAG_SIZE, CM_JIT_FLAG_SIZE, "%s", stepstr);
351 if (numJitFlags >= CM_RT_JITTER_MAX_NUM_FLAGS)
352 {
353 CM_ASSERTMESSAGE("Error: Invalid jitter user flags number.");
354 hr = CM_FAILURE;
355 goto finish;
356 }
357 jitFlags[numJitFlags] = flagStepInfo;
358 numJitFlags++;
359 }
360 else
361 {
362 CM_ASSERTMESSAGE("Error: Out of system memory.");
363 MosSafeDeleteArray(m_options);
364 return CM_OUT_OF_HOST_MEMORY;
365 }
366 }
367 }
368
369 if (useVisaApi)
370 {
371 m_kernelCount = header->getNumKernels();
372 }
373 else
374 {
375 bytePos = 4 + 1 + 1; // m_cisaMagicNumber:sizeof (uint32_t) + m_cisaMajorVersion:sizeof(uint8_t) + m_cisaMinorVersion:sizeof(uint8_t)
376 unsigned short numKernels;
377 READ_FIELD_FROM_BUF(numKernels, unsigned short);
378 m_kernelCount = numKernels;
379 }
380
381 #ifdef _DEBUG
382 if(m_isJitterEnabled)
383 CM_NORMALMESSAGE("Jitter Compiling...");
384 #endif
385
386 for (uint32_t i = 0; i < m_kernelCount; i++)
387 {
388 CM_KERNEL_INFO* kernInfo = new (std::nothrow) CM_KERNEL_INFO;
389 if (!kernInfo)
390 {
391 CM_ASSERTMESSAGE("Error: Out of system memory.");
392 MosSafeDeleteArray(flagStepInfo);
393 MosSafeDeleteArray(m_options);
394 hr = CM_OUT_OF_HOST_MEMORY;
395 goto finish;
396 }
397 CmSafeMemSet(kernInfo, 0, sizeof(CM_KERNEL_INFO));
398
399 vISA::Kernel *kernel = nullptr;
400 uint16_t nameLen = 0;
401 if (useVisaApi)
402 {
403 kernel = header->getKernelInfo()[i];
404 if (getVersionAsInt(m_cisaMajorVersion, m_cisaMinorVersion) < getVersionAsInt(3, 7))
405 {
406 nameLen = (uint16_t) kernel->getNameLen_Ver306();
407 }
408 else
409 {
410 nameLen = kernel->getNameLen();
411 }
412 CmSafeMemCopy(kernInfo->kernelName, kernel->getName(), nameLen);
413 }
414 else
415 {
416 READ_FIELD_FROM_BUF(nameLen, uint8_t);
417 CmSafeMemCopy(kernInfo->kernelName, buf + bytePos, nameLen);
418 // move bytePos to the right index
419 bytePos += nameLen;
420 }
421
422 if(m_isJitterEnabled)
423 {
424 if (useVisaApi)
425 {
426 kernInfo->kernelIsaOffset = kernel->getOffset();
427 kernInfo->kernelIsaSize = kernel->getSize();
428 kernInfo->inputCountOffset = kernel->getInputOffset();
429 }
430 else
431 {
432 uint32_t kernelIsaOffset;
433 uint32_t kernelIsaSize;
434 uint32_t inputCountOffset;
435
436 READ_FIELD_FROM_BUF(kernelIsaOffset, uint32_t); //read kernel isa offset
437 READ_FIELD_FROM_BUF(kernelIsaSize, uint32_t); //read kernel isa size
438 READ_FIELD_FROM_BUF(inputCountOffset, uint32_t);
439
440 kernInfo->kernelIsaOffset = kernelIsaOffset;
441 kernInfo->kernelIsaSize = kernelIsaSize;
442 kernInfo->inputCountOffset = inputCountOffset;
443
444 // read relocation symbols
445 unsigned short numVarSyms = 0, num_func_syms = 0;
446 unsigned char numGenBinaries = 0;
447
448 READ_FIELD_FROM_BUF(numVarSyms, uint16_t);
449 // CISA layout of var_syms is symbolic_index[0], resolved_index[0],
450 // ...resolved_index[n-1]. Skip all.
451 bytePos += sizeof(uint16_t) * numVarSyms;
452 bytePos += sizeof(uint16_t) * numVarSyms;
453
454 READ_FIELD_FROM_BUF(num_func_syms, uint16_t);
455 // CISA layout of func_syms is symbolic_index[0], resolved_index[0],
456 // ...resolved_index[n-1]. Skip all.
457 bytePos += sizeof(uint16_t) * num_func_syms;
458 bytePos += sizeof(uint16_t) * num_func_syms;
459
460 // numGenBinaries
461 READ_FIELD_FROM_BUF(numGenBinaries, uint8_t);
462 // CISA layout of gen_binaries is genPlatform[0], binary offset[0],
463 // binary size[0]...binary size[n-1]. Skip all.
464 // skip all genPlatform
465 bytePos += sizeof(uint8_t) * numGenBinaries;
466 // skip all binary offset
467 bytePos += sizeof(uint32_t) * numGenBinaries;
468 // skip all binary size
469 bytePos += sizeof(uint32_t) * numGenBinaries;
470
471 // bytePos should point to nameLen of next kernel
472 }
473 }
474 else // non jitting
475 {
476 uint8_t numGenBinaries = 0;
477 if (useVisaApi)
478 {
479 kernInfo->kernelIsaOffset = kernel->getOffset();
480 kernInfo->inputCountOffset = kernel->getInputOffset();
481 numGenBinaries = kernel->getNumGenBinaries();
482 }
483 else
484 {
485 uint32_t kernelIsaOffset;
486 READ_FIELD_FROM_BUF(kernelIsaOffset, uint32_t); //read kernel isa offset
487 kernInfo->kernelIsaOffset = kernelIsaOffset;
488
489 // skipping kernelIsaSize
490 bytePos += 4;
491
492 uint32_t inputCountOffset;
493 READ_FIELD_FROM_BUF(inputCountOffset, uint32_t);
494 kernInfo->inputCountOffset = inputCountOffset;
495
496 // read relocation symbols
497 unsigned short numVarSyms = 0, num_func_syms = 0;
498
499 READ_FIELD_FROM_BUF(numVarSyms, uint16_t);
500 // CISA layout of var_syms is symbolic_index[0], resolved_index[0],
501 // ...resolved_index[n-1]. Skip all.
502 bytePos += sizeof(uint16_t) * numVarSyms;
503 bytePos += sizeof(uint16_t) * numVarSyms;
504
505 READ_FIELD_FROM_BUF(num_func_syms, uint16_t);
506 // CISA layout of func_syms is symbolic_index[0], resolved_index[0],
507 // ...resolved_index[n-1]. Skip all.
508 bytePos += sizeof(uint16_t) * num_func_syms;
509 bytePos += sizeof(uint16_t) * num_func_syms;
510
511 // numGenBinaries
512 READ_FIELD_FROM_BUF(numGenBinaries, uint8_t);
513 }
514
515 uint32_t genxBinaryOffset = 0;
516 uint32_t genxBinarySize = 0;
517 for (int j = 0; j < numGenBinaries; j++)
518 {
519 vISA::GenBinary *genBinary = nullptr;
520 uint8_t genPlatform = 0;
521 uint32_t offset = 0;
522 uint32_t size = 0;
523
524 if (useVisaApi)
525 {
526 genBinary = kernel->getGenBinaryInfo()[j];
527 genPlatform = genBinary->getGenPlatform();
528 offset = genBinary->getBinaryOffset();
529 size = genBinary->getBinarySize();
530 }
531 else
532 {
533 // genPlatform
534 READ_FIELD_FROM_BUF(genPlatform, uint8_t);
535 // binary offset
536 READ_FIELD_FROM_BUF(offset, uint32_t);
537 // binary size
538 READ_FIELD_FROM_BUF(size, uint32_t);
539 }
540
541 if (cmHalState->cmHalInterface->IsCisaIDSupported((uint32_t)genPlatform))
542 {
543 // assign correct offset/size based on platform
544 genxBinaryOffset = offset;
545 genxBinarySize = size;
546 }
547 else
548 {
549 MosSafeDeleteArray(flagStepInfo);
550 MosSafeDeleteArray(m_options);
551 CmSafeDelete(kernInfo);
552 return CM_INVALID_GENX_BINARY;
553 }
554 }
555
556 kernInfo->genxBinaryOffset = genxBinaryOffset;
557 kernInfo->genxBinarySize = genxBinarySize;
558
559 if ( kernInfo->genxBinarySize == 0 || kernInfo->genxBinaryOffset == 0 )
560 {
561 CM_ASSERTMESSAGE("Error: Invalid genx binary.");
562 MosSafeDeleteArray(flagStepInfo);
563 MosSafeDeleteArray(m_options);
564 CmSafeDelete(kernInfo);
565 return CM_INVALID_GENX_BINARY;
566 }
567 }
568
569 if(m_isJitterEnabled)
570 {
571 kernInfo->jitBinaryCode = 0;
572 void* jitBinary = 0;// = (void**)malloc(m_kernelCount*sizeof(void*));
573 uint32_t jitBinarySize = 0;// = (uint32_t*)malloc(m_kernelCount*sizeof(uint32_t*));
574 char* errorMsg = (char*)malloc(CM_JIT_ERROR_MESSAGE_SIZE);
575 if (errorMsg == nullptr)
576 {
577 CM_ASSERTMESSAGE("Error: Out of system memory.");
578 CmSafeDelete(kernInfo);
579 hr = CM_OUT_OF_HOST_MEMORY;
580 goto finish;
581 }
582 CmSafeMemSet( errorMsg, 0, CM_JIT_ERROR_MESSAGE_SIZE );
583
584 FINALIZER_INFO *jitProfInfo = (FINALIZER_INFO *)malloc(CM_JIT_PROF_INFO_SIZE);
585 if(jitProfInfo == nullptr)
586 {
587 CM_ASSERTMESSAGE("Error: Out of system memory.");
588 free(errorMsg);
589 CmSafeDelete(kernInfo);
590 hr = CM_OUT_OF_HOST_MEMORY;
591 goto finish;
592 }
593 CmSafeMemSet( jitProfInfo, 0, CM_JIT_PROF_INFO_SIZE );
594
595 void *extra_info = nullptr;
596 CmNotifierGroup *notifiers = m_device->GetNotifiers();
597 if (notifiers)
598 {
599 notifiers->NotifyCallingJitter(&extra_info);
600 }
601
602 if (m_fJITCompile_v2)
603 {
604 result = m_fJITCompile_v2( kernInfo->kernelName, (uint8_t*)cisaCode, cisaCodeSize,
605 jitBinary, jitBinarySize, platform, m_cisaMajorVersion, m_cisaMinorVersion, numJitFlags, jitFlags, errorMsg, jitProfInfo, extra_info );
606 }
607 else
608 {
609 result = m_fJITCompile( kernInfo->kernelName, (uint8_t*)cisaCode, cisaCodeSize,
610 jitBinary, jitBinarySize, platform, m_cisaMajorVersion, m_cisaMinorVersion, numJitFlags, jitFlags, errorMsg, jitProfInfo );
611 }
612
613 //if error code returned or error message not nullptr
614 if(result != CM_SUCCESS)// || errorMsg[0])
615 {
616 CM_NORMALMESSAGE("%s.", errorMsg);
617 free(errorMsg);
618 CmSafeDelete(kernInfo);
619 hr = CM_JIT_COMPILE_FAILURE;
620 goto finish;
621 }
622
623 // if spill code exists and scrach space disabled, return error to user
624 if( jitProfInfo->isSpill && m_device->IsScratchSpaceDisabled())
625 {
626 CmSafeDelete(kernInfo);
627 free(errorMsg);
628 return CM_INVALID_KERNEL_SPILL_CODE;
629 }
630
631 free(errorMsg);
632
633 kernInfo->jitBinaryCode = jitBinary;
634 kernInfo->jitBinarySize = jitBinarySize;
635 kernInfo->jitInfo = jitProfInfo;
636
637 #if USE_EXTENSION_CODE
638 if ( m_isHwDebugEnabled )
639 {
640 NotifyKernelBinary(this->m_device,
641 this,
642 kernInfo->kernelName,
643 jitBinary,
644 jitBinarySize,
645 jitProfInfo->genDebugInfo,
646 jitProfInfo->genDebugInfoSize,
647 nullptr,
648 m_device->GetDriverStoreFlag());
649 }
650 #endif
651 }
652
653 m_kernelInfo.SetElement( i, kernInfo );
654 this->AcquireKernelInfo(i);
655 }
656
657 #ifdef _DEBUG
658 if(m_isJitterEnabled)
659 CM_NORMALMESSAGE("Jitter Done.");
660 #endif
661
662 // now bytePos index to the start of common isa body;
663 // compute the code size for common isa
664 m_programCodeSize = cisaCodeSize;
665 m_programCode = MOS_NewArray(uint8_t, m_programCodeSize);
666 if( !m_programCode )
667 {
668 CM_ASSERTMESSAGE("Error: Out of system memory.");
669 hr = CM_OUT_OF_HOST_MEMORY;
670 goto finish;
671 }
672
673 //Copy CISA content
674 CmFastMemCopy((void *)m_programCode, cisaCode, cisaCodeSize);
675
676 //Caculate hash value for each kernel
677 for (uint32_t i = 0; i < m_kernelCount; i++)
678 {
679 CM_KERNEL_INFO *kernelInfo = (CM_KERNEL_INFO *)m_kernelInfo.GetElement(i);
680 CM_CHK_NULL_GOTOFINISH_CMERROR(kernelInfo);
681 // higher 32bit is the order of kernel in LoadProgram in the device
682 // lower 32bit is the hash value of kernel info
683 kernelInfo->hashValue = GetKernelInfoHash(kernelInfo) | ((uint64_t)m_device->KernelsLoaded() << 32);
684 ++ m_device->KernelsLoaded();
685 }
686
687 hr = CM_SUCCESS;
688
689 finish:
690 MosSafeDeleteArray(flagStepInfo);
691 if(hr != CM_SUCCESS )
692 {
693 MosSafeDeleteArray(m_options);
694 MosSafeDeleteArray(m_programCode);
695 }
696 return hr;
697 }
698
699 //*-----------------------------------------------------------------------------
700 //| Purpose: Get size and address of Common Isa
701 //| Returns: Result of the operation.
702 //*-----------------------------------------------------------------------------
GetCommonISACode(void * & commonISACode,uint32_t & size)703 int32_t CmProgramRT::GetCommonISACode( void* & commonISACode, uint32_t & size )
704 {
705 commonISACode = (void *)m_programCode;
706 size = m_programCodeSize;
707
708 return CM_SUCCESS;
709 }
710
711 //*-----------------------------------------------------------------------------
712 //| Purpose: Get the count of kernel
713 //| Returns: Result of the operation.
714 //*-----------------------------------------------------------------------------
GetKernelCount(uint32_t & kernelCount)715 int32_t CmProgramRT::GetKernelCount( uint32_t& kernelCount )
716 {
717 kernelCount = m_kernelCount;
718 return CM_SUCCESS;
719 }
720
721 //*-----------------------------------------------------------------------------
722 //| Purpose: Get the Kernel's Infomation
723 //| Returns: Result of the operation.
724 //*-----------------------------------------------------------------------------
GetKernelInfo(uint32_t index,CM_KERNEL_INFO * & kernelInfo)725 int32_t CmProgramRT::GetKernelInfo( uint32_t index, CM_KERNEL_INFO*& kernelInfo )
726 {
727 if( index < m_kernelCount )
728 {
729 kernelInfo = (CM_KERNEL_INFO*)m_kernelInfo.GetElement( index ) ;
730 return CM_SUCCESS;
731 }
732 else
733 {
734 kernelInfo = nullptr;
735 return CM_FAILURE;
736 }
737 }
738
739 //*-----------------------------------------------------------------------------
740 //| Purpose: Get the name of ISA file
741 //| Returns: Result of the operation.
742 //*-----------------------------------------------------------------------------
GetIsaFileName(char * & isaFileName)743 int32_t CmProgramRT::GetIsaFileName( char* & isaFileName )
744 {
745 isaFileName = m_isaFileName;
746 return CM_SUCCESS;
747 }
748
749 //*-----------------------------------------------------------------------------
750 //| Purpose: Get Kernel's options
751 //| Returns: Result of the operation.
752 //*-----------------------------------------------------------------------------
GetKernelOptions(char * & kernelOptions)753 int32_t CmProgramRT::GetKernelOptions( char* & kernelOptions )
754 {
755 kernelOptions = m_options;
756 return CM_SUCCESS;
757 }
758
759 //*-----------------------------------------------------------------------------
760 //| Purpose: Get the number of Surfaces
761 //| Returns: Result of the operation.
762 //*-----------------------------------------------------------------------------
GetSurfaceCount(void)763 uint32_t CmProgramRT::GetSurfaceCount(void)
764 {
765 return m_surfaceCount;
766 }
767
768 //*-----------------------------------------------------------------------------
769 //| Purpose: Set Program's surface count
770 //| Returns: Result of the operation.
771 //*-----------------------------------------------------------------------------
SetSurfaceCount(uint32_t count)772 int32_t CmProgramRT::SetSurfaceCount(uint32_t count)
773 {
774 m_surfaceCount = count;
775 return CM_SUCCESS;
776 }
777
778 //*-----------------------------------------------------------------------------
779 //| Purpose: Acquire Kernel Info
780 //| Returns: Result of the operation.
781 //*-----------------------------------------------------------------------------
AcquireKernelInfo(uint32_t index)782 uint32_t CmProgramRT::AcquireKernelInfo(uint32_t index)
783 {
784 CM_KERNEL_INFO* kernelInfo = nullptr;
785
786 if( index < m_kernelCount )
787 {
788 kernelInfo = (CM_KERNEL_INFO *)m_kernelInfo.GetElement( index ) ;
789 if (kernelInfo)
790 {
791 CM_ASSERT( (int32_t)kernelInfo->kernelInfoRefCount >= 0 );
792 CM_ASSERT( kernelInfo->kernelInfoRefCount < UINT_MAX );
793
794 ++ kernelInfo->kernelInfoRefCount;
795 return kernelInfo->kernelInfoRefCount;
796 }
797 else
798 {
799 return 0;
800 }
801 }
802 else
803 {
804 return 0;
805 }
806 }
807
808 //*-----------------------------------------------------------------------------
809 //| Purpose: Release Kernel Info
810 //| Returns: Result of the operation.
811 //*-----------------------------------------------------------------------------
ReleaseKernelInfo(uint32_t index)812 uint32_t CmProgramRT::ReleaseKernelInfo(uint32_t index)
813 {
814 CM_KERNEL_INFO* kernelInfo = nullptr;
815
816 if( index < m_kernelCount )
817 {
818 kernelInfo = (CM_KERNEL_INFO *)m_kernelInfo.GetElement( index ) ;
819 if (kernelInfo)
820 {
821 CM_ASSERT( kernelInfo->kernelInfoRefCount > 0 );
822
823 -- kernelInfo->kernelInfoRefCount;
824
825 if (kernelInfo->kernelInfoRefCount == 1)
826 {
827 /////////////////////////////////////////////////////////////
828 //Free global string memory space, Start
829 for (int i = 0; i < (int) kernelInfo->globalStringCount; i ++)
830 {
831 if (kernelInfo->globalStrings[i])
832 {
833 free((void *)kernelInfo->globalStrings[i]);
834 }
835 }
836 if (kernelInfo->globalStrings)
837 {
838 free((void *)kernelInfo->globalStrings);
839 kernelInfo->globalStrings = nullptr;
840 kernelInfo->globalStringCount = 0;
841 }
842 //Free global string memory space, End
843 /////////////////////////////////////////////////////////////
844
845 for (uint32_t i = 0; i < kernelInfo->surfaceCount; i++) {
846 if (kernelInfo->surface[i].attributeCount && kernelInfo->surface[i].attributes) {
847 free(kernelInfo->surface[i].attributes);
848 }
849 }
850 if (kernelInfo->surface) {
851 free(kernelInfo->surface);
852 kernelInfo->surface = nullptr;
853 kernelInfo->surfaceCount = 0;
854 }
855
856 return 1;
857 }
858
859 else if (kernelInfo->kernelInfoRefCount == 0)
860 {
861 if(m_isJitterEnabled)
862 {
863 if(kernelInfo->jitBinaryCode)
864 m_fFreeBlock(kernelInfo->jitBinaryCode);
865 if(kernelInfo->jitInfo)
866 {
867 if (kernelInfo->jitInfo->freeGRFInfo)
868 {
869 m_fFreeBlock(kernelInfo->jitInfo->freeGRFInfo);
870 }
871 free(kernelInfo->jitInfo);
872 }
873 }
874
875 /////////////////////////////////////////////////////////////
876 //Free global string memory space, Start
877 for (int i = 0; i < (int) kernelInfo->globalStringCount; i ++)
878 {
879 if (kernelInfo->globalStrings[i])
880 {
881 free((void *)kernelInfo->globalStrings[i]);
882 }
883 }
884 if (kernelInfo->globalStrings)
885 {
886 free((void *)kernelInfo->globalStrings);
887 }
888 //Free global string memory space, End
889 /////////////////////////////////////////////////////////////
890
891 for (uint32_t i = 0; i < kernelInfo->surfaceCount; i++) {
892 if (kernelInfo->surface[i].attributeCount && kernelInfo->surface[i].attributes) {
893 free(kernelInfo->surface[i].attributes);
894 }
895 }
896 if (kernelInfo->surface) {
897 free(kernelInfo->surface);
898 }
899
900 CmSafeDelete( kernelInfo );
901
902 m_kernelInfo.SetElement(index, nullptr);
903
904 return 0;
905 }
906 else
907 {
908 return kernelInfo->kernelInfoRefCount;
909 }
910 }
911 else
912 {
913 return 0;
914 }
915 }
916 else
917 {
918 return 0;
919 }
920 }
921
GetKernelInfoRefCount(uint32_t index,uint32_t & refCount)922 int32_t CmProgramRT::GetKernelInfoRefCount(uint32_t index, uint32_t& refCount)
923 {
924 CM_KERNEL_INFO* kernelInfo = nullptr;
925
926 refCount = 0;
927
928 if( index < m_kernelCount )
929 {
930 kernelInfo =(CM_KERNEL_INFO *) m_kernelInfo.GetElement( index ) ;
931 if (kernelInfo)
932 {
933 refCount = kernelInfo->kernelInfoRefCount;
934 return CM_SUCCESS;
935 }
936 else
937 {
938 return CM_FAILURE;
939 }
940 }
941 else
942 {
943 return CM_FAILURE;
944 }
945 }
946
GetCISAVersion(uint32_t & majorVersion,uint32_t & minorVersion)947 int32_t CmProgramRT::GetCISAVersion(uint32_t& majorVersion, uint32_t& minorVersion)
948 {
949 majorVersion = m_cisaMajorVersion;
950 minorVersion = m_cisaMinorVersion;
951
952 return CM_SUCCESS;
953 }
954
GetProgramIndex()955 uint32_t CmProgramRT::GetProgramIndex()
956 {
957 return m_programIndex;
958 }
959
getISAfile()960 vISA::ISAfile *CmProgramRT::getISAfile()
961 {
962 return m_isaFile;
963 }
964
965 template <typename T>
hashCombine(uint32_t & res,const T & field)966 inline void hashCombine(uint32_t &res, const T &field)
967 {
968 std::hash<T> hasher;
969 res ^= hasher(field) + 0x9e3779b9 + (res << 6) + (res >> 2);
970 }
971
hashCombineString(uint32_t & res,char * str)972 inline void hashCombineString(uint32_t &res, char *str)
973 {
974 uint32_t strHash = std::hash<std::string>{}(std::string(str));
975 hashCombine(res, strHash);
976 }
977
GetKernelInfoHash(CM_KERNEL_INFO * kernelInfo)978 uint32_t CmProgramRT::GetKernelInfoHash(CM_KERNEL_INFO *kernelInfo)
979 {
980 uint32_t value = 0;
981 hashCombineString(value, kernelInfo->kernelName);
982 hashCombine(value, kernelInfo->inputCountOffset);
983 hashCombine(value, kernelInfo->kernelIsaOffset);
984 hashCombine(value, kernelInfo->kernelIsaSize);
985 uint8_t *kernelBin = nullptr;
986 uint32_t kernelSize = 0;
987 if (m_isJitterEnabled)
988 {
989 kernelBin = (uint8_t *)kernelInfo->jitBinaryCode;
990 kernelSize = kernelInfo->jitBinarySize;
991 }
992 else
993 {
994 kernelBin = m_programCode + kernelInfo->genxBinaryOffset;
995 kernelSize = kernelInfo->genxBinarySize;
996 }
997 uint32_t *kernelBinDW = (uint32_t *)kernelBin;
998 uint32_t kernelSizeDW = kernelSize / 4;
999 double step = (double)kernelSizeDW/64.0;
1000
1001 for (int i = 0; i < 256; i ++)
1002 {
1003 int index = (int)(kernelSizeDW - 1 - i*step);
1004 if (index < 0)
1005 {
1006 index = 0;
1007 }
1008 hashCombine(value, kernelBinDW[index]);
1009 }
1010 hashCombine(value, kernelSize);
1011
1012 return value;
1013 }
1014
1015 }
1016