1 //
2 // Copyright (c) 2023 The Khronos Group Inc.
3 //
4 // Licensed under the Apache License, Version 2.0 (the "License");
5 // you may not use this file except in compliance with the License.
6 // You may obtain a copy of the License at
7 //
8 // http://www.apache.org/licenses/LICENSE-2.0
9 //
10 // Unless required by applicable law or agreed to in writing, software
11 // distributed under the License is distributed on an "AS IS" BASIS,
12 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 // See the License for the specific language governing permissions and
14 // limitations under the License.
15 //
16 #include "harness/compat.h"
17
18 #include <stdio.h>
19 #include <stdlib.h>
20 #include <string.h>
21 #include <sys/types.h>
22 #include <sys/stat.h>
23 #include <vector>
24
25 #include "procs.h"
26 #include "harness/conversions.h"
27
28 // clang-format off
29
30 static const char *async_strided_global_to_local_kernel =
31 "%s\n" // optional pragma string
32 "__kernel void test_fn( const __global %s *src, __global %s *dst, __local %s *localBuffer, int copiesPerWorkgroup, int copiesPerWorkItem, int stride )\n"
33 "{\n"
34 " int i;\n"
35 // Zero the local storage first
36 " for(i=0; i<copiesPerWorkItem; i++)\n"
37 " localBuffer[ get_local_id( 0 )*copiesPerWorkItem+i ] = (%s)(%s)0;\n"
38 // Do this to verify all kernels are done zeroing the local buffer before we try the copy
39 " barrier( CLK_LOCAL_MEM_FENCE );\n"
40 " event_t event;\n"
41 " event = async_work_group_strided_copy( (__local %s*)localBuffer, (__global const %s*)(src+copiesPerWorkgroup*stride*get_group_id(0)), (size_t)copiesPerWorkgroup, (size_t)stride, 0 );\n"
42 // Wait for the copy to complete, then verify by manually copying to the dest
43 " wait_group_events( 1, &event );\n"
44 " for(i=0; i<copiesPerWorkItem; i++)\n"
45 " dst[ get_global_id( 0 )*copiesPerWorkItem*stride+i*stride ] = localBuffer[ get_local_id( 0 )*copiesPerWorkItem+i ];\n"
46 "}\n" ;
47
48 static const char *async_strided_local_to_global_kernel =
49 "%s\n" // optional pragma string
50 "__kernel void test_fn( const __global %s *src, __global %s *dst, __local %s *localBuffer, int copiesPerWorkgroup, int copiesPerWorkItem, int stride )\n"
51 "{\n"
52 " int i;\n"
53 // Zero the local storage first
54 " for(i=0; i<copiesPerWorkItem; i++)\n"
55 " localBuffer[ get_local_id( 0 )*copiesPerWorkItem+i ] = (%s)(%s)0;\n"
56 // Do this to verify all kernels are done zeroing the local buffer before we try the copy
57 " barrier( CLK_LOCAL_MEM_FENCE );\n"
58 " for(i=0; i<copiesPerWorkItem; i++)\n"
59 " localBuffer[ get_local_id( 0 )*copiesPerWorkItem+i ] = src[ get_global_id( 0 )*copiesPerWorkItem*stride+i*stride ];\n"
60 // Do this to verify all kernels are done copying to the local buffer before we try the copy
61 " barrier( CLK_LOCAL_MEM_FENCE );\n"
62 " event_t event;\n"
63 " event = async_work_group_strided_copy((__global %s*)(dst+copiesPerWorkgroup*stride*get_group_id(0)), (__local const %s*)localBuffer, (size_t)copiesPerWorkgroup, (size_t)stride, 0 );\n"
64 " wait_group_events( 1, &event );\n"
65 "}\n" ;
66
67 // clang-format on
68
test_strided_copy(cl_device_id deviceID,cl_context context,cl_command_queue queue,const char * kernelCode,ExplicitType vecType,int vecSize,int stride)69 int test_strided_copy(cl_device_id deviceID, cl_context context, cl_command_queue queue, const char *kernelCode, ExplicitType vecType, int vecSize, int stride)
70 {
71 int error;
72 clProgramWrapper program;
73 clKernelWrapper kernel;
74 clMemWrapper streams[ 2 ];
75 size_t threads[ 1 ], localThreads[ 1 ];
76 MTdataHolder d(gRandomSeed);
77 char vecNameString[64]; vecNameString[0] = 0;
78
79 if (vecSize == 1)
80 sprintf(vecNameString, "%s", get_explicit_type_name(vecType));
81 else
82 sprintf(vecNameString, "%s%d", get_explicit_type_name(vecType), vecSize);
83
84
85 log_info("Testing %s\n", vecNameString);
86
87 cl_long max_local_mem_size;
88 error = clGetDeviceInfo(deviceID, CL_DEVICE_LOCAL_MEM_SIZE, sizeof(max_local_mem_size), &max_local_mem_size, NULL);
89 test_error( error, "clGetDeviceInfo for CL_DEVICE_LOCAL_MEM_SIZE failed.");
90
91 unsigned int num_of_compute_devices;
92 error = clGetDeviceInfo(deviceID, CL_DEVICE_MAX_COMPUTE_UNITS, sizeof(num_of_compute_devices), &num_of_compute_devices, NULL);
93 test_error( error, "clGetDeviceInfo for CL_DEVICE_MAX_COMPUTE_UNITS failed.");
94
95 char programSource[4096]; programSource[0]=0;
96 char *programPtr;
97
98 std::string extStr = "";
99 if (vecType == kDouble)
100 extStr = "#pragma OPENCL EXTENSION cl_khr_fp64 : enable";
101 else if (vecType == kHalf)
102 extStr = "#pragma OPENCL EXTENSION cl_khr_fp16 : enable";
103
104 sprintf(programSource, kernelCode, extStr.c_str(), vecNameString,
105 vecNameString, vecNameString, vecNameString,
106 get_explicit_type_name(vecType), vecNameString, vecNameString);
107 //log_info("program: %s\n", programSource);
108 programPtr = programSource;
109
110 error = create_single_kernel_helper( context, &program, &kernel, 1, (const char **)&programPtr, "test_fn" );
111 test_error( error, "Unable to create testing kernel" );
112
113 size_t max_workgroup_size;
114 error = clGetKernelWorkGroupInfo(kernel, deviceID, CL_KERNEL_WORK_GROUP_SIZE, sizeof(max_workgroup_size), &max_workgroup_size, NULL);
115 test_error (error, "clGetKernelWorkGroupInfo failed for CL_KERNEL_WORK_GROUP_SIZE.");
116
117 size_t max_local_workgroup_size[3];
118 error = clGetDeviceInfo(deviceID, CL_DEVICE_MAX_WORK_ITEM_SIZES, sizeof(max_local_workgroup_size), max_local_workgroup_size, NULL);
119 test_error (error, "clGetDeviceInfo failed for CL_DEVICE_MAX_WORK_ITEM_SIZES");
120
121 // Pick the minimum of the device and the kernel
122 if (max_workgroup_size > max_local_workgroup_size[0])
123 max_workgroup_size = max_local_workgroup_size[0];
124
125 size_t elementSize = get_explicit_type_size(vecType)* ((vecSize == 3) ? 4 : vecSize);
126
127 cl_ulong max_global_mem_size;
128 error = clGetDeviceInfo(deviceID, CL_DEVICE_GLOBAL_MEM_SIZE, sizeof(max_global_mem_size), &max_global_mem_size, NULL);
129 test_error (error, "clGetDeviceInfo failed for CL_DEVICE_GLOBAL_MEM_SIZE");
130
131 if (max_global_mem_size > (cl_ulong)SIZE_MAX) {
132 max_global_mem_size = (cl_ulong)SIZE_MAX;
133 }
134
135 cl_bool unified_mem;
136 error = clGetDeviceInfo(deviceID, CL_DEVICE_HOST_UNIFIED_MEMORY, sizeof(unified_mem), &unified_mem, NULL);
137 test_error (error, "clGetDeviceInfo failed for CL_DEVICE_HOST_UNIFIED_MEMORY");
138
139 int number_of_global_mem_buffers = (unified_mem) ? 4 : 2;
140
141 size_t numberOfCopiesPerWorkitem = 3;
142 size_t localStorageSpacePerWorkitem = numberOfCopiesPerWorkitem*elementSize;
143 size_t maxLocalWorkgroupSize = (((int)max_local_mem_size/2)/localStorageSpacePerWorkitem);
144
145 size_t localWorkgroupSize = maxLocalWorkgroupSize;
146 if (maxLocalWorkgroupSize > max_workgroup_size)
147 localWorkgroupSize = max_workgroup_size;
148
149 size_t localBufferSize = localWorkgroupSize*elementSize*numberOfCopiesPerWorkitem;
150 size_t numberOfLocalWorkgroups = 579;//1111;
151
152 // Reduce the numberOfLocalWorkgroups so that no more than 1/2 of CL_DEVICE_GLOBAL_MEM_SIZE is consumed
153 // by the allocated buffer. This is done to avoid resource errors resulting from address space fragmentation.
154 size_t numberOfLocalWorkgroupsLimit = max_global_mem_size / (2 * number_of_global_mem_buffers * localBufferSize * stride);
155 if (numberOfLocalWorkgroups > numberOfLocalWorkgroupsLimit) numberOfLocalWorkgroups = numberOfLocalWorkgroupsLimit;
156
157 size_t globalBufferSize = numberOfLocalWorkgroups*localBufferSize*stride;
158 size_t globalWorkgroupSize = numberOfLocalWorkgroups*localWorkgroupSize;
159
160 std::vector<unsigned char> inBuffer(globalBufferSize);
161 std::vector<unsigned char> outBuffer(globalBufferSize);
162 memset(outBuffer.data(), 0, globalBufferSize);
163
164 cl_int copiesPerWorkItemInt, copiesPerWorkgroup;
165 copiesPerWorkItemInt = (int)numberOfCopiesPerWorkitem;
166 copiesPerWorkgroup = (int)(numberOfCopiesPerWorkitem*localWorkgroupSize);
167
168 log_info("Global: %d, local %d, local buffer %db, global buffer %db, copy stride %d, each work group will copy %d elements and each work item item will copy %d elements.\n",
169 (int) globalWorkgroupSize, (int)localWorkgroupSize, (int)localBufferSize, (int)globalBufferSize, (int)stride, copiesPerWorkgroup, copiesPerWorkItemInt);
170
171 threads[0] = globalWorkgroupSize;
172 localThreads[0] = localWorkgroupSize;
173
174 generate_random_data(vecType,
175 globalBufferSize / get_explicit_type_size(vecType), d,
176 inBuffer.data());
177
178 streams[0] = clCreateBuffer(context, CL_MEM_COPY_HOST_PTR, globalBufferSize,
179 inBuffer.data(), &error);
180 test_error( error, "Unable to create input buffer" );
181 streams[1] = clCreateBuffer(context, CL_MEM_COPY_HOST_PTR, globalBufferSize,
182 outBuffer.data(), &error);
183 test_error( error, "Unable to create output buffer" );
184
185 error = clSetKernelArg( kernel, 0, sizeof( streams[ 0 ] ), &streams[ 0 ] );
186 test_error( error, "Unable to set kernel argument" );
187 error = clSetKernelArg( kernel, 1, sizeof( streams[ 1 ] ), &streams[ 1 ] );
188 test_error( error, "Unable to set kernel argument" );
189 error = clSetKernelArg( kernel, 2, localBufferSize, NULL );
190 test_error( error, "Unable to set kernel argument" );
191 error = clSetKernelArg( kernel, 3, sizeof(copiesPerWorkgroup), &copiesPerWorkgroup );
192 test_error( error, "Unable to set kernel argument" );
193 error = clSetKernelArg( kernel, 4, sizeof(copiesPerWorkItemInt), &copiesPerWorkItemInt );
194 test_error( error, "Unable to set kernel argument" );
195 error = clSetKernelArg( kernel, 5, sizeof(stride), &stride );
196 test_error( error, "Unable to set kernel argument" );
197
198 // Enqueue
199 error = clEnqueueNDRangeKernel( queue, kernel, 1, NULL, threads, localThreads, 0, NULL, NULL );
200 test_error( error, "Unable to queue kernel" );
201
202 // Read
203 error = clEnqueueReadBuffer(queue, streams[1], CL_TRUE, 0, globalBufferSize,
204 outBuffer.data(), 0, NULL, NULL);
205 test_error( error, "Unable to read results" );
206
207 // Verify
208 size_t typeSize = get_explicit_type_size(vecType)* vecSize;
209 for (int i=0; i<(int)globalBufferSize; i+=(int)elementSize*(int)stride)
210 {
211 if (memcmp(&inBuffer.at(i), &outBuffer.at(i), typeSize) != 0)
212 {
213 unsigned char *inchar =
214 static_cast<unsigned char *>(&inBuffer.at(i));
215 unsigned char *outchar =
216 static_cast<unsigned char *>(&outBuffer.at(i));
217 char values[4096];
218 values[0] = 0;
219
220 log_error( "ERROR: Results of copy did not validate!\n" );
221 sprintf(values + strlen( values), "%d -> [", i);
222 for (int j=0; j<(int)elementSize; j++)
223 sprintf(values + strlen( values), "%2x ", inchar[j]);
224 sprintf(values + strlen(values), "] != [");
225 for (int j=0; j<(int)elementSize; j++)
226 sprintf(values + strlen( values), "%2x ", outchar[j]);
227 sprintf(values + strlen(values), "]");
228 log_error("%s\n", values);
229 return -1;
230 }
231 }
232
233 return 0;
234 }
235
test_strided_copy_all_types(cl_device_id deviceID,cl_context context,cl_command_queue queue,const char * kernelCode)236 int test_strided_copy_all_types(cl_device_id deviceID, cl_context context, cl_command_queue queue, const char *kernelCode)
237 {
238 const std::vector<ExplicitType> vecType = { kChar, kUChar, kShort, kUShort,
239 kInt, kUInt, kLong, kULong,
240 kFloat, kHalf, kDouble };
241 const unsigned int vecSizes[] = { 1, 2, 3, 4, 8, 16, 0 };
242 const unsigned int strideSizes[] = { 1, 3, 4, 5, 0 };
243 unsigned int size, typeIndex, stride;
244
245 int errors = 0;
246
247 bool fp16Support = is_extension_available(deviceID, "cl_khr_fp16");
248 bool fp64Support = is_extension_available(deviceID, "cl_khr_fp64");
249
250 for (typeIndex = 0; typeIndex < vecType.size(); typeIndex++)
251 {
252 if (( vecType[ typeIndex ] == kLong || vecType[ typeIndex ] == kULong ) && !gHasLong )
253 continue;
254 else if (vecType[typeIndex] == kDouble && !fp64Support)
255 continue;
256 else if (vecType[typeIndex] == kHalf && !fp16Support)
257 continue;
258
259 for( size = 0; vecSizes[ size ] != 0; size++ )
260 {
261 for( stride = 0; strideSizes[ stride ] != 0; stride++)
262 {
263 if (test_strided_copy( deviceID, context, queue, kernelCode, vecType[typeIndex], vecSizes[size], strideSizes[stride] ))
264 {
265 errors++;
266 }
267 }
268 }
269 }
270 if (errors)
271 return -1;
272 return 0;
273 }
274
test_async_strided_copy_global_to_local(cl_device_id deviceID,cl_context context,cl_command_queue queue,int num_elements)275 int test_async_strided_copy_global_to_local(cl_device_id deviceID, cl_context context, cl_command_queue queue, int num_elements)
276 {
277 return test_strided_copy_all_types( deviceID, context, queue, async_strided_global_to_local_kernel );
278 }
279
test_async_strided_copy_local_to_global(cl_device_id deviceID,cl_context context,cl_command_queue queue,int num_elements)280 int test_async_strided_copy_local_to_global(cl_device_id deviceID, cl_context context, cl_command_queue queue, int num_elements)
281 {
282 return test_strided_copy_all_types( deviceID, context, queue, async_strided_local_to_global_kernel );
283 }
284
285