1 #include <gtest/gtest.h>
2
3 #include <ATen/ATen.h>
4 #include <ATen/core/Reduction.h>
5 #include <torch/cuda.h>
6 #include <ATen/test/test_assert.h>
7 #include <c10/util/irange.h>
8 #include <c10/util/CallOnce.h>
9
10 // for TH compat test only...
11 struct THFloatTensor;
12
13 #include <iostream>
14 #include <chrono>
15 // NOLINTNEXTLINE(modernize-deprecated-headers)
16 #include <string.h>
17 #include <sstream>
18 #include <thread>
19 #include <mutex>
20
21 #define ASSERT_EQ_RESOLVED(X, Y) \
22 { \
23 bool isEQ = X == Y; \
24 ASSERT_TRUE(isEQ); \
25 }
26
27 using namespace at;
28
TestResize(DeprecatedTypeProperties & type)29 void TestResize(DeprecatedTypeProperties& type) {
30 auto a = at::empty({0}, type.options());
31 a.resize_({3, 4});
32 ASSERT_EQ_RESOLVED(a.numel(), 12);
33 a.resize_({5, 7});
34 ASSERT_EQ_RESOLVED(a.numel(), 35);
35 }
36
TestOnesAndDot(DeprecatedTypeProperties & type)37 void TestOnesAndDot(DeprecatedTypeProperties& type) {
38 Tensor b0 = ones({1, 1}, type);
39 ASSERT_EQ_RESOLVED((b0 + b0).sum().item<double>(), 2);
40
41 Tensor b1 = ones({1, 2}, type);
42 ASSERT_EQ_RESOLVED((b1 + b1).sum().item<double>(), 4);
43
44 Tensor b = ones({3, 4}, type);
45 ASSERT_EQ_RESOLVED((b + b).sum().item<double>(), 24);
46 ASSERT_EQ_RESOLVED(b.numel(), 12);
47 if (type.backend() != Backend::CPU || type.scalarType() != kHalf) {
48 ASSERT_EQ_RESOLVED(b.view(-1).dot(b.view(-1)).item<double>(), 12);
49 }
50 }
51
TestSort(DeprecatedTypeProperties & type)52 void TestSort(DeprecatedTypeProperties& type) {
53 Tensor b = rand({3, 4}, type);
54
55 auto z = b.sort(1);
56 auto z_sorted = std::get<0>(z);
57
58 bool isLT = z_sorted[0][0].item<float>() < z_sorted[0][1].item<float>();
59 ASSERT_TRUE(isLT);
60 }
61
TestRandperm(DeprecatedTypeProperties & type)62 void TestRandperm(DeprecatedTypeProperties& type) {
63 if (type.backend() != Backend::CUDA) {
64 Tensor b = randperm(15, type);
65 auto [rv, ri] = sort(b, 0);
66 bool isLE = (rv[0].item<float>() <= rv[1].item<float>());
67 ASSERT_TRUE(isLE);
68 }
69 }
70
SendContext()71 void SendContext() {
72 std::stringstream ss;
73 ss << "context: " << std::hex << (int64_t)&globalContext() << std::endl;
74 }
75
TestAdd(DeprecatedTypeProperties & type)76 void TestAdd(DeprecatedTypeProperties& type) {
77 Tensor a = rand({3, 4}, type);
78 Tensor b = rand({3, 4}, type);
79 Tensor c = add(a, add(a, b));
80 // TODO:0-dim Tensor d(3.f);
81 Scalar d = 3.f;
82 if (type.backend() == Backend::CPU && type.scalarType() == kHalf) {
83 ASSERT_TRUE(add(c, d).allclose(a + a + b + d, 1e-2));
84 } else {
85 ASSERT_TRUE(add(c, d).allclose(a + a + b + d));
86 }
87 }
88
TestZeros(DeprecatedTypeProperties & type)89 void TestZeros(DeprecatedTypeProperties& type) {
90 auto begin = std::chrono::high_resolution_clock::now();
91 Tensor a = zeros({1024, 1024}, type);
92 for (C10_UNUSED const auto i : c10::irange(1, 1000)) {
93 a = zeros({128, 128}, type);
94 }
95 auto end = std::chrono::high_resolution_clock::now();
96 std::cout << std::dec << " "
97 << std::chrono::duration_cast<std::chrono::milliseconds>(
98 end - begin)
99 .count()
100 << " ms" << std::endl;
101
102 std::srand(std::time(nullptr));
103 ASSERT_EQ(norm(a).item<double>(), 0.0);
104 }
105
TestLoadsOfAdds(DeprecatedTypeProperties & type)106 void TestLoadsOfAdds(DeprecatedTypeProperties& type) {
107 auto begin = std::chrono::high_resolution_clock::now();
108 Tensor d = ones({3, 4}, type);
109 Tensor r = zeros({3, 4}, type);
110 for (C10_UNUSED const auto i : c10::irange(1000)) {
111 add_out(r, r, d);
112 }
113 auto end = std::chrono::high_resolution_clock::now();
114 // TODO TEST PERF?
115 std::cout << std::dec << " "
116 << std::chrono::duration_cast<std::chrono::milliseconds>(
117 end - begin)
118 .count()
119 << " ms" << std::endl;
120 ASSERT_EQ_RESOLVED(norm(1000 * d).item<double>(), norm(r).item<double>());
121 }
122
TestLoadOfAddsWithCopy(DeprecatedTypeProperties & type)123 void TestLoadOfAddsWithCopy(DeprecatedTypeProperties& type) {
124 auto begin = std::chrono::high_resolution_clock::now();
125 Tensor d = ones({3, 4}, type);
126 Tensor r = zeros({3, 4}, type);
127 for (C10_UNUSED const auto i : c10::irange(1000)) {
128 r = add(r, d);
129 }
130 auto end = std::chrono::high_resolution_clock::now();
131 // TODO TEST PERF?
132 std::cout << std::dec << " "
133 << std::chrono::duration_cast<std::chrono::milliseconds>(
134 end - begin)
135 .count()
136 << " ms" << std::endl;
137 ASSERT_EQ_RESOLVED(norm(1000 * d).item<double>(), norm(r).item<double>());
138 }
139
TestIsContiguous(DeprecatedTypeProperties & type)140 void TestIsContiguous(DeprecatedTypeProperties& type) {
141 Tensor a = rand({3, 4}, type);
142 ASSERT_TRUE(a.is_contiguous());
143 a = a.transpose(0, 1);
144 ASSERT_FALSE(a.is_contiguous());
145 }
146
TestPermute(DeprecatedTypeProperties & type)147 void TestPermute(DeprecatedTypeProperties& type) {
148 Tensor a = rand({3, 4, 5}, type);
149 Tensor b = a.permute({1, 2, 0});
150 ASSERT_TRUE(b.sizes().equals({4, 5, 3}));
151 ASSERT_TRUE(b.strides().equals({5, 1, 20}));
152 }
153
TestMm(DeprecatedTypeProperties & type)154 void TestMm(DeprecatedTypeProperties& type) {
155 if (type.backend() != Backend::CPU || type.scalarType() != kHalf) {
156 Tensor a = rand({3, 4}, type);
157 Tensor b = rand({4}, type);
158 Tensor c = mv(a, b);
159 ASSERT_TRUE(c.equal(addmv(zeros({3}, type), a, b, 0, 1)));
160 }
161 }
162
TestSqueeze(DeprecatedTypeProperties & type)163 void TestSqueeze(DeprecatedTypeProperties& type) {
164 Tensor a = rand({2, 1}, type);
165 Tensor b = squeeze(a);
166 ASSERT_EQ_RESOLVED(b.dim(), 1);
167 a = rand({1}, type);
168 b = squeeze(a);
169 // TODO 0-dim squeeze
170 ASSERT_TRUE(a[0].equal(b));
171 }
172
TestCopy(DeprecatedTypeProperties & type)173 void TestCopy(DeprecatedTypeProperties& type) {
174 Tensor a = zeros({4, 3}, type);
175 Tensor e = rand({4, 3}, type);
176 a.copy_(e);
177 ASSERT_TRUE(a.equal(e));
178 }
179
TestCopyBroadcasting(DeprecatedTypeProperties & type)180 void TestCopyBroadcasting(DeprecatedTypeProperties& type) {
181 Tensor a = zeros({4, 3}, type);
182 Tensor e = rand({3}, type);
183 a.copy_(e);
184 for (const auto i : c10::irange(4)) {
185 ASSERT_TRUE(a[i].equal(e));
186 }
187 }
TestAbsValue(DeprecatedTypeProperties & type)188 void TestAbsValue(DeprecatedTypeProperties& type) {
189 Tensor r = at::abs(at::scalar_tensor(-3, type.options()));
190 ASSERT_EQ_RESOLVED(r.item<int32_t>(), 3);
191 }
192 /*
193 TODO(zach): operator overloads
194 #if 0
195 {
196 std::cout << "eq (value):" << std::endl;
197 Tensor a = Tensor(10.f);
198 std::cout << (a == 11_i64) << " -- should be 0" << std::endl;
199 std::cout << (a == 10_i64) << " -- should be 1" << std::endl;
200 std::cout << (a == 10.) << " -- should be 1" << std::endl;
201 }
202 #endif
203 */
204
TestAddingAValueWithScalar(DeprecatedTypeProperties & type)205 void TestAddingAValueWithScalar(DeprecatedTypeProperties& type) {
206 Tensor a = rand({4, 3}, type);
207 ASSERT_TRUE((ones({4, 3}, type) + a).equal(add(a, 1)));
208 }
209
TestSelect(DeprecatedTypeProperties & type)210 void TestSelect(DeprecatedTypeProperties& type) {
211 Tensor a = rand({3, 7}, type);
212 auto a_13 = select(a, 1, 3);
213 auto a_13_02 = select(select(a, 1, 3), 0, 2);
214 ASSERT_TRUE(a[0][3].equal(a_13[0]));
215 ASSERT_TRUE(a[2][3].equal(a_13_02));
216 }
217
TestZeroDim(DeprecatedTypeProperties & type)218 void TestZeroDim(DeprecatedTypeProperties& type) {
219 Tensor a = at::scalar_tensor(4, type.options()); // rand(type, {1});
220
221 Tensor b = rand({3, 4}, type);
222 ASSERT_EQ_RESOLVED((a + a).dim(), 0);
223 ASSERT_EQ_RESOLVED((1 + a).dim(), 0);
224 ASSERT_EQ_RESOLVED((b + a).dim(), 2);
225 ASSERT_EQ_RESOLVED((a + b).dim(), 2);
226 auto c = rand({3, 4}, type);
227 ASSERT_EQ_RESOLVED(c[1][2].dim(), 0);
228
229 auto f = rand({3, 4}, type);
230 f[2] = zeros({4}, type);
231 f[1][0] = -1;
232 ASSERT_EQ_RESOLVED(f[2][0].item<double>(), 0);
233 }
234
TestToCFloat()235 void TestToCFloat() {
236 Tensor a = zeros({3, 4});
237 Tensor b = ones({3, 7});
238 Tensor c = cat({a, b}, 1);
239 ASSERT_EQ_RESOLVED(c.size(1), 11);
240
241 Tensor e = rand({});
242 ASSERT_EQ_RESOLVED(*e.data_ptr<float>(), e.sum().item<float>());
243 }
TestToString()244 void TestToString() {
245 Tensor b = ones({3, 7}) * .0000001f;
246 std::stringstream s;
247 s << b << "\n";
248 std::string expect = "1e-07 *";
249 ASSERT_EQ_RESOLVED(s.str().substr(0, expect.size()), expect);
250 }
251
TestIndexingByScalar()252 void TestIndexingByScalar() {
253 Tensor tensor = arange(0, 10, kInt);
254 Tensor one = ones({}, kInt);
255 for (const auto i : c10::irange(tensor.numel())) {
256 ASSERT_TRUE(tensor[i].equal(one * i));
257 }
258 for (size_t i = 0; i < static_cast<uint64_t>(tensor.numel()); ++i) {
259 ASSERT_TRUE(tensor[i].equal(one * static_cast<int64_t>(i)));
260 }
261 for (const auto i : c10::irange(tensor.numel())) {
262 ASSERT_TRUE(tensor[i].equal(one * i));
263 }
264 // NOLINTNEXTLINE(bugprone-too-small-loop-variable)
265 for (int16_t i = 0; i < tensor.numel(); ++i) {
266 ASSERT_TRUE(tensor[i].equal(one * i));
267 }
268 // NOLINTNEXTLINE(bugprone-too-small-loop-variable)
269 for (int8_t i = 0; i < tensor.numel(); ++i) {
270 ASSERT_TRUE(tensor[i].equal(one * i));
271 }
272 // Throw StartsWith("Can only index tensors with integral scalars")
273 // NOLINTNEXTLINE(hicpp-avoid-goto,cppcoreguidelines-avoid-magic-numbers,cppcoreguidelines-avoid-goto)
274 ASSERT_ANY_THROW(tensor[Scalar(3.14)].equal(one));
275 }
276
TestIndexingByZerodimTensor()277 void TestIndexingByZerodimTensor() {
278 Tensor tensor = arange(0, 10, kInt);
279 Tensor one = ones({}, kInt);
280 for (const auto i : c10::irange(tensor.numel())) {
281 ASSERT_TRUE(tensor[one * i].equal(one * i));
282 }
283 // Throw StartsWith(
284 // "Can only index tensors with integral scalars")
285 // NOLINTNEXTLINE(hicpp-avoid-goto,cppcoreguidelines-avoid-magic-numbers,cppcoreguidelines-avoid-goto)
286 ASSERT_ANY_THROW(tensor[ones({}) * 3.14].equal(one));
287 // Throw StartsWith("Can only index with tensors that are defined")
288 // NOLINTNEXTLINE(hicpp-avoid-goto,cppcoreguidelines-avoid-goto)
289 ASSERT_ANY_THROW(tensor[Tensor()].equal(one));
290 // Throw StartsWith("Can only index with tensors that are scalars (zero-dim)")
291 // NOLINTNEXTLINE(hicpp-avoid-goto,cppcoreguidelines-avoid-goto)
292 ASSERT_ANY_THROW(tensor[ones({2, 3, 4}, kInt)].equal(one));
293 }
TestIndexingMixedDevice(DeprecatedTypeProperties & type)294 void TestIndexingMixedDevice(DeprecatedTypeProperties& type) {
295 Tensor tensor = randn({20, 20}, type);
296 Tensor index = arange(10, kLong).cpu();
297 Tensor result = tensor.index({index});
298 ASSERT_TRUE(result[0].equal(tensor[0]));
299 }
TestDispatch()300 void TestDispatch() {
301 Tensor tensor = randn({20, 20});
302 Tensor other = randn({20, 20});
303 auto result = tensor.m(relu).m(mse_loss, other, at::Reduction::Mean);
304 ASSERT_TRUE(result.allclose(mse_loss(relu(tensor), other)));
305 }
306
TestNegativeDim(DeprecatedTypeProperties & type)307 void TestNegativeDim(DeprecatedTypeProperties& type) {
308 // NOLINTNEXTLINE(hicpp-avoid-goto,cppcoreguidelines-avoid-goto)
309 ASSERT_ANY_THROW(empty({5, -5, 5}, type.options()));
310 // NOLINTNEXTLINE(hicpp-avoid-goto,cppcoreguidelines-avoid-goto)
311 ASSERT_ANY_THROW(empty({5, -5, -5}, type.options()));
312 Tensor tensor = empty({5, 5}, type.options());
313 // NOLINTNEXTLINE(hicpp-avoid-goto,cppcoreguidelines-avoid-goto)
314 ASSERT_ANY_THROW(tensor.reshape({-5, -5}));
315 }
316
TestView(DeprecatedTypeProperties & type)317 void TestView(DeprecatedTypeProperties& type) {
318 // Testing the tensor view path, which is different from
319 // the Variable view path, see https://github.com/pytorch/pytorch/pull/23452
320 // for details
321 Tensor tensor = randn({3, 4}, type);;
322 Tensor viewed = tensor.view({3, 4});
323 tensor.resize_({6, 2});
324 ASSERT_TRUE(tensor.sizes().equals({6, 2}));
325 ASSERT_TRUE(viewed.sizes().equals({3, 4}));
326 }
327
TestIntArrayRefExpansion(DeprecatedTypeProperties & type)328 void TestIntArrayRefExpansion(DeprecatedTypeProperties& type) {
329 if (type.backend() != Backend::CPU || type.scalarType() != kHalf) {
330 max_pool2d(randn({3, 3, 3, 3}, type.options()), 2, 1, 1, 1);
331 max_pool3d(randn({3, 3, 3, 3, 3}, type.options()), 2, 1, 1, 1);
332 avg_pool2d(randn({3, 3, 3, 3}, type.options()), 2, 1, 1);
333 avg_pool3d(randn({3, 3, 3, 3, 3}, type.options()), 2, 1, 1);
334 }
335 }
336
test(DeprecatedTypeProperties & type)337 void test(DeprecatedTypeProperties& type) {
338 TestResize(type);
339 TestOnesAndDot(type);
340
341 TestSort(type);
342 TestRandperm(type);
343 TestAdd(type);
344 TestZeros(type);
345 TestLoadsOfAdds(type);
346 TestLoadOfAddsWithCopy(type);
347 TestIsContiguous(type);
348 TestPermute(type);
349 TestMm(type);
350 TestSqueeze(type);
351 TestCopy(type);
352 TestCopyBroadcasting(type);
353 TestAbsValue(type);
354 TestAddingAValueWithScalar(type);
355 TestSelect(type);
356 TestZeroDim(type);
357 TestToCFloat();
358 TestToString();
359 TestIndexingByScalar();
360 TestIndexingByZerodimTensor();
361 TestIndexingMixedDevice(type);
362 TestDispatch();
363 TestNegativeDim(type);
364 TestView(type);
365 TestIntArrayRefExpansion(type);
366 }
367
TEST(BasicTest,BasicTestCPU)368 TEST(BasicTest, BasicTestCPU) {
369 manual_seed(123);
370
371 test(CPU(kFloat));
372 }
373
TEST(BasicTest,BasicTestHalfCPU)374 TEST(BasicTest, BasicTestHalfCPU) {
375 manual_seed(234);
376
377 test(CPU(kHalf));
378 }
379
TEST(BasicTest,BasicTestCUDA)380 TEST(BasicTest, BasicTestCUDA) {
381 manual_seed(123);
382
383 if (at::hasCUDA()) {
384 test(CUDA(kFloat));
385 }
386 }
387
TEST(BasicTest,FactoryMethodsTest)388 TEST(BasicTest, FactoryMethodsTest) {
389 // Test default values
390 at::Tensor tensor0 = at::empty({4});
391 ASSERT_EQ(tensor0.dtype(), at::kFloat);
392 ASSERT_EQ(tensor0.layout(), at::kStrided);
393 ASSERT_EQ(tensor0.device(), at::kCPU);
394 ASSERT_FALSE(tensor0.requires_grad());
395 ASSERT_FALSE(tensor0.is_pinned());
396
397 // Test setting requires_grad to false.
398 tensor0 = at::empty({4}, at::TensorOptions().requires_grad(false));
399 ASSERT_EQ(tensor0.dtype(), at::kFloat);
400 ASSERT_EQ(tensor0.layout(), at::kStrided);
401 ASSERT_EQ(tensor0.device(), at::kCPU);
402 ASSERT_FALSE(tensor0.requires_grad());
403 ASSERT_FALSE(tensor0.is_pinned());
404
405 // Test setting requires_grad to true.
406 // This is a bug. Requires_grad was set to TRUE but this is not implemented.
407 // NOLINTNEXTLINE(hicpp-avoid-goto,cppcoreguidelines-avoid-goto)
408 EXPECT_ANY_THROW(at::empty({4}, at::TensorOptions().requires_grad(true)));
409
410 // Test setting dtype
411 at::Tensor tensor1 = at::empty({4}, at::TensorOptions().dtype(at::kHalf));
412 ASSERT_EQ(tensor1.dtype(), at::kHalf);
413 ASSERT_EQ(tensor1.layout(), at::kStrided);
414 ASSERT_EQ(tensor1.device(), at::kCPU);
415 ASSERT_FALSE(tensor1.requires_grad());
416 ASSERT_FALSE(tensor1.is_pinned());
417
418 // Sparse tensor CPU test to avoid requiring CUDA to catch simple bugs.
419 // Sparse tensors do not work with static CPU dispatch.
420 #ifndef ATEN_CPU_STATIC_DISPATCH
421 tensor1 = at::empty({4}, at::TensorOptions().dtype(at::kHalf).layout(at::kSparse));
422 ASSERT_EQ(tensor1.dtype(), at::kHalf);
423 ASSERT_EQ(tensor1.layout(), at::kSparse);
424 ASSERT_EQ(tensor1.device(), at::kCPU);
425 ASSERT_FALSE(tensor1.requires_grad());
426 // NOLINTNEXTLINE(hicpp-avoid-goto,cppcoreguidelines-avoid-goto)
427 ASSERT_FALSE(tensor1.is_pinned());
428 #endif // ATEN_CPU_STATIC_DISPATCH
429
430 if (torch::cuda::is_available()) {
431 // Test setting pin memory
432 tensor1 = at::empty({4}, at::TensorOptions().pinned_memory(true));
433 ASSERT_EQ(tensor1.dtype(), at::kFloat);
434 ASSERT_EQ(tensor1.layout(), at::kStrided);
435 ASSERT_EQ(tensor1.device(), at::kCPU);
436 ASSERT_EQ(tensor1.requires_grad(), false);
437 ASSERT_FALSE(tensor1.device().is_cuda());
438 ASSERT_TRUE(tensor1.is_pinned());
439
440 // Test setting device
441 tensor1 = at::empty({4}, at::TensorOptions().device(at::kCUDA));
442 ASSERT_EQ(tensor1.dtype(), at::kFloat);
443 ASSERT_EQ(tensor1.layout(), at::kStrided);
444 ASSERT_TRUE(tensor1.device().is_cuda());
445 ASSERT_FALSE(tensor1.requires_grad());
446 ASSERT_FALSE(tensor1.is_pinned());
447
448 // Test set everything
449 tensor1 = at::empty({4}, at::TensorOptions().dtype(at::kHalf).device(at::kCUDA).layout(at::kSparse).requires_grad(false));
450 ASSERT_EQ(tensor1.dtype(), at::kHalf);
451 ASSERT_EQ(tensor1.layout(), at::kSparse);
452 ASSERT_TRUE(tensor1.device().is_cuda());
453 ASSERT_THROWS(tensor1.nbytes());
454
455 // This is a bug
456 // Issue https://github.com/pytorch/pytorch/issues/30405
457 ASSERT_FALSE(tensor1.requires_grad());
458 ASSERT_FALSE(tensor1.is_pinned());
459 }
460
461 // Test _like variants
462 if (torch::cuda::is_available()) {
463 // Issue https://github.com/pytorch/pytorch/issues/28093
464 at::Tensor proto = at::empty({1}, at::kDouble);
465 tensor0 = at::empty_like(proto, at::kCUDA);
466 ASSERT_EQ(tensor0.dtype(), at::kDouble);
467 ASSERT_EQ(tensor0.layout(), at::kStrided);
468 ASSERT_TRUE(tensor0.device().is_cuda());
469 ASSERT_FALSE(tensor0.requires_grad());
470 ASSERT_FALSE(tensor0.is_pinned());
471 }
472 }
473
TEST(BasicTest,BasicStdTestCPU)474 TEST(BasicTest, BasicStdTestCPU) {
475 c10::once_flag flag1, flag2;
476
477 auto simple_do_once = [&]()
478 {
479 c10::call_once(flag1, [](){ std::cout << "Simple example: called once\n"; });
480 };
481
482 auto may_throw_function = [&](bool do_throw)
483 {
484 if (do_throw) {
485 std::cout << "throw: call_once will retry\n"; // this may appear more than once
486 TORCH_CHECK(false, "throw exception");
487 }
488 std::cout << "Didn't throw, call_once will not attempt again\n"; // guaranteed once
489 };
490
491 auto do_once = [&](bool do_throw)
492 {
493 try {
494 c10::call_once(flag2, may_throw_function, do_throw);
495 }
496 catch (...) {
497 }
498 };
499
500 std::thread st1(simple_do_once);
501 std::thread st2(simple_do_once);
502 std::thread st3(simple_do_once);
503 std::thread st4(simple_do_once);
504 st1.join();
505 st2.join();
506 st3.join();
507 st4.join();
508
509 std::thread t1(do_once, true);
510 std::thread t2(do_once, true);
511 std::thread t3(do_once, false);
512 std::thread t4(do_once, true);
513 t1.join();
514 t2.join();
515 t3.join();
516 t4.join();
517 }
518