xref: /aosp_15_r20/external/pytorch/aten/src/ATen/test/basic.cpp (revision da0073e96a02ea20f0ac840b70461e3646d07c45)
1 #include <gtest/gtest.h>
2 
3 #include <ATen/ATen.h>
4 #include <ATen/core/Reduction.h>
5 #include <torch/cuda.h>
6 #include <ATen/test/test_assert.h>
7 #include <c10/util/irange.h>
8 #include <c10/util/CallOnce.h>
9 
10 // for TH compat test only...
11 struct THFloatTensor;
12 
13 #include <iostream>
14 #include <chrono>
15 // NOLINTNEXTLINE(modernize-deprecated-headers)
16 #include <string.h>
17 #include <sstream>
18 #include <thread>
19 #include <mutex>
20 
21 #define ASSERT_EQ_RESOLVED(X, Y) \
22   {                              \
23     bool isEQ = X == Y;          \
24     ASSERT_TRUE(isEQ);           \
25   }
26 
27 using namespace at;
28 
TestResize(DeprecatedTypeProperties & type)29 void TestResize(DeprecatedTypeProperties& type) {
30   auto a = at::empty({0}, type.options());
31   a.resize_({3, 4});
32   ASSERT_EQ_RESOLVED(a.numel(), 12);
33   a.resize_({5, 7});
34   ASSERT_EQ_RESOLVED(a.numel(), 35);
35 }
36 
TestOnesAndDot(DeprecatedTypeProperties & type)37 void TestOnesAndDot(DeprecatedTypeProperties& type) {
38   Tensor b0 = ones({1, 1}, type);
39   ASSERT_EQ_RESOLVED((b0 + b0).sum().item<double>(), 2);
40 
41   Tensor b1 = ones({1, 2}, type);
42   ASSERT_EQ_RESOLVED((b1 + b1).sum().item<double>(), 4);
43 
44   Tensor b = ones({3, 4}, type);
45   ASSERT_EQ_RESOLVED((b + b).sum().item<double>(), 24);
46   ASSERT_EQ_RESOLVED(b.numel(), 12);
47   if (type.backend() != Backend::CPU || type.scalarType() != kHalf) {
48     ASSERT_EQ_RESOLVED(b.view(-1).dot(b.view(-1)).item<double>(), 12);
49   }
50 }
51 
TestSort(DeprecatedTypeProperties & type)52 void TestSort(DeprecatedTypeProperties& type) {
53   Tensor b = rand({3, 4}, type);
54 
55   auto z = b.sort(1);
56   auto z_sorted = std::get<0>(z);
57 
58   bool isLT = z_sorted[0][0].item<float>() < z_sorted[0][1].item<float>();
59   ASSERT_TRUE(isLT);
60 }
61 
TestRandperm(DeprecatedTypeProperties & type)62 void TestRandperm(DeprecatedTypeProperties& type) {
63   if (type.backend() != Backend::CUDA) {
64     Tensor b = randperm(15, type);
65     auto [rv, ri] = sort(b, 0);
66     bool isLE = (rv[0].item<float>() <= rv[1].item<float>());
67     ASSERT_TRUE(isLE);
68   }
69 }
70 
SendContext()71 void SendContext() {
72   std::stringstream ss;
73   ss << "context: " << std::hex << (int64_t)&globalContext() << std::endl;
74 }
75 
TestAdd(DeprecatedTypeProperties & type)76 void TestAdd(DeprecatedTypeProperties& type) {
77   Tensor a = rand({3, 4}, type);
78   Tensor b = rand({3, 4}, type);
79   Tensor c = add(a, add(a, b));
80   // TODO:0-dim Tensor d(3.f);
81   Scalar d = 3.f;
82   if (type.backend() == Backend::CPU && type.scalarType() == kHalf) {
83       ASSERT_TRUE(add(c, d).allclose(a + a + b + d, 1e-2));
84   } else {
85       ASSERT_TRUE(add(c, d).allclose(a + a + b + d));
86   }
87 }
88 
TestZeros(DeprecatedTypeProperties & type)89 void TestZeros(DeprecatedTypeProperties& type) {
90   auto begin = std::chrono::high_resolution_clock::now();
91   Tensor a = zeros({1024, 1024}, type);
92   for (C10_UNUSED const auto i : c10::irange(1, 1000)) {
93     a = zeros({128, 128}, type);
94   }
95   auto end = std::chrono::high_resolution_clock::now();
96   std::cout << std::dec << "   "
97             << std::chrono::duration_cast<std::chrono::milliseconds>(
98                    end - begin)
99                    .count()
100             << " ms" << std::endl;
101 
102    std::srand(std::time(nullptr));
103    ASSERT_EQ(norm(a).item<double>(), 0.0);
104 }
105 
TestLoadsOfAdds(DeprecatedTypeProperties & type)106 void TestLoadsOfAdds(DeprecatedTypeProperties& type) {
107   auto begin = std::chrono::high_resolution_clock::now();
108   Tensor d = ones({3, 4}, type);
109   Tensor r = zeros({3, 4}, type);
110   for (C10_UNUSED const auto i : c10::irange(1000)) {
111     add_out(r, r, d);
112   }
113   auto end = std::chrono::high_resolution_clock::now();
114   // TODO TEST PERF?
115   std::cout << std::dec << "   "
116             << std::chrono::duration_cast<std::chrono::milliseconds>(
117                    end - begin)
118                    .count()
119             << " ms" << std::endl;
120   ASSERT_EQ_RESOLVED(norm(1000 * d).item<double>(), norm(r).item<double>());
121 }
122 
TestLoadOfAddsWithCopy(DeprecatedTypeProperties & type)123 void TestLoadOfAddsWithCopy(DeprecatedTypeProperties& type) {
124   auto begin = std::chrono::high_resolution_clock::now();
125   Tensor d = ones({3, 4}, type);
126   Tensor r = zeros({3, 4}, type);
127   for (C10_UNUSED const auto i : c10::irange(1000)) {
128     r = add(r, d);
129   }
130   auto end = std::chrono::high_resolution_clock::now();
131   // TODO TEST PERF?
132   std::cout << std::dec << "   "
133             << std::chrono::duration_cast<std::chrono::milliseconds>(
134                    end - begin)
135                    .count()
136             << " ms" << std::endl;
137   ASSERT_EQ_RESOLVED(norm(1000 * d).item<double>(), norm(r).item<double>());
138 }
139 
TestIsContiguous(DeprecatedTypeProperties & type)140 void TestIsContiguous(DeprecatedTypeProperties& type) {
141   Tensor a = rand({3, 4}, type);
142   ASSERT_TRUE(a.is_contiguous());
143   a = a.transpose(0, 1);
144   ASSERT_FALSE(a.is_contiguous());
145 }
146 
TestPermute(DeprecatedTypeProperties & type)147 void TestPermute(DeprecatedTypeProperties& type) {
148   Tensor a = rand({3, 4, 5}, type);
149   Tensor b = a.permute({1, 2, 0});
150   ASSERT_TRUE(b.sizes().equals({4, 5, 3}));
151   ASSERT_TRUE(b.strides().equals({5, 1, 20}));
152 }
153 
TestMm(DeprecatedTypeProperties & type)154 void TestMm(DeprecatedTypeProperties& type) {
155   if (type.backend() != Backend::CPU || type.scalarType() != kHalf) {
156     Tensor a = rand({3, 4}, type);
157     Tensor b = rand({4}, type);
158     Tensor c = mv(a, b);
159     ASSERT_TRUE(c.equal(addmv(zeros({3}, type), a, b, 0, 1)));
160   }
161 }
162 
TestSqueeze(DeprecatedTypeProperties & type)163 void TestSqueeze(DeprecatedTypeProperties& type) {
164   Tensor a = rand({2, 1}, type);
165   Tensor b = squeeze(a);
166   ASSERT_EQ_RESOLVED(b.dim(), 1);
167   a = rand({1}, type);
168   b = squeeze(a);
169   // TODO 0-dim squeeze
170   ASSERT_TRUE(a[0].equal(b));
171 }
172 
TestCopy(DeprecatedTypeProperties & type)173 void TestCopy(DeprecatedTypeProperties& type) {
174   Tensor a = zeros({4, 3}, type);
175   Tensor e = rand({4, 3}, type);
176   a.copy_(e);
177   ASSERT_TRUE(a.equal(e));
178 }
179 
TestCopyBroadcasting(DeprecatedTypeProperties & type)180 void TestCopyBroadcasting(DeprecatedTypeProperties& type) {
181   Tensor a = zeros({4, 3}, type);
182   Tensor e = rand({3}, type);
183   a.copy_(e);
184   for (const auto i : c10::irange(4)) {
185     ASSERT_TRUE(a[i].equal(e));
186   }
187 }
TestAbsValue(DeprecatedTypeProperties & type)188 void TestAbsValue(DeprecatedTypeProperties& type) {
189   Tensor r = at::abs(at::scalar_tensor(-3, type.options()));
190   ASSERT_EQ_RESOLVED(r.item<int32_t>(), 3);
191 }
192 /*
193    TODO(zach): operator overloads
194 #if 0
195 {
196 std::cout << "eq (value):" << std::endl;
197 Tensor a = Tensor(10.f);
198 std::cout << (a == 11_i64) << " -- should be 0" << std::endl;
199 std::cout << (a == 10_i64) << " -- should be 1" << std::endl;
200 std::cout << (a == 10.) << " -- should be 1" << std::endl;
201 }
202 #endif
203 */
204 
TestAddingAValueWithScalar(DeprecatedTypeProperties & type)205 void TestAddingAValueWithScalar(DeprecatedTypeProperties& type) {
206   Tensor a = rand({4, 3}, type);
207   ASSERT_TRUE((ones({4, 3}, type) + a).equal(add(a, 1)));
208 }
209 
TestSelect(DeprecatedTypeProperties & type)210 void TestSelect(DeprecatedTypeProperties& type) {
211   Tensor a = rand({3, 7}, type);
212   auto a_13 = select(a, 1, 3);
213   auto a_13_02 = select(select(a, 1, 3), 0, 2);
214   ASSERT_TRUE(a[0][3].equal(a_13[0]));
215   ASSERT_TRUE(a[2][3].equal(a_13_02));
216 }
217 
TestZeroDim(DeprecatedTypeProperties & type)218 void TestZeroDim(DeprecatedTypeProperties& type) {
219   Tensor a = at::scalar_tensor(4, type.options()); // rand(type, {1});
220 
221   Tensor b = rand({3, 4}, type);
222   ASSERT_EQ_RESOLVED((a + a).dim(), 0);
223   ASSERT_EQ_RESOLVED((1 + a).dim(), 0);
224   ASSERT_EQ_RESOLVED((b + a).dim(), 2);
225   ASSERT_EQ_RESOLVED((a + b).dim(), 2);
226   auto c = rand({3, 4}, type);
227   ASSERT_EQ_RESOLVED(c[1][2].dim(), 0);
228 
229   auto f = rand({3, 4}, type);
230   f[2] = zeros({4}, type);
231   f[1][0] = -1;
232   ASSERT_EQ_RESOLVED(f[2][0].item<double>(), 0);
233 }
234 
TestToCFloat()235 void TestToCFloat() {
236   Tensor a = zeros({3, 4});
237   Tensor b = ones({3, 7});
238   Tensor c = cat({a, b}, 1);
239   ASSERT_EQ_RESOLVED(c.size(1), 11);
240 
241   Tensor e = rand({});
242   ASSERT_EQ_RESOLVED(*e.data_ptr<float>(), e.sum().item<float>());
243 }
TestToString()244 void TestToString() {
245   Tensor b = ones({3, 7}) * .0000001f;
246   std::stringstream s;
247   s << b << "\n";
248   std::string expect = "1e-07 *";
249   ASSERT_EQ_RESOLVED(s.str().substr(0, expect.size()), expect);
250 }
251 
TestIndexingByScalar()252 void TestIndexingByScalar() {
253   Tensor tensor = arange(0, 10, kInt);
254   Tensor one = ones({}, kInt);
255   for (const auto i : c10::irange(tensor.numel())) {
256     ASSERT_TRUE(tensor[i].equal(one * i));
257   }
258   for (size_t i = 0; i < static_cast<uint64_t>(tensor.numel()); ++i) {
259     ASSERT_TRUE(tensor[i].equal(one * static_cast<int64_t>(i)));
260   }
261   for (const auto i : c10::irange(tensor.numel())) {
262     ASSERT_TRUE(tensor[i].equal(one * i));
263   }
264   // NOLINTNEXTLINE(bugprone-too-small-loop-variable)
265   for (int16_t i = 0; i < tensor.numel(); ++i) {
266     ASSERT_TRUE(tensor[i].equal(one * i));
267   }
268   // NOLINTNEXTLINE(bugprone-too-small-loop-variable)
269   for (int8_t i = 0; i < tensor.numel(); ++i) {
270     ASSERT_TRUE(tensor[i].equal(one * i));
271   }
272   // Throw StartsWith("Can only index tensors with integral scalars")
273   // NOLINTNEXTLINE(hicpp-avoid-goto,cppcoreguidelines-avoid-magic-numbers,cppcoreguidelines-avoid-goto)
274   ASSERT_ANY_THROW(tensor[Scalar(3.14)].equal(one));
275 }
276 
TestIndexingByZerodimTensor()277 void TestIndexingByZerodimTensor() {
278   Tensor tensor = arange(0, 10, kInt);
279   Tensor one = ones({}, kInt);
280   for (const auto i : c10::irange(tensor.numel())) {
281     ASSERT_TRUE(tensor[one * i].equal(one * i));
282   }
283   // Throw StartsWith(
284   //            "Can only index tensors with integral scalars")
285   // NOLINTNEXTLINE(hicpp-avoid-goto,cppcoreguidelines-avoid-magic-numbers,cppcoreguidelines-avoid-goto)
286   ASSERT_ANY_THROW(tensor[ones({}) * 3.14].equal(one));
287   // Throw StartsWith("Can only index with tensors that are defined")
288   // NOLINTNEXTLINE(hicpp-avoid-goto,cppcoreguidelines-avoid-goto)
289   ASSERT_ANY_THROW(tensor[Tensor()].equal(one));
290   // Throw StartsWith("Can only index with tensors that are scalars (zero-dim)")
291   // NOLINTNEXTLINE(hicpp-avoid-goto,cppcoreguidelines-avoid-goto)
292   ASSERT_ANY_THROW(tensor[ones({2, 3, 4}, kInt)].equal(one));
293 }
TestIndexingMixedDevice(DeprecatedTypeProperties & type)294 void TestIndexingMixedDevice(DeprecatedTypeProperties& type) {
295   Tensor tensor = randn({20, 20}, type);
296   Tensor index = arange(10, kLong).cpu();
297   Tensor result = tensor.index({index});
298   ASSERT_TRUE(result[0].equal(tensor[0]));
299 }
TestDispatch()300 void TestDispatch() {
301   Tensor tensor = randn({20, 20});
302   Tensor other = randn({20, 20});
303   auto result = tensor.m(relu).m(mse_loss, other, at::Reduction::Mean);
304   ASSERT_TRUE(result.allclose(mse_loss(relu(tensor), other)));
305 }
306 
TestNegativeDim(DeprecatedTypeProperties & type)307 void TestNegativeDim(DeprecatedTypeProperties& type) {
308   // NOLINTNEXTLINE(hicpp-avoid-goto,cppcoreguidelines-avoid-goto)
309   ASSERT_ANY_THROW(empty({5, -5, 5}, type.options()));
310   // NOLINTNEXTLINE(hicpp-avoid-goto,cppcoreguidelines-avoid-goto)
311   ASSERT_ANY_THROW(empty({5, -5, -5}, type.options()));
312   Tensor tensor = empty({5, 5}, type.options());
313   // NOLINTNEXTLINE(hicpp-avoid-goto,cppcoreguidelines-avoid-goto)
314   ASSERT_ANY_THROW(tensor.reshape({-5, -5}));
315 }
316 
TestView(DeprecatedTypeProperties & type)317 void TestView(DeprecatedTypeProperties& type) {
318   // Testing the tensor view path, which is different from
319   // the Variable view path, see https://github.com/pytorch/pytorch/pull/23452
320   // for details
321   Tensor tensor = randn({3, 4}, type);;
322   Tensor viewed = tensor.view({3, 4});
323   tensor.resize_({6, 2});
324   ASSERT_TRUE(tensor.sizes().equals({6, 2}));
325   ASSERT_TRUE(viewed.sizes().equals({3, 4}));
326 }
327 
TestIntArrayRefExpansion(DeprecatedTypeProperties & type)328 void TestIntArrayRefExpansion(DeprecatedTypeProperties& type) {
329   if (type.backend() != Backend::CPU || type.scalarType() != kHalf) {
330     max_pool2d(randn({3, 3, 3, 3}, type.options()), 2, 1, 1, 1);
331     max_pool3d(randn({3, 3, 3, 3, 3}, type.options()), 2, 1, 1, 1);
332     avg_pool2d(randn({3, 3, 3, 3}, type.options()), 2, 1, 1);
333     avg_pool3d(randn({3, 3, 3, 3, 3}, type.options()), 2, 1, 1);
334   }
335 }
336 
test(DeprecatedTypeProperties & type)337 void test(DeprecatedTypeProperties& type) {
338   TestResize(type);
339   TestOnesAndDot(type);
340 
341   TestSort(type);
342   TestRandperm(type);
343   TestAdd(type);
344   TestZeros(type);
345   TestLoadsOfAdds(type);
346   TestLoadOfAddsWithCopy(type);
347   TestIsContiguous(type);
348   TestPermute(type);
349   TestMm(type);
350   TestSqueeze(type);
351   TestCopy(type);
352   TestCopyBroadcasting(type);
353   TestAbsValue(type);
354   TestAddingAValueWithScalar(type);
355   TestSelect(type);
356   TestZeroDim(type);
357   TestToCFloat();
358   TestToString();
359   TestIndexingByScalar();
360   TestIndexingByZerodimTensor();
361   TestIndexingMixedDevice(type);
362   TestDispatch();
363   TestNegativeDim(type);
364   TestView(type);
365   TestIntArrayRefExpansion(type);
366 }
367 
TEST(BasicTest,BasicTestCPU)368 TEST(BasicTest, BasicTestCPU) {
369   manual_seed(123);
370 
371   test(CPU(kFloat));
372 }
373 
TEST(BasicTest,BasicTestHalfCPU)374 TEST(BasicTest, BasicTestHalfCPU) {
375   manual_seed(234);
376 
377   test(CPU(kHalf));
378 }
379 
TEST(BasicTest,BasicTestCUDA)380 TEST(BasicTest, BasicTestCUDA) {
381   manual_seed(123);
382 
383   if (at::hasCUDA()) {
384     test(CUDA(kFloat));
385   }
386 }
387 
TEST(BasicTest,FactoryMethodsTest)388 TEST(BasicTest, FactoryMethodsTest) {
389   // Test default values
390   at::Tensor tensor0 = at::empty({4});
391   ASSERT_EQ(tensor0.dtype(), at::kFloat);
392   ASSERT_EQ(tensor0.layout(), at::kStrided);
393   ASSERT_EQ(tensor0.device(), at::kCPU);
394   ASSERT_FALSE(tensor0.requires_grad());
395   ASSERT_FALSE(tensor0.is_pinned());
396 
397   // Test setting requires_grad to false.
398   tensor0 = at::empty({4}, at::TensorOptions().requires_grad(false));
399   ASSERT_EQ(tensor0.dtype(), at::kFloat);
400   ASSERT_EQ(tensor0.layout(), at::kStrided);
401   ASSERT_EQ(tensor0.device(), at::kCPU);
402   ASSERT_FALSE(tensor0.requires_grad());
403   ASSERT_FALSE(tensor0.is_pinned());
404 
405   // Test setting requires_grad to true.
406   // This is a bug. Requires_grad was set to TRUE but this is not implemented.
407   // NOLINTNEXTLINE(hicpp-avoid-goto,cppcoreguidelines-avoid-goto)
408   EXPECT_ANY_THROW(at::empty({4}, at::TensorOptions().requires_grad(true)));
409 
410   // Test setting dtype
411   at::Tensor tensor1 = at::empty({4}, at::TensorOptions().dtype(at::kHalf));
412   ASSERT_EQ(tensor1.dtype(), at::kHalf);
413   ASSERT_EQ(tensor1.layout(), at::kStrided);
414   ASSERT_EQ(tensor1.device(), at::kCPU);
415   ASSERT_FALSE(tensor1.requires_grad());
416   ASSERT_FALSE(tensor1.is_pinned());
417 
418   // Sparse tensor CPU test to avoid requiring CUDA to catch simple bugs.
419   // Sparse tensors do not work with static CPU dispatch.
420 #ifndef ATEN_CPU_STATIC_DISPATCH
421   tensor1 = at::empty({4}, at::TensorOptions().dtype(at::kHalf).layout(at::kSparse));
422   ASSERT_EQ(tensor1.dtype(), at::kHalf);
423   ASSERT_EQ(tensor1.layout(), at::kSparse);
424   ASSERT_EQ(tensor1.device(), at::kCPU);
425   ASSERT_FALSE(tensor1.requires_grad());
426   // NOLINTNEXTLINE(hicpp-avoid-goto,cppcoreguidelines-avoid-goto)
427   ASSERT_FALSE(tensor1.is_pinned());
428 #endif // ATEN_CPU_STATIC_DISPATCH
429 
430   if (torch::cuda::is_available()) {
431     // Test setting pin memory
432     tensor1 = at::empty({4}, at::TensorOptions().pinned_memory(true));
433     ASSERT_EQ(tensor1.dtype(), at::kFloat);
434     ASSERT_EQ(tensor1.layout(), at::kStrided);
435     ASSERT_EQ(tensor1.device(), at::kCPU);
436     ASSERT_EQ(tensor1.requires_grad(), false);
437     ASSERT_FALSE(tensor1.device().is_cuda());
438     ASSERT_TRUE(tensor1.is_pinned());
439 
440     // Test setting device
441     tensor1 = at::empty({4}, at::TensorOptions().device(at::kCUDA));
442     ASSERT_EQ(tensor1.dtype(), at::kFloat);
443     ASSERT_EQ(tensor1.layout(), at::kStrided);
444     ASSERT_TRUE(tensor1.device().is_cuda());
445     ASSERT_FALSE(tensor1.requires_grad());
446     ASSERT_FALSE(tensor1.is_pinned());
447 
448     // Test set everything
449     tensor1 = at::empty({4}, at::TensorOptions().dtype(at::kHalf).device(at::kCUDA).layout(at::kSparse).requires_grad(false));
450     ASSERT_EQ(tensor1.dtype(), at::kHalf);
451     ASSERT_EQ(tensor1.layout(), at::kSparse);
452     ASSERT_TRUE(tensor1.device().is_cuda());
453     ASSERT_THROWS(tensor1.nbytes());
454 
455     // This is a bug
456     // Issue https://github.com/pytorch/pytorch/issues/30405
457     ASSERT_FALSE(tensor1.requires_grad());
458     ASSERT_FALSE(tensor1.is_pinned());
459   }
460 
461   // Test _like variants
462   if (torch::cuda::is_available()) {
463     // Issue https://github.com/pytorch/pytorch/issues/28093
464     at::Tensor proto = at::empty({1}, at::kDouble);
465     tensor0 = at::empty_like(proto, at::kCUDA);
466     ASSERT_EQ(tensor0.dtype(), at::kDouble);
467     ASSERT_EQ(tensor0.layout(), at::kStrided);
468     ASSERT_TRUE(tensor0.device().is_cuda());
469     ASSERT_FALSE(tensor0.requires_grad());
470     ASSERT_FALSE(tensor0.is_pinned());
471   }
472 }
473 
TEST(BasicTest,BasicStdTestCPU)474 TEST(BasicTest, BasicStdTestCPU) {
475   c10::once_flag flag1, flag2;
476 
477   auto simple_do_once = [&]()
478   {
479       c10::call_once(flag1, [](){ std::cout << "Simple example: called once\n"; });
480   };
481 
482   auto may_throw_function = [&](bool do_throw)
483   {
484     if (do_throw) {
485       std::cout << "throw: call_once will retry\n"; // this may appear more than once
486       TORCH_CHECK(false, "throw exception");
487     }
488     std::cout << "Didn't throw, call_once will not attempt again\n"; // guaranteed once
489   };
490 
491   auto do_once = [&](bool do_throw)
492   {
493     try {
494       c10::call_once(flag2, may_throw_function, do_throw);
495     }
496     catch (...) {
497     }
498   };
499 
500   std::thread st1(simple_do_once);
501   std::thread st2(simple_do_once);
502   std::thread st3(simple_do_once);
503   std::thread st4(simple_do_once);
504   st1.join();
505   st2.join();
506   st3.join();
507   st4.join();
508 
509   std::thread t1(do_once, true);
510   std::thread t2(do_once, true);
511   std::thread t3(do_once, false);
512   std::thread t4(do_once, true);
513   t1.join();
514   t2.join();
515   t3.join();
516   t4.join();
517 }
518