1 /* 2 * Copyright 2022 The gRPC Authors 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package io.grpc.xds; 18 19 import static com.google.common.truth.Truth.assertThat; 20 import static org.junit.Assert.fail; 21 22 import com.github.xds.type.v3.TypedStruct; 23 import com.google.common.collect.ImmutableList; 24 import com.google.common.collect.ImmutableMap; 25 import com.google.common.collect.Lists; 26 import com.google.protobuf.Any; 27 import com.google.protobuf.BoolValue; 28 import com.google.protobuf.Duration; 29 import com.google.protobuf.FloatValue; 30 import com.google.protobuf.Struct; 31 import com.google.protobuf.UInt32Value; 32 import com.google.protobuf.UInt64Value; 33 import com.google.protobuf.Value; 34 import io.envoyproxy.envoy.config.cluster.v3.Cluster; 35 import io.envoyproxy.envoy.config.cluster.v3.Cluster.LbPolicy; 36 import io.envoyproxy.envoy.config.cluster.v3.Cluster.LeastRequestLbConfig; 37 import io.envoyproxy.envoy.config.cluster.v3.Cluster.RingHashLbConfig; 38 import io.envoyproxy.envoy.config.cluster.v3.Cluster.RingHashLbConfig.HashFunction; 39 import io.envoyproxy.envoy.config.cluster.v3.LoadBalancingPolicy; 40 import io.envoyproxy.envoy.config.cluster.v3.LoadBalancingPolicy.Policy; 41 import io.envoyproxy.envoy.config.core.v3.TypedExtensionConfig; 42 import io.envoyproxy.envoy.extensions.load_balancing_policies.client_side_weighted_round_robin.v3.ClientSideWeightedRoundRobin; 43 import io.envoyproxy.envoy.extensions.load_balancing_policies.least_request.v3.LeastRequest; 44 import io.envoyproxy.envoy.extensions.load_balancing_policies.pick_first.v3.PickFirst; 45 import io.envoyproxy.envoy.extensions.load_balancing_policies.ring_hash.v3.RingHash; 46 import io.envoyproxy.envoy.extensions.load_balancing_policies.round_robin.v3.RoundRobin; 47 import io.envoyproxy.envoy.extensions.load_balancing_policies.wrr_locality.v3.WrrLocality; 48 import io.grpc.LoadBalancer; 49 import io.grpc.LoadBalancer.Helper; 50 import io.grpc.LoadBalancerProvider; 51 import io.grpc.LoadBalancerRegistry; 52 import io.grpc.internal.JsonUtil; 53 import io.grpc.internal.ServiceConfigUtil; 54 import io.grpc.internal.ServiceConfigUtil.LbConfig; 55 import io.grpc.xds.XdsClientImpl.ResourceInvalidException; 56 import java.util.List; 57 import org.junit.After; 58 import org.junit.Test; 59 import org.junit.runner.RunWith; 60 import org.junit.runners.JUnit4; 61 62 /** 63 * Unit test for {@link LoadBalancerConfigFactory}. 64 */ 65 @RunWith(JUnit4.class) 66 public class LoadBalancerConfigFactoryTest { 67 68 private static final Policy ROUND_ROBIN_POLICY = Policy.newBuilder().setTypedExtensionConfig( 69 TypedExtensionConfig.newBuilder().setTypedConfig( 70 Any.pack(RoundRobin.getDefaultInstance()))).build(); 71 72 private static final long RING_HASH_MIN_RING_SIZE = 1; 73 private static final long RING_HASH_MAX_RING_SIZE = 2; 74 private static final Policy RING_HASH_POLICY = Policy.newBuilder().setTypedExtensionConfig( 75 TypedExtensionConfig.newBuilder().setTypedConfig(Any.pack( 76 RingHash.newBuilder().setMinimumRingSize(UInt64Value.of(RING_HASH_MIN_RING_SIZE)) 77 .setMaximumRingSize(UInt64Value.of(RING_HASH_MAX_RING_SIZE)) 78 .setHashFunction(RingHash.HashFunction.XX_HASH).build()))).build(); 79 80 private static final int LEAST_REQUEST_CHOICE_COUNT = 10; 81 private static final Policy LEAST_REQUEST_POLICY = Policy.newBuilder().setTypedExtensionConfig( 82 TypedExtensionConfig.newBuilder().setTypedConfig(Any.pack( 83 LeastRequest.newBuilder().setChoiceCount(UInt32Value.of(LEAST_REQUEST_CHOICE_COUNT)) 84 .build()))).build(); 85 86 private static final Policy PICK_FIRST_POLICY = Policy.newBuilder().setTypedExtensionConfig( 87 TypedExtensionConfig.newBuilder().setTypedConfig(Any.pack( 88 PickFirst.newBuilder().setShuffleAddressList(true) 89 .build()))).build(); 90 91 private static final Policy WRR_POLICY = Policy.newBuilder() 92 .setTypedExtensionConfig(TypedExtensionConfig.newBuilder() 93 .setName("backend") 94 .setTypedConfig( 95 Any.pack(ClientSideWeightedRoundRobin.newBuilder() 96 .setBlackoutPeriod(Duration.newBuilder().setSeconds(287).build()) 97 .setEnableOobLoadReport( 98 BoolValue.newBuilder().setValue(true).build()) 99 .setErrorUtilizationPenalty( 100 FloatValue.newBuilder().setValue(1.75F).build()) 101 .build())) 102 .build()) 103 .build(); 104 private static final String CUSTOM_POLICY_NAME = "myorg.MyCustomLeastRequestPolicy"; 105 private static final String CUSTOM_POLICY_FIELD_KEY = "choiceCount"; 106 private static final double CUSTOM_POLICY_FIELD_VALUE = 2; 107 private static final Policy CUSTOM_POLICY = Policy.newBuilder().setTypedExtensionConfig( 108 TypedExtensionConfig.newBuilder().setTypedConfig(Any.pack( 109 TypedStruct.newBuilder().setTypeUrl( 110 "type.googleapis.com/" + CUSTOM_POLICY_NAME).setValue( 111 Struct.newBuilder().putFields(CUSTOM_POLICY_FIELD_KEY, 112 Value.newBuilder().setNumberValue(CUSTOM_POLICY_FIELD_VALUE).build())) 113 .build()))).build(); 114 private static final Policy CUSTOM_POLICY_UDPA = Policy.newBuilder().setTypedExtensionConfig( 115 TypedExtensionConfig.newBuilder().setTypedConfig(Any.pack( 116 com.github.udpa.udpa.type.v1.TypedStruct.newBuilder().setTypeUrl( 117 "type.googleapis.com/" + CUSTOM_POLICY_NAME).setValue( 118 Struct.newBuilder().putFields(CUSTOM_POLICY_FIELD_KEY, 119 Value.newBuilder().setNumberValue(CUSTOM_POLICY_FIELD_VALUE).build())) 120 .build()))).build(); 121 private static final FakeCustomLoadBalancerProvider CUSTOM_POLICY_PROVIDER 122 = new FakeCustomLoadBalancerProvider(); 123 124 private static final LbConfig VALID_ROUND_ROBIN_CONFIG = new LbConfig("wrr_locality_experimental", 125 ImmutableMap.of("childPolicy", 126 ImmutableList.of(ImmutableMap.of("round_robin", ImmutableMap.of())))); 127 128 private static final LbConfig VALID_WRR_CONFIG = new LbConfig("wrr_locality_experimental", 129 ImmutableMap.of("childPolicy", ImmutableList.of( 130 ImmutableMap.of("weighted_round_robin", 131 ImmutableMap.of("blackoutPeriod","287s", "enableOobLoadReport", true, 132 "errorUtilizationPenalty", 1.75F ))))); 133 private static final LbConfig VALID_RING_HASH_CONFIG = new LbConfig("ring_hash_experimental", 134 ImmutableMap.of("minRingSize", (double) RING_HASH_MIN_RING_SIZE, "maxRingSize", 135 (double) RING_HASH_MAX_RING_SIZE)); 136 private static final LbConfig VALID_CUSTOM_CONFIG = new LbConfig(CUSTOM_POLICY_NAME, 137 ImmutableMap.of(CUSTOM_POLICY_FIELD_KEY, CUSTOM_POLICY_FIELD_VALUE)); 138 private static final LbConfig VALID_CUSTOM_CONFIG_IN_WRR = new LbConfig( 139 "wrr_locality_experimental", ImmutableMap.of("childPolicy", ImmutableList.of( 140 ImmutableMap.of(VALID_CUSTOM_CONFIG.getPolicyName(), 141 VALID_CUSTOM_CONFIG.getRawConfigValue())))); 142 private static final LbConfig VALID_LEAST_REQUEST_CONFIG = new LbConfig( 143 "least_request_experimental", 144 ImmutableMap.of("choiceCount", (double) LEAST_REQUEST_CHOICE_COUNT)); 145 private static final LbConfig VALID_PICK_FIRST_CONFIG = new LbConfig( 146 "pick_first", 147 ImmutableMap.of("shuffleAddressList", true)); 148 149 @After deregisterCustomProvider()150 public void deregisterCustomProvider() { 151 LoadBalancerRegistry.getDefaultRegistry().deregister(CUSTOM_POLICY_PROVIDER); 152 } 153 154 @Test roundRobin()155 public void roundRobin() throws ResourceInvalidException { 156 Cluster cluster = newCluster(buildWrrPolicy(ROUND_ROBIN_POLICY)); 157 158 assertThat(newLbConfig(cluster, true, true, true)).isEqualTo(VALID_ROUND_ROBIN_CONFIG); 159 } 160 161 @Test weightedRoundRobin()162 public void weightedRoundRobin() throws ResourceInvalidException { 163 Cluster cluster = newCluster(buildWrrPolicy(WRR_POLICY)); 164 165 assertThat(newLbConfig(cluster, true, true, true)).isEqualTo(VALID_WRR_CONFIG); 166 } 167 168 @Test weightedRoundRobin_invalid()169 public void weightedRoundRobin_invalid() throws ResourceInvalidException { 170 Cluster cluster = newCluster(buildWrrPolicy(Policy.newBuilder() 171 .setTypedExtensionConfig(TypedExtensionConfig.newBuilder() 172 .setName("backend") 173 .setTypedConfig( 174 Any.pack(ClientSideWeightedRoundRobin.newBuilder() 175 .setBlackoutPeriod(Duration.newBuilder().setNanos(1000000000).build()) 176 .setEnableOobLoadReport( 177 BoolValue.newBuilder().setValue(true).build()) 178 .build())) 179 .build()) 180 .build())); 181 182 assertResourceInvalidExceptionThrown(cluster, true, true, true, 183 "Invalid duration in weighted round robin config"); 184 } 185 186 @Test weightedRoundRobin_fallback_roundrobin()187 public void weightedRoundRobin_fallback_roundrobin() throws ResourceInvalidException { 188 Cluster cluster = newCluster(buildWrrPolicy(WRR_POLICY, ROUND_ROBIN_POLICY)); 189 190 assertThat(newLbConfig(cluster, true, false, true)).isEqualTo(VALID_ROUND_ROBIN_CONFIG); 191 } 192 193 @Test roundRobin_legacy()194 public void roundRobin_legacy() throws ResourceInvalidException { 195 Cluster cluster = Cluster.newBuilder().setLbPolicy(LbPolicy.ROUND_ROBIN).build(); 196 197 assertThat(newLbConfig(cluster, true, true, true)).isEqualTo(VALID_ROUND_ROBIN_CONFIG); 198 } 199 200 @Test ringHash()201 public void ringHash() throws ResourceInvalidException { 202 Cluster cluster = Cluster.newBuilder() 203 .setLoadBalancingPolicy(LoadBalancingPolicy.newBuilder().addPolicies(RING_HASH_POLICY)) 204 .build(); 205 206 assertThat(newLbConfig(cluster, true, true, true)).isEqualTo(VALID_RING_HASH_CONFIG); 207 } 208 209 @Test ringHash_legacy()210 public void ringHash_legacy() throws ResourceInvalidException { 211 Cluster cluster = Cluster.newBuilder().setLbPolicy(LbPolicy.RING_HASH).setRingHashLbConfig( 212 RingHashLbConfig.newBuilder().setMinimumRingSize(UInt64Value.of(RING_HASH_MIN_RING_SIZE)) 213 .setMaximumRingSize(UInt64Value.of(RING_HASH_MAX_RING_SIZE)) 214 .setHashFunction(HashFunction.XX_HASH)).build(); 215 216 assertThat(newLbConfig(cluster, true, true, true)).isEqualTo(VALID_RING_HASH_CONFIG); 217 } 218 219 @Test ringHash_invalidHash()220 public void ringHash_invalidHash() { 221 Cluster cluster = newCluster( 222 Policy.newBuilder().setTypedExtensionConfig(TypedExtensionConfig.newBuilder() 223 .setTypedConfig(Any.pack( 224 RingHash.newBuilder().setMinimumRingSize(UInt64Value.of(RING_HASH_MIN_RING_SIZE)) 225 .setMaximumRingSize(UInt64Value.of(RING_HASH_MAX_RING_SIZE)) 226 .setHashFunction(RingHash.HashFunction.MURMUR_HASH_2).build()))).build()); 227 228 assertResourceInvalidExceptionThrown(cluster, true, true, true, "Invalid ring hash function"); 229 } 230 231 @Test ringHash_invalidHash_legacy()232 public void ringHash_invalidHash_legacy() { 233 Cluster cluster = Cluster.newBuilder().setLbPolicy(LbPolicy.RING_HASH).setRingHashLbConfig( 234 RingHashLbConfig.newBuilder().setHashFunction(HashFunction.MURMUR_HASH_2)).build(); 235 236 assertResourceInvalidExceptionThrown(cluster, true, true, true, "invalid ring hash function"); 237 } 238 239 @Test leastRequest()240 public void leastRequest() throws ResourceInvalidException { 241 Cluster cluster = Cluster.newBuilder() 242 .setLoadBalancingPolicy(LoadBalancingPolicy.newBuilder().addPolicies(LEAST_REQUEST_POLICY)) 243 .build(); 244 245 assertThat(newLbConfig(cluster, true, true, true)).isEqualTo(VALID_LEAST_REQUEST_CONFIG); 246 } 247 248 @Test leastRequest_legacy()249 public void leastRequest_legacy() throws ResourceInvalidException { 250 System.setProperty("io.grpc.xds.experimentalEnableLeastRequest", "true"); 251 252 Cluster cluster = Cluster.newBuilder().setLbPolicy(LbPolicy.LEAST_REQUEST) 253 .setLeastRequestLbConfig( 254 LeastRequestLbConfig.newBuilder() 255 .setChoiceCount(UInt32Value.of(LEAST_REQUEST_CHOICE_COUNT))).build(); 256 257 LbConfig lbConfig = newLbConfig(cluster, true, true, true); 258 assertThat(lbConfig.getPolicyName()).isEqualTo("wrr_locality_experimental"); 259 260 List<LbConfig> childConfigs = ServiceConfigUtil.unwrapLoadBalancingConfigList( 261 JsonUtil.getListOfObjects(lbConfig.getRawConfigValue(), "childPolicy")); 262 assertThat(childConfigs.get(0).getPolicyName()).isEqualTo("least_request_experimental"); 263 assertThat( 264 JsonUtil.getNumberAsLong(childConfigs.get(0).getRawConfigValue(), "choiceCount")).isEqualTo( 265 LEAST_REQUEST_CHOICE_COUNT); 266 } 267 268 @Test leastRequest_notEnabled()269 public void leastRequest_notEnabled() { 270 Cluster cluster = Cluster.newBuilder().setLbPolicy(LbPolicy.LEAST_REQUEST).build(); 271 272 assertResourceInvalidExceptionThrown(cluster, false, true, true, "unsupported lb policy"); 273 } 274 275 @Test pickFirst()276 public void pickFirst() throws ResourceInvalidException { 277 Cluster cluster = Cluster.newBuilder() 278 .setLoadBalancingPolicy(LoadBalancingPolicy.newBuilder().addPolicies(PICK_FIRST_POLICY)) 279 .build(); 280 281 assertThat(newLbConfig(cluster, true, true, true)).isEqualTo(VALID_PICK_FIRST_CONFIG); 282 } 283 284 @Test pickFirst_notEnabled()285 public void pickFirst_notEnabled() throws ResourceInvalidException { 286 Cluster cluster = Cluster.newBuilder() 287 .setLoadBalancingPolicy(LoadBalancingPolicy.newBuilder().addPolicies(PICK_FIRST_POLICY)) 288 .build(); 289 290 assertResourceInvalidExceptionThrown(cluster, true, true, false, "Invalid LoadBalancingPolicy"); 291 } 292 293 @Test customRootLb_providerRegistered()294 public void customRootLb_providerRegistered() throws ResourceInvalidException { 295 LoadBalancerRegistry.getDefaultRegistry().register(CUSTOM_POLICY_PROVIDER); 296 297 assertThat(newLbConfig(newCluster(CUSTOM_POLICY), false, 298 true, true)).isEqualTo(VALID_CUSTOM_CONFIG); 299 } 300 301 @Test customRootLb_providerNotRegistered()302 public void customRootLb_providerNotRegistered() throws ResourceInvalidException { 303 Cluster cluster = Cluster.newBuilder() 304 .setLoadBalancingPolicy(LoadBalancingPolicy.newBuilder().addPolicies(CUSTOM_POLICY)) 305 .build(); 306 307 assertResourceInvalidExceptionThrown(cluster, false, true, true, "Invalid LoadBalancingPolicy"); 308 } 309 310 // When a provider for the endpoint picking custom policy is available, the configuration should 311 // use it. 312 @Test customLbInWrr_providerRegistered()313 public void customLbInWrr_providerRegistered() throws ResourceInvalidException { 314 LoadBalancerRegistry.getDefaultRegistry().register(CUSTOM_POLICY_PROVIDER); 315 316 Cluster cluster = Cluster.newBuilder().setLoadBalancingPolicy(LoadBalancingPolicy.newBuilder() 317 .addPolicies(buildWrrPolicy(CUSTOM_POLICY, ROUND_ROBIN_POLICY))).build(); 318 319 assertThat(newLbConfig(cluster, false, true, true)).isEqualTo(VALID_CUSTOM_CONFIG_IN_WRR); 320 } 321 322 // When a provider for the endpoint picking custom policy is available, the configuration should 323 // use it. This one uses the legacy UDPA TypedStruct that is also supported. 324 @Test customLbInWrr_providerRegistered_udpa()325 public void customLbInWrr_providerRegistered_udpa() throws ResourceInvalidException { 326 LoadBalancerRegistry.getDefaultRegistry().register(CUSTOM_POLICY_PROVIDER); 327 328 Cluster cluster = Cluster.newBuilder().setLoadBalancingPolicy(LoadBalancingPolicy.newBuilder() 329 .addPolicies(buildWrrPolicy(CUSTOM_POLICY_UDPA, ROUND_ROBIN_POLICY))).build(); 330 331 assertThat(newLbConfig(cluster, false, true, true)).isEqualTo(VALID_CUSTOM_CONFIG_IN_WRR); 332 } 333 334 // When a provider for the custom wrr_locality child policy is NOT available, we should fall back 335 // to the round_robin that is also provided. 336 @Test customLbInWrr_providerNotRegistered()337 public void customLbInWrr_providerNotRegistered() throws ResourceInvalidException { 338 Cluster cluster = Cluster.newBuilder().setLoadBalancingPolicy(LoadBalancingPolicy.newBuilder() 339 .addPolicies(buildWrrPolicy(CUSTOM_POLICY, ROUND_ROBIN_POLICY))).build(); 340 341 assertThat(newLbConfig(cluster, false, true, true)).isEqualTo(VALID_ROUND_ROBIN_CONFIG); 342 } 343 344 // When a provider for the custom wrr_locality child policy is NOT available and no alternative 345 // child policy is provided, the configuration is invalid. 346 @Test customLbInWrr_providerNotRegistered_noFallback()347 public void customLbInWrr_providerNotRegistered_noFallback() throws ResourceInvalidException { 348 Cluster cluster = Cluster.newBuilder().setLoadBalancingPolicy( 349 LoadBalancingPolicy.newBuilder().addPolicies(buildWrrPolicy(CUSTOM_POLICY))).build(); 350 351 assertResourceInvalidExceptionThrown(cluster, false, true, true, "Invalid LoadBalancingPolicy"); 352 } 353 354 @Test maxRecursion()355 public void maxRecursion() { 356 Cluster cluster = Cluster.newBuilder() 357 .setLoadBalancingPolicy( 358 LoadBalancingPolicy.newBuilder().addPolicies( 359 buildWrrPolicy( // Wheee... 360 buildWrrPolicy( // ...eee... 361 buildWrrPolicy( // ...eee! 362 buildWrrPolicy( 363 buildWrrPolicy( 364 buildWrrPolicy( 365 buildWrrPolicy( 366 buildWrrPolicy( 367 buildWrrPolicy( 368 buildWrrPolicy( 369 buildWrrPolicy( 370 buildWrrPolicy( 371 buildWrrPolicy( 372 buildWrrPolicy( 373 buildWrrPolicy( 374 buildWrrPolicy( 375 buildWrrPolicy( 376 ROUND_ROBIN_POLICY))))))))))))))))))).build(); 377 378 assertResourceInvalidExceptionThrown(cluster, false, true, true, 379 "Maximum LB config recursion depth reached"); 380 } 381 newCluster(Policy... policies)382 private Cluster newCluster(Policy... policies) { 383 return Cluster.newBuilder().setLoadBalancingPolicy( 384 LoadBalancingPolicy.newBuilder().addAllPolicies(Lists.newArrayList(policies))).build(); 385 } 386 buildWrrPolicy(Policy... childPolicy)387 private static Policy buildWrrPolicy(Policy... childPolicy) { 388 return Policy.newBuilder().setTypedExtensionConfig(TypedExtensionConfig.newBuilder() 389 .setTypedConfig(Any.pack(WrrLocality.newBuilder().setEndpointPickingPolicy( 390 LoadBalancingPolicy.newBuilder().addAllPolicies(Lists.newArrayList(childPolicy))) 391 .build()))).build(); 392 } 393 newLbConfig(Cluster cluster, boolean enableLeastRequest, boolean enableWrr, boolean enablePickFirst)394 private LbConfig newLbConfig(Cluster cluster, boolean enableLeastRequest, boolean enableWrr, 395 boolean enablePickFirst) 396 throws ResourceInvalidException { 397 return ServiceConfigUtil.unwrapLoadBalancingConfig( 398 LoadBalancerConfigFactory.newConfig(cluster, enableLeastRequest, 399 enableWrr, enablePickFirst)); 400 } 401 assertResourceInvalidExceptionThrown(Cluster cluster, boolean enableLeastRequest, boolean enableWrr, boolean enablePickFirst, String expectedMessage)402 private void assertResourceInvalidExceptionThrown(Cluster cluster, boolean enableLeastRequest, 403 boolean enableWrr, boolean enablePickFirst, String expectedMessage) { 404 try { 405 newLbConfig(cluster, enableLeastRequest, enableWrr, enablePickFirst); 406 } catch (ResourceInvalidException e) { 407 assertThat(e).hasMessageThat().contains(expectedMessage); 408 return; 409 } 410 fail("ResourceInvalidException not thrown"); 411 } 412 413 private static class FakeCustomLoadBalancerProvider extends LoadBalancerProvider { 414 415 @Override newLoadBalancer(Helper helper)416 public LoadBalancer newLoadBalancer(Helper helper) { 417 return null; 418 } 419 420 @Override isAvailable()421 public boolean isAvailable() { 422 return true; 423 } 424 425 @Override getPriority()426 public int getPriority() { 427 return 5; 428 } 429 430 @Override getPolicyName()431 public String getPolicyName() { 432 return CUSTOM_POLICY_NAME; 433 } 434 } 435 } 436