1 /* 2 * Copyright 2022 The gRPC Authors 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package io.grpc.xds; 18 19 import static io.grpc.xds.XdsTestControlPlaneService.ADS_TYPE_URL_CDS; 20 import static io.grpc.xds.XdsTestControlPlaneService.ADS_TYPE_URL_EDS; 21 import static io.grpc.xds.XdsTestControlPlaneService.ADS_TYPE_URL_LDS; 22 import static io.grpc.xds.XdsTestControlPlaneService.ADS_TYPE_URL_RDS; 23 24 import com.google.common.collect.ImmutableMap; 25 import com.google.protobuf.Any; 26 import com.google.protobuf.Message; 27 import com.google.protobuf.UInt32Value; 28 import io.envoyproxy.envoy.config.cluster.v3.Cluster; 29 import io.envoyproxy.envoy.config.core.v3.Address; 30 import io.envoyproxy.envoy.config.core.v3.AggregatedConfigSource; 31 import io.envoyproxy.envoy.config.core.v3.ConfigSource; 32 import io.envoyproxy.envoy.config.core.v3.HealthStatus; 33 import io.envoyproxy.envoy.config.core.v3.SocketAddress; 34 import io.envoyproxy.envoy.config.core.v3.TrafficDirection; 35 import io.envoyproxy.envoy.config.endpoint.v3.ClusterLoadAssignment; 36 import io.envoyproxy.envoy.config.endpoint.v3.Endpoint; 37 import io.envoyproxy.envoy.config.endpoint.v3.LbEndpoint; 38 import io.envoyproxy.envoy.config.endpoint.v3.LocalityLbEndpoints; 39 import io.envoyproxy.envoy.config.listener.v3.ApiListener; 40 import io.envoyproxy.envoy.config.listener.v3.Filter; 41 import io.envoyproxy.envoy.config.listener.v3.FilterChain; 42 import io.envoyproxy.envoy.config.listener.v3.FilterChainMatch; 43 import io.envoyproxy.envoy.config.listener.v3.Listener; 44 import io.envoyproxy.envoy.config.route.v3.NonForwardingAction; 45 import io.envoyproxy.envoy.config.route.v3.Route; 46 import io.envoyproxy.envoy.config.route.v3.RouteAction; 47 import io.envoyproxy.envoy.config.route.v3.RouteConfiguration; 48 import io.envoyproxy.envoy.config.route.v3.RouteMatch; 49 import io.envoyproxy.envoy.config.route.v3.VirtualHost; 50 import io.envoyproxy.envoy.extensions.filters.http.router.v3.Router; 51 import io.envoyproxy.envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager; 52 import io.envoyproxy.envoy.extensions.filters.network.http_connection_manager.v3.HttpFilter; 53 import io.envoyproxy.envoy.extensions.filters.network.http_connection_manager.v3.Rds; 54 import io.grpc.NameResolverRegistry; 55 import io.grpc.Server; 56 import io.grpc.netty.NettyServerBuilder; 57 import java.util.Collections; 58 import java.util.Map; 59 import java.util.UUID; 60 import java.util.concurrent.TimeUnit; 61 import java.util.logging.Level; 62 import java.util.logging.Logger; 63 import org.junit.rules.TestWatcher; 64 import org.junit.runner.Description; 65 66 /** 67 * Starts a control plane server and sets up the test to use it. Initialized with a default 68 * configuration, but also provides methods for updating the configuration. 69 */ 70 public class ControlPlaneRule extends TestWatcher { 71 private static final Logger logger = Logger.getLogger(ControlPlaneRule.class.getName()); 72 73 private static final String SCHEME = "test-xds"; 74 private static final String RDS_NAME = "route-config.googleapis.com"; 75 private static final String CLUSTER_NAME = "cluster0"; 76 private static final String EDS_NAME = "eds-service-0"; 77 private static final String SERVER_LISTENER_TEMPLATE_NO_REPLACEMENT = 78 "grpc/server?udpa.resource.listening_address="; 79 private static final String HTTP_CONNECTION_MANAGER_TYPE_URL = 80 "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3" 81 + ".HttpConnectionManager"; 82 83 private String serverHostName; 84 private Server server; 85 private XdsTestControlPlaneService controlPlaneService; 86 private XdsTestLoadReportingService loadReportingService; 87 private XdsNameResolverProvider nameResolverProvider; 88 ControlPlaneRule()89 public ControlPlaneRule() { 90 serverHostName = "test-server"; 91 } 92 setServerHostName(String serverHostName)93 public ControlPlaneRule setServerHostName(String serverHostName) { 94 this.serverHostName = serverHostName; 95 return this; 96 } 97 98 /** 99 * Returns the test control plane service interface. 100 */ getService()101 public XdsTestControlPlaneService getService() { 102 return controlPlaneService; 103 } 104 105 /** 106 * Returns the server instance. 107 */ getServer()108 public Server getServer() { 109 return server; 110 } 111 starting(Description description)112 @Override protected void starting(Description description) { 113 // Start the control plane server. 114 try { 115 controlPlaneService = new XdsTestControlPlaneService(); 116 loadReportingService = new XdsTestLoadReportingService(); 117 NettyServerBuilder controlPlaneServerBuilder = NettyServerBuilder.forPort(0) 118 .addService(controlPlaneService) 119 .addService(loadReportingService); 120 server = controlPlaneServerBuilder.build().start(); 121 } catch (Exception e) { 122 throw new AssertionError("unable to start the control plane server", e); 123 } 124 125 // Configure and register an xDS name resolver so that gRPC knows how to connect to the server. 126 nameResolverProvider = XdsNameResolverProvider.createForTest(SCHEME, 127 defaultBootstrapOverride()); 128 NameResolverRegistry.getDefaultRegistry().register(nameResolverProvider); 129 } 130 finished(Description description)131 @Override protected void finished(Description description) { 132 if (server != null) { 133 server.shutdownNow(); 134 try { 135 if (!server.awaitTermination(5, TimeUnit.SECONDS)) { 136 logger.log(Level.SEVERE, "Timed out waiting for server shutdown"); 137 } 138 } catch (InterruptedException e) { 139 throw new AssertionError("unable to shut down control plane server", e); 140 } 141 } 142 NameResolverRegistry.getDefaultRegistry().deregister(nameResolverProvider); 143 } 144 145 /** 146 * For test purpose, use boostrapOverride to programmatically provide bootstrap info. 147 */ defaultBootstrapOverride()148 public Map<String, ?> defaultBootstrapOverride() { 149 return ImmutableMap.of( 150 "node", ImmutableMap.of( 151 "id", UUID.randomUUID().toString(), 152 "cluster", "cluster0"), 153 "xds_servers", Collections.singletonList( 154 155 ImmutableMap.of( 156 "server_uri", "localhost:" + server.getPort(), 157 "channel_creds", Collections.singletonList( 158 ImmutableMap.of("type", "insecure") 159 ), 160 "server_features", Collections.singletonList("xds_v3") 161 ) 162 ), 163 "server_listener_resource_name_template", SERVER_LISTENER_TEMPLATE_NO_REPLACEMENT 164 ); 165 } 166 setLdsConfig(Listener serverListener, Listener clientListener)167 void setLdsConfig(Listener serverListener, Listener clientListener) { 168 getService().setXdsConfig(ADS_TYPE_URL_LDS, 169 ImmutableMap.of(SERVER_LISTENER_TEMPLATE_NO_REPLACEMENT, serverListener, 170 serverHostName, clientListener)); 171 } 172 setRdsConfig(RouteConfiguration routeConfiguration)173 void setRdsConfig(RouteConfiguration routeConfiguration) { 174 getService().setXdsConfig(ADS_TYPE_URL_RDS, ImmutableMap.of(RDS_NAME, routeConfiguration)); 175 } 176 setCdsConfig(Cluster cluster)177 void setCdsConfig(Cluster cluster) { 178 getService().setXdsConfig(ADS_TYPE_URL_CDS, 179 ImmutableMap.<String, Message>of(CLUSTER_NAME, cluster)); 180 } 181 setEdsConfig(ClusterLoadAssignment clusterLoadAssignment)182 void setEdsConfig(ClusterLoadAssignment clusterLoadAssignment) { 183 getService().setXdsConfig(ADS_TYPE_URL_EDS, 184 ImmutableMap.<String, Message>of(EDS_NAME, clusterLoadAssignment)); 185 } 186 187 /** 188 * Builds a new default RDS configuration. 189 */ buildRouteConfiguration(String authority)190 static RouteConfiguration buildRouteConfiguration(String authority) { 191 io.envoyproxy.envoy.config.route.v3.VirtualHost virtualHost = VirtualHost.newBuilder() 192 .addDomains(authority) 193 .addRoutes( 194 Route.newBuilder() 195 .setMatch( 196 RouteMatch.newBuilder().setPrefix("/").build()) 197 .setRoute( 198 RouteAction.newBuilder().setCluster(CLUSTER_NAME).build()).build()).build(); 199 return RouteConfiguration.newBuilder().setName(RDS_NAME).addVirtualHosts(virtualHost).build(); 200 } 201 202 /** 203 * Builds a new default CDS configuration. 204 */ buildCluster()205 static Cluster buildCluster() { 206 return Cluster.newBuilder() 207 .setName(CLUSTER_NAME) 208 .setType(Cluster.DiscoveryType.EDS) 209 .setEdsClusterConfig( 210 Cluster.EdsClusterConfig.newBuilder() 211 .setServiceName(EDS_NAME) 212 .setEdsConfig( 213 ConfigSource.newBuilder() 214 .setAds(AggregatedConfigSource.newBuilder().build()) 215 .build()) 216 .build()) 217 .setLbPolicy(Cluster.LbPolicy.ROUND_ROBIN) 218 .build(); 219 } 220 221 /** 222 * Builds a new default EDS configuration. 223 */ buildClusterLoadAssignment(String hostName, int port)224 static ClusterLoadAssignment buildClusterLoadAssignment(String hostName, int port) { 225 Address address = Address.newBuilder() 226 .setSocketAddress( 227 SocketAddress.newBuilder().setAddress(hostName).setPortValue(port).build()).build(); 228 LocalityLbEndpoints endpoints = LocalityLbEndpoints.newBuilder() 229 .setLoadBalancingWeight(UInt32Value.of(10)) 230 .setPriority(0) 231 .addLbEndpoints( 232 LbEndpoint.newBuilder() 233 .setEndpoint( 234 Endpoint.newBuilder().setAddress(address).build()) 235 .setHealthStatus(HealthStatus.HEALTHY) 236 .build()).build(); 237 return ClusterLoadAssignment.newBuilder() 238 .setClusterName(EDS_NAME) 239 .addEndpoints(endpoints) 240 .build(); 241 } 242 243 /** 244 * Builds a new client listener. 245 */ buildClientListener(String name)246 static Listener buildClientListener(String name) { 247 HttpFilter httpFilter = HttpFilter.newBuilder() 248 .setName("terminal-filter") 249 .setTypedConfig(Any.pack(Router.newBuilder().build())) 250 .setIsOptional(true) 251 .build(); 252 ApiListener apiListener = ApiListener.newBuilder().setApiListener(Any.pack( 253 io.envoyproxy.envoy.extensions.filters.network.http_connection_manager.v3 254 .HttpConnectionManager.newBuilder() 255 .setRds( 256 Rds.newBuilder() 257 .setRouteConfigName(RDS_NAME) 258 .setConfigSource( 259 ConfigSource.newBuilder() 260 .setAds(AggregatedConfigSource.getDefaultInstance()))) 261 .addAllHttpFilters(Collections.singletonList(httpFilter)) 262 .build(), 263 HTTP_CONNECTION_MANAGER_TYPE_URL)).build(); 264 return Listener.newBuilder() 265 .setName(name) 266 .setApiListener(apiListener).build(); 267 } 268 269 /** 270 * Builds a new server listener. 271 */ buildServerListener()272 static Listener buildServerListener() { 273 HttpFilter routerFilter = HttpFilter.newBuilder() 274 .setName("terminal-filter") 275 .setTypedConfig( 276 Any.pack(Router.newBuilder().build())) 277 .setIsOptional(true) 278 .build(); 279 VirtualHost virtualHost = io.envoyproxy.envoy.config.route.v3.VirtualHost.newBuilder() 280 .setName("virtual-host-0") 281 .addDomains("*") 282 .addRoutes( 283 Route.newBuilder() 284 .setMatch( 285 RouteMatch.newBuilder().setPrefix("/").build()) 286 .setNonForwardingAction(NonForwardingAction.newBuilder().build()) 287 .build()).build(); 288 RouteConfiguration routeConfig = RouteConfiguration.newBuilder() 289 .addVirtualHosts(virtualHost) 290 .build(); 291 io.envoyproxy.envoy.config.listener.v3.Filter filter = Filter.newBuilder() 292 .setName("network-filter-0") 293 .setTypedConfig( 294 Any.pack( 295 HttpConnectionManager.newBuilder() 296 .setRouteConfig(routeConfig) 297 .addAllHttpFilters(Collections.singletonList(routerFilter)) 298 .build())).build(); 299 FilterChainMatch filterChainMatch = FilterChainMatch.newBuilder() 300 .setSourceType(FilterChainMatch.ConnectionSourceType.ANY) 301 .build(); 302 FilterChain filterChain = FilterChain.newBuilder() 303 .setName("filter-chain-0") 304 .setFilterChainMatch(filterChainMatch) 305 .addFilters(filter) 306 .build(); 307 return Listener.newBuilder() 308 .setName(SERVER_LISTENER_TEMPLATE_NO_REPLACEMENT) 309 .setTrafficDirection(TrafficDirection.INBOUND) 310 .addFilterChains(filterChain) 311 .build(); 312 } 313 } 314