1 /* 2 * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"). 5 * You may not use this file except in compliance with the License. 6 * A copy of the License is located at 7 * 8 * http://aws.amazon.com/apache2.0 9 * 10 * or in the "license" file accompanying this file. This file is distributed 11 * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 * express or implied. See the License for the specific language governing 13 * permissions and limitations under the License. 14 */ 15 16 package software.amazon.awssdk.utils.http; 17 18 import static java.util.stream.Collectors.groupingBy; 19 import static java.util.stream.Collectors.mapping; 20 import static java.util.stream.Collectors.toList; 21 import static software.amazon.awssdk.utils.FunctionalUtils.invokeSafely; 22 import static software.amazon.awssdk.utils.StringUtils.isEmpty; 23 24 import java.io.UnsupportedEncodingException; 25 import java.net.URI; 26 import java.net.URLDecoder; 27 import java.net.URLEncoder; 28 import java.util.ArrayList; 29 import java.util.Arrays; 30 import java.util.Collection; 31 import java.util.Collections; 32 import java.util.LinkedHashMap; 33 import java.util.List; 34 import java.util.Map; 35 import java.util.Map.Entry; 36 import java.util.Optional; 37 import java.util.Set; 38 import java.util.function.UnaryOperator; 39 import java.util.stream.Collectors; 40 import java.util.stream.Stream; 41 import software.amazon.awssdk.annotations.SdkProtectedApi; 42 import software.amazon.awssdk.utils.ProxyEnvironmentSetting; 43 import software.amazon.awssdk.utils.ProxySystemSetting; 44 import software.amazon.awssdk.utils.StringUtils; 45 import software.amazon.awssdk.utils.Validate; 46 47 /** 48 * A set of utilities that assist with HTTP message-related interactions. 49 */ 50 @SdkProtectedApi 51 public final class SdkHttpUtils { 52 private static final String DEFAULT_ENCODING = "UTF-8"; 53 54 /** 55 * Characters that we need to fix up after URLEncoder.encode(). 56 */ 57 private static final String[] ENCODED_CHARACTERS_WITH_SLASHES = new String[] {"+", "*", "%7E", "%2F"}; 58 private static final String[] ENCODED_CHARACTERS_WITH_SLASHES_REPLACEMENTS = new String[] {"%20", "%2A", "~", "/"}; 59 60 private static final String[] ENCODED_CHARACTERS_WITHOUT_SLASHES = new String[] {"+", "*", "%7E"}; 61 private static final String[] ENCODED_CHARACTERS_WITHOUT_SLASHES_REPLACEMENTS = new String[] {"%20", "%2A", "~"}; 62 63 // List of headers that may appear only once in a request; i.e. is not a list of values. 64 // Taken from https://github.com/apache/httpcomponents-client/blob/81c1bc4dc3ca5a3134c5c60e8beff08be2fd8792/httpclient5-cache/src/test/java/org/apache/hc/client5/http/impl/cache/HttpTestUtils.java#L69-L85 with modifications: 65 // removed: accept-ranges, if-match, if-none-match, vary since it looks like they're defined as lists 66 private static final Set<String> SINGLE_HEADERS = Stream.of("age", "authorization", 67 "content-length", "content-location", "content-md5", "content-range", "content-type", 68 "date", "etag", "expires", "from", "host", "if-modified-since", "if-range", 69 "if-unmodified-since", "last-modified", "location", "max-forwards", 70 "proxy-authorization", "range", "referer", "retry-after", "server", "user-agent") 71 .collect(Collectors.toSet()); 72 73 SdkHttpUtils()74 private SdkHttpUtils() { 75 } 76 77 /** 78 * Encode a string according to RFC 3986: encoding for URI paths, query strings, etc. 79 */ urlEncode(String value)80 public static String urlEncode(String value) { 81 return urlEncode(value, false); 82 } 83 84 /** 85 * Encode a string according to RFC 3986, but ignore "/" characters. This is useful for encoding the components of a path, 86 * without encoding the path separators. 87 */ urlEncodeIgnoreSlashes(String value)88 public static String urlEncodeIgnoreSlashes(String value) { 89 return urlEncode(value, true); 90 } 91 92 /** 93 * Encode a string according to RFC 1630: encoding for form data. 94 */ formDataEncode(String value)95 public static String formDataEncode(String value) { 96 return value == null ? null : invokeSafely(() -> URLEncoder.encode(value, DEFAULT_ENCODING)); 97 } 98 99 /** 100 * Decode the string according to RFC 3986: encoding for URI paths, query strings, etc. 101 * <p> 102 * Assumes the decoded string is UTF-8 encoded. 103 * 104 * @param value The string to decode. 105 * @return The decoded string. 106 */ urlDecode(String value)107 public static String urlDecode(String value) { 108 if (value == null) { 109 return null; 110 } 111 try { 112 return URLDecoder.decode(value, DEFAULT_ENCODING); 113 } catch (UnsupportedEncodingException e) { 114 throw new RuntimeException("Unable to decode value", e); 115 } 116 } 117 118 /** 119 * Encode each of the keys and values in the provided query parameters using {@link #urlEncode(String)}. 120 */ encodeQueryParameters(Map<String, List<String>> rawQueryParameters)121 public static Map<String, List<String>> encodeQueryParameters(Map<String, List<String>> rawQueryParameters) { 122 return encodeMapOfLists(rawQueryParameters, SdkHttpUtils::urlEncode); 123 } 124 125 /** 126 * Encode each of the keys and values in the provided form data using {@link #formDataEncode(String)}. 127 */ encodeFormData(Map<String, List<String>> rawFormData)128 public static Map<String, List<String>> encodeFormData(Map<String, List<String>> rawFormData) { 129 return encodeMapOfLists(rawFormData, SdkHttpUtils::formDataEncode); 130 } 131 encodeMapOfLists(Map<String, List<String>> map, UnaryOperator<String> encoder)132 private static Map<String, List<String>> encodeMapOfLists(Map<String, List<String>> map, UnaryOperator<String> encoder) { 133 Validate.notNull(map, "Map must not be null."); 134 135 Map<String, List<String>> result = new LinkedHashMap<>(); 136 137 for (Entry<String, List<String>> queryParameter : map.entrySet()) { 138 String key = queryParameter.getKey(); 139 String encodedKey = encoder.apply(key); 140 141 List<String> value = queryParameter.getValue(); 142 List<String> encodedValue = value == null 143 ? null 144 : queryParameter.getValue().stream().map(encoder).collect(Collectors.toList()); 145 146 result.put(encodedKey, encodedValue); 147 } 148 149 return result; 150 } 151 152 /** 153 * Encode a string for use in the path of a URL; uses URLEncoder.encode, 154 * (which encodes a string for use in the query portion of a URL), then 155 * applies some postfilters to fix things up per the RFC. Can optionally 156 * handle strings which are meant to encode a path (ie include '/'es 157 * which should NOT be escaped). 158 * 159 * @param value the value to encode 160 * @param ignoreSlashes true if the value is intended to represent a path 161 * @return the encoded value 162 */ urlEncode(String value, boolean ignoreSlashes)163 private static String urlEncode(String value, boolean ignoreSlashes) { 164 if (value == null) { 165 return null; 166 } 167 168 String encoded = invokeSafely(() -> URLEncoder.encode(value, DEFAULT_ENCODING)); 169 170 if (!ignoreSlashes) { 171 return StringUtils.replaceEach(encoded, 172 ENCODED_CHARACTERS_WITHOUT_SLASHES, 173 ENCODED_CHARACTERS_WITHOUT_SLASHES_REPLACEMENTS); 174 } 175 176 return StringUtils.replaceEach(encoded, ENCODED_CHARACTERS_WITH_SLASHES, ENCODED_CHARACTERS_WITH_SLASHES_REPLACEMENTS); 177 } 178 179 /** 180 * Encode the provided query parameters using {@link #encodeQueryParameters(Map)} and then flatten them into a string that 181 * can be used as the query string in a URL. The result is not prepended with "?". 182 */ encodeAndFlattenQueryParameters(Map<String, List<String>> rawQueryParameters)183 public static Optional<String> encodeAndFlattenQueryParameters(Map<String, List<String>> rawQueryParameters) { 184 return encodeAndFlatten(rawQueryParameters, SdkHttpUtils::urlEncode); 185 } 186 187 /** 188 * Encode the provided form data using {@link #encodeFormData(Map)} and then flatten them into a string that 189 * can be used as the body of a form data request. 190 */ encodeAndFlattenFormData(Map<String, List<String>> rawFormData)191 public static Optional<String> encodeAndFlattenFormData(Map<String, List<String>> rawFormData) { 192 return encodeAndFlatten(rawFormData, SdkHttpUtils::formDataEncode); 193 } 194 encodeAndFlatten(Map<String, List<String>> data, UnaryOperator<String> encoder)195 private static Optional<String> encodeAndFlatten(Map<String, List<String>> data, UnaryOperator<String> encoder) { 196 Validate.notNull(data, "Map must not be null."); 197 198 if (data.isEmpty()) { 199 return Optional.empty(); 200 } 201 202 StringBuilder queryString = new StringBuilder(); 203 data.forEach((key, values) -> { 204 String encodedKey = encoder.apply(key); 205 206 if (values != null) { 207 values.forEach(value -> { 208 if (queryString.length() > 0) { 209 queryString.append('&'); 210 } 211 queryString.append(encodedKey); 212 if (value != null) { 213 queryString.append('=').append(encoder.apply(value)); 214 } 215 }); 216 } 217 }); 218 219 return Optional.of(queryString.toString()); 220 } 221 222 /** 223 * Flatten the provided query parameters into a string that can be used as the query string in a URL. The result is not 224 * prepended with "?". This is useful when you have already-encoded query parameters you wish to flatten. 225 */ flattenQueryParameters(Map<String, List<String>> toFlatten)226 public static Optional<String> flattenQueryParameters(Map<String, List<String>> toFlatten) { 227 if (toFlatten.isEmpty()) { 228 return Optional.empty(); 229 } 230 231 StringBuilder result = new StringBuilder(); 232 flattenQueryParameters(result, toFlatten); 233 return Optional.of(result.toString()); 234 } 235 236 /** 237 * Flatten the provided query parameters into a string that can be used as the query string in a URL. The result is not 238 * prepended with "?". This is useful when you have already-encoded query parameters you wish to flatten. 239 */ flattenQueryParameters(StringBuilder result, Map<String, List<String>> toFlatten)240 public static void flattenQueryParameters(StringBuilder result, Map<String, List<String>> toFlatten) { 241 if (toFlatten.isEmpty()) { 242 return; 243 } 244 245 boolean first = true; 246 for (Entry<String, List<String>> encodedQueryParameter : toFlatten.entrySet()) { 247 String key = encodedQueryParameter.getKey(); 248 249 List<String> values = Optional.ofNullable(encodedQueryParameter.getValue()).orElseGet(Collections::emptyList); 250 251 for (String value : values) { 252 if (!first) { 253 result.append('&'); 254 } else { 255 first = false; 256 } 257 result.append(key); 258 if (value != null) { 259 result.append('='); 260 result.append(value); 261 } 262 } 263 } 264 } 265 266 /** 267 * Returns true if the specified port is the standard port for the given protocol. (i.e. 80 for HTTP or 443 for HTTPS). 268 * 269 * Null or -1 ports (to simplify interaction with {@link URI}'s default value) are treated as standard ports. 270 * 271 * @return True if the specified port is standard for the specified protocol, otherwise false. 272 */ isUsingStandardPort(String protocol, Integer port)273 public static boolean isUsingStandardPort(String protocol, Integer port) { 274 Validate.paramNotNull(protocol, "protocol"); 275 Validate.isTrue(protocol.equals("http") || protocol.equals("https"), 276 "Protocol must be 'http' or 'https', but was '%s'.", protocol); 277 278 String scheme = StringUtils.lowerCase(protocol); 279 280 return port == null || port == -1 || 281 (scheme.equals("http") && port == 80) || 282 (scheme.equals("https") && port == 443); 283 } 284 285 /** 286 * Retrieve the standard port for the provided protocol. 287 */ standardPort(String protocol)288 public static int standardPort(String protocol) { 289 if (protocol.equalsIgnoreCase("http")) { 290 return 80; 291 } else if (protocol.equalsIgnoreCase("https")) { 292 return 443; 293 } else { 294 throw new IllegalArgumentException("Unknown protocol: " + protocol); 295 } 296 } 297 298 /** 299 * Append the given path to the given baseUri, separating them with a slash, if required. The result will preserve the 300 * trailing slash of the provided path. 301 */ appendUri(String baseUri, String path)302 public static String appendUri(String baseUri, String path) { 303 Validate.paramNotNull(baseUri, "baseUri"); 304 StringBuilder resultUri = new StringBuilder(baseUri); 305 306 if (!StringUtils.isEmpty(path)) { 307 if (!baseUri.endsWith("/")) { 308 resultUri.append("/"); 309 } 310 311 resultUri.append(path.startsWith("/") ? path.substring(1) : path); 312 } 313 314 return resultUri.toString(); 315 } 316 317 /** 318 * Perform a case-insensitive search for a particular header in the provided map of headers. 319 * 320 * @param headers The headers to search. 321 * @param header The header to search for (case insensitively). 322 * @return A stream providing the values for the headers that matched the requested header. 323 * @deprecated Use {@code SdkHttpHeaders#matchingHeaders} 324 */ 325 @Deprecated allMatchingHeaders(Map<String, List<String>> headers, String header)326 public static Stream<String> allMatchingHeaders(Map<String, List<String>> headers, String header) { 327 return headers.entrySet().stream() 328 .filter(e -> e.getKey().equalsIgnoreCase(header)) 329 .flatMap(e -> e.getValue() != null ? e.getValue().stream() : Stream.empty()); 330 } 331 332 /** 333 * Perform a case-insensitive search for a particular header in the provided map of headers. 334 * 335 * @param headersToSearch The headers to search. 336 * @param headersToFind The headers to search for (case insensitively). 337 * @return A stream providing the values for the headers that matched the requested header. 338 * @deprecated Use {@code SdkHttpHeaders#matchingHeaders} 339 */ 340 @Deprecated allMatchingHeadersFromCollection(Map<String, List<String>> headersToSearch, Collection<String> headersToFind)341 public static Stream<String> allMatchingHeadersFromCollection(Map<String, List<String>> headersToSearch, 342 Collection<String> headersToFind) { 343 return headersToSearch.entrySet().stream() 344 .filter(e -> headersToFind.stream() 345 .anyMatch(headerToFind -> e.getKey().equalsIgnoreCase(headerToFind))) 346 .flatMap(e -> e.getValue() != null ? e.getValue().stream() : Stream.empty()); 347 } 348 349 /** 350 * Perform a case-insensitive search for a particular header in the provided map of headers, returning the first matching 351 * header, if one is found. 352 * <br> 353 * This is useful for headers like 'Content-Type' or 'Content-Length' of which there is expected to be only one value present. 354 * 355 * @param headers The headers to search. 356 * @param header The header to search for (case insensitively). 357 * @return The first header that matched the requested one, or empty if one was not found. 358 * @deprecated Use {@code SdkHttpHeaders#firstMatchingHeader} 359 */ 360 @Deprecated firstMatchingHeader(Map<String, List<String>> headers, String header)361 public static Optional<String> firstMatchingHeader(Map<String, List<String>> headers, String header) { 362 for (Entry<String, List<String>> headerEntry : headers.entrySet()) { 363 if (headerEntry.getKey().equalsIgnoreCase(header) && 364 headerEntry.getValue() != null && 365 !headerEntry.getValue().isEmpty()) { 366 return Optional.of(headerEntry.getValue().get(0)); 367 } 368 } 369 return Optional.empty(); 370 } 371 372 /** 373 * Perform a case-insensitive search for a set of headers in the provided map of headers, returning the first matching 374 * header, if one is found. 375 * 376 * @param headersToSearch The headers to search. 377 * @param headersToFind The header to search for (case insensitively). 378 * @return The first header that matched a requested one, or empty if one was not found. 379 * @deprecated Use {@code SdkHttpHeaders#firstMatchingHeader} 380 */ 381 @Deprecated firstMatchingHeaderFromCollection(Map<String, List<String>> headersToSearch, Collection<String> headersToFind)382 public static Optional<String> firstMatchingHeaderFromCollection(Map<String, List<String>> headersToSearch, 383 Collection<String> headersToFind) { 384 for (Entry<String, List<String>> headerEntry : headersToSearch.entrySet()) { 385 for (String headerToFind : headersToFind) { 386 if (headerEntry.getKey().equalsIgnoreCase(headerToFind) && 387 headerEntry.getValue() != null && 388 !headerEntry.getValue().isEmpty()) { 389 return Optional.of(headerEntry.getValue().get(0)); 390 } 391 } 392 } 393 394 return Optional.empty(); 395 } 396 isSingleHeader(String h)397 public static boolean isSingleHeader(String h) { 398 return SINGLE_HEADERS.contains(StringUtils.lowerCase(h)); 399 } 400 401 /** 402 * Extracts query parameters from the given URI 403 */ uriParams(URI uri)404 public static Map<String, List<String>> uriParams(URI uri) { 405 return splitQueryString(uri.getRawQuery()) 406 .stream() 407 .map(s -> s.split("=")) 408 .map(s -> s.length == 1 ? new String[] { s[0], null } : s) 409 .collect(groupingBy(a -> urlDecode(a[0]), mapping(a -> urlDecode(a[1]), toList()))); 410 } 411 splitQueryString(String queryString)412 public static List<String> splitQueryString(String queryString) { 413 List<String> results = new ArrayList<>(); 414 StringBuilder result = new StringBuilder(); 415 for (int i = 0; i < queryString.length(); i++) { 416 char character = queryString.charAt(i); 417 if (character != '&') { 418 result.append(character); 419 } else { 420 results.add(StringUtils.trimToEmpty(result.toString())); 421 result.setLength(0); 422 } 423 } 424 results.add(StringUtils.trimToEmpty(result.toString())); 425 return results; 426 } 427 428 /** 429 * Returns the Java system property for nonProxyHosts as set of Strings. 430 * See http://docs.oracle.com/javase/7/docs/api/java/net/doc-files/net-properties.html 431 */ parseNonProxyHostsProperty()432 public static Set<String> parseNonProxyHostsProperty() { 433 String systemNonProxyHosts = ProxySystemSetting.NON_PROXY_HOSTS.getStringValue().orElse(null); 434 return extractNonProxyHosts(systemNonProxyHosts); 435 } 436 extractNonProxyHosts(String nonProxyHosts)437 private static Set<String> extractNonProxyHosts(String nonProxyHosts) { 438 if (nonProxyHosts != null && !isEmpty(nonProxyHosts)) { 439 return Arrays.stream(nonProxyHosts.split("\\|")) 440 .map(String::toLowerCase) 441 .map(s -> StringUtils.replace(s, "*", ".*?")) 442 .collect(Collectors.toSet()); 443 } 444 return Collections.emptySet(); 445 } 446 parseNonProxyHostsEnvironmentVariable()447 public static Set<String> parseNonProxyHostsEnvironmentVariable() { 448 String hosts = ProxyEnvironmentSetting.NO_PROXY.getStringValue() 449 .map(noProxyHost -> noProxyHost.replace(",", "|")) 450 .orElse(null); 451 return extractNonProxyHosts(hosts); 452 } 453 } 454