1use std::io::{self, Cursor, Read};
2use std::net::SocketAddr;
3use std::num::NonZeroUsize;
4use std::str::FromStr;
5use std::{fmt, io::BufRead};
6
7use log::debug;
8use url::Url;
9
10use crate::body::SizedReader;
11use crate::chunked::Decoder as ChunkDecoder;
12use crate::error::{Error, ErrorKind::BadStatus};
13use crate::header::{get_all_headers, get_header, Header, HeaderLine};
14use crate::pool::{PoolReturnRead, PoolReturner};
15use crate::stream::{DeadlineStream, ReadOnlyStream, Stream};
16use crate::unit::Unit;
17use crate::{stream, Agent, ErrorKind};
18
19#[cfg(feature = "json")]
20use serde::de::DeserializeOwned;
21
22#[cfg(feature = "charset")]
23use encoding_rs::Encoding;
24
25#[cfg(feature = "gzip")]
26use flate2::read::MultiGzDecoder;
27
28#[cfg(feature = "brotli")]
29use brotli_decompressor::Decompressor as BrotliDecoder;
30
31pub const DEFAULT_CONTENT_TYPE: &str = "text/plain";
32pub const DEFAULT_CHARACTER_SET: &str = "utf-8";
33const INTO_STRING_LIMIT: usize = 10 * 1_024 * 1_024;
34// Follow the example of curl and limit a single header to 100kB:
35// https://curl.se/libcurl/c/CURLOPT_HEADERFUNCTION.html
36const MAX_HEADER_SIZE: usize = 100 * 1_024;
37const MAX_HEADER_COUNT: usize = 100;
38
39#[derive(Copy, Clone, Debug, PartialEq)]
40enum ConnectionOption {
41 KeepAlive,
42 Close,
43}
44
45#[derive(Copy, Clone, Debug, PartialEq)]
46enum BodyType {
47 LengthDelimited(usize),
48 Chunked,
49 CloseDelimited,
50}
51
52/// Response instances are created as results of firing off requests.
53///
54/// The `Response` is used to read response headers and decide what to do with the body.
55/// Note that the socket connection is open and the body not read until one of
56/// [`into_reader()`](#method.into_reader), [`into_json()`](#method.into_json), or
57/// [`into_string()`](#method.into_string) consumes the response.
58///
59/// When dropping a `Response` instance, one one of two things can happen. If
60/// the response has unread bytes, the underlying socket cannot be reused,
61/// and the connection is closed. If there are no unread bytes, the connection
62/// is returned to the [`Agent`] connection pool used (notice there is always
63/// an agent present, even when not explicitly configured by the user).
64///
65/// ```
66/// # fn main() -> Result<(), ureq::Error> {
67/// # ureq::is_test(true);
68/// let response = ureq::get("http://example.com/").call()?;
69///
70/// // socket is still open and the response body has not been read.
71///
72/// let text = response.into_string()?;
73///
74/// // response is consumed, and body has been read.
75/// # Ok(())
76/// # }
77/// ```
78pub struct Response {
79 pub(crate) url: Url,
80 pub(crate) status_line: String,
81 pub(crate) index: ResponseStatusIndex,
82 pub(crate) status: u16,
83 pub(crate) headers: Vec<Header>,
84 pub(crate) reader: Box<dyn Read + Send + Sync + 'static>,
85 /// The socket address of the server that sent the response.
86 pub(crate) remote_addr: SocketAddr,
87 /// The socket address of the client that sent the request.
88 pub(crate) local_addr: SocketAddr,
89 /// The redirect history of this response, if any. The history starts with
90 /// the first response received and ends with the response immediately
91 /// previous to this one.
92 ///
93 /// If this response was not redirected, the history is empty.
94 pub(crate) history: Vec<Url>,
95}
96
97/// index into status_line where we split: HTTP/1.1 200 OK
98#[derive(Debug, Clone, Copy, Eq, PartialEq)]
99pub(crate) struct ResponseStatusIndex {
100 pub(crate) http_version: usize,
101 pub(crate) response_code: usize,
102}
103
104impl fmt::Debug for Response {
105 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
106 write!(
107 f,
108 "Response[status: {}, status_text: {}, url: {}]",
109 self.status(),
110 self.status_text(),
111 self.url,
112 )
113 }
114}
115
116impl Response {
117 /// Construct a response with a status, status text and a string body.
118 ///
119 /// This is hopefully useful for unit tests.
120 ///
121 /// Example:
122 ///
123 /// ```
124 /// # fn main() -> Result<(), ureq::Error> {
125 /// # ureq::is_test(true);
126 /// let resp = ureq::Response::new(401, "Authorization Required", "Please log in")?;
127 ///
128 /// assert_eq!(resp.status(), 401);
129 /// # Ok(())
130 /// # }
131 /// ```
132 pub fn new(status: u16, status_text: &str, body: &str) -> Result<Response, Error> {
133 let r = format!("HTTP/1.1 {} {}\r\n\r\n{}", status, status_text, body);
134 (r.as_ref() as &str).parse()
135 }
136
137 /// The URL we ended up at. This can differ from the request url when
138 /// we have followed redirects.
139 pub fn get_url(&self) -> &str {
140 &self.url[..]
141 }
142
143 /// The http version: `HTTP/1.1`
144 pub fn http_version(&self) -> &str {
145 &self.status_line.as_str()[0..self.index.http_version]
146 }
147
148 /// The status as a u16: `200`
149 pub fn status(&self) -> u16 {
150 self.status
151 }
152
153 /// The status text: `OK`
154 ///
155 /// The HTTP spec allows for non-utf8 status texts. This uses from_utf8_lossy to
156 /// convert such lines to &str.
157 pub fn status_text(&self) -> &str {
158 self.status_line.as_str()[self.index.response_code + 1..].trim()
159 }
160
161 /// The header value for the given name, or None if not found.
162 ///
163 /// For historical reasons, the HTTP spec allows for header values
164 /// to be encoded using encodings like iso-8859-1. Such encodings
165 /// means the values are not possible to interpret as utf-8.
166 ///
167 /// In case the header value can't be read as utf-8, this function
168 /// returns `None` (while the name is visible in [`Response::headers_names()`]).
169 pub fn header(&self, name: &str) -> Option<&str> {
170 get_header(&self.headers, name)
171 }
172
173 /// A list of the header names in this response.
174 /// Lowercased to be uniform.
175 ///
176 /// It's possible for a header name to be returned by this function, and
177 /// still give a `None` value. See [`Response::header()`] for an explanation
178 /// as to why.
179 pub fn headers_names(&self) -> Vec<String> {
180 self.headers
181 .iter()
182 .map(|h| h.name().to_lowercase())
183 .collect()
184 }
185
186 /// Tells if the response has the named header.
187 pub fn has(&self, name: &str) -> bool {
188 self.header(name).is_some()
189 }
190
191 /// All headers corresponding values for the give name, or empty vector.
192 pub fn all(&self, name: &str) -> Vec<&str> {
193 get_all_headers(&self.headers, name)
194 }
195
196 /// The content type part of the "Content-Type" header without
197 /// the charset.
198 ///
199 /// Example:
200 ///
201 /// ```
202 /// # fn main() -> Result<(), ureq::Error> {
203 /// # ureq::is_test(true);
204 /// let resp = ureq::get("http://example.com/charset/iso").call()?;
205 /// assert_eq!(resp.header("content-type"), Some("text/html; charset=ISO-8859-1"));
206 /// assert_eq!("text/html", resp.content_type());
207 /// # Ok(())
208 /// # }
209 /// ```
210 pub fn content_type(&self) -> &str {
211 self.header("content-type")
212 .map(|header| {
213 header
214 .find(';')
215 .map(|index| &header[0..index])
216 .unwrap_or(header)
217 })
218 .unwrap_or(DEFAULT_CONTENT_TYPE)
219 }
220
221 /// The character set part of the "Content-Type".
222 ///
223 /// Example:
224 ///
225 /// ```
226 /// # fn main() -> Result<(), ureq::Error> {
227 /// # ureq::is_test(true);
228 /// let resp = ureq::get("http://example.com/charset/iso").call()?;
229 /// assert_eq!(resp.header("content-type"), Some("text/html; charset=ISO-8859-1"));
230 /// assert_eq!("ISO-8859-1", resp.charset());
231 /// # Ok(())
232 /// # }
233 /// ```
234 pub fn charset(&self) -> &str {
235 charset_from_content_type(self.header("content-type"))
236 }
237
238 /// The socket address of the server that sent the response.
239 pub fn remote_addr(&self) -> SocketAddr {
240 self.remote_addr
241 }
242
243 /// The local address the request was made from.
244 pub fn local_addr(&self) -> SocketAddr {
245 self.local_addr
246 }
247
248 /// Turn this response into a `impl Read` of the body.
249 ///
250 /// 1. If `Transfer-Encoding: chunked`, the returned reader will unchunk it
251 /// and any `Content-Length` header is ignored.
252 /// 2. If `Content-Length` is set, the returned reader is limited to this byte
253 /// length regardless of how many bytes the server sends.
254 /// 3. If no length header, the reader is until server stream end.
255 ///
256 /// Note: If you use `read_to_end()` on the resulting reader, a malicious
257 /// server might return enough bytes to exhaust available memory. If you're
258 /// making requests to untrusted servers, you should use `.take()` to
259 /// limit the response bytes read.
260 ///
261 /// Example:
262 ///
263 /// ```
264 /// use std::io::Read;
265 /// # fn main() -> Result<(), Box<dyn std::error::Error>> {
266 /// # ureq::is_test(true);
267 /// let resp = ureq::get("http://httpbin.org/bytes/100")
268 /// .call()?;
269 ///
270 /// assert!(resp.has("Content-Length"));
271 /// let len: usize = resp.header("Content-Length")
272 /// .unwrap()
273 /// .parse()?;
274 ///
275 /// let mut bytes: Vec<u8> = Vec::with_capacity(len);
276 /// resp.into_reader()
277 /// .take(10_000_000)
278 /// .read_to_end(&mut bytes)?;
279 ///
280 /// assert_eq!(bytes.len(), len);
281 /// # Ok(())
282 /// # }
283 /// ```
284 pub fn into_reader(self) -> Box<dyn Read + Send + Sync + 'static> {
285 self.reader
286 }
287
288 // Determine what to do with the connection after we've read the body.
289 fn connection_option(
290 response_version: &str,
291 connection_header: Option<&str>,
292 ) -> ConnectionOption {
293 // https://datatracker.ietf.org/doc/html/rfc9112#name-tear-down
294 // "A client that receives a "close" connection option MUST cease sending requests on that
295 // connection and close the connection after reading the response message containing the "close"
296 // connection option"
297 //
298 // Per https://www.rfc-editor.org/rfc/rfc2068#section-19.7.1, an HTTP/1.0 response can explicitly
299 // say "Connection: keep-alive" in response to a request with "Connection: keep-alive". We don't
300 // send "Connection: keep-alive" in the request but are willing to accept in the response anyhow.
301 use ConnectionOption::*;
302 let is_http10 = response_version.eq_ignore_ascii_case("HTTP/1.0");
303 match (is_http10, connection_header) {
304 (true, Some(c)) if c.eq_ignore_ascii_case("keep-alive") => KeepAlive,
305 (true, _) => Close,
306 (false, Some(c)) if c.eq_ignore_ascii_case("close") => Close,
307 (false, _) => KeepAlive,
308 }
309 }
310
311 /// Determine how the body should be read, based on
312 /// <https://datatracker.ietf.org/doc/html/rfc9112#name-message-body-length>
313 fn body_type(
314 request_method: &str,
315 response_status: u16,
316 response_version: &str,
317 headers: &[Header],
318 ) -> BodyType {
319 let is_http10 = response_version.eq_ignore_ascii_case("HTTP/1.0");
320
321 let is_head = request_method.eq_ignore_ascii_case("head");
322 let has_no_body = is_head
323 || match response_status {
324 204 | 304 => true,
325 _ => false,
326 };
327
328 if has_no_body {
329 return BodyType::LengthDelimited(0);
330 }
331
332 let is_chunked = get_header(headers, "transfer-encoding")
333 .map(|enc| !enc.is_empty()) // whatever it says, do chunked
334 .unwrap_or(false);
335
336 // https://www.rfc-editor.org/rfc/rfc2068#page-161
337 // > a persistent connection with an HTTP/1.0 client cannot make
338 // > use of the chunked transfer-coding
339 let use_chunked = !is_http10 && is_chunked;
340
341 if use_chunked {
342 return BodyType::Chunked;
343 }
344
345 let length = get_header(headers, "content-length").and_then(|v| v.parse::<usize>().ok());
346
347 match length {
348 Some(n) => BodyType::LengthDelimited(n),
349 None => BodyType::CloseDelimited,
350 }
351 }
352
353 fn stream_to_reader(
354 mut stream: DeadlineStream,
355 unit: &Unit,
356 body_type: BodyType,
357 compression: Option<Compression>,
358 connection_option: ConnectionOption,
359 ) -> Box<dyn Read + Send + Sync + 'static> {
360 if connection_option == ConnectionOption::Close {
361 stream.inner_mut().set_unpoolable();
362 }
363 let inner = stream.inner_ref();
364 let result = inner.set_read_timeout(unit.agent.config.timeout_read);
365 if let Err(e) = result {
366 return Box::new(ErrorReader(e));
367 }
368 let buffer_len = inner.buffer().len();
369
370 let body_reader: Box<dyn Read + Send + Sync> = match body_type {
371 // Chunked responses have an unknown length, but do have an end of body
372 // marker. When we encounter the marker, we can return the underlying stream
373 // to the connection pool.
374 BodyType::Chunked => {
375 debug!("Chunked body in response");
376 Box::new(PoolReturnRead::new(ChunkDecoder::new(stream)))
377 }
378 // Responses with a content-length header means we should limit the reading
379 // of the body to the number of bytes in the header. Once done, we can
380 // return the underlying stream to the connection pool.
381 BodyType::LengthDelimited(len) => {
382 match NonZeroUsize::new(len) {
383 None => {
384 debug!("zero-length body returning stream directly to pool");
385 let stream: Stream = stream.into();
386 // TODO: This expect can actually panic if we get an error when
387 // returning the stream to the pool. We reset the read timeouts
388 // when we do that, and since that's a syscall it can fail.
389 stream.return_to_pool().expect("returning stream to pool");
390 Box::new(std::io::empty())
391 }
392 Some(len) => {
393 let mut limited_read = LimitedRead::new(stream, len);
394
395 if len.get() <= buffer_len {
396 debug!("Body entirely buffered (length: {})", len);
397 let mut buf = vec![0; len.get()];
398 // TODO: This expect can actually panic if we get an error when
399 // returning the stream to the pool. We reset the read timeouts
400 // when we do that, and since that's a syscall it can fail.
401 limited_read
402 .read_exact(&mut buf)
403 .expect("failed to read exact buffer length from stream");
404 Box::new(Cursor::new(buf))
405 } else {
406 debug!("Streaming body until content-length: {}", len);
407 Box::new(limited_read)
408 }
409 }
410 }
411 }
412 BodyType::CloseDelimited => {
413 debug!("Body of unknown size - read until socket close");
414 Box::new(stream)
415 }
416 };
417
418 match compression {
419 None => body_reader,
420 Some(c) => c.wrap_reader(body_reader),
421 }
422 }
423
424 /// Turn this response into a String of the response body. By default uses `utf-8`,
425 /// but can work with charset, see below.
426 ///
427 /// This is potentially memory inefficient for large bodies since the
428 /// implementation first reads the reader to end into a `Vec<u8>` and then
429 /// attempts to decode it using the charset.
430 ///
431 /// If the response is larger than 10 megabytes, this will return an error.
432 ///
433 /// Example:
434 ///
435 /// ```
436 /// # fn main() -> Result<(), ureq::Error> {
437 /// # ureq::is_test(true);
438 /// let text = ureq::get("http://httpbin.org/get?success")
439 /// .call()?
440 /// .into_string()?;
441 ///
442 /// assert!(text.contains("success"));
443 /// # Ok(())
444 /// # }
445 /// ```
446 ///
447 /// ## Charset support
448 ///
449 /// If you enable feature `ureq = { version = "*", features = ["charset"] }`, into_string()
450 /// attempts to respect the character encoding of the `Content-Type` header. If there is no
451 /// Content-Type header, or the Content-Type header does not specify a charset, into_string()
452 /// uses `utf-8`.
453 ///
454 /// I.e. `Content-Type: text/plain; charset=iso-8859-1` would be decoded in latin-1.
455 ///
456 pub fn into_string(self) -> io::Result<String> {
457 #[cfg(feature = "charset")]
458 let encoding = Encoding::for_label(self.charset().as_bytes())
459 .or_else(|| Encoding::for_label(DEFAULT_CHARACTER_SET.as_bytes()))
460 .unwrap();
461
462 let mut buf: Vec<u8> = vec![];
463 self.into_reader()
464 .take((INTO_STRING_LIMIT + 1) as u64)
465 .read_to_end(&mut buf)?;
466 if buf.len() > INTO_STRING_LIMIT {
467 return Err(io::Error::new(
468 io::ErrorKind::Other,
469 "response too big for into_string",
470 ));
471 }
472
473 #[cfg(feature = "charset")]
474 {
475 let (text, _, _) = encoding.decode(&buf);
476 Ok(text.into_owned())
477 }
478 #[cfg(not(feature = "charset"))]
479 {
480 Ok(String::from_utf8_lossy(&buf).to_string())
481 }
482 }
483
484 /// Read the body of this response into a serde_json::Value, or any other type that
485 /// implements the [serde::Deserialize] trait.
486 ///
487 /// You must use either a type annotation as shown below (`message: Message`), or the
488 /// [turbofish operator] (`::<Type>`) so Rust knows what type you are trying to read.
489 ///
490 /// [turbofish operator]: https://matematikaadit.github.io/posts/rust-turbofish.html
491 ///
492 /// Example:
493 ///
494 /// ```
495 /// # fn main() -> Result<(), ureq::Error> {
496 /// # ureq::is_test(true);
497 /// // This import requires the `derive` feature on `serde`.
498 /// // Put this in Cargo.toml: serde = { version = "1", features = ["derive"] }
499 /// use serde::{Deserialize};
500 ///
501 /// #[derive(Deserialize)]
502 /// struct Message {
503 /// text: String,
504 /// }
505 ///
506 /// let message: Message =
507 /// ureq::get("http://example.com/get/hello_world.json")
508 /// .call()?
509 /// .into_json()?;
510 ///
511 /// assert_eq!(message.text, "Ok");
512 /// # Ok(())
513 /// # }
514 /// ```
515 ///
516 /// Or, if you don't want to define a struct to read your JSON into, you can
517 /// use the convenient `serde_json::Value` type to parse arbitrary or unknown
518 /// JSON.
519 ///
520 /// ```
521 /// # fn main() -> Result<(), ureq::Error> {
522 /// # ureq::is_test(true);
523 /// let json: serde_json::Value = ureq::get("http://example.com/get/hello_world.json")
524 /// .call()?
525 /// .into_json()?;
526 ///
527 /// assert_eq!(json["text"], "Ok");
528 /// # Ok(())
529 /// # }
530 /// ```
531 #[cfg(feature = "json")]
532 pub fn into_json<T: DeserializeOwned>(self) -> io::Result<T> {
533 use crate::stream::io_err_timeout;
534
535 let reader = self.into_reader();
536 serde_json::from_reader(reader).map_err(|e| {
537 // This is to unify TimedOut io::Error in the API.
538 if let Some(kind) = e.io_error_kind() {
539 if kind == io::ErrorKind::TimedOut {
540 return io_err_timeout(e.to_string());
541 }
542 }
543
544 io::Error::new(
545 io::ErrorKind::InvalidData,
546 format!("Failed to read JSON: {}", e),
547 )
548 })
549 }
550
551 /// Create a response from a Read trait impl.
552 ///
553 /// This is hopefully useful for unit tests.
554 ///
555 /// Example:
556 ///
557 /// use std::io::Cursor;
558 ///
559 /// let text = "HTTP/1.1 401 Authorization Required\r\n\r\nPlease log in\n";
560 /// let read = Cursor::new(text.to_string().into_bytes());
561 /// let resp = ureq::Response::do_from_read(read);
562 ///
563 /// assert_eq!(resp.status(), 401);
564 pub(crate) fn do_from_stream(stream: Stream, unit: Unit) -> Result<Response, Error> {
565 let remote_addr = stream.remote_addr;
566
567 let local_addr = match stream.socket() {
568 Some(sock) => sock.local_addr().map_err(Error::from)?,
569 None => std::net::SocketAddrV4::new(std::net::Ipv4Addr::new(127, 0, 0, 1), 0).into(),
570 };
571
572 //
573 // HTTP/1.1 200 OK\r\n
574 let mut stream = stream::DeadlineStream::new(stream, unit.deadline);
575
576 // The status line we can ignore non-utf8 chars and parse as_str_lossy().
577 let status_line = read_next_line(&mut stream, "the status line")?.into_string_lossy();
578 let (index, status) = parse_status_line(status_line.as_str())?;
579 let http_version = &status_line.as_str()[0..index.http_version];
580
581 let mut headers: Vec<Header> = Vec::new();
582 while headers.len() <= MAX_HEADER_COUNT {
583 let line = read_next_line(&mut stream, "a header")?;
584 if line.is_empty() {
585 break;
586 }
587 if let Ok(header) = line.into_header() {
588 headers.push(header);
589 }
590 }
591
592 if headers.len() > MAX_HEADER_COUNT {
593 return Err(ErrorKind::BadHeader.msg(
594 format!("more than {} header fields in response", MAX_HEADER_COUNT).as_str(),
595 ));
596 }
597
598 let compression =
599 get_header(&headers, "content-encoding").and_then(Compression::from_header_value);
600
601 let connection_option =
602 Self::connection_option(http_version, get_header(&headers, "connection"));
603
604 let body_type = Self::body_type(&unit.method, status, http_version, &headers);
605
606 // remove Content-Encoding and length due to automatic decompression
607 if compression.is_some() {
608 headers.retain(|h| !h.is_name("content-encoding") && !h.is_name("content-length"));
609 }
610
611 let reader =
612 Self::stream_to_reader(stream, &unit, body_type, compression, connection_option);
613
614 let url = unit.url.clone();
615
616 let response = Response {
617 url,
618 status_line,
619 index,
620 status,
621 headers,
622 reader,
623 remote_addr,
624 local_addr,
625 history: vec![],
626 };
627 Ok(response)
628 }
629
630 #[cfg(test)]
631 pub fn set_url(&mut self, url: Url) {
632 self.url = url;
633 }
634
635 #[cfg(test)]
636 pub fn history_from_previous(&mut self, previous: Response) {
637 self.history = previous.history;
638 self.history.push(previous.url);
639 }
640}
641
642#[derive(Copy, Clone, Debug, PartialEq, Eq)]
643pub(crate) enum Compression {
644 #[cfg(feature = "brotli")]
645 Brotli,
646 #[cfg(feature = "gzip")]
647 Gzip,
648}
649
650impl Compression {
651 /// Convert a string like "br" to an enum value
652 fn from_header_value(value: &str) -> Option<Compression> {
653 match value {
654 #[cfg(feature = "brotli")]
655 "br" => Some(Compression::Brotli),
656 #[cfg(feature = "gzip")]
657 "gzip" | "x-gzip" => Some(Compression::Gzip),
658 _ => None,
659 }
660 }
661
662 /// Wrap the raw reader with a decompressing reader
663 #[allow(unused_variables)] // when no features enabled, reader is unused (unreachable)
664 pub(crate) fn wrap_reader(
665 self,
666 reader: Box<dyn Read + Send + Sync + 'static>,
667 ) -> Box<dyn Read + Send + Sync + 'static> {
668 match self {
669 #[cfg(feature = "brotli")]
670 Compression::Brotli => Box::new(BrotliDecoder::new(reader, 4096)),
671 #[cfg(feature = "gzip")]
672 Compression::Gzip => Box::new(MultiGzDecoder::new(reader)),
673 }
674 }
675}
676
677/// parse a line like: HTTP/1.1 200 OK\r\n
678fn parse_status_line(line: &str) -> Result<(ResponseStatusIndex, u16), Error> {
679 //
680
681 if !line.is_ascii() {
682 return Err(BadStatus.msg("Status line not ASCII"));
683 }
684 // https://tools.ietf.org/html/rfc7230#section-3.1.2
685 // status-line = HTTP-version SP status-code SP reason-phrase CRLF
686 let mut split: Vec<&str> = line.splitn(3, ' ').collect();
687 if split.len() == 2 {
688 // As a special case, we are lenient parsing lines without a space after the code.
689 // This is technically against spec. "HTTP/1.1 200\r\n"
690 split.push("");
691 }
692 if split.len() != 3 {
693 return Err(BadStatus.msg("Wrong number of tokens in status line"));
694 }
695
696 // https://tools.ietf.org/html/rfc7230#appendix-B
697 // HTTP-name = %x48.54.54.50 ; HTTP
698 // HTTP-version = HTTP-name "/" DIGIT "." DIGIT
699 let http_version = split[0];
700 if !http_version.starts_with("HTTP/") {
701 return Err(BadStatus.msg("HTTP version did not start with HTTP/"));
702 }
703 if http_version.len() != 8 {
704 return Err(BadStatus.msg("HTTP version was wrong length"));
705 }
706 if !http_version.as_bytes()[5].is_ascii_digit() || !http_version.as_bytes()[7].is_ascii_digit()
707 {
708 return Err(BadStatus.msg("HTTP version did not match format"));
709 }
710
711 let status_str: &str = split[1];
712 // status-code = 3DIGIT
713 if status_str.len() != 3 {
714 return Err(BadStatus.msg("Status code was wrong length"));
715 }
716
717 let status: u16 = status_str
718 .parse()
719 .map_err(|_| BadStatus.msg(format!("unable to parse status as u16 ({})", status_str)))?;
720
721 Ok((
722 ResponseStatusIndex {
723 http_version: http_version.len(),
724 response_code: http_version.len() + status_str.len(),
725 },
726 status,
727 ))
728}
729
730impl FromStr for Response {
731 type Err = Error;
732 /// Parse a response from a string.
733 ///
734 /// Example:
735 /// ```
736 /// # fn main() -> Result<(), ureq::Error> {
737 /// let s = "HTTP/1.1 200 OK\r\n\
738 /// X-Forwarded-For: 1.2.3.4\r\n\
739 /// Content-Type: text/plain\r\n\
740 /// \r\n\
741 /// Hello World!!!";
742 /// let resp: ureq::Response = s.parse()?;
743 /// assert!(resp.has("X-Forwarded-For"));
744 /// let body = resp.into_string()?;
745 /// assert_eq!(body, "Hello World!!!");
746 /// # Ok(())
747 /// # }
748 /// ```
749 fn from_str(s: &str) -> Result<Self, Self::Err> {
750 let remote_addr = "0.0.0.0:0".parse().unwrap();
751 let stream = Stream::new(
752 ReadOnlyStream::new(s.into()),
753 remote_addr,
754 PoolReturner::none(),
755 );
756 let request_url = "https://example.com".parse().unwrap();
757 let request_reader = SizedReader {
758 size: crate::body::BodySize::Empty,
759 reader: Box::new(std::io::empty()),
760 };
761 let unit = Unit::new(
762 &Agent::new(),
763 "GET",
764 &request_url,
765 vec![],
766 &request_reader,
767 None,
768 );
769 Self::do_from_stream(stream, unit)
770 }
771}
772
773fn read_next_line(reader: &mut impl BufRead, context: &str) -> io::Result<HeaderLine> {
774 let mut buf = Vec::new();
775 let result = reader
776 .take((MAX_HEADER_SIZE + 1) as u64)
777 .read_until(b'\n', &mut buf);
778
779 match result {
780 Ok(0) => Err(io::Error::new(
781 io::ErrorKind::ConnectionAborted,
782 "Unexpected EOF",
783 )),
784 Ok(n) if n > MAX_HEADER_SIZE => Err(io::Error::new(
785 io::ErrorKind::Other,
786 format!("header field longer than {} bytes", MAX_HEADER_SIZE),
787 )),
788 Ok(_) => Ok(()),
789 Err(e) => {
790 // Provide context to errors encountered while reading the line.
791 let reason = format!("Error encountered in {}", context);
792
793 let kind = e.kind();
794
795 // Use an intermediate wrapper type which carries the error message
796 // as well as a .source() reference to the original error.
797 let wrapper = Error::new(ErrorKind::Io, Some(reason)).src(e);
798
799 Err(io::Error::new(kind, wrapper))
800 }
801 }?;
802
803 if !buf.ends_with(b"\n") {
804 return Err(io::Error::new(
805 io::ErrorKind::InvalidInput,
806 format!("Header field didn't end with \\n: {:?}", buf),
807 ));
808 }
809
810 buf.pop();
811 if buf.ends_with(b"\r") {
812 buf.pop();
813 }
814
815 Ok(buf.into())
816}
817
818/// Limits a `Read` to a content size (as set by a "Content-Length" header).
819pub(crate) struct LimitedRead<R> {
820 reader: Option<R>,
821 limit: usize,
822 position: usize,
823}
824
825impl<R: Read + Sized + Into<Stream>> LimitedRead<R> {
826 pub(crate) fn new(reader: R, limit: NonZeroUsize) -> Self {
827 LimitedRead {
828 reader: Some(reader),
829 limit: limit.get(),
830 position: 0,
831 }
832 }
833
834 pub(crate) fn remaining(&self) -> usize {
835 self.limit - self.position
836 }
837
838 fn return_stream_to_pool(&mut self) -> io::Result<()> {
839 if let Some(reader: R) = self.reader.take() {
840 // Convert back to a stream. If return_to_pool fails, the stream will
841 // drop and the connection will be closed.
842 let stream: Stream = reader.into();
843 stream.return_to_pool()?;
844 }
845
846 Ok(())
847 }
848}
849
850impl<R: Read + Sized + Into<Stream>> Read for LimitedRead<R> {
851 fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
852 if self.remaining() == 0 {
853 return Ok(0);
854 }
855 let from = if self.remaining() < buf.len() {
856 &mut buf[0..self.remaining()]
857 } else {
858 buf
859 };
860 let reader = match self.reader.as_mut() {
861 // If the reader has already been taken, return Ok(0) to all reads.
862 None => return Ok(0),
863 Some(r) => r,
864 };
865 match reader.read(from) {
866 // https://tools.ietf.org/html/rfc7230#page-33
867 // If the sender closes the connection or
868 // the recipient times out before the indicated number of octets are
869 // received, the recipient MUST consider the message to be
870 // incomplete and close the connection.
871 // TODO: actually close the connection by dropping the stream
872 Ok(0) => Err(io::Error::new(
873 io::ErrorKind::UnexpectedEof,
874 "response body closed before all bytes were read",
875 )),
876 Ok(amount) => {
877 self.position += amount;
878 if self.remaining() == 0 {
879 self.return_stream_to_pool()?;
880 }
881 Ok(amount)
882 }
883 Err(e) => Err(e),
884 }
885 }
886}
887
888/// Extract the charset from a "Content-Type" header.
889///
890/// "Content-Type: text/plain; charset=iso8859-1" -> "iso8859-1"
891///
892/// *Internal API*
893pub(crate) fn charset_from_content_type(header: Option<&str>) -> &str {
894 headerOption<&str>
895 .and_then(|header: &str| {
896 header.find(';').and_then(|semi: usize| {
897 headerOption[semi + 1..]
898 .find('=')
899 .map(|equal: usize| header[semi + equal + 2..].trim())
900 })
901 })
902 .unwrap_or(DEFAULT_CHARACTER_SET)
903}
904
905// ErrorReader returns an error for every read.
906// The error is as close to a clone of the underlying
907// io::Error as we can get.
908struct ErrorReader(io::Error);
909
910impl Read for ErrorReader {
911 fn read(&mut self, _buf: &mut [u8]) -> io::Result<usize> {
912 Err(io::Error::new(self.0.kind(), self.0.to_string()))
913 }
914}
915
916#[cfg(test)]
917mod tests {
918 use std::io::Cursor;
919
920 use crate::{body::Payload, pool::PoolKey};
921
922 use super::*;
923
924 #[test]
925 fn short_read() {
926 use std::io::Cursor;
927 let test_stream = crate::test::TestStream::new(Cursor::new(vec![b'a'; 3]), std::io::sink());
928 let stream = Stream::new(
929 test_stream,
930 "1.1.1.1:4343".parse().unwrap(),
931 PoolReturner::none(),
932 );
933 let mut lr = LimitedRead::new(stream, std::num::NonZeroUsize::new(10).unwrap());
934 let mut buf = vec![0; 1000];
935 let result = lr.read_to_end(&mut buf);
936 assert!(result.err().unwrap().kind() == io::ErrorKind::UnexpectedEof);
937 }
938
939 #[test]
940 fn content_type_without_charset() {
941 let s = "HTTP/1.1 200 OK\r\n\
942 Content-Type: application/json\r\n\
943 \r\n\
944 OK";
945 let resp = s.parse::<Response>().unwrap();
946 assert_eq!("application/json", resp.content_type());
947 }
948
949 #[test]
950 fn content_type_without_cr() {
951 let s = "HTTP/1.1 200 OK\r\n\
952 Content-Type: application/json\n\
953 \r\n\
954 OK";
955 let resp = s.parse::<Response>().unwrap();
956 assert_eq!("application/json", resp.content_type());
957 }
958
959 #[test]
960 fn content_type_with_charset() {
961 let s = "HTTP/1.1 200 OK\r\n\
962 Content-Type: application/json; charset=iso-8859-4\r\n\
963 \r\n\
964 OK";
965 let resp = s.parse::<Response>().unwrap();
966 assert_eq!("application/json", resp.content_type());
967 }
968
969 #[test]
970 fn content_type_default() {
971 let s = "HTTP/1.1 200 OK\r\n\r\nOK";
972 let resp = s.parse::<Response>().unwrap();
973 assert_eq!("text/plain", resp.content_type());
974 }
975
976 #[test]
977 fn charset() {
978 let s = "HTTP/1.1 200 OK\r\n\
979 Content-Type: application/json; charset=iso-8859-4\r\n\
980 \r\n\
981 OK";
982 let resp = s.parse::<Response>().unwrap();
983 assert_eq!("iso-8859-4", resp.charset());
984 }
985
986 #[test]
987 fn charset_default() {
988 let s = "HTTP/1.1 200 OK\r\n\
989 Content-Type: application/json\r\n\
990 \r\n\
991 OK";
992 let resp = s.parse::<Response>().unwrap();
993 assert_eq!("utf-8", resp.charset());
994 }
995
996 #[test]
997 fn chunked_transfer() {
998 let s = "HTTP/1.1 200 OK\r\n\
999 Transfer-Encoding: Chunked\r\n\
1000 \r\n\
1001 3\r\n\
1002 hel\r\n\
1003 b\r\n\
1004 lo world!!!\r\n\
1005 0\r\n\
1006 \r\n";
1007 let resp = s.parse::<Response>().unwrap();
1008 assert_eq!("hello world!!!", resp.into_string().unwrap());
1009 }
1010
1011 #[test]
1012 fn into_string_large() {
1013 const LEN: usize = INTO_STRING_LIMIT + 1;
1014 let s = format!(
1015 "HTTP/1.1 200 OK\r\n\
1016 Content-Length: {}\r\n
1017 \r\n
1018 {}",
1019 LEN,
1020 "A".repeat(LEN),
1021 );
1022 let result = s.parse::<Response>().unwrap();
1023 let err = result
1024 .into_string()
1025 .expect_err("didn't error with too-long body");
1026 assert_eq!(err.to_string(), "response too big for into_string");
1027 assert_eq!(err.kind(), io::ErrorKind::Other);
1028 }
1029
1030 #[test]
1031 #[cfg(feature = "json")]
1032 fn parse_simple_json() {
1033 let s = "HTTP/1.1 200 OK\r\n\
1034 \r\n\
1035 {\"hello\":\"world\"}";
1036 let resp = s.parse::<Response>().unwrap();
1037 let v: serde_json::Value = resp.into_json().unwrap();
1038 let compare = "{\"hello\":\"world\"}"
1039 .parse::<serde_json::Value>()
1040 .unwrap();
1041 assert_eq!(v, compare);
1042 }
1043
1044 #[test]
1045 #[cfg(feature = "json")]
1046 fn parse_deserialize_json() {
1047 use serde::Deserialize;
1048
1049 #[derive(Deserialize)]
1050 struct Hello {
1051 hello: String,
1052 }
1053
1054 let s = "HTTP/1.1 200 OK\r\n\
1055 \r\n\
1056 {\"hello\":\"world\"}";
1057 let resp = s.parse::<Response>().unwrap();
1058 let v: Hello = resp.into_json::<Hello>().unwrap();
1059 assert_eq!(v.hello, "world");
1060 }
1061
1062 #[test]
1063 fn parse_borked_header() {
1064 let s = "HTTP/1.1 BORKED\r\n".to_string();
1065 let err = s.parse::<Response>().unwrap_err();
1066 assert_eq!(err.kind(), BadStatus);
1067 }
1068
1069 #[test]
1070 fn parse_header_without_reason() {
1071 let s = "HTTP/1.1 302\r\n\r\n".to_string();
1072 let resp = s.parse::<Response>().unwrap();
1073 assert_eq!(resp.status_text(), "");
1074 }
1075
1076 #[test]
1077 fn read_next_line_large() {
1078 const LEN: usize = MAX_HEADER_SIZE + 1;
1079 let s = format!("Long-Header: {}\r\n", "A".repeat(LEN),);
1080 let mut cursor = Cursor::new(s);
1081 let result = read_next_line(&mut cursor, "some context");
1082 let err = result.expect_err("did not error on too-large header");
1083 assert_eq!(err.kind(), io::ErrorKind::Other);
1084 assert_eq!(
1085 err.to_string(),
1086 format!("header field longer than {} bytes", MAX_HEADER_SIZE)
1087 );
1088 }
1089
1090 #[test]
1091 fn too_many_headers() {
1092 const LEN: usize = MAX_HEADER_COUNT + 1;
1093 let s = format!(
1094 "HTTP/1.1 200 OK\r\n\
1095 {}
1096 \r\n
1097 hi",
1098 "Header: value\r\n".repeat(LEN),
1099 );
1100 let err = s
1101 .parse::<Response>()
1102 .expect_err("did not error on too many headers");
1103 assert_eq!(err.kind(), ErrorKind::BadHeader);
1104 assert_eq!(
1105 err.to_string(),
1106 format!(
1107 "Bad Header: more than {} header fields in response",
1108 MAX_HEADER_COUNT
1109 )
1110 );
1111 }
1112
1113 #[test]
1114 #[cfg(feature = "charset")]
1115 fn read_next_line_non_ascii_reason() {
1116 let (cow, _, _) =
1117 encoding_rs::WINDOWS_1252.encode("HTTP/1.1 302 Déplacé Temporairement\r\n");
1118 let bytes = cow.to_vec();
1119 let mut reader = io::BufReader::new(io::Cursor::new(bytes));
1120 let r = read_next_line(&mut reader, "test status line");
1121 let h = r.unwrap();
1122 assert_eq!(h.to_string(), "HTTP/1.1 302 D�plac� Temporairement");
1123 }
1124
1125 #[test]
1126 #[cfg(feature = "charset")]
1127 fn parse_header_with_non_utf8() {
1128 let (cow, _, _) = encoding_rs::WINDOWS_1252.encode(
1129 "HTTP/1.1 200 OK\r\n\
1130 x-geo-header: gött mos!\r\n\
1131 \r\n\
1132 OK",
1133 );
1134 let v = cow.to_vec();
1135 let s = Stream::new(
1136 ReadOnlyStream::new(v),
1137 crate::stream::remote_addr_for_test(),
1138 PoolReturner::none(),
1139 );
1140 let request_url = "https://example.com".parse().unwrap();
1141 let request_reader = SizedReader {
1142 size: crate::body::BodySize::Empty,
1143 reader: Box::new(std::io::empty()),
1144 };
1145 let unit = Unit::new(
1146 &Agent::new(),
1147 "GET",
1148 &request_url,
1149 vec![],
1150 &request_reader,
1151 None,
1152 );
1153 let resp = Response::do_from_stream(s.into(), unit).unwrap();
1154 assert_eq!(resp.status(), 200);
1155 assert_eq!(resp.header("x-geo-header"), None);
1156 }
1157
1158 #[test]
1159 fn history() {
1160 let mut response0 = Response::new(302, "Found", "").unwrap();
1161 response0.set_url("http://1.example.com/".parse().unwrap());
1162 assert!(response0.history.is_empty());
1163
1164 let mut response1 = Response::new(302, "Found", "").unwrap();
1165 response1.set_url("http://2.example.com/".parse().unwrap());
1166 response1.history_from_previous(response0);
1167
1168 let mut response2 = Response::new(404, "NotFound", "").unwrap();
1169 response2.set_url("http://2.example.com/".parse().unwrap());
1170 response2.history_from_previous(response1);
1171
1172 let hist: Vec<String> = response2.history.iter().map(|r| r.to_string()).collect();
1173 assert_eq!(hist, ["http://1.example.com/", "http://2.example.com/"])
1174 }
1175
1176 #[test]
1177 fn response_implements_send_and_sync() {
1178 let _response: Box<dyn Send> = Box::new(Response::new(302, "Found", "").unwrap());
1179 let _response: Box<dyn Sync> = Box::new(Response::new(302, "Found", "").unwrap());
1180 }
1181
1182 #[test]
1183 fn ensure_response_size() {
1184 // This is platform dependent, so we can't be too strict or precise.
1185 let size = std::mem::size_of::<Response>();
1186 println!("Response size: {}", size);
1187 assert!(size < 400); // 200 on Macbook M1
1188 }
1189
1190 // Test that a stream gets returned to the pool immediately for a zero-length response, and
1191 // that reads from the response's body consistently return Ok(0).
1192 #[test]
1193 fn zero_length_body_immediate_return() {
1194 use std::io::Cursor;
1195 let response_bytes = "HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n"
1196 .as_bytes()
1197 .to_vec();
1198 let test_stream =
1199 crate::test::TestStream::new(Cursor::new(response_bytes), std::io::sink());
1200 let agent = Agent::new();
1201 let agent2 = agent.clone();
1202 let stream = Stream::new(
1203 test_stream,
1204 "1.1.1.1:4343".parse().unwrap(),
1205 PoolReturner::new(&agent, PoolKey::from_parts("https", "example.com", 443)),
1206 );
1207 Response::do_from_stream(
1208 stream,
1209 Unit::new(
1210 &agent,
1211 "GET",
1212 &"https://example.com/".parse().unwrap(),
1213 vec![],
1214 &Payload::Empty.into_read(),
1215 None,
1216 ),
1217 )
1218 .unwrap();
1219 assert_eq!(agent2.state.pool.len(), 1);
1220 }
1221
1222 #[test]
1223 #[cfg(feature = "gzip")]
1224 fn gzip_content_length() {
1225 use std::io::Cursor;
1226 let response_bytes =
1227 b"HTTP/1.1 200 OK\r\nContent-Encoding: gzip\r\nContent-Length: 23\r\n\r\n\
1228\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\x03\xcb\xc8\xe4\x02\x00\x7a\x7a\x6f\xed\x03\x00\x00\x00";
1229 // Follow the response with an infinite stream of 0 bytes, so the content-length
1230 // is important.
1231 let reader = Cursor::new(response_bytes).chain(std::io::repeat(0u8));
1232 let test_stream = crate::test::TestStream::new(reader, std::io::sink());
1233 let agent = Agent::new();
1234 let stream = Stream::new(
1235 test_stream,
1236 "1.1.1.1:4343".parse().unwrap(),
1237 PoolReturner::none(),
1238 );
1239 let resp = Response::do_from_stream(
1240 stream,
1241 Unit::new(
1242 &agent,
1243 "GET",
1244 &"https://example.com/".parse().unwrap(),
1245 vec![],
1246 &Payload::Empty.into_read(),
1247 None,
1248 ),
1249 )
1250 .unwrap();
1251 let body = resp.into_string().unwrap();
1252 assert_eq!(body, "hi\n");
1253 }
1254
1255 #[test]
1256 fn connection_option() {
1257 use ConnectionOption::*;
1258 assert_eq!(Response::connection_option("HTTP/1.0", None), Close);
1259 assert_eq!(Response::connection_option("HtTp/1.0", None), Close);
1260 assert_eq!(Response::connection_option("HTTP/1.0", Some("blah")), Close);
1261 assert_eq!(
1262 Response::connection_option("HTTP/1.0", Some("keep-ALIVE")),
1263 KeepAlive
1264 );
1265 assert_eq!(
1266 Response::connection_option("http/1.0", Some("keep-alive")),
1267 KeepAlive
1268 );
1269
1270 assert_eq!(Response::connection_option("http/1.1", None), KeepAlive);
1271 assert_eq!(
1272 Response::connection_option("http/1.1", Some("blah")),
1273 KeepAlive
1274 );
1275 assert_eq!(
1276 Response::connection_option("http/1.1", Some("keep-alive")),
1277 KeepAlive
1278 );
1279 assert_eq!(
1280 Response::connection_option("http/1.1", Some("CLOSE")),
1281 Close
1282 );
1283 }
1284
1285 #[test]
1286 fn body_type() {
1287 use BodyType::*;
1288 assert_eq!(
1289 Response::body_type("GET", 200, "HTTP/1.1", &[]),
1290 CloseDelimited
1291 );
1292 assert_eq!(
1293 Response::body_type("HEAD", 200, "HTTP/1.1", &[]),
1294 LengthDelimited(0)
1295 );
1296 assert_eq!(
1297 Response::body_type("hEaD", 200, "HTTP/1.1", &[]),
1298 LengthDelimited(0)
1299 );
1300 assert_eq!(
1301 Response::body_type("head", 200, "HTTP/1.1", &[]),
1302 LengthDelimited(0)
1303 );
1304 assert_eq!(
1305 Response::body_type("GET", 304, "HTTP/1.1", &[]),
1306 LengthDelimited(0)
1307 );
1308 assert_eq!(
1309 Response::body_type("GET", 204, "HTTP/1.1", &[]),
1310 LengthDelimited(0)
1311 );
1312 assert_eq!(
1313 Response::body_type(
1314 "GET",
1315 200,
1316 "HTTP/1.1",
1317 &[Header::new("Transfer-Encoding", "chunked"),]
1318 ),
1319 Chunked
1320 );
1321 assert_eq!(
1322 Response::body_type(
1323 "GET",
1324 200,
1325 "HTTP/1.1",
1326 &[Header::new("Content-Length", "123"),]
1327 ),
1328 LengthDelimited(123)
1329 );
1330 assert_eq!(
1331 Response::body_type(
1332 "GET",
1333 200,
1334 "HTTP/1.1",
1335 &[
1336 Header::new("Content-Length", "123"),
1337 Header::new("Transfer-Encoding", "chunked"),
1338 ]
1339 ),
1340 Chunked
1341 );
1342 assert_eq!(
1343 Response::body_type(
1344 "GET",
1345 200,
1346 "HTTP/1.1",
1347 &[
1348 Header::new("Transfer-Encoding", "chunked"),
1349 Header::new("Content-Length", "123"),
1350 ]
1351 ),
1352 Chunked
1353 );
1354 assert_eq!(
1355 Response::body_type(
1356 "HEAD",
1357 200,
1358 "HTTP/1.1",
1359 &[
1360 Header::new("Transfer-Encoding", "chunked"),
1361 Header::new("Content-Length", "123"),
1362 ]
1363 ),
1364 LengthDelimited(0)
1365 );
1366 assert_eq!(
1367 Response::body_type(
1368 "GET",
1369 200,
1370 "HTTP/1.0",
1371 &[Header::new("Transfer-Encoding", "chunked"),]
1372 ),
1373 CloseDelimited,
1374 "HTTP/1.0 did not support chunked encoding"
1375 );
1376 assert_eq!(
1377 Response::body_type(
1378 "GET",
1379 200,
1380 "HTTP/1.0",
1381 &[Header::new("Content-Length", "123"),]
1382 ),
1383 LengthDelimited(123)
1384 );
1385 }
1386}
1387