From db0035e89de4411282318f3d593c52504fa74517 Mon Sep 17 00:00:00 2001 From: Jay McCarthy Date: Sat, 14 Dec 2013 07:58:53 -0700 Subject: [PATCH] Fix PR14247 Add "Connection: close" to requests that we know will end after one request. Read upto 1K bytes at a time on requests rather than just 1 at a time. Note: edited to remove abandon -> close changes (cherry picked from commit 87135b110bdf147655e9bd73d68880905834c6d4) --- .../net-test/tests/net/http-client.rkt | 36 +++++++++---------- racket/collects/net/http-client.rkt | 24 +++++++++---- 2 files changed, 36 insertions(+), 24 deletions(-) diff --git a/pkgs/net-pkgs/net-test/tests/net/http-client.rkt b/pkgs/net-pkgs/net-test/tests/net/http-client.rkt index d416dc06ca..6db83159f1 100644 --- a/pkgs/net-pkgs/net-test/tests/net/http-client.rkt +++ b/pkgs/net-pkgs/net-test/tests/net/http-client.rkt @@ -116,110 +116,110 @@ (tests ["HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nTransfer-Encoding: chunked\r\n\r\n24\r\nThis is the data in the first chunk \r\n1A\r\nand this is the second one\r\n0\r\n" - #"GET / HTTP/1.1\r\nHost: localhost:REDACTED\r\nUser-Agent: Racket/REDACTED (net/http-client)\r\nAccept-Encoding: gzip\r\n\r\n" + #"GET / HTTP/1.1\r\nHost: localhost:REDACTED\r\nUser-Agent: Racket/REDACTED (net/http-client)\r\nAccept-Encoding: gzip\r\nConnection: close\r\n\r\n" #"HTTP/1.1 200 OK" '(#"Content-Type: text/plain" #"Transfer-Encoding: chunked") #"This is the data in the first chunk and this is the second one"] ["HTTP/1.0 200 OK\r\nContent-Type: text/plain\r\n\r\nThis is the data in the first chunk and this is the second one" - #"GET / HTTP/1.1\r\nHost: localhost:REDACTED\r\nUser-Agent: Racket/REDACTED (net/http-client)\r\nAccept-Encoding: gzip\r\n\r\n" + #"GET / HTTP/1.1\r\nHost: localhost:REDACTED\r\nUser-Agent: Racket/REDACTED (net/http-client)\r\nAccept-Encoding: gzip\r\nConnection: close\r\n\r\n" #"HTTP/1.0 200 OK" '(#"Content-Type: text/plain") #"This is the data in the first chunk and this is the second one"] ["HTTP/1.0 200 OK\nContent-Type: text/plain\n\nThis is the data in the first chunk and this is the second one" - #"GET / HTTP/1.1\r\nHost: localhost:REDACTED\r\nUser-Agent: Racket/REDACTED (net/http-client)\r\nAccept-Encoding: gzip\r\n\r\n" + #"GET / HTTP/1.1\r\nHost: localhost:REDACTED\r\nUser-Agent: Racket/REDACTED (net/http-client)\r\nAccept-Encoding: gzip\r\nConnection: close\r\n\r\n" #"HTTP/1.0 200 OK" '(#"Content-Type: text/plain") #"This is the data in the first chunk and this is the second one"] ["HTTP/1.0 200 OK\r\nContent-Type: text/plain\r\nContent-Length: 62\r\n\r\nThis is the data in the first chunk and this is the second one" - #"GET / HTTP/1.1\r\nHost: localhost:REDACTED\r\nUser-Agent: Racket/REDACTED (net/http-client)\r\nAccept-Encoding: gzip\r\n\r\n" + #"GET / HTTP/1.1\r\nHost: localhost:REDACTED\r\nUser-Agent: Racket/REDACTED (net/http-client)\r\nAccept-Encoding: gzip\r\nConnection: close\r\n\r\n" #"HTTP/1.0 200 OK" '(#"Content-Type: text/plain" #"Content-Length: 62") #"This is the data in the first chunk and this is the second one"] ["HTTP/1.0 200 OK\r\nContent-Type: text/plain\r\n\r\nThis is the data in the first chunk and this is the second one" - #"GET / HTTP/1.1\r\nHost: localhost:REDACTED\r\nUser-Agent: Racket/REDACTED (net/http-client)\r\nAccept-Encoding: gzip\r\n\r\n" + #"GET / HTTP/1.1\r\nHost: localhost:REDACTED\r\nUser-Agent: Racket/REDACTED (net/http-client)\r\nAccept-Encoding: gzip\r\nConnection: close\r\n\r\n" #"HTTP/1.0 200 OK" '(#"Content-Type: text/plain") #"This is the data in the first chunk and this is the second one"] ["HTTP/1.0 200 OK\r\nContent-Type: text/plain\r\ncontent-length: 62\r\n\r\nThis is the data in the first chunk and this is the second one" - #"GET / HTTP/1.1\r\nHost: localhost:REDACTED\r\nUser-Agent: Racket/REDACTED (net/http-client)\r\nAccept-Encoding: gzip\r\n\r\n" + #"GET / HTTP/1.1\r\nHost: localhost:REDACTED\r\nUser-Agent: Racket/REDACTED (net/http-client)\r\nAccept-Encoding: gzip\r\nConnection: close\r\n\r\n" #"HTTP/1.0 200 OK" '(#"Content-Type: text/plain" #"content-length: 62") #"This is the data in the first chunk and this is the second one"] ["HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nTransfer-Encoding: chunked\r\n\r\n20\r\nThis is the data in the first ch\r\n21\r\nand this is the second oneXXXXXXX\r\n0\r\n" - #"GET / HTTP/1.1\r\nHost: localhost:REDACTED\r\nUser-Agent: Racket/REDACTED (net/http-client)\r\nAccept-Encoding: gzip\r\n\r\n" + #"GET / HTTP/1.1\r\nHost: localhost:REDACTED\r\nUser-Agent: Racket/REDACTED (net/http-client)\r\nAccept-Encoding: gzip\r\nConnection: close\r\n\r\n" #"HTTP/1.1 200 OK" '(#"Content-Type: text/plain" #"Transfer-Encoding: chunked") #"This is the data in the first chand this is the second oneXXXXXXX"] ["HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nTransfer-Encoding: chunked\r\n\r\n24\r\nThis is the data in the first chunk \r\n1A\r\nand this is the second one\r\n0\r\n" - #"GET / HTTP/1.1\r\nHost: localhost:REDACTED\r\nUser-Agent: Racket/REDACTED (net/http-client)\r\nAccept-Encoding: gzip\r\n\r\n" + #"GET / HTTP/1.1\r\nHost: localhost:REDACTED\r\nUser-Agent: Racket/REDACTED (net/http-client)\r\nAccept-Encoding: gzip\r\nConnection: close\r\n\r\n" #"HTTP/1.1 200 OK" '(#"Content-Type: text/plain" #"Transfer-Encoding: chunked") #"This is the data in the first chunk and this is the second one"] ["HTTP/1.0 200 OK\r\nContent-Type: text/plain\r\n\r\nThis is the data in the first chunk and this is the second one" - #"GET / HTTP/1.1\r\nHost: localhost:REDACTED\r\nUser-Agent: Racket/REDACTED (net/http-client)\r\nAccept-Encoding: gzip\r\n\r\n" + #"GET / HTTP/1.1\r\nHost: localhost:REDACTED\r\nUser-Agent: Racket/REDACTED (net/http-client)\r\nAccept-Encoding: gzip\r\nConnection: close\r\n\r\n" #"HTTP/1.0 200 OK" '(#"Content-Type: text/plain") #"This is the data in the first chunk and this is the second one"] ["HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nTransfer-Encoding: chunked\r\n\r\n20\r\nThis is the data in the first ch\r\n21\r\nand this is the second oneXXXXXXX\r\n0\r\n" - #"GET / HTTP/1.1\r\nHost: localhost:REDACTED\r\nUser-Agent: Racket/REDACTED (net/http-client)\r\nAccept-Encoding: gzip\r\n\r\n" + #"GET / HTTP/1.1\r\nHost: localhost:REDACTED\r\nUser-Agent: Racket/REDACTED (net/http-client)\r\nAccept-Encoding: gzip\r\nConnection: close\r\n\r\n" #"HTTP/1.1 200 OK" '(#"Content-Type: text/plain" #"Transfer-Encoding: chunked") #"This is the data in the first chand this is the second oneXXXXXXX"] ["HTTP/1.0 200 OK\r\nContent-Type: text/plain\r\n\r\nThis is the data in the first chunk and this is the second one\r\n" - #"GET / HTTP/1.1\r\nHost: localhost:REDACTED\r\nUser-Agent: Racket/REDACTED (net/http-client)\r\nAccept-Encoding: gzip\r\n\r\n" + #"GET / HTTP/1.1\r\nHost: localhost:REDACTED\r\nUser-Agent: Racket/REDACTED (net/http-client)\r\nAccept-Encoding: gzip\r\nConnection: close\r\n\r\n" #"HTTP/1.0 200 OK" '(#"Content-Type: text/plain") #"This is the data in the first chunk and this is the second one\r\n"] ["HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nTransfer-Encoding: chunked\r\n\r\n24\r\nThis is the data in the first chunk \r\n1A\r\nand this is the second one\r\n0\r\n" - #"GET / HTTP/1.1\r\nHost: localhost:REDACTED\r\nUser-Agent: Racket/REDACTED (net/http-client)\r\nAccept-Encoding: gzip\r\n\r\n" + #"GET / HTTP/1.1\r\nHost: localhost:REDACTED\r\nUser-Agent: Racket/REDACTED (net/http-client)\r\nAccept-Encoding: gzip\r\nConnection: close\r\n\r\n" #"HTTP/1.1 200 OK" '(#"Content-Type: text/plain" #"Transfer-Encoding: chunked") #"This is the data in the first chunk and this is the second one"] ["HTTP/1.0 200 OK\r\nContent-Type: text/plain\r\n\r\nThis is the data in the first chunk and this is the second one" - #"GET / HTTP/1.1\r\nHost: localhost:REDACTED\r\nUser-Agent: Racket/REDACTED (net/http-client)\r\nAccept-Encoding: gzip\r\n\r\n" + #"GET / HTTP/1.1\r\nHost: localhost:REDACTED\r\nUser-Agent: Racket/REDACTED (net/http-client)\r\nAccept-Encoding: gzip\r\nConnection: close\r\n\r\n" #"HTTP/1.0 200 OK" '(#"Content-Type: text/plain") #"This is the data in the first chunk and this is the second one"] ["HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nTransfer-Encoding: chunked\r\n\r\n20\r\nThis is the data in the first ch\r\n21\r\nand this is the second oneXXXXXXX\r\n0\r\n" - #"GET / HTTP/1.1\r\nHost: localhost:REDACTED\r\nUser-Agent: Racket/REDACTED (net/http-client)\r\nAccept-Encoding: gzip\r\n\r\n" + #"GET / HTTP/1.1\r\nHost: localhost:REDACTED\r\nUser-Agent: Racket/REDACTED (net/http-client)\r\nAccept-Encoding: gzip\r\nConnection: close\r\n\r\n" #"HTTP/1.1 200 OK" '(#"Content-Type: text/plain" #"Transfer-Encoding: chunked") #"This is the data in the first chand this is the second oneXXXXXXX"] ["HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nTransfer-Encoding: chunked\r\nAnother-Header: ta-daa\r\n\r\n20\r\nThis is the data in the first ch\r\n21\r\nand this is the second oneXXXXXXX\r\n0\r\n" - #"GET / HTTP/1.1\r\nHost: localhost:REDACTED\r\nUser-Agent: Racket/REDACTED (net/http-client)\r\nAccept-Encoding: gzip\r\n\r\n" + #"GET / HTTP/1.1\r\nHost: localhost:REDACTED\r\nUser-Agent: Racket/REDACTED (net/http-client)\r\nAccept-Encoding: gzip\r\nConnection: close\r\n\r\n" #"HTTP/1.1 200 OK" '(#"Content-Type: text/plain" #"Transfer-Encoding: chunked" #"Another-Header: ta-daa") #"This is the data in the first chand this is the second oneXXXXXXX"] ["HTTP/1.1 301 Moved Permanently\r\nLocation: http://localhost:9002/whatever\r\n\r\nstuff" - #"GET / HTTP/1.1\r\nHost: localhost:REDACTED\r\nUser-Agent: Racket/REDACTED (net/http-client)\r\nAccept-Encoding: gzip\r\n\r\n" + #"GET / HTTP/1.1\r\nHost: localhost:REDACTED\r\nUser-Agent: Racket/REDACTED (net/http-client)\r\nAccept-Encoding: gzip\r\nConnection: close\r\n\r\n" #"HTTP/1.1 301 Moved Permanently" '(#"Location: http://localhost:9002/whatever") #"stuff"] ["HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nTransfer-Encoding: chunked\r\nAnother-Header: ta-daa\r\n\r\n20\r\nThis is the data in the first ch\r\n21\r\nand this is the second oneXXXXXXX\r\n0\r\n" - #"GET / HTTP/1.1\r\nHost: localhost:REDACTED\r\nUser-Agent: Racket/REDACTED (net/http-client)\r\nAccept-Encoding: gzip\r\n\r\n" + #"GET / HTTP/1.1\r\nHost: localhost:REDACTED\r\nUser-Agent: Racket/REDACTED (net/http-client)\r\nAccept-Encoding: gzip\r\nConnection: close\r\n\r\n" #"HTTP/1.1 200 OK" '(#"Content-Type: text/plain" #"Transfer-Encoding: chunked" #"Another-Header: ta-daa") #"This is the data in the first chand this is the second oneXXXXXXX"] ["HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nTransfer-Encoding: chunked\r\nAnother-Header: ta-daa\r\n\r\nbb \r\n\n\t\t\t\t\t \n\t\t\t\t\t ABCNANOTECH Co., LTD.\n\t\t\t\t\t \n\t\t\t\t\t \n\t\t\t\t\t \n\t\t\t\t\t \r\n0\r\n\r\n" - #"GET / HTTP/1.1\r\nHost: localhost:REDACTED\r\nUser-Agent: Racket/REDACTED (net/http-client)\r\nAccept-Encoding: gzip\r\n\r\n" + #"GET / HTTP/1.1\r\nHost: localhost:REDACTED\r\nUser-Agent: Racket/REDACTED (net/http-client)\r\nAccept-Encoding: gzip\r\nConnection: close\r\n\r\n" #"HTTP/1.1 200 OK" '(#"Content-Type: text/plain" #"Transfer-Encoding: chunked" #"Another-Header: ta-daa") #"\n\t\t\t\t\t \n\t\t\t\t\t ABCNANOTECH Co., LTD.\n\t\t\t\t\t \n\t\t\t\t\t \n\t\t\t\t\t \n\t\t\t\t\t "])) diff --git a/racket/collects/net/http-client.rkt b/racket/collects/net/http-client.rkt index 22e66d3607..d445616104 100644 --- a/racket/collects/net/http-client.rkt +++ b/racket/collects/net/http-client.rkt @@ -96,6 +96,7 @@ (define (http-conn-send! hc url-bs #:version [version-bs #"1.1"] #:method [method-bss #"GET"] + #:close? [close? #f] #:headers [headers-bs empty] #:content-decode [decodes '(gzip)] ;; xxx maybe support other kinds of data (ports and writing functions) @@ -120,6 +121,9 @@ (when data (unless (regexp-member #rx"^(?i:Content-Length:) +.+$" headers-bs) (fprintf to "Content-Length: ~a\r\n" (bytes-length data)))) + (when close? + (unless (regexp-member #rx"^(?i:Connection:) +.+$" headers-bs) + (fprintf to "Connection: close\r\n"))) (for ([h (in-list headers-bs)]) (fprintf to "~a\r\n" h)) (fprintf to "\r\n") @@ -137,13 +141,19 @@ empty (cons top (http-conn-headers! hc)))) -;; xxx read more at a time +(define BUFFER-SIZE 1024) (define (copy-bytes in out count) - (unless (zero? count) - (define b (read-byte in)) - (unless (eof-object? b) - (write-byte b out) - (copy-bytes in out (sub1 count))))) + (define buffer (make-bytes BUFFER-SIZE)) + (let loop ([count count]) + (when (positive? count) + (define r + (read-bytes-avail! buffer in 0 + (if (< count BUFFER-SIZE) + count + BUFFER-SIZE))) + (unless (eof-object? r) + (write-bytes buffer out 0 r) + (loop (- count r)))))) (define (http-conn-response-port/rest! hc) (http-conn-response-port/length! hc +inf.0 #:close? #t)) @@ -247,6 +257,7 @@ (http-conn-send! hc url-bs #:version version-bs #:method method-bss + #:close? close? #:headers headers-bs #:content-decode decodes #:data data-bsf) @@ -296,6 +307,7 @@ (http-conn-live? (or/c bytes? string?)) (#:version (or/c bytes? string?) #:method (or/c bytes? string? symbol?) + #:close? boolean? #:headers (listof (or/c bytes? string?)) #:content-decode (listof symbol?) #:data (or/c false/c bytes? string?))