mirrors/dnscrypt-proxy
1
0
Fork 0
mirror of https://github.com/DNSCrypt/dnscrypt-proxy.git synced 2025-03-04 02:14:40 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

Support gzip compression to fetch source files

Browse source
This commit is contained in:
Frank Denis 2024-04-25 12:43:29 +02:00
parent 987ae216e3
commit 249dba391d
2 changed files with 27 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
dnscrypt-proxy/sources.go
View file

@ -131,7 +131,7 @@ func (source *Source) parseURLs(urls []string) {
} }
func fetchFromURL(xTransport *XTransport, u *url.URL) ([]byte, error) { func fetchFromURL(xTransport *XTransport, u *url.URL) ([]byte, error) {
bin, _, _, _, err := xTransport.Get(u, "", DefaultTimeout) bin, _, _, _, err := xTransport.GetWithCompression(u, "", DefaultTimeout)
return bin, err return bin, err
} }

29
dnscrypt-proxy/xtransport.go
View file

@ -2,6 +2,7 @@ package main
import ( import (
"bytes" "bytes"
"compress/gzip"
"context" "context"
"crypto/sha512" "crypto/sha512"
"crypto/tls" "crypto/tls"
@ -482,6 +483,7 @@ func (xTransport *XTransport) Fetch(
contentType string, contentType string,
body *[]byte, body *[]byte,
timeout time.Duration, timeout time.Duration,
compress bool,
) ([]byte, int, *tls.ConnectionState, time.Duration, error) { ) ([]byte, int, *tls.ConnectionState, time.Duration, error) {
if timeout <= 0 { if timeout <= 0 {
timeout = xTransport.timeout timeout = xTransport.timeout
@ -530,6 +532,9 @@ func (xTransport *XTransport) Fetch(
) )
return nil, 0, nil, 0, err return nil, 0, nil, 0, err
} }
if compress && body == nil {
header["Accept-Encoding"] = []string{"gzip"}
}
req := &http.Request{ req := &http.Request{
Method: method, Method: method,
URL: url, URL: url,
@ -596,7 +601,17 @@ func (xTransport *XTransport) Fetch(
} }
} }
tls := resp.TLS tls := resp.TLS
bin, err := io.ReadAll(io.LimitReader(resp.Body, MaxHTTPBodyLength))
var bodyReader io.ReadCloser = resp.Body
if compress && resp.Header.Get("Content-Encoding") == "gzip" {
bodyReader, err = gzip.NewReader(io.LimitReader(resp.Body, MaxHTTPBodyLength))
if err != nil {
return nil, statusCode, tls, rtt, err
}
defer bodyReader.Close()
}
bin, err := io.ReadAll(io.LimitReader(bodyReader, MaxHTTPBodyLength))
if err != nil { if err != nil {
return nil, statusCode, tls, rtt, err return nil, statusCode, tls, rtt, err
} }
@ -604,12 +619,20 @@ func (xTransport *XTransport) Fetch(
return bin, statusCode, tls, rtt, err return bin, statusCode, tls, rtt, err
} }
func (xTransport *XTransport) GetWithCompression(
url *url.URL,
accept string,
timeout time.Duration,
) ([]byte, int, *tls.ConnectionState, time.Duration, error) {
return xTransport.Fetch("GET", url, accept, "", nil, timeout, true)
}
func (xTransport *XTransport) Get( func (xTransport *XTransport) Get(
url *url.URL, url *url.URL,
accept string, accept string,
timeout time.Duration, timeout time.Duration,
) ([]byte, int, *tls.ConnectionState, time.Duration, error) { ) ([]byte, int, *tls.ConnectionState, time.Duration, error) {
return xTransport.Fetch("GET", url, accept, "", nil, timeout) return xTransport.Fetch("GET", url, accept, "", nil, timeout, false)
} }
func (xTransport *XTransport) Post( func (xTransport *XTransport) Post(
@ -619,7 +642,7 @@ func (xTransport *XTransport) Post(
body *[]byte, body *[]byte,
timeout time.Duration, timeout time.Duration,
) ([]byte, int, *tls.ConnectionState, time.Duration, error) { ) ([]byte, int, *tls.ConnectionState, time.Duration, error) {
return xTransport.Fetch("POST", url, accept, contentType, body, timeout) return xTransport.Fetch("POST", url, accept, contentType, body, timeout, false)
} }
func (xTransport *XTransport) dohLikeQuery( func (xTransport *XTransport) dohLikeQuery(