upd/http.go

package main

import (
	"crypto/sha256"
	"crypto/x509"
	"encoding/hex"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"net"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"time"
)

var userAgent = "upd/" + version + " (go)"

var (
	client     *http.Client
	clientMode caMode = -1 // forces a build on first use
)

// One client per verification mode. The timeout covers connect, TLS and the
// wait for the response header - not the body, because a download may
// legitimately take longer than that.
func httpClient() (*http.Client, error) {
	if client != nil && clientMode == caCurrent {
		return client, nil
	}
	cfg, err := tlsConfig(caCurrent)
	if err != nil {
		return nil, err
	}
	to := time.Duration(opt.timeout) * time.Second
	client = &http.Client{
		Transport: &http.Transport{
			Proxy:                 http.ProxyFromEnvironment,
			DialContext:           (&net.Dialer{Timeout: to, KeepAlive: 30 * time.Second}).DialContext,
			TLSClientConfig:       cfg,
			TLSHandshakeTimeout:   to,
			ResponseHeaderTimeout: to,
			ExpectContinueTimeout: time.Second,
			ForceAttemptHTTP2:     true,
		},
	}
	clientMode = caCurrent
	return client, nil
}

type response struct {
	status int
	header http.Header
	body   []byte // empty when the body went to dst
}

// A writer that can be rewound, so a retry does not append to a half-written
// file. *os.File satisfies it.
type resettable interface {
	io.Writer
	Truncate(int64) error
	Seek(int64, int) (int64, error)
}

// httpGet retries what curl's --retry covers: connection failures and server
// side errors. A 4xx is an answer, not a hiccup, and is returned as is.
func httpGet(url string, headers map[string]string, dst io.Writer) (*response, error) {
	const attempts = 3
	var lastErr error
	retryNow := false // set when the next attempt changes something itself

	for i := range attempts {
		if i > 0 {
			if r, ok := dst.(resettable); ok {
				if _, err := r.Seek(0, io.SeekStart); err != nil {
					return nil, err
				}
				if err := r.Truncate(0); err != nil {
					return nil, err
				}
			}
			if !retryNow {
				time.Sleep(time.Duration(i) * time.Second)
				verbose("retrying (%d/%d): %s", i, attempts-1, url)
			}
		}
		retryNow = false

		// A CA bundle that cannot be read is a configuration error, not a
		// network one - repeating it would not help.
		if _, err := httpClient(); err != nil {
			return nil, err
		}

		resp, err := httpTry(url, headers, dst)
		if err != nil {
			lastErr = err
			// A store that does not know the issuer is not a hiccup, but it is
			// the one certificate failure a fresh root list can fix - so try
			// the bundled one straight away, and say so.
			if isUnknownAuthority(err) && caCurrent == caSystem && bundledPool() != nil {
				caCurrent = caBundled
				retryNow = true
				fmt.Fprintf(os.Stderr,
					"Note: the system CA store does not know this issuer, using the bundled CA list.\n"+
						"      Update the ca-certificates package to make this permanent.\n")
				continue
			}
			if isPermanent(err) {
				return nil, err
			}
			continue
		}
		if resp.status >= 500 && i < attempts-1 {
			lastErr = fmt.Errorf("HTTP %d: %s", resp.status, url)
			continue
		}
		return resp, nil
	}
	return nil, lastErr
}

func httpTry(url string, headers map[string]string, dst io.Writer) (*response, error) {
	req, err := http.NewRequest(http.MethodGet, url, nil)
	if err != nil {
		return nil, err
	}
	req.Header.Set("User-Agent", userAgent)
	for _, k := range sortedKeys(headers) {
		req.Header.Set(k, headers[k])
	}

	c, err := httpClient()
	if err != nil {
		return nil, err
	}
	resp, err := c.Do(req)
	if err != nil {
		return nil, transportError(err, url)
	}
	defer resp.Body.Close()

	out := &response{status: resp.StatusCode, header: resp.Header}
	if dst != nil && resp.StatusCode == http.StatusOK {
		if _, err := io.Copy(dst, resp.Body); err != nil {
			return nil, fmt.Errorf("download interrupted: %w", err)
		}
		return out, nil
	}
	// Everything that is not a download: API answers and the error bodies both
	// forges explain themselves in. A release list with 50 entries runs into
	// megabytes, so the cap is only there to bound a runaway response - and it
	// says so instead of handing on half a document.
	const maxBody = 32 << 20
	body, err := io.ReadAll(io.LimitReader(resp.Body, maxBody+1))
	if err != nil {
		return nil, err
	}
	if len(body) > maxBody {
		return nil, fmt.Errorf("response from %s is larger than %s", url, humanSize(maxBody))
	}
	out.body = body
	return out, nil
}

// The Go TLS stack speaks whatever the server does, so the old "your OpenSSL
// is too old" class of failure is gone - but DNS, proxies, clocks and private
// CAs are still there. Name the one that hit.
func transportError(err error, url string) error {
	var (
		dns      *net.DNSError
		hostname x509.HostnameError
		invalid  x509.CertificateInvalidError
	)
	msg := ""
	switch {
	case errors.As(err, &dns):
		msg = "the host could not be resolved - check DNS and $https_proxy"
	case isUnknownAuthority(err):
		msg = "the issuer is unknown even to the bundled CA list - for a private CA " +
			"pass --cacert <file>, or --insecure to skip verification"
	case errors.As(err, &invalid):
		msg = "the certificate is outside its validity period - check the system clock"
	case errors.As(err, &hostname):
		msg = "the certificate does not match the host name"
	case errors.Is(err, os.ErrDeadlineExceeded) || strings.Contains(err.Error(), "timeout"):
		msg = "timed out - raise --timeout"
	case strings.Contains(err.Error(), "connection refused"):
		msg = "connection refused - check the port and any firewall"
	}
	if msg == "" {
		return fmt.Errorf("request failed: %s\n  %w", url, err)
	}
	return fmt.Errorf("request failed: %s\n  %w\n  %s", url, err, msg)
}

// ==============================================================================
// API requests with an ETag cache
// ==============================================================================

type cacheEntry struct {
	URL  string `json:"url"`
	ETag string `json:"etag"`
	Body string `json:"body"`
}

func cacheDir() string {
	base := os.Getenv("XDG_CACHE_HOME")
	if base == "" {
		base = filepath.Join(homeDir(), ".cache")
	}
	return filepath.Join(base, "upd")
}

func cacheFile(url string) string {
	sum := sha256.Sum256([]byte(url))
	return filepath.Join(cacheDir(), hex.EncodeToString(sum[:])[:16]+".json")
}

// apiGet fetches a JSON endpoint, revalidating a cached copy via ETag.
// soft turns a 404 into (nil, nil) instead of an error.
func (c *forgeCtx) apiGet(url string, soft bool) (json.RawMessage, error) {
	hdr := c.authHeaders()
	hdr["Accept"] = "application/json"
	if c.forge == "github" {
		hdr["Accept"] = "application/vnd.github+json"
		hdr["X-GitHub-Api-Version"] = "2022-11-28"
	}

	var cached cacheEntry
	cf := cacheFile(url)
	if err := readJSON(cf, &cached); err == nil && cached.ETag != "" {
		hdr["If-None-Match"] = cached.ETag
	}

	resp, err := httpGet(url, hdr, nil)
	if err != nil {
		return nil, err
	}

	if resp.status == http.StatusNotModified && cached.Body != "" {
		verbose("304 not modified, using cached %s", url)
		return validJSON([]byte(cached.Body), url, c)
	}
	if soft && resp.status == http.StatusNotFound {
		return nil, nil
	}
	if resp.status == http.StatusForbidden || resp.status == http.StatusTooManyRequests {
		if left := resp.header.Get("X-RateLimit-Remaining"); left == "0" {
			return nil, fmt.Errorf("%s rate limit reached (remaining: %s).\n"+
				"  Set a token via --token or $GITHUB_TOKEN", c.forge, left)
		}
	}
	// Both forges answer 404 for a repository the caller may not see, so a
	// repository that is missing and one that is merely private look alike.
	if resp.status == http.StatusUnauthorized || resp.status == http.StatusForbidden ||
		resp.status == http.StatusNotFound {
		hint := "a private repository needs --token or $UPD_TOKEN"
		if c.token != "" {
			hint = "the token does not grant access to this repository"
		}
		return nil, fmt.Errorf("HTTP %d%s: %s\n  %s", resp.status, apiMessage(resp.body), url, hint)
	}
	if resp.status != http.StatusOK {
		return nil, fmt.Errorf("HTTP %d: %s", resp.status, url)
	}

	data, err := validJSON(resp.body, url, c)
	if err != nil {
		return nil, err
	}
	if etag := resp.header.Get("ETag"); etag != "" {
		if err := os.MkdirAll(cacheDir(), 0o755); err == nil {
			writeJSON(cf, cacheEntry{URL: url, ETag: etag, Body: string(resp.body)})
		}
	}
	return data, nil
}

func validJSON(body []byte, url string, c *forgeCtx) (json.RawMessage, error) {
	if !json.Valid(body) {
		return nil, fmt.Errorf("response from %s is not JSON (is this really a %s instance?)", url, c.forge)
	}
	if msg := apiMessage(body); msg != "" {
		return nil, fmt.Errorf("%s error:%s", c.forge, msg)
	}
	return body, nil
}

// Both forges explain themselves in a JSON "message" field.
func apiMessage(body []byte) string {
	var m struct {
		Message string `json:"message"`
	}
	if err := json.Unmarshal(body, &m); err != nil || m.Message == "" {
		return ""
	}
	return " (" + m.Message + ")"
}

// ==============================================================================
// Downloads
// ==============================================================================

func downloadTo(path, url string, headers map[string]string, size int64) error {
	fh, err := os.Create(path)
	if err != nil {
		return err
	}
	defer fh.Close()

	var dst io.Writer = fh
	if bar := newProgress(size); bar != nil {
		defer bar.finish()
		dst = &progressWriter{file: fh, bar: bar}
	}

	resp, err := httpGet(url, headers, dst)
	if err != nil {
		return err
	}
	if resp.status != http.StatusOK {
		return fmt.Errorf("download failed (HTTP %d%s): %s", resp.status, apiMessage(resp.body), url)
	}
	return fh.Sync()
}

// progressWriter keeps the file a resettable writer for the retry path.
type progressWriter struct {
	file *os.File
	bar  *progress
}

func (w *progressWriter) Write(p []byte) (int, error) {
	n, err := w.file.Write(p)
	w.bar.add(int64(n))
	return n, err
}

func (w *progressWriter) Truncate(n int64) error { w.bar.reset(); return w.file.Truncate(n) }
func (w *progressWriter) Seek(off int64, whence int) (int64, error) {
	return w.file.Seek(off, whence)
}

type progress struct {
	total, got int64
	last       time.Time
}

func newProgress(total int64) *progress {
	if opt.quiet || !isTerminal(os.Stdout) {
		return nil
	}
	return &progress{total: total}
}

func (p *progress) reset() { p.got = 0 }

func (p *progress) add(n int64) {
	p.got += n
	if time.Since(p.last) < 100*time.Millisecond {
		return
	}
	p.last = time.Now()
	p.draw()
}

func (p *progress) draw() {
	const width = 40
	if p.total <= 0 {
		fmt.Printf("\r  %s", humanSize(p.got))
		return
	}
	pct := float64(p.got) / float64(p.total)
	if pct > 1 {
		pct = 1
	}
	filled := int(pct * width)
	fmt.Printf("\r  [%s%s] %5.1f%%  %s",
		strings.Repeat("#", filled), strings.Repeat(" ", width-filled),
		pct*100, humanSize(p.total))
}

func (p *progress) finish() {
	if p == nil {
		return
	}
	p.draw()
	fmt.Print("\n")
}

func isTerminal(f *os.File) bool {
	st, err := f.Stat()
	return err == nil && st.Mode()&os.ModeCharDevice != 0
}