package main import ( "crypto/sha256" "crypto/x509" "encoding/hex" "encoding/json" "errors" "fmt" "io" "net" "net/http" "os" "path/filepath" "strings" "time" ) var userAgent = "upd/" + version + " (go)" var ( client *http.Client clientMode caMode = -1 // forces a build on first use ) // One client per verification mode. The timeout covers connect, TLS and the // wait for the response header - not the body, because a download may // legitimately take longer than that. func httpClient() (*http.Client, error) { if client != nil && clientMode == caCurrent { return client, nil } cfg, err := tlsConfig(caCurrent) if err != nil { return nil, err } to := time.Duration(opt.timeout) * time.Second client = &http.Client{ Transport: &http.Transport{ Proxy: http.ProxyFromEnvironment, DialContext: (&net.Dialer{Timeout: to, KeepAlive: 30 * time.Second}).DialContext, TLSClientConfig: cfg, TLSHandshakeTimeout: to, ResponseHeaderTimeout: to, ExpectContinueTimeout: time.Second, ForceAttemptHTTP2: true, }, } clientMode = caCurrent return client, nil } type response struct { status int header http.Header body []byte // empty when the body went to dst } // A writer that can be rewound, so a retry does not append to a half-written // file. *os.File satisfies it. type resettable interface { io.Writer Truncate(int64) error Seek(int64, int) (int64, error) } // httpGet retries what curl's --retry covers: connection failures and server // side errors. A 4xx is an answer, not a hiccup, and is returned as is. func httpGet(url string, headers map[string]string, dst io.Writer) (*response, error) { const attempts = 3 var lastErr error retryNow := false // set when the next attempt changes something itself for i := range attempts { if i > 0 { if r, ok := dst.(resettable); ok { if _, err := r.Seek(0, io.SeekStart); err != nil { return nil, err } if err := r.Truncate(0); err != nil { return nil, err } } if !retryNow { time.Sleep(time.Duration(i) * time.Second) verbose("retrying (%d/%d): %s", i, attempts-1, url) } } retryNow = false // A CA bundle that cannot be read is a configuration error, not a // network one - repeating it would not help. if _, err := httpClient(); err != nil { return nil, err } resp, err := httpTry(url, headers, dst) if err != nil { lastErr = err // A store that does not know the issuer is not a hiccup, but it is // the one certificate failure a fresh root list can fix - so try // the bundled one straight away, and say so. if isUnknownAuthority(err) && caCurrent == caSystem && bundledPool() != nil { caCurrent = caBundled retryNow = true fmt.Fprintf(os.Stderr, "Note: the system CA store does not know this issuer, using the bundled CA list.\n"+ " Update the ca-certificates package to make this permanent.\n") continue } if isPermanent(err) { return nil, err } continue } if resp.status >= 500 && i < attempts-1 { lastErr = fmt.Errorf("HTTP %d: %s", resp.status, url) continue } return resp, nil } return nil, lastErr } func httpTry(url string, headers map[string]string, dst io.Writer) (*response, error) { req, err := http.NewRequest(http.MethodGet, url, nil) if err != nil { return nil, err } req.Header.Set("User-Agent", userAgent) for _, k := range sortedKeys(headers) { req.Header.Set(k, headers[k]) } c, err := httpClient() if err != nil { return nil, err } resp, err := c.Do(req) if err != nil { return nil, transportError(err, url) } defer resp.Body.Close() out := &response{status: resp.StatusCode, header: resp.Header} if dst != nil && resp.StatusCode == http.StatusOK { if _, err := io.Copy(dst, resp.Body); err != nil { return nil, fmt.Errorf("download interrupted: %w", err) } return out, nil } // Everything that is not a download: API answers and the error bodies both // forges explain themselves in. A release list with 50 entries runs into // megabytes, so the cap is only there to bound a runaway response - and it // says so instead of handing on half a document. const maxBody = 32 << 20 body, err := io.ReadAll(io.LimitReader(resp.Body, maxBody+1)) if err != nil { return nil, err } if len(body) > maxBody { return nil, fmt.Errorf("response from %s is larger than %s", url, humanSize(maxBody)) } out.body = body return out, nil } // The Go TLS stack speaks whatever the server does, so the old "your OpenSSL // is too old" class of failure is gone - but DNS, proxies, clocks and private // CAs are still there. Name the one that hit. func transportError(err error, url string) error { var ( dns *net.DNSError hostname x509.HostnameError invalid x509.CertificateInvalidError ) msg := "" switch { case errors.As(err, &dns): msg = "the host could not be resolved - check DNS and $https_proxy" case isUnknownAuthority(err): msg = "the issuer is unknown even to the bundled CA list - for a private CA " + "pass --cacert , or --insecure to skip verification" case errors.As(err, &invalid): msg = "the certificate is outside its validity period - check the system clock" case errors.As(err, &hostname): msg = "the certificate does not match the host name" case errors.Is(err, os.ErrDeadlineExceeded) || strings.Contains(err.Error(), "timeout"): msg = "timed out - raise --timeout" case strings.Contains(err.Error(), "connection refused"): msg = "connection refused - check the port and any firewall" } if msg == "" { return fmt.Errorf("request failed: %s\n %w", url, err) } return fmt.Errorf("request failed: %s\n %w\n %s", url, err, msg) } // ============================================================================== // API requests with an ETag cache // ============================================================================== type cacheEntry struct { URL string `json:"url"` ETag string `json:"etag"` Body string `json:"body"` } func cacheDir() string { base := os.Getenv("XDG_CACHE_HOME") if base == "" { base = filepath.Join(homeDir(), ".cache") } return filepath.Join(base, "upd") } func cacheFile(url string) string { sum := sha256.Sum256([]byte(url)) return filepath.Join(cacheDir(), hex.EncodeToString(sum[:])[:16]+".json") } // apiGet fetches a JSON endpoint, revalidating a cached copy via ETag. // soft turns a 404 into (nil, nil) instead of an error. func (c *forgeCtx) apiGet(url string, soft bool) (json.RawMessage, error) { hdr := c.authHeaders() hdr["Accept"] = "application/json" if c.forge == "github" { hdr["Accept"] = "application/vnd.github+json" hdr["X-GitHub-Api-Version"] = "2022-11-28" } var cached cacheEntry cf := cacheFile(url) if err := readJSON(cf, &cached); err == nil && cached.ETag != "" { hdr["If-None-Match"] = cached.ETag } resp, err := httpGet(url, hdr, nil) if err != nil { return nil, err } if resp.status == http.StatusNotModified && cached.Body != "" { verbose("304 not modified, using cached %s", url) return validJSON([]byte(cached.Body), url, c) } if soft && resp.status == http.StatusNotFound { return nil, nil } if resp.status == http.StatusForbidden || resp.status == http.StatusTooManyRequests { if left := resp.header.Get("X-RateLimit-Remaining"); left == "0" { return nil, fmt.Errorf("%s rate limit reached (remaining: %s).\n"+ " Set a token via --token or $GITHUB_TOKEN", c.forge, left) } } // Both forges answer 404 for a repository the caller may not see, so a // repository that is missing and one that is merely private look alike. if resp.status == http.StatusUnauthorized || resp.status == http.StatusForbidden || resp.status == http.StatusNotFound { hint := "a private repository needs --token or $UPD_TOKEN" if c.token != "" { hint = "the token does not grant access to this repository" } return nil, fmt.Errorf("HTTP %d%s: %s\n %s", resp.status, apiMessage(resp.body), url, hint) } if resp.status != http.StatusOK { return nil, fmt.Errorf("HTTP %d: %s", resp.status, url) } data, err := validJSON(resp.body, url, c) if err != nil { return nil, err } if etag := resp.header.Get("ETag"); etag != "" { if err := os.MkdirAll(cacheDir(), 0o755); err == nil { writeJSON(cf, cacheEntry{URL: url, ETag: etag, Body: string(resp.body)}) } } return data, nil } func validJSON(body []byte, url string, c *forgeCtx) (json.RawMessage, error) { if !json.Valid(body) { return nil, fmt.Errorf("response from %s is not JSON (is this really a %s instance?)", url, c.forge) } if msg := apiMessage(body); msg != "" { return nil, fmt.Errorf("%s error:%s", c.forge, msg) } return body, nil } // Both forges explain themselves in a JSON "message" field. func apiMessage(body []byte) string { var m struct { Message string `json:"message"` } if err := json.Unmarshal(body, &m); err != nil || m.Message == "" { return "" } return " (" + m.Message + ")" } // ============================================================================== // Downloads // ============================================================================== func downloadTo(path, url string, headers map[string]string, size int64) error { fh, err := os.Create(path) if err != nil { return err } defer fh.Close() var dst io.Writer = fh if bar := newProgress(size); bar != nil { defer bar.finish() dst = &progressWriter{file: fh, bar: bar} } resp, err := httpGet(url, headers, dst) if err != nil { return err } if resp.status != http.StatusOK { return fmt.Errorf("download failed (HTTP %d%s): %s", resp.status, apiMessage(resp.body), url) } return fh.Sync() } // progressWriter keeps the file a resettable writer for the retry path. type progressWriter struct { file *os.File bar *progress } func (w *progressWriter) Write(p []byte) (int, error) { n, err := w.file.Write(p) w.bar.add(int64(n)) return n, err } func (w *progressWriter) Truncate(n int64) error { w.bar.reset(); return w.file.Truncate(n) } func (w *progressWriter) Seek(off int64, whence int) (int64, error) { return w.file.Seek(off, whence) } type progress struct { total, got int64 last time.Time } func newProgress(total int64) *progress { if opt.quiet || !isTerminal(os.Stdout) { return nil } return &progress{total: total} } func (p *progress) reset() { p.got = 0 } func (p *progress) add(n int64) { p.got += n if time.Since(p.last) < 100*time.Millisecond { return } p.last = time.Now() p.draw() } func (p *progress) draw() { const width = 40 if p.total <= 0 { fmt.Printf("\r %s", humanSize(p.got)) return } pct := float64(p.got) / float64(p.total) if pct > 1 { pct = 1 } filled := int(pct * width) fmt.Printf("\r [%s%s] %5.1f%% %s", strings.Repeat("#", filled), strings.Repeat(" ", width-filled), pct*100, humanSize(p.total)) } func (p *progress) finish() { if p == nil { return } p.draw() fmt.Print("\n") } func isTerminal(f *os.File) bool { st, err := f.Stat() return err == nil && st.Mode()&os.ModeCharDevice != 0 }