initial commit [141.14.140.180,mike]
This commit is contained in:
@@ -0,0 +1,397 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"crypto/x509"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
var userAgent = "upd/" + version + " (go)"
|
||||
|
||||
var (
|
||||
client *http.Client
|
||||
clientMode caMode = -1 // forces a build on first use
|
||||
)
|
||||
|
||||
// One client per verification mode. The timeout covers connect, TLS and the
|
||||
// wait for the response header - not the body, because a download may
|
||||
// legitimately take longer than that.
|
||||
func httpClient() (*http.Client, error) {
|
||||
if client != nil && clientMode == caCurrent {
|
||||
return client, nil
|
||||
}
|
||||
cfg, err := tlsConfig(caCurrent)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
to := time.Duration(opt.timeout) * time.Second
|
||||
client = &http.Client{
|
||||
Transport: &http.Transport{
|
||||
Proxy: http.ProxyFromEnvironment,
|
||||
DialContext: (&net.Dialer{Timeout: to, KeepAlive: 30 * time.Second}).DialContext,
|
||||
TLSClientConfig: cfg,
|
||||
TLSHandshakeTimeout: to,
|
||||
ResponseHeaderTimeout: to,
|
||||
ExpectContinueTimeout: time.Second,
|
||||
ForceAttemptHTTP2: true,
|
||||
},
|
||||
}
|
||||
clientMode = caCurrent
|
||||
return client, nil
|
||||
}
|
||||
|
||||
type response struct {
|
||||
status int
|
||||
header http.Header
|
||||
body []byte // empty when the body went to dst
|
||||
}
|
||||
|
||||
// A writer that can be rewound, so a retry does not append to a half-written
|
||||
// file. *os.File satisfies it.
|
||||
type resettable interface {
|
||||
io.Writer
|
||||
Truncate(int64) error
|
||||
Seek(int64, int) (int64, error)
|
||||
}
|
||||
|
||||
// httpGet retries what curl's --retry covers: connection failures and server
|
||||
// side errors. A 4xx is an answer, not a hiccup, and is returned as is.
|
||||
func httpGet(url string, headers map[string]string, dst io.Writer) (*response, error) {
|
||||
const attempts = 3
|
||||
var lastErr error
|
||||
retryNow := false // set when the next attempt changes something itself
|
||||
|
||||
for i := range attempts {
|
||||
if i > 0 {
|
||||
if r, ok := dst.(resettable); ok {
|
||||
if _, err := r.Seek(0, io.SeekStart); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := r.Truncate(0); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if !retryNow {
|
||||
time.Sleep(time.Duration(i) * time.Second)
|
||||
verbose("retrying (%d/%d): %s", i, attempts-1, url)
|
||||
}
|
||||
}
|
||||
retryNow = false
|
||||
|
||||
// A CA bundle that cannot be read is a configuration error, not a
|
||||
// network one - repeating it would not help.
|
||||
if _, err := httpClient(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
resp, err := httpTry(url, headers, dst)
|
||||
if err != nil {
|
||||
lastErr = err
|
||||
// A store that does not know the issuer is not a hiccup, but it is
|
||||
// the one certificate failure a fresh root list can fix - so try
|
||||
// the bundled one straight away, and say so.
|
||||
if isUnknownAuthority(err) && caCurrent == caSystem && bundledPool() != nil {
|
||||
caCurrent = caBundled
|
||||
retryNow = true
|
||||
fmt.Fprintf(os.Stderr,
|
||||
"Note: the system CA store does not know this issuer, using the bundled CA list.\n"+
|
||||
" Update the ca-certificates package to make this permanent.\n")
|
||||
continue
|
||||
}
|
||||
if isPermanent(err) {
|
||||
return nil, err
|
||||
}
|
||||
continue
|
||||
}
|
||||
if resp.status >= 500 && i < attempts-1 {
|
||||
lastErr = fmt.Errorf("HTTP %d: %s", resp.status, url)
|
||||
continue
|
||||
}
|
||||
return resp, nil
|
||||
}
|
||||
return nil, lastErr
|
||||
}
|
||||
|
||||
func httpTry(url string, headers map[string]string, dst io.Writer) (*response, error) {
|
||||
req, err := http.NewRequest(http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set("User-Agent", userAgent)
|
||||
for _, k := range sortedKeys(headers) {
|
||||
req.Header.Set(k, headers[k])
|
||||
}
|
||||
|
||||
c, err := httpClient()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
resp, err := c.Do(req)
|
||||
if err != nil {
|
||||
return nil, transportError(err, url)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
out := &response{status: resp.StatusCode, header: resp.Header}
|
||||
if dst != nil && resp.StatusCode == http.StatusOK {
|
||||
if _, err := io.Copy(dst, resp.Body); err != nil {
|
||||
return nil, fmt.Errorf("download interrupted: %w", err)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
// Everything that is not a download: API answers and the error bodies both
|
||||
// forges explain themselves in. A release list with 50 entries runs into
|
||||
// megabytes, so the cap is only there to bound a runaway response - and it
|
||||
// says so instead of handing on half a document.
|
||||
const maxBody = 32 << 20
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, maxBody+1))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(body) > maxBody {
|
||||
return nil, fmt.Errorf("response from %s is larger than %s", url, humanSize(maxBody))
|
||||
}
|
||||
out.body = body
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// The Go TLS stack speaks whatever the server does, so the old "your OpenSSL
|
||||
// is too old" class of failure is gone - but DNS, proxies, clocks and private
|
||||
// CAs are still there. Name the one that hit.
|
||||
func transportError(err error, url string) error {
|
||||
var (
|
||||
dns *net.DNSError
|
||||
hostname x509.HostnameError
|
||||
invalid x509.CertificateInvalidError
|
||||
)
|
||||
msg := ""
|
||||
switch {
|
||||
case errors.As(err, &dns):
|
||||
msg = "the host could not be resolved - check DNS and $https_proxy"
|
||||
case isUnknownAuthority(err):
|
||||
msg = "the issuer is unknown even to the bundled CA list - for a private CA " +
|
||||
"pass --cacert <file>, or --insecure to skip verification"
|
||||
case errors.As(err, &invalid):
|
||||
msg = "the certificate is outside its validity period - check the system clock"
|
||||
case errors.As(err, &hostname):
|
||||
msg = "the certificate does not match the host name"
|
||||
case errors.Is(err, os.ErrDeadlineExceeded) || strings.Contains(err.Error(), "timeout"):
|
||||
msg = "timed out - raise --timeout"
|
||||
case strings.Contains(err.Error(), "connection refused"):
|
||||
msg = "connection refused - check the port and any firewall"
|
||||
}
|
||||
if msg == "" {
|
||||
return fmt.Errorf("request failed: %s\n %w", url, err)
|
||||
}
|
||||
return fmt.Errorf("request failed: %s\n %w\n %s", url, err, msg)
|
||||
}
|
||||
|
||||
// ==============================================================================
|
||||
// API requests with an ETag cache
|
||||
// ==============================================================================
|
||||
|
||||
type cacheEntry struct {
|
||||
URL string `json:"url"`
|
||||
ETag string `json:"etag"`
|
||||
Body string `json:"body"`
|
||||
}
|
||||
|
||||
func cacheDir() string {
|
||||
base := os.Getenv("XDG_CACHE_HOME")
|
||||
if base == "" {
|
||||
base = filepath.Join(homeDir(), ".cache")
|
||||
}
|
||||
return filepath.Join(base, "upd")
|
||||
}
|
||||
|
||||
func cacheFile(url string) string {
|
||||
sum := sha256.Sum256([]byte(url))
|
||||
return filepath.Join(cacheDir(), hex.EncodeToString(sum[:])[:16]+".json")
|
||||
}
|
||||
|
||||
// apiGet fetches a JSON endpoint, revalidating a cached copy via ETag.
|
||||
// soft turns a 404 into (nil, nil) instead of an error.
|
||||
func (c *forgeCtx) apiGet(url string, soft bool) (json.RawMessage, error) {
|
||||
hdr := c.authHeaders()
|
||||
hdr["Accept"] = "application/json"
|
||||
if c.forge == "github" {
|
||||
hdr["Accept"] = "application/vnd.github+json"
|
||||
hdr["X-GitHub-Api-Version"] = "2022-11-28"
|
||||
}
|
||||
|
||||
var cached cacheEntry
|
||||
cf := cacheFile(url)
|
||||
if err := readJSON(cf, &cached); err == nil && cached.ETag != "" {
|
||||
hdr["If-None-Match"] = cached.ETag
|
||||
}
|
||||
|
||||
resp, err := httpGet(url, hdr, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if resp.status == http.StatusNotModified && cached.Body != "" {
|
||||
verbose("304 not modified, using cached %s", url)
|
||||
return validJSON([]byte(cached.Body), url, c)
|
||||
}
|
||||
if soft && resp.status == http.StatusNotFound {
|
||||
return nil, nil
|
||||
}
|
||||
if resp.status == http.StatusForbidden || resp.status == http.StatusTooManyRequests {
|
||||
if left := resp.header.Get("X-RateLimit-Remaining"); left == "0" {
|
||||
return nil, fmt.Errorf("%s rate limit reached (remaining: %s).\n"+
|
||||
" Set a token via --token or $GITHUB_TOKEN", c.forge, left)
|
||||
}
|
||||
}
|
||||
// Both forges answer 404 for a repository the caller may not see, so a
|
||||
// repository that is missing and one that is merely private look alike.
|
||||
if resp.status == http.StatusUnauthorized || resp.status == http.StatusForbidden ||
|
||||
resp.status == http.StatusNotFound {
|
||||
hint := "a private repository needs --token or $UPD_TOKEN"
|
||||
if c.token != "" {
|
||||
hint = "the token does not grant access to this repository"
|
||||
}
|
||||
return nil, fmt.Errorf("HTTP %d%s: %s\n %s", resp.status, apiMessage(resp.body), url, hint)
|
||||
}
|
||||
if resp.status != http.StatusOK {
|
||||
return nil, fmt.Errorf("HTTP %d: %s", resp.status, url)
|
||||
}
|
||||
|
||||
data, err := validJSON(resp.body, url, c)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if etag := resp.header.Get("ETag"); etag != "" {
|
||||
if err := os.MkdirAll(cacheDir(), 0o755); err == nil {
|
||||
writeJSON(cf, cacheEntry{URL: url, ETag: etag, Body: string(resp.body)})
|
||||
}
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
|
||||
func validJSON(body []byte, url string, c *forgeCtx) (json.RawMessage, error) {
|
||||
if !json.Valid(body) {
|
||||
return nil, fmt.Errorf("response from %s is not JSON (is this really a %s instance?)", url, c.forge)
|
||||
}
|
||||
if msg := apiMessage(body); msg != "" {
|
||||
return nil, fmt.Errorf("%s error:%s", c.forge, msg)
|
||||
}
|
||||
return body, nil
|
||||
}
|
||||
|
||||
// Both forges explain themselves in a JSON "message" field.
|
||||
func apiMessage(body []byte) string {
|
||||
var m struct {
|
||||
Message string `json:"message"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &m); err != nil || m.Message == "" {
|
||||
return ""
|
||||
}
|
||||
return " (" + m.Message + ")"
|
||||
}
|
||||
|
||||
// ==============================================================================
|
||||
// Downloads
|
||||
// ==============================================================================
|
||||
|
||||
func downloadTo(path, url string, headers map[string]string, size int64) error {
|
||||
fh, err := os.Create(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer fh.Close()
|
||||
|
||||
var dst io.Writer = fh
|
||||
if bar := newProgress(size); bar != nil {
|
||||
defer bar.finish()
|
||||
dst = &progressWriter{file: fh, bar: bar}
|
||||
}
|
||||
|
||||
resp, err := httpGet(url, headers, dst)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if resp.status != http.StatusOK {
|
||||
return fmt.Errorf("download failed (HTTP %d%s): %s", resp.status, apiMessage(resp.body), url)
|
||||
}
|
||||
return fh.Sync()
|
||||
}
|
||||
|
||||
// progressWriter keeps the file a resettable writer for the retry path.
|
||||
type progressWriter struct {
|
||||
file *os.File
|
||||
bar *progress
|
||||
}
|
||||
|
||||
func (w *progressWriter) Write(p []byte) (int, error) {
|
||||
n, err := w.file.Write(p)
|
||||
w.bar.add(int64(n))
|
||||
return n, err
|
||||
}
|
||||
|
||||
func (w *progressWriter) Truncate(n int64) error { w.bar.reset(); return w.file.Truncate(n) }
|
||||
func (w *progressWriter) Seek(off int64, whence int) (int64, error) {
|
||||
return w.file.Seek(off, whence)
|
||||
}
|
||||
|
||||
type progress struct {
|
||||
total, got int64
|
||||
last time.Time
|
||||
}
|
||||
|
||||
func newProgress(total int64) *progress {
|
||||
if opt.quiet || !isTerminal(os.Stdout) {
|
||||
return nil
|
||||
}
|
||||
return &progress{total: total}
|
||||
}
|
||||
|
||||
func (p *progress) reset() { p.got = 0 }
|
||||
|
||||
func (p *progress) add(n int64) {
|
||||
p.got += n
|
||||
if time.Since(p.last) < 100*time.Millisecond {
|
||||
return
|
||||
}
|
||||
p.last = time.Now()
|
||||
p.draw()
|
||||
}
|
||||
|
||||
func (p *progress) draw() {
|
||||
const width = 40
|
||||
if p.total <= 0 {
|
||||
fmt.Printf("\r %s", humanSize(p.got))
|
||||
return
|
||||
}
|
||||
pct := float64(p.got) / float64(p.total)
|
||||
if pct > 1 {
|
||||
pct = 1
|
||||
}
|
||||
filled := int(pct * width)
|
||||
fmt.Printf("\r [%s%s] %5.1f%% %s",
|
||||
strings.Repeat("#", filled), strings.Repeat(" ", width-filled),
|
||||
pct*100, humanSize(p.total))
|
||||
}
|
||||
|
||||
func (p *progress) finish() {
|
||||
if p == nil {
|
||||
return
|
||||
}
|
||||
p.draw()
|
||||
fmt.Print("\n")
|
||||
}
|
||||
|
||||
func isTerminal(f *os.File) bool {
|
||||
st, err := f.Stat()
|
||||
return err == nil && st.Mode()&os.ModeCharDevice != 0
|
||||
}
|
||||
Reference in New Issue
Block a user