initial commit [141.14.140.180,mike]

This commit is contained in:
2026-08-14 09:09:59 +02:00
commit 6a991a25a6
19 changed files with 6691 additions and 0 deletions
+13
View File
@@ -0,0 +1,13 @@
.DS_Store
.AppleDouble
.LSOverride
._*
.DocumentRevisions-V100
.fseventsd
.Spotlight-V100
.TemporaryItems
.Trashes
bin/
upd
upd-*-*
.mgshrc
+104
View File
@@ -0,0 +1,104 @@
# upd (Go)
A port of the Perl `upd` one directory up. Same command line, same config file,
same state and cache files - the two can be used interchangeably on the same
machine.
## Build
```sh
./build.sh
```
builds every platform into `./bin`, statically linked and stripped, and bumps
the patch version by 0.0.1 on each run. `version.txt` holds the version just
built; the same number goes into the binaries via `-ldflags -X main.version`,
so `upd --version` and the release tag always agree. One platform only:
```sh
PLATFORMS="linux/amd64" ./build.sh
```
The asset names in `./bin` - `upd-<goos>-<goarch>` - are exactly what
`--update` looks for in a release, so a release is `./bin` uploaded as it is.
A plain `go build -o ~/bin/upd .` still works; it just reports the fallback
version from `main.go`.
`go test ./...` runs the asset selection against the release JSON in
`../t/corpus`, the same corpus `t/select.t` uses, and fails if the two
implementations start disagreeing.
## Updating itself
upd installs itself the same way it installs everything else - same forge
layer, same CA fallback, same checksum check, same atomic replace:
```sh
upd --version # what is running
upd --check-update # look, change nothing
upd --update # download the newest release and replace the binary
```
`--update` replaces the file the running binary actually is, following a
symlink into `./bin` to the file behind it. It refuses before downloading if
that directory is not writable, and it runs the downloaded binary once with
`--version` before letting it take over, so a truncated file or one for the
wrong platform never replaces a working one. `--force` reinstalls the current
version, `--dry-run` says what it would do.
Beyond that, an ordinary run looks for a new release once a day, in the
background, and mentions it on stderr:
```
Note: upd 2.0.7 is available, run 'upd --update'.
```
The look never happens in the foreground - the run itself is never slowed down
or made to depend on the network - and never when stderr is not a terminal, so
cron and pipelines stay silent. `UPD_NO_UPDATE_CHECK=1` switches it off
altogether. The note lives in `~/.cache/upd/selfupdate.json`.
The mechanism is the one in `dx`; here it goes through upd's own machinery
instead of bringing its own HTTP client.
## Why a port
The Perl version needs curl or wget for HTTPS, which is exactly what breaks on
an older system: a curl linked against OpenSSL 1.0.x cannot complete a
handshake with a server that requires TLS 1.2, and there is nothing upd can do
about it beyond falling back to wget. The Go binary brings its own TLS stack,
so the transport question disappears - along with `--stderr` juggling, exit
code translation and the wget fallback.
## Certificates
Verification uses the system CA store. If that store does not know the issuer -
the usual case on a machine whose `ca-certificates` package predates Let's
Encrypt's ISRG roots - upd says so once and retries with the Mozilla CA list
embedded in `ca-bundle.pem`, so the binary stays self-sufficient. Refresh that
file with:
```sh
curl -o ca-bundle.pem https://curl.se/ca/cacert.pem
```
`--cacert FILE` (or `$UPD_CACERT`) replaces the system store, for a private CA.
`--insecure` skips verification altogether and warns on every run. Both switch
the automatic fallback off: an explicit choice stays the choice.
## Differences
| | Perl | Go |
|---|---|---|
| TLS | curl / wget / IO::Socket::SSL | built in |
| CA roots | whatever the system has | system, with the Mozilla list as fallback |
| `--cacert`, `--insecure` | – | yes |
| `tar`, `unzip` | external | in-process |
| `.gz`, `.bz2` | external | in-process |
| `.xz`, `.zst`, `.lz4`, `.7z` | external | external (unchanged) |
| Archive paths | trusted to `tar` | checked against traversal |
| `version-flag=` in the config file | overwritten by its own default | honoured |
Behaviour that deliberately stayed identical: asset scoring (same picks on the
whole corpus), the `--install` file-or-directory rule, state and cache file
layout, exit codes 0/1/2/10.
+355
View File
@@ -0,0 +1,355 @@
package main
import (
"archive/tar"
"archive/zip"
"bytes"
"compress/bzip2"
"compress/gzip"
"fmt"
"io"
"os"
"os/exec"
"path/filepath"
"strings"
)
// Single-file compression that Go does not carry in its standard library. Each
// entry lists the tools that can expand it, in order of preference; the first
// one found wins. All of them write to stdout with the flags given.
var external = map[string]struct {
tools []string
args []string
}{
"xz": {[]string{"xz", "unxz"}, []string{"-dc"}},
"lzma": {[]string{"xz", "unxz", "lzma"}, []string{"-dc"}},
"zst": {[]string{"zstd", "unzstd"}, []string{"-dcq"}},
"lz4": {[]string{"lz4", "unlz4"}, []string{"-dcq"}},
}
type decomp struct {
prog string
args []string
}
// decompressor reports how to expand ext, or nil if this machine cannot.
// gz and bz2 are handled in-process and need no helper at all.
func decompressor(ext string) *decomp {
switch ext {
case "gz", "bz2":
return &decomp{} // stdlib, no external program
}
e, ok := external[ext]
if !ok {
return nil
}
for _, t := range e.tools {
if p, err := exec.LookPath(t); err == nil {
return &decomp{prog: p, args: e.args}
}
}
return nil
}
func sevenZip() string {
for _, t := range []string{"7zz", "7z", "7za"} {
if p, err := exec.LookPath(t); err == nil {
return p
}
}
return ""
}
// reader wraps f in whatever ext needs, shelling out only when the standard
// library has no codec for it.
func decompressReader(ext, path string, f *os.File) (io.Reader, func() error, error) {
nop := func() error { return nil }
switch ext {
case "gz":
zr, err := gzip.NewReader(f)
if err != nil {
return nil, nil, err
}
return zr, zr.Close, nil
case "bz2":
return bzip2.NewReader(f), nop, nil
}
d := decompressor(ext)
if d == nil {
return nil, nil, fmt.Errorf("cannot unpack .%s - install %s or pick another asset with --asset/--pattern",
ext, strings.Join(external[ext].tools, "/"))
}
cmd := exec.Command(d.prog, append(append([]string{}, d.args...), path)...)
cmd.Stdin = nil
cmd.Stderr = os.Stderr
out, err := cmd.StdoutPipe()
if err != nil {
return nil, nil, err
}
if err := cmd.Start(); err != nil {
return nil, nil, err
}
return out, func() error {
io.Copy(io.Discard, out) // drain, or the child gets EPIPE
if err := cmd.Wait(); err != nil {
return fmt.Errorf("failed to decompress %s (%s: %w)", filepath.Base(path), filepath.Base(d.prog), err)
}
return nil
}, nil
}
// extractIfArchive unpacks into a fresh directory below tmpdir and returns its
// path, or "" if the asset is a bare binary (then the download is the binary).
func extractIfArchive(file, tmpdir string) (string, error) {
lc := strings.ToLower(filepath.Base(file))
out := filepath.Join(tmpdir, "x")
switch {
case tarballRe.MatchString(lc):
if err := os.Mkdir(out, 0o755); err != nil {
return "", err
}
return out, untar(file, out, tarCodec(lc))
case strings.HasSuffix(lc, ".zip"):
if err := os.Mkdir(out, 0o755); err != nil {
return "", err
}
return out, unzip(file, out)
case strings.HasSuffix(lc, ".7z"):
z := sevenZip()
if z == "" {
return "", fmt.Errorf("7z not found, but needed for %s", lc)
}
if err := os.Mkdir(out, 0o755); err != nil {
return "", err
}
cmd := exec.Command(z, "x", "-y", "-o"+out, file)
cmd.Stdout, cmd.Stderr = io.Discard, os.Stderr
if err := cmd.Run(); err != nil {
return "", fmt.Errorf("failed to extract %s: %w", lc, err)
}
return out, nil
}
// A single compressed file, e.g. restic_0.19.1_darwin_arm64.bz2 - the
// binary itself, just squeezed.
if m := singleRe.FindStringSubmatch(lc); m != nil {
ext := m[1]
if err := os.Mkdir(out, 0o755); err != nil {
return "", err
}
stem := strings.TrimSuffix(filepath.Base(file), filepath.Ext(file))
plain := filepath.Join(out, stem)
if err := decompressFile(ext, file, plain); err != nil {
return "", err
}
return out, nil
}
return "", nil // bare binary
}
// tarCodec maps a tarball suffix to the compression wrapped around the tar.
func tarCodec(lc string) string {
switch {
case strings.HasSuffix(lc, ".tar.gz"), strings.HasSuffix(lc, ".tgz"):
return "gz"
case strings.HasSuffix(lc, ".tar.bz2"), strings.HasSuffix(lc, ".tbz"):
return "bz2"
case strings.HasSuffix(lc, ".tar.xz"), strings.HasSuffix(lc, ".txz"):
return "xz"
case strings.HasSuffix(lc, ".tar.zst"), strings.HasSuffix(lc, ".tzst"):
return "zst"
}
return "" // plain .tar
}
func decompressFile(ext, src, dst string) error {
f, err := os.Open(src)
if err != nil {
return err
}
defer f.Close()
r, closer, err := decompressReader(ext, src, f)
if err != nil {
return err
}
out, err := os.Create(dst)
if err != nil {
return err
}
defer out.Close()
if _, err := io.Copy(out, r); err != nil {
return err
}
return closer()
}
func untar(file, dest, codec string) error {
f, err := os.Open(file)
if err != nil {
return err
}
defer f.Close()
var r io.Reader = f
closer := func() error { return nil }
if codec != "" {
r, closer, err = decompressReader(codec, file, f)
if err != nil {
return err
}
}
tr := tar.NewReader(r)
for {
h, err := tr.Next()
if err == io.EOF {
break
}
if err != nil {
return fmt.Errorf("failed to extract %s: %w", filepath.Base(file), err)
}
path, ok := safeJoin(dest, h.Name)
if !ok {
verbose("skipping %s (path escapes the archive)", h.Name)
continue
}
switch h.Typeflag {
case tar.TypeDir:
if err := os.MkdirAll(path, 0o755); err != nil {
return err
}
case tar.TypeReg:
if err := writeFileFrom(path, tr, os.FileMode(h.Mode).Perm()); err != nil {
return err
}
default:
// Symlinks and devices are never the binary we are after.
verbose("skipping %s (not a regular file)", h.Name)
}
}
return closer()
}
func unzip(file, dest string) error {
zr, err := zip.OpenReader(file)
if err != nil {
return fmt.Errorf("failed to extract %s: %w", filepath.Base(file), err)
}
defer zr.Close()
for _, f := range zr.File {
path, ok := safeJoin(dest, f.Name)
if !ok {
verbose("skipping %s (path escapes the archive)", f.Name)
continue
}
if f.FileInfo().IsDir() {
if err := os.MkdirAll(path, 0o755); err != nil {
return err
}
continue
}
if !f.Mode().IsRegular() {
continue
}
rc, err := f.Open()
if err != nil {
return err
}
err = writeFileFrom(path, rc, f.Mode().Perm())
rc.Close()
if err != nil {
return err
}
}
return nil
}
// safeJoin keeps an archive from writing outside its own directory.
func safeJoin(dest, name string) (string, bool) {
clean := filepath.Clean(filepath.FromSlash(name))
if filepath.IsAbs(clean) || clean == ".." || strings.HasPrefix(clean, ".."+string(os.PathSeparator)) {
return "", false
}
return filepath.Join(dest, clean), true
}
func writeFileFrom(path string, r io.Reader, mode os.FileMode) error {
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
return err
}
if mode == 0 {
mode = 0o644
}
f, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, mode)
if err != nil {
return err
}
defer f.Close()
_, err = io.Copy(f, r)
return err
}
// findInTree looks for a file called name; failing that, for the single
// executable in the tree.
func findInTree(root, name string) string {
var files []string
filepath.WalkDir(root, func(p string, d os.DirEntry, err error) error {
if err == nil && d.Type().IsRegular() {
files = append(files, p)
}
return nil
})
for _, p := range files {
if b := filepath.Base(p); b == name || b == name+".exe" {
return p
}
}
var execs []string
for _, p := range files {
if looksExecutable(p) {
execs = append(execs, p)
}
}
if len(execs) == 1 {
return execs[0]
}
verbose("archive contains %d files, %d of them executable", len(files), len(execs))
return ""
}
// Magic bytes beat guessing by file size: ELF, Mach-O (incl. fat), PE, script.
func looksExecutable(path string) bool {
f, err := os.Open(path)
if err != nil {
return false
}
defer f.Close()
m := make([]byte, 4)
n, _ := io.ReadFull(f, m)
if n < 2 {
return false
}
m = m[:n]
switch {
case bytes.HasPrefix(m, []byte("\x7fELF")):
return true
case bytes.HasPrefix(m, []byte{0xcf, 0xfa, 0xed, 0xfe}), // Mach-O 64
bytes.HasPrefix(m, []byte{0xce, 0xfa, 0xed, 0xfe}), // Mach-O 32
bytes.HasPrefix(m, []byte{0xca, 0xfe, 0xba, 0xbe}), // fat
bytes.HasPrefix(m, []byte{0xbe, 0xba, 0xfe, 0xca}):
return true
case bytes.HasPrefix(m, []byte("MZ")):
return true
case bytes.HasPrefix(m, []byte("#!")):
return true
}
return false
}
+238
View File
@@ -0,0 +1,238 @@
package main
import (
"fmt"
"os"
"os/exec"
"regexp"
"sort"
"strings"
"sync"
)
// Aliases commonly used in release asset names, including Rust target triples.
func osAlias(o string) []string {
switch o {
case "darwin":
return []string{"darwin", "macos", "osx", "mac", "apple"}
case "windows":
return []string{"windows", "win"}
}
return []string{o}
}
func archAlias(a string) []string {
switch a {
case "amd64":
return []string{"amd64", "x86_64", "x64", "64bit"}
case "arm64":
return []string{"arm64", "aarch64"}
case "386":
return []string{"386", "i386", "i686", "x86", "32bit"}
case "arm":
return []string{"arm", "armv7", "armv6", "armhf"}
case "riscv64":
return []string{"riscv64", "riscv64gc"}
}
return []string{a}
}
var (
// Checksums, signatures and OS packages are not plain binaries.
notBinaryRe = regexp.MustCompile(`\.(sha\d*|sha256sum|md5|asc|sig|pem|sbom|json|txt|deb|rpm|apk|dmg|pkg|msi|snap|flatpak|appimage)$`)
// Cross-compile targets whose triples contain a host OS token:
// aarch64-linux-android is Android, not Linux.
foreignRe = regexp.MustCompile(`(^|[^a-z0-9])(android\w*|ios|wasi|wasm\w*|emscripten)([^a-z0-9]|$)`)
archiveRe = regexp.MustCompile(`\.(tar\.gz|tgz|zip|tar\.xz|tar\.bz2|tar\.zst|tzst)$`)
noiseRe = regexp.MustCompile(`debug|symbols|static-pie|profile`)
tarballRe = regexp.MustCompile(`\.(tar\.gz|tgz|tar\.bz2|tbz|tar\.xz|txz|tar\.zst|tzst|tar)$`)
singleRe = regexp.MustCompile(`\.(gz|bz2|xz|zst|lz4|lzma)$`)
refuseRe = regexp.MustCompile(`\.(rar|dmg|pkg|msi|deb|rpm)$`)
widerArm = regexp.MustCompile(`arm64|aarch64`)
wider386 = regexp.MustCompile(`x86[_-]?64|amd64`)
win64Re = regexp.MustCompile(`(^|[^a-z0-9])win64($|[^a-z0-9])`)
win32Re = regexp.MustCompile(`(^|[^a-z0-9])win32($|[^a-z0-9])`)
)
// Compound tokens that name OS and architecture in one word. Rewritten before
// matching so the normal token rules apply. "win32" is Node-speak for Windows
// in general (pnpm-win32-x64.zip), so it contributes no architecture.
func normalizeTokens(n string) string {
n = win64Re.ReplaceAllString(n, "${1}windows-amd64${2}")
n = win32Re.ReplaceAllString(n, "${1}windows${2}")
return n
}
// tokenRe matches any of the words as a token, i.e. not glued to letters or
// digits. RE2 has no look-around, so the boundaries are ordinary characters.
func tokenRe(words []string) *regexp.Regexp {
quoted := make([]string, len(words))
for i, w := range words {
quoted[i] = regexp.QuoteMeta(w)
}
return regexp.MustCompile(`(^|[^a-z0-9])(` + strings.Join(quoted, "|") + `)([^a-z0-9]|$)`)
}
// Can this machine turn the asset into a binary? tar, zip, gzip and bzip2 are
// handled in-process; the rest needs a helper on $PATH.
func unpackable(n string) bool {
n = strings.ToLower(n)
switch {
case regexp.MustCompile(`\.(tar\.gz|tgz|tar\.bz2|tbz|tar|zip)$`).MatchString(n):
return true
case regexp.MustCompile(`\.(tar\.xz|txz)$`).MatchString(n):
return decompressor("xz") != nil
case regexp.MustCompile(`\.(tar\.zst|tzst)$`).MatchString(n):
return decompressor("zst") != nil
case strings.HasSuffix(n, ".7z"):
return sevenZip() != ""
case singleRe.MatchString(n):
return decompressor(singleRe.FindStringSubmatch(n)[1]) != nil
case refuseRe.MatchString(n):
return false
}
return true // bare binary or an unversioned name
}
// Glibc vs musl builds: pick what this system actually runs.
var muslOnce = sync.OnceValue(func() bool {
if _, err := os.Stat("/etc/alpine-release"); err == nil {
return true
}
ldd, err := exec.LookPath("ldd")
if err != nil {
return false
}
out, _ := exec.Command(ldd, "--version").CombinedOutput()
return strings.Contains(strings.ToLower(string(out)), "musl")
})
type candidate struct {
score int
pref int
length int
asset *Asset
}
// pickAsset mirrors the Perl scoring one to one - the corpus test in
// asset_test.go checks that the two keep agreeing.
func pickAsset(assets []Asset, bin, wantOS, wantArch string, spec *Spec) (*Asset, error) {
if spec.Asset != "" {
for i := range assets {
if assets[i].Name == spec.Asset {
return &assets[i], nil
}
}
return nil, fmt.Errorf("asset %q is not part of the release", spec.Asset)
}
if spec.Pattern != "" {
re, err := regexp.Compile(spec.Pattern)
if err != nil {
return nil, fmt.Errorf("invalid --pattern: %w", err)
}
for i := range assets {
if re.MatchString(assets[i].Name) {
return &assets[i], nil
}
}
return nil, nil
}
arches := archAlias(wantArch)
// Universal macOS builds serve both architectures.
if wantOS == "darwin" {
arches = append(arches, "universal", "universal2")
}
osRe, archRe := tokenRe(osAlias(wantOS)), tokenRe(arches)
var cands []candidate
var rejected []string
for i := range assets {
as := &assets[i]
n := strings.ToLower(as.Name)
m := normalizeTokens(n)
switch {
case notBinaryRe.MatchString(n):
rejected = append(rejected, n+" (not a binary)")
continue
case foreignRe.MatchString(m):
rejected = append(rejected, n+" (foreign target)")
continue
case !osRe.MatchString(m):
rejected = append(rejected, n+" (os)")
continue
case !archRe.MatchString(m):
rejected = append(rejected, n+" (arch)")
continue
// Keep 32-bit "arm" off "arm64" and "386" off "x86_64".
case wantArch == "arm" && widerArm.MatchString(m),
wantArch == "386" && wider386.MatchString(m),
wantArch == "amd64" && widerArm.MatchString(m):
rejected = append(rejected, n+" (wider arch)")
continue
}
score := 0
if bin != "" && strings.HasPrefix(n, strings.ToLower(bin)) {
score += 10 // named after the binary
}
if archiveRe.MatchString(n) || !strings.Contains(n, ".") {
score += 3
}
if noiseRe.MatchString(n) {
score -= 5
}
// Only avoid formats this machine has no tool for - restic, for one,
// ships nothing but .bz2, and that is handled in-process.
if !unpackable(n) {
score -= 8
}
// Toolchain preference is a tie-break, not a penalty: a project that
// ships musl only (or mingw only) must not be downranked for it.
pref := 0
switch {
case wantOS == "linux" && (strings.Contains(n, "musl") || strings.Contains(n, "gnu")):
libc := "gnu"
if muslOnce() {
libc = "musl"
}
if !strings.Contains(n, libc) {
pref = 1
}
case wantOS == "windows" && (strings.Contains(n, "msvc") || strings.Contains(n, "gnu")):
if !strings.Contains(n, "msvc") { // msvc is the normal Windows build
pref = 1
}
}
cands = append(cands, candidate{score: score, pref: pref, length: len(n), asset: as})
}
// Best score, then preferred toolchain, then the shortest name (which
// avoids special variants like -baseline or -static).
sort.SliceStable(cands, func(i, j int) bool {
a, b := cands[i], cands[j]
if a.score != b.score {
return a.score > b.score
}
if a.pref != b.pref {
return a.pref < b.pref
}
return a.length < b.length
})
if opt.verbose {
for _, c := range cands {
verbose("candidate score=%-3d pref=%d %s", c.score, c.pref, c.asset.Name)
}
for _, r := range rejected {
verbose("rejected: %s", r)
}
}
if len(cands) == 0 {
return nil, nil
}
return cands[0].asset, nil
}
+310
View File
@@ -0,0 +1,310 @@
package main
import (
"encoding/json"
"fmt"
"os"
"path/filepath"
"regexp"
"sort"
"strings"
"testing"
)
// Run the asset selection against the real release JSON in t/corpus/, offline.
// Every pick is checked for tokens that contradict the requested platform, so
// a wrong choice fails loudly instead of just looking plausible. This is the
// Go twin of t/select.t and must reach the same verdict.
var platforms = [][2]string{
{"darwin", "arm64"}, {"darwin", "amd64"}, {"linux", "amd64"},
{"linux", "arm64"}, {"windows", "amd64"},
}
// Tokens that must never appear in a pick for the given platform.
var badOS = map[string][]string{
"darwin": {"linux", "windows", "freebsd", "netbsd", "openbsd", "solaris", "aix"},
"linux": {"darwin", "macos", "osx", "apple", "windows", "freebsd", "netbsd", "openbsd", "solaris", "aix"},
"windows": {"linux", "darwin", "macos", "osx", "apple", "freebsd", "netbsd", "openbsd", "solaris", "aix"},
}
var badArch = map[string][]string{
"amd64": {"arm64", "aarch64", "armv7", "armhf", "i686", "i386", "riscv", "riscv64", "ppc64", "ppc64le", "s390x", "sparc64", "mips", "mips64"},
"arm64": {"amd64", "x86_64", "i686", "i386", "riscv", "riscv64", "ppc64", "ppc64le", "s390x", "sparc64", "mips", "mips64"},
}
// Cross-compile targets that are never the host.
var foreign = []string{"android", "androideabi", "ios", "wasi", "wasm", "wasm32", "emscripten"}
var notABinary = regexp.MustCompile(`\.(deb|rpm|pkg|dmg|msi|apk|snap|appimage|asc|sig|sha256)$`)
func corpusDir(t *testing.T) string {
dir := filepath.Join("..", "t", "corpus")
if _, err := os.Stat(dir); err != nil {
t.Skipf("no corpus at %s", dir)
}
return dir
}
func tok(name, word string) bool {
return regexp.MustCompile(`(^|[^a-z0-9])` + regexp.QuoteMeta(word) + `([^a-z0-9]|$)`).MatchString(name)
}
func TestCorpusSelection(t *testing.T) {
dir := corpusDir(t)
bins := map[string]string{}
if body, err := os.ReadFile(filepath.Join(dir, "binaries.json")); err == nil {
json.Unmarshal(body, &bins)
}
files, err := filepath.Glob(filepath.Join(dir, "*__*.json"))
if err != nil || len(files) == 0 {
t.Fatalf("no corpus files in %s", dir)
}
sort.Strings(files)
stats := map[string]int{}
gaps := map[string][]string{}
for _, file := range files {
slug := strings.TrimSuffix(filepath.Base(file), ".json")
repo := strings.Replace(slug, "__", "/", 1)
body, err := os.ReadFile(file)
if err != nil {
t.Fatalf("%s: %v", file, err)
}
var rel Release
if err := json.Unmarshal(body, &rel); err != nil {
t.Fatalf("%s: %v", file, err)
}
if len(rel.Assets) == 0 {
t.Errorf("%s: no assets in release", repo)
continue
}
bin := bins[repo]
if bin == "" {
bin = repo[strings.LastIndex(repo, "/")+1:]
}
for _, p := range platforms {
goos, goarch := p[0], p[1]
hit, err := pickAsset(rel.Assets, bin, goos, goarch, &Spec{})
if err != nil {
t.Errorf("%s %s/%s: %v", repo, goos, goarch, err)
continue
}
if hit == nil {
stats["no match"]++
gaps[repo] = append(gaps[repo], goos+"/"+goarch)
continue
}
name := strings.ToLower(hit.Name)
flag := ""
for _, w := range badOS[goos] {
if tok(name, w) {
flag = "WRONG OS (" + w + ")"
}
}
for _, w := range badArch[goarch] {
if tok(name, w) {
flag = "WRONG ARCH (" + w + ")"
}
}
for _, w := range foreign {
if tok(name, w) {
flag = "FOREIGN TARGET (" + w + ")"
}
}
if flag == "" && goos != "windows" && strings.HasSuffix(name, ".exe") {
flag = "EXE ON UNIX"
}
if flag == "" && notABinary.MatchString(name) {
flag = "NOT A BINARY"
}
if flag == "" && !unpackable(name) {
flag = "UNPACKABLE?"
}
if flag != "" {
t.Errorf("%s %s/%s -> %s [%s]", repo, goos, goarch, hit.Name, flag)
stats[flag]++
continue
}
stats["ok"]++
}
}
var summary []string
for _, k := range sortedKeys(stats) {
summary = append(summary, fmt.Sprintf("%s=%d", k, stats[k]))
}
t.Logf("Totals: %s", strings.Join(summary, ", "))
if len(gaps) > 0 {
var lines []string
for _, repo := range sortedKeys(gaps) {
lines = append(lines, fmt.Sprintf("%s (%s)", repo, strings.Join(gaps[repo], " ")))
}
t.Logf("Unmatched: %s", strings.Join(lines, ", "))
}
}
// The Perl version reaches these totals on the same corpus; a change here is a
// change in behaviour and wants a look, not a blind update.
func TestCorpusTotalsMatchPerl(t *testing.T) {
dir := corpusDir(t)
files, _ := filepath.Glob(filepath.Join(dir, "*__*.json"))
bins := map[string]string{}
if body, err := os.ReadFile(filepath.Join(dir, "binaries.json")); err == nil {
json.Unmarshal(body, &bins)
}
ok, misses := 0, 0
for _, file := range files {
slug := strings.TrimSuffix(filepath.Base(file), ".json")
repo := strings.Replace(slug, "__", "/", 1)
body, _ := os.ReadFile(file)
var rel Release
if json.Unmarshal(body, &rel) != nil {
continue
}
bin := bins[repo]
if bin == "" {
bin = repo[strings.LastIndex(repo, "/")+1:]
}
for _, p := range platforms {
hit, err := pickAsset(rel.Assets, bin, p[0], p[1], &Spec{})
switch {
case err != nil:
t.Fatalf("%s: %v", repo, err)
case hit == nil:
misses++
default:
ok++
}
}
}
if ok != 141 || misses != 9 {
t.Errorf("corpus totals drifted: ok=%d no-match=%d, want ok=141 no-match=9", ok, misses)
}
}
func TestParseRepo(t *testing.T) {
cases := []struct{ in, base, owner, repo string }{
{"https://github.com/sxyazi/yazi", "https://github.com", "sxyazi", "yazi"},
{"https://git.micw.org/mike/dns", "https://git.micw.org", "mike", "dns"},
{"https://git.micw.org/mike/dns.git", "https://git.micw.org", "mike", "dns"},
{"https://git.micw.org/mike/dns/", "https://git.micw.org", "mike", "dns"},
{"git.micw.org/mike/dns", "https://git.micw.org", "mike", "dns"},
{"https://example.org/gitea/mike/dns", "https://example.org/gitea", "mike", "dns"},
{"http://localhost:3000/mike/dns", "http://localhost:3000", "mike", "dns"},
}
for _, c := range cases {
base, owner, repo, err := parseRepo(c.in)
if err != nil {
t.Errorf("%s: %v", c.in, err)
continue
}
if base != c.base || owner != c.owner || repo != c.repo {
t.Errorf("%s -> %s %s %s, want %s %s %s", c.in, base, owner, repo, c.base, c.owner, c.repo)
}
}
for _, bad := range []string{"https://github.com/onlyowner", "https://github.com"} {
if _, _, _, err := parseRepo(bad); err == nil {
t.Errorf("%s: expected an error", bad)
}
}
}
func TestForgeDetection(t *testing.T) {
cases := []struct{ base, forge, api string }{
{"https://github.com", "github", "https://api.github.com"},
{"https://git.micw.org", "gitea", "https://git.micw.org/api/v1"},
{"https://example.org/gitea", "gitea", "https://example.org/gitea/api/v1"},
{"https://notgithub.com", "gitea", "https://notgithub.com/api/v1"},
}
for _, c := range cases {
f, err := detectForge(c.base, "")
if err != nil || f != c.forge {
t.Errorf("%s -> %q (%v), want %q", c.base, f, err, c.forge)
}
if got := apiBase(c.base, f); got != c.api {
t.Errorf("%s -> %s, want %s", c.base, got, c.api)
}
}
if _, err := detectForge("https://x", "gitlab"); err == nil {
t.Error("expected an error for an unknown forge")
}
// GitHub Enterprise needs the override and lands on /api/v3.
if got := apiBase("https://gh.corp.example", "github"); got != "https://gh.corp.example/api/v3" {
t.Errorf("enterprise API base: %s", got)
}
}
func TestParseNames(t *testing.T) {
cases := []struct {
list, fallback string
want []name
}{
{"", "yazi", []name{{"yazi", "yazi"}}},
{"yazi,ya", "x", []name{{"yazi", "yazi"}, {"ya", "ya"}}},
{"yazi:yazi-nightly", "x", []name{{"yazi", "yazi-nightly"}}},
{"a , b", "x", []name{{"a", "a"}, {"b", "b"}}},
}
for _, c := range cases {
got := parseNames(c.list, c.fallback)
if len(got) != len(c.want) {
t.Errorf("%q -> %v, want %v", c.list, got, c.want)
continue
}
for i := range got {
if got[i] != c.want[i] {
t.Errorf("%q -> %v, want %v", c.list, got, c.want)
}
}
}
}
func TestSplitTokens(t *testing.T) {
got := splitTokens(`https://x/y install=~/bin name="a b" pre`)
want := []string{"https://x/y", "install=~/bin", "name=a b", "pre"}
if strings.Join(got, "|") != strings.Join(want, "|") {
t.Errorf("got %q, want %q", got, want)
}
}
func TestNormalizeTokens(t *testing.T) {
cases := [][2]string{
{"pnpm-win32-x64.zip", "pnpm-windows-x64.zip"},
{"tool-win64.zip", "tool-windows-amd64.zip"},
{"winsomething-x64", "winsomething-x64"}, // not a token, left alone
}
for _, c := range cases {
if got := normalizeTokens(c[0]); got != c[1] {
t.Errorf("%s -> %s, want %s", c[0], got, c[1])
}
}
}
func TestStateFileLayout(t *testing.T) {
t.Setenv("XDG_STATE_HOME", "/tmp/state")
// Same recipe as the Perl version: sanitised path, dot, 8 hex characters.
got := filepath.Base(stateFile("/home/mike/bin/fzf"))
if !regexp.MustCompile(`^home_mike_bin_fzf\.[0-9a-f]{8}\.json$`).MatchString(got) {
t.Errorf("unexpected state file name: %s", got)
}
}
func TestSafeJoin(t *testing.T) {
for _, bad := range []string{"../evil", "/etc/passwd", "a/../../evil"} {
if _, ok := safeJoin("/dest", bad); ok {
t.Errorf("%q should have been rejected", bad)
}
}
if p, ok := safeJoin("/dest", "sub/bin/tool"); !ok || p != filepath.Join("/dest", "sub/bin/tool") {
t.Errorf("safeJoin rejected a normal path: %s %v", p, ok)
}
}
Executable
+63
View File
@@ -0,0 +1,63 @@
#!/bin/sh
# Build upd for the usual platforms into ./bin, auto-incrementing the patch
# version by 0.0.1 on every build.
#
# version.txt holds the currently built version. Each run increments the patch
# component, then builds every platform with that one version injected via
# -ldflags, and writes it back. So version.txt always reflects the version of
# the binaries just built, and all of them carry the same one.
#
# The asset names ./bin ends up with are exactly what selfupdate.go looks for
# in a release: upd-<goos>-<goarch>. Upload the directory as it is.
#
# Override the platform list to build just one, or to add a platform:
# PLATFORMS="linux/amd64" ./build.sh
# PLATFORMS="linux/386 linux/arm64" ./build.sh
#
# Windows is not in the list: the run is wrapped in a SIGHUP handler and the
# install path rules assume a Unix $PATH, so shipping it would promise more
# than has been tested. PLATFORMS can add it.
#
# -s -w drops the symbol table and DWARF info, -trimpath keeps build paths out
# of the binary; together they roughly halve it. Neither affects a panic trace.
set -e
cd "$(dirname "$0")"
PLATFORMS=${PLATFORMS:-"darwin/arm64 darwin/amd64 linux/amd64 linux/arm64"}
V=$(cat version.txt 2>/dev/null || echo 2.0.0)
# split MAJOR.MINOR.PATCH and increment PATCH (no carry: 2.0.9 -> 2.0.10)
MAJOR=${V%%.*}
REST=${V#*.}
MINOR=${REST%%.*}
PATCH=${REST#*.}
PATCH=$((PATCH + 1))
NV="$MAJOR.$MINOR.$PATCH"
mkdir -p bin
HOST="$(go env GOOS)/$(go env GOARCH)"
for p in $PLATFORMS; do
os=${p%/*}
arch=${p#*/}
out="bin/upd-$os-$arch"
# CGO_ENABLED=0 throughout: it makes the cross builds work without a
# toolchain per target and the binaries static - which is the point on the
# old machines upd exists for. It also settles the one thing cgo would
# change here: name resolution goes through Go's own resolver, not the
# system one, and TLS never touched the C library to begin with.
CGO_ENABLED=0 GOOS="$os" GOARCH="$arch" \
go build -trimpath -ldflags "-s -w -X main.version=$NV" -o "$out" .
if [ "$p" = "$HOST" ]; then
ln -sf "upd-$os-$arch" bin/upd # the one for this machine
echo " $out -> bin/upd"
else
echo " $out"
fi
done
echo "$NV" > version.txt
echo "built upd v$NV"
+2950
View File
File diff suppressed because it is too large Load Diff
+250
View File
@@ -0,0 +1,250 @@
package main
import (
"encoding/json"
"fmt"
"net/url"
"os"
"regexp"
"strings"
)
// Gitea and GitHub expose the same release JSON, only under different API
// roots and with different auth headers.
type forgeCtx struct {
base string // scheme, host and any sub-path Gitea is mounted under
owner string
repo string
forge string // "github" or "gitea"
token string
api string // .../repos/<owner>/<repo>
}
type Asset struct {
ID int64 `json:"id"`
Name string `json:"name"`
Size int64 `json:"size"`
BrowserDownloadURL string `json:"browser_download_url"`
Digest string `json:"digest"`
CreatedAt string `json:"created_at"`
UpdatedAt string `json:"updated_at"`
}
type Release struct {
TagName string `json:"tag_name"`
HTMLURL string `json:"html_url"`
PublishedAt string `json:"published_at"`
Draft bool `json:"draft"`
Prerelease bool `json:"prerelease"`
Assets []Asset `json:"assets"`
}
func newForgeCtx(spec *Spec) (*forgeCtx, error) {
base, owner, repo, err := parseRepo(spec.Repo)
if err != nil {
return nil, err
}
forge, err := detectForge(base, spec.Forge)
if err != nil {
return nil, err
}
c := &forgeCtx{
base: base,
owner: owner,
repo: repo,
forge: forge,
token: resolveToken(forge, spec.Token),
}
c.api = apiBase(base, forge) + "/repos/" + url.PathEscape(owner) + "/" + url.PathEscape(repo)
return c, nil
}
func parseRepo(repo string) (base, owner, name string, err error) {
repo = strings.TrimRight(repo, "/")
repo = strings.TrimSuffix(repo, ".git")
if !strings.HasPrefix(repo, "http://") && !strings.HasPrefix(repo, "https://") {
repo = "https://" + repo
}
u, e := url.Parse(repo)
if e != nil || u.Host == "" {
return "", "", "", fmt.Errorf("cannot parse repository URL: %s", repo)
}
// Anything before <owner>/<repo> is a base path (Gitea under a sub-path).
seg := strings.FieldsFunc(u.Path, func(r rune) bool { return r == '/' })
if len(seg) < 2 {
return "", "", "", fmt.Errorf("repository URL needs <owner>/<repo>: %s", repo)
}
owner, name = seg[len(seg)-2], seg[len(seg)-1]
base = u.Scheme + "://" + u.Host
if prefix := seg[:len(seg)-2]; len(prefix) > 0 {
base += "/" + strings.Join(prefix, "/")
}
return base, owner, name, nil
}
var githubHost = regexp.MustCompile(`(^|\.)github\.com$`)
func hostOf(base string) string {
u, err := url.Parse(base)
if err != nil {
return ""
}
return strings.ToLower(u.Hostname())
}
func detectForge(base, override string) (string, error) {
if override != "" {
f := strings.ToLower(override)
if f != "github" && f != "gitea" {
return "", fmt.Errorf("unknown forge %q (use github or gitea)", override)
}
return f, nil
}
if githubHost.MatchString(hostOf(base)) {
return "github", nil
}
return "gitea", nil
}
func apiBase(base, forge string) string {
if forge != "github" {
return base + "/api/v1"
}
if githubHost.MatchString(hostOf(base)) {
return "https://api.github.com"
}
return base + "/api/v3" // GitHub Enterprise
}
func resolveToken(forge, explicit string) string {
if explicit != "" {
return explicit
}
env := []string{"UPD_TOKEN", "GITEA_TOKEN"}
if forge == "github" {
env = []string{"UPD_TOKEN", "GITHUB_TOKEN", "GH_TOKEN"}
}
for _, k := range env {
if v := os.Getenv(k); v != "" {
return v
}
}
return ""
}
func (c *forgeCtx) authHeaders() map[string]string {
if c.token == "" {
return map[string]string{}
}
// GitHub wants "Bearer", Gitea wants "token".
if c.forge == "github" {
return map[string]string{"Authorization": "Bearer " + c.token}
}
return map[string]string{"Authorization": "token " + c.token}
}
// Private GitHub assets are only reachable through the API URL; the browser
// URL redirects to storage, where the Authorization header is dropped.
func (c *forgeCtx) assetURL(a *Asset) (string, map[string]string) {
if c.forge == "github" && c.token != "" && a.ID != 0 {
return fmt.Sprintf("%s/releases/assets/%d", c.api, a.ID),
map[string]string{"Accept": "application/octet-stream"}
}
return a.BrowserDownloadURL, map[string]string{}
}
func (c *forgeCtx) releasesURL() string {
if c.forge == "github" {
return c.api + "/releases?per_page=50"
}
return c.api + "/releases?limit=50&draft=false"
}
// ==============================================================================
// Release selection
// ==============================================================================
func (c *forgeCtx) fetchRelease(spec *Spec) (*Release, []Release, error) {
// A specific tag: both forges have a direct endpoint. Fall back to the list
// so "26.5.6" also finds a tag named "v26.5.6".
if spec.Tag != "" {
body, err := c.apiGet(c.api+"/releases/tags/"+url.PathEscape(spec.Tag), true)
if err != nil {
return nil, nil, err
}
if body != nil {
var rel Release
if json.Unmarshal(body, &rel) == nil && rel.TagName != "" {
return &rel, nil, nil
}
}
list, err := c.releaseList()
if err != nil {
return nil, nil, err
}
for i := range list {
if list[i].TagName == spec.Tag {
return &list[i], nil, nil
}
}
for i := range list {
if normVer(list[i].TagName) == normVer(spec.Tag) {
return &list[i], nil, nil
}
}
return nil, nil, fmt.Errorf("release %q not found (--list shows all)", spec.Tag)
}
// The plain case is one request: the forge already knows its latest
// non-draft, non-prerelease release.
if !spec.Pre && !opt.list {
body, err := c.apiGet(c.api+"/releases/latest", true)
if err != nil {
return nil, nil, err
}
if body != nil {
var rel Release
if json.Unmarshal(body, &rel) == nil && rel.TagName != "" {
return &rel, nil, nil
}
}
verbose("no /releases/latest, falling back to the release list")
}
list, err := c.releaseList()
if err != nil {
return nil, nil, err
}
if len(list) == 0 {
return nil, nil, fmt.Errorf("no releases found in %s/%s", c.owner, c.repo)
}
if opt.list {
return nil, list, nil
}
for i := range list {
if !list[i].Draft && (spec.Pre || !list[i].Prerelease) {
return &list[i], nil, nil
}
}
return nil, nil, fmt.Errorf("no suitable release found (try --pre)")
}
func (c *forgeCtx) releaseList() ([]Release, error) {
body, err := c.apiGet(c.releasesURL(), false)
if err != nil {
return nil, err
}
var list []Release
if err := json.Unmarshal(body, &list); err != nil {
return nil, fmt.Errorf("release list from %s is not an array: %w", c.releasesURL(), err)
}
return list, nil
}
func normVer(v string) string {
v = strings.TrimSpace(v)
if v != "" && (v[0] == 'v' || v[0] == 'V') {
v = v[1:]
}
return strings.TrimSpace(v)
}
+3
View File
@@ -0,0 +1,3 @@
module git.micw.org/mike/upd
go 1.22
+397
View File
@@ -0,0 +1,397 @@
package main
import (
"crypto/sha256"
"crypto/x509"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"net"
"net/http"
"os"
"path/filepath"
"strings"
"time"
)
var userAgent = "upd/" + version + " (go)"
var (
client *http.Client
clientMode caMode = -1 // forces a build on first use
)
// One client per verification mode. The timeout covers connect, TLS and the
// wait for the response header - not the body, because a download may
// legitimately take longer than that.
func httpClient() (*http.Client, error) {
if client != nil && clientMode == caCurrent {
return client, nil
}
cfg, err := tlsConfig(caCurrent)
if err != nil {
return nil, err
}
to := time.Duration(opt.timeout) * time.Second
client = &http.Client{
Transport: &http.Transport{
Proxy: http.ProxyFromEnvironment,
DialContext: (&net.Dialer{Timeout: to, KeepAlive: 30 * time.Second}).DialContext,
TLSClientConfig: cfg,
TLSHandshakeTimeout: to,
ResponseHeaderTimeout: to,
ExpectContinueTimeout: time.Second,
ForceAttemptHTTP2: true,
},
}
clientMode = caCurrent
return client, nil
}
type response struct {
status int
header http.Header
body []byte // empty when the body went to dst
}
// A writer that can be rewound, so a retry does not append to a half-written
// file. *os.File satisfies it.
type resettable interface {
io.Writer
Truncate(int64) error
Seek(int64, int) (int64, error)
}
// httpGet retries what curl's --retry covers: connection failures and server
// side errors. A 4xx is an answer, not a hiccup, and is returned as is.
func httpGet(url string, headers map[string]string, dst io.Writer) (*response, error) {
const attempts = 3
var lastErr error
retryNow := false // set when the next attempt changes something itself
for i := range attempts {
if i > 0 {
if r, ok := dst.(resettable); ok {
if _, err := r.Seek(0, io.SeekStart); err != nil {
return nil, err
}
if err := r.Truncate(0); err != nil {
return nil, err
}
}
if !retryNow {
time.Sleep(time.Duration(i) * time.Second)
verbose("retrying (%d/%d): %s", i, attempts-1, url)
}
}
retryNow = false
// A CA bundle that cannot be read is a configuration error, not a
// network one - repeating it would not help.
if _, err := httpClient(); err != nil {
return nil, err
}
resp, err := httpTry(url, headers, dst)
if err != nil {
lastErr = err
// A store that does not know the issuer is not a hiccup, but it is
// the one certificate failure a fresh root list can fix - so try
// the bundled one straight away, and say so.
if isUnknownAuthority(err) && caCurrent == caSystem && bundledPool() != nil {
caCurrent = caBundled
retryNow = true
fmt.Fprintf(os.Stderr,
"Note: the system CA store does not know this issuer, using the bundled CA list.\n"+
" Update the ca-certificates package to make this permanent.\n")
continue
}
if isPermanent(err) {
return nil, err
}
continue
}
if resp.status >= 500 && i < attempts-1 {
lastErr = fmt.Errorf("HTTP %d: %s", resp.status, url)
continue
}
return resp, nil
}
return nil, lastErr
}
func httpTry(url string, headers map[string]string, dst io.Writer) (*response, error) {
req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
return nil, err
}
req.Header.Set("User-Agent", userAgent)
for _, k := range sortedKeys(headers) {
req.Header.Set(k, headers[k])
}
c, err := httpClient()
if err != nil {
return nil, err
}
resp, err := c.Do(req)
if err != nil {
return nil, transportError(err, url)
}
defer resp.Body.Close()
out := &response{status: resp.StatusCode, header: resp.Header}
if dst != nil && resp.StatusCode == http.StatusOK {
if _, err := io.Copy(dst, resp.Body); err != nil {
return nil, fmt.Errorf("download interrupted: %w", err)
}
return out, nil
}
// Everything that is not a download: API answers and the error bodies both
// forges explain themselves in. A release list with 50 entries runs into
// megabytes, so the cap is only there to bound a runaway response - and it
// says so instead of handing on half a document.
const maxBody = 32 << 20
body, err := io.ReadAll(io.LimitReader(resp.Body, maxBody+1))
if err != nil {
return nil, err
}
if len(body) > maxBody {
return nil, fmt.Errorf("response from %s is larger than %s", url, humanSize(maxBody))
}
out.body = body
return out, nil
}
// The Go TLS stack speaks whatever the server does, so the old "your OpenSSL
// is too old" class of failure is gone - but DNS, proxies, clocks and private
// CAs are still there. Name the one that hit.
func transportError(err error, url string) error {
var (
dns *net.DNSError
hostname x509.HostnameError
invalid x509.CertificateInvalidError
)
msg := ""
switch {
case errors.As(err, &dns):
msg = "the host could not be resolved - check DNS and $https_proxy"
case isUnknownAuthority(err):
msg = "the issuer is unknown even to the bundled CA list - for a private CA " +
"pass --cacert <file>, or --insecure to skip verification"
case errors.As(err, &invalid):
msg = "the certificate is outside its validity period - check the system clock"
case errors.As(err, &hostname):
msg = "the certificate does not match the host name"
case errors.Is(err, os.ErrDeadlineExceeded) || strings.Contains(err.Error(), "timeout"):
msg = "timed out - raise --timeout"
case strings.Contains(err.Error(), "connection refused"):
msg = "connection refused - check the port and any firewall"
}
if msg == "" {
return fmt.Errorf("request failed: %s\n %w", url, err)
}
return fmt.Errorf("request failed: %s\n %w\n %s", url, err, msg)
}
// ==============================================================================
// API requests with an ETag cache
// ==============================================================================
type cacheEntry struct {
URL string `json:"url"`
ETag string `json:"etag"`
Body string `json:"body"`
}
func cacheDir() string {
base := os.Getenv("XDG_CACHE_HOME")
if base == "" {
base = filepath.Join(homeDir(), ".cache")
}
return filepath.Join(base, "upd")
}
func cacheFile(url string) string {
sum := sha256.Sum256([]byte(url))
return filepath.Join(cacheDir(), hex.EncodeToString(sum[:])[:16]+".json")
}
// apiGet fetches a JSON endpoint, revalidating a cached copy via ETag.
// soft turns a 404 into (nil, nil) instead of an error.
func (c *forgeCtx) apiGet(url string, soft bool) (json.RawMessage, error) {
hdr := c.authHeaders()
hdr["Accept"] = "application/json"
if c.forge == "github" {
hdr["Accept"] = "application/vnd.github+json"
hdr["X-GitHub-Api-Version"] = "2022-11-28"
}
var cached cacheEntry
cf := cacheFile(url)
if err := readJSON(cf, &cached); err == nil && cached.ETag != "" {
hdr["If-None-Match"] = cached.ETag
}
resp, err := httpGet(url, hdr, nil)
if err != nil {
return nil, err
}
if resp.status == http.StatusNotModified && cached.Body != "" {
verbose("304 not modified, using cached %s", url)
return validJSON([]byte(cached.Body), url, c)
}
if soft && resp.status == http.StatusNotFound {
return nil, nil
}
if resp.status == http.StatusForbidden || resp.status == http.StatusTooManyRequests {
if left := resp.header.Get("X-RateLimit-Remaining"); left == "0" {
return nil, fmt.Errorf("%s rate limit reached (remaining: %s).\n"+
" Set a token via --token or $GITHUB_TOKEN", c.forge, left)
}
}
// Both forges answer 404 for a repository the caller may not see, so a
// repository that is missing and one that is merely private look alike.
if resp.status == http.StatusUnauthorized || resp.status == http.StatusForbidden ||
resp.status == http.StatusNotFound {
hint := "a private repository needs --token or $UPD_TOKEN"
if c.token != "" {
hint = "the token does not grant access to this repository"
}
return nil, fmt.Errorf("HTTP %d%s: %s\n %s", resp.status, apiMessage(resp.body), url, hint)
}
if resp.status != http.StatusOK {
return nil, fmt.Errorf("HTTP %d: %s", resp.status, url)
}
data, err := validJSON(resp.body, url, c)
if err != nil {
return nil, err
}
if etag := resp.header.Get("ETag"); etag != "" {
if err := os.MkdirAll(cacheDir(), 0o755); err == nil {
writeJSON(cf, cacheEntry{URL: url, ETag: etag, Body: string(resp.body)})
}
}
return data, nil
}
func validJSON(body []byte, url string, c *forgeCtx) (json.RawMessage, error) {
if !json.Valid(body) {
return nil, fmt.Errorf("response from %s is not JSON (is this really a %s instance?)", url, c.forge)
}
if msg := apiMessage(body); msg != "" {
return nil, fmt.Errorf("%s error:%s", c.forge, msg)
}
return body, nil
}
// Both forges explain themselves in a JSON "message" field.
func apiMessage(body []byte) string {
var m struct {
Message string `json:"message"`
}
if err := json.Unmarshal(body, &m); err != nil || m.Message == "" {
return ""
}
return " (" + m.Message + ")"
}
// ==============================================================================
// Downloads
// ==============================================================================
func downloadTo(path, url string, headers map[string]string, size int64) error {
fh, err := os.Create(path)
if err != nil {
return err
}
defer fh.Close()
var dst io.Writer = fh
if bar := newProgress(size); bar != nil {
defer bar.finish()
dst = &progressWriter{file: fh, bar: bar}
}
resp, err := httpGet(url, headers, dst)
if err != nil {
return err
}
if resp.status != http.StatusOK {
return fmt.Errorf("download failed (HTTP %d%s): %s", resp.status, apiMessage(resp.body), url)
}
return fh.Sync()
}
// progressWriter keeps the file a resettable writer for the retry path.
type progressWriter struct {
file *os.File
bar *progress
}
func (w *progressWriter) Write(p []byte) (int, error) {
n, err := w.file.Write(p)
w.bar.add(int64(n))
return n, err
}
func (w *progressWriter) Truncate(n int64) error { w.bar.reset(); return w.file.Truncate(n) }
func (w *progressWriter) Seek(off int64, whence int) (int64, error) {
return w.file.Seek(off, whence)
}
type progress struct {
total, got int64
last time.Time
}
func newProgress(total int64) *progress {
if opt.quiet || !isTerminal(os.Stdout) {
return nil
}
return &progress{total: total}
}
func (p *progress) reset() { p.got = 0 }
func (p *progress) add(n int64) {
p.got += n
if time.Since(p.last) < 100*time.Millisecond {
return
}
p.last = time.Now()
p.draw()
}
func (p *progress) draw() {
const width = 40
if p.total <= 0 {
fmt.Printf("\r %s", humanSize(p.got))
return
}
pct := float64(p.got) / float64(p.total)
if pct > 1 {
pct = 1
}
filled := int(pct * width)
fmt.Printf("\r [%s%s] %5.1f%% %s",
strings.Repeat("#", filled), strings.Repeat(" ", width-filled),
pct*100, humanSize(p.total))
}
func (p *progress) finish() {
if p == nil {
return
}
p.draw()
fmt.Print("\n")
}
func isTerminal(f *os.File) bool {
st, err := f.Stat()
return err == nil && st.Mode()&os.ModeCharDevice != 0
}
+372
View File
@@ -0,0 +1,372 @@
package main
import (
"context"
"crypto/sha256"
"encoding/hex"
"fmt"
"io"
"os"
"os/exec"
"path/filepath"
"regexp"
"runtime"
"slices"
"strings"
"sync"
"time"
)
// target is one file to write, plus the name to look for inside the asset.
type target struct {
name string // inside the archive
out string // file name on disk
dest string // full path
}
func resolveTargets(spec *Spec, names []name, goos string) ([]target, error) {
want := spec.Install
if want == "" {
want = os.Getenv("UPD_INSTALL")
}
want = expandTilde(want)
ext := ""
if goos == "windows" {
ext = ".exe"
}
// --install is a directory unless it clearly points at a single binary:
// an existing file, or a last segment that is one of the binary names.
// Everything else is a directory (and gets created) - otherwise a config
// line like "install=~/bin" would create a *file* called bin.
var dir, single string
if want != "" {
leaf := filepath.Base(want)
isBin := false
for _, n := range names {
if leaf == n.out || leaf == n.out+ext {
isBin = true
}
}
st, err := os.Stat(want)
isDir := err == nil && st.IsDir()
isFile := err == nil && st.Mode().IsRegular()
if !isDir && (isBin || isFile) {
if len(names) > 1 {
return nil, fmt.Errorf("--install points at the file %q but %d binaries were requested; give a directory instead", want, len(names))
}
single = want
dir = filepath.Dir(want)
} else {
dir = strings.TrimRight(want, string(os.PathSeparator))
if dir == "" {
dir = want
}
}
} else {
dir = defaultDir(names[0].out)
}
if err := makeDir(dir); err != nil {
return nil, err
}
var targets []target
for _, n := range names {
dest := single
if dest == "" {
dest = filepath.Join(dir, n.out+ext)
}
// Replacing a symlink would silently break the link, so follow it.
if st, err := os.Lstat(dest); err == nil && st.Mode()&os.ModeSymlink != 0 {
if real, err := filepath.EvalSymlinks(dest); err == nil {
info("Note: %s is a symlink, installing to %s", dest, real)
dest = real
}
}
targets = append(targets, target{name: n.src, out: n.out, dest: dest})
}
return targets, nil
}
// Without --install: replace the binary already on PATH, else ~/.local/bin.
func defaultDir(bin string) string {
for _, dir := range filepath.SplitList(os.Getenv("PATH")) {
if dir == "" {
continue
}
p := filepath.Join(dir, bin)
if st, err := os.Stat(p); err == nil && st.Mode()&0o111 != 0 && writable(dir) {
return dir
}
}
return filepath.Join(homeDir(), ".local", "bin")
}
// Actually try it: permission bits say nothing about a read-only mount, and
// this runs right before we would write there anyway.
func writable(dir string) bool {
f, err := os.CreateTemp(dir, ".upd-*")
if err != nil {
return false
}
f.Close()
os.Remove(f.Name())
return true
}
func makeDir(dir string) error {
if opt.dryRun || opt.list || opt.check {
return nil
}
if st, err := os.Stat(dir); err == nil && st.IsDir() {
return nil
}
if err := os.MkdirAll(dir, 0o755); err != nil {
return fmt.Errorf("cannot create target directory %s: %w", dir, err)
}
return nil
}
func inPath(dir string) bool {
return slices.Contains(filepath.SplitList(os.Getenv("PATH")), dir)
}
// ==============================================================================
// State
// ==============================================================================
func stateDir() string {
base := os.Getenv("XDG_STATE_HOME")
if base == "" {
base = filepath.Join(homeDir(), ".local", "state")
}
return filepath.Join(base, "upd")
}
var unsafeChars = regexp.MustCompile(`[^A-Za-z0-9._-]+`)
// One state file per installed path; readable name plus a hash for uniqueness.
// The layout matches the Perl version so both can share a state directory.
func stateFile(dest string) string {
key := unsafeChars.ReplaceAllString(dest, "_")
key = strings.Trim(key, "_")
if len(key) > 70 {
key = key[len(key)-70:]
}
sum := sha256.Sum256([]byte(dest))
return filepath.Join(stateDir(), key+"."+hex.EncodeToString(sum[:])[:8]+".json")
}
// state is written with the keys in the order Perl's canonical JSON produces,
// so a file written by either version reads the same in a diff.
type state struct {
Asset string `json:"asset"`
AssetSize int64 `json:"asset_size"`
Binary string `json:"binary"`
BinarySHA256 string `json:"binary_sha256"`
Dest string `json:"dest"`
Forge string `json:"forge"`
InstalledAt string `json:"installed_at"`
Repo string `json:"repo"`
Stamp string `json:"stamp"`
Tag string `json:"tag"`
}
// Identity of a release asset: the digest if the forge publishes one (GitHub
// does), otherwise its upload time - this is what makes rolling tags such as
// "nightly" detectable.
func assetStamp(a *Asset) string {
switch {
case a.Digest != "":
return a.Digest
case a.UpdatedAt != "":
return "t:" + a.UpdatedAt
case a.CreatedAt != "":
return "t:" + a.CreatedAt
}
return fmt.Sprintf("s:%d", a.Size)
}
func targetCurrent(t target, tag string, a *Asset, spec *Spec) bool {
if st, err := os.Stat(t.dest); err != nil || !st.Mode().IsRegular() {
return false
}
var s state
if err := readJSON(stateFile(t.dest), &s); err == nil && s.Tag != "" {
if s.Tag != tag || s.Asset != a.Name || s.Stamp != assetStamp(a) {
return false
}
// A locally replaced binary counts as out of date.
same := s.BinarySHA256 == fileSHA256(t.dest)
if !same {
verbose("%s: state matches but binary differs", t.dest)
}
return same
}
// No state yet (first run after an install by other means): ask the binary.
v := installedVersion(t.dest, spec)
verbose("%s: no state file, binary reports %q", t.dest, v)
return v != "" && v == normVer(tag)
}
func installedTag(t target) string {
var s state
if err := readJSON(stateFile(t.dest), &s); err == nil && s.Tag != "" {
return s.Tag
}
if st, err := os.Stat(t.dest); err != nil || !st.Mode().IsRegular() {
return "-"
}
if v := installedVersion(t.dest, &Spec{}); v != "" {
return v
}
return "unknown"
}
func writeState(t target, c *forgeCtx, tag string, a *Asset) error {
if err := os.MkdirAll(stateDir(), 0o755); err != nil {
return fmt.Errorf("cannot create %s: %w", stateDir(), err)
}
return writeJSON(stateFile(t.dest), state{
Asset: a.Name,
AssetSize: a.Size,
Binary: t.out,
BinarySHA256: fileSHA256(t.dest),
Dest: t.dest,
Forge: c.forge,
InstalledAt: time.Now().UTC().Format("2006-01-02T15:04:05Z"),
Repo: fmt.Sprintf("%s/%s/%s", c.base, c.owner, c.repo),
Stamp: assetStamp(a),
Tag: tag,
})
}
var versionRe = regexp.MustCompile(`\d+\.\d+(\.\d+)*([-+][\w.]+)?`)
// Legacy fallback: ask the binary for its version (used when no state exists).
func installedVersion(path string, spec *Spec) string {
if st, err := os.Stat(path); err != nil || st.Mode()&0o111 == 0 {
return ""
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
cmd := exec.CommandContext(ctx, path, spec.versionFlag())
cmd.Stdin = nil
out, _ := cmd.CombinedOutput() // a non-zero exit still often prints the version
return normVer(versionRe.FindString(string(out)))
}
// ==============================================================================
// Installing
// ==============================================================================
// Leftovers from an interrupted install must not linger next to the binary.
var temps struct {
sync.Mutex
paths []string
}
func addTemp(p string) {
temps.Lock()
defer temps.Unlock()
temps.paths = append(temps.paths, p)
}
func dropTemp(p string) {
temps.Lock()
defer temps.Unlock()
for i, q := range temps.paths {
if q == p {
temps.paths = append(temps.paths[:i], temps.paths[i+1:]...)
return
}
}
}
func cleanupTemps() {
temps.Lock()
defer temps.Unlock()
for _, p := range temps.paths {
os.Remove(p)
}
temps.paths = nil
}
func installAtomic(src, dest string) error {
dir := filepath.Dir(dest)
if st, err := os.Stat(dir); err != nil || !st.IsDir() {
return fmt.Errorf("target directory %s does not exist", dir)
}
if !writable(dir) {
return fmt.Errorf("no write permission in %s (use sudo, or pick another --install path)", dir)
}
tmp := fmt.Sprintf("%s.new.%d", dest, os.Getpid())
addTemp(tmp)
if err := copyFile(src, tmp, 0o755); err != nil {
os.Remove(tmp)
dropTemp(tmp)
return err
}
// rename is atomic and works even while dest is currently running.
if err := os.Rename(tmp, dest); err != nil {
os.Remove(tmp)
dropTemp(tmp)
return fmt.Errorf("cannot replace %s: %w", dest, err)
}
dropTemp(tmp)
return nil
}
func copyFile(src, dst string, mode os.FileMode) error {
in, err := os.Open(src)
if err != nil {
return fmt.Errorf("cannot read %s: %w", src, err)
}
defer in.Close()
out, err := os.OpenFile(dst, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, mode)
if err != nil {
return fmt.Errorf("cannot write %s: %w", dst, err)
}
if _, err := io.Copy(out, in); err != nil {
out.Close()
return fmt.Errorf("error writing %s: %w", dst, err)
}
if err := out.Close(); err != nil {
return fmt.Errorf("error writing %s: %w", dst, err)
}
return os.Chmod(dst, mode)
}
func stripQuarantine(dest string) {
if runtime.GOOS != "darwin" {
return
}
if _, err := os.Stat("/usr/bin/xattr"); err != nil {
return
}
cmd := exec.Command("/usr/bin/xattr", "-d", "com.apple.quarantine", dest)
cmd.Stdout, cmd.Stderr = io.Discard, io.Discard
cmd.Run() // absent attribute is not an error worth reporting
}
func fileSHA256(path string) string {
f, err := os.Open(path)
if err != nil {
return ""
}
defer f.Close()
h := sha256.New()
if _, err := io.Copy(h, f); err != nil {
return ""
}
return hex.EncodeToString(h.Sum(nil))
}
+473
View File
@@ -0,0 +1,473 @@
// upd - download the matching binary from the latest Gitea/GitHub release.
//
// upd https://git.micw.org/mike/dns
// upd https://github.com/sxyazi/yazi --install ~/bin --name yazi,ya
// upd --all # everything listed in the config file
// upd --all --check # exit 10 if any update is available
//
// A port of the Perl original. The one visible difference is the transport:
// TLS comes from the Go runtime, so there is no curl, wget or IO::Socket::SSL
// in the picture and no old system library to trip over. State and cache files
// keep the Perl layout, so both versions can share ~/.local/state/upd.
package main
import (
"errors"
"flag"
"fmt"
"io"
"os"
"os/signal"
"path/filepath"
"sort"
"strings"
"syscall"
)
// version is a var, not a const, so build.sh can put the built number in via
// -ldflags "-X main.version=...". The value here only ever shows up in a bare
// `go build`; the version actually built is the one in version.txt.
var version = "2.0.0"
// Exit codes: 0 ok, 1 usage, 2 error, 10 update available (--check only).
const (
exOK = 0
exUsage = 1
exError = 2
exOutdated = 10
)
// Options that belong to a single repository. Anything else is global and
// applies to every entry of an --all run.
var specKeys = []string{
"install", "name", "tag", "asset", "pattern", "os", "arch",
"forge", "token", "version-flag", "pre",
}
type options struct {
repo string
tag string
install string
name string
asset string
pattern string
os string
arch string
token string
forge string
versionFlag string
timeout int
config string
cacert string
insecure bool
all bool
check bool
pre bool
force bool
list bool
dryRun bool
quiet bool
verbose bool
help bool
// upd on itself, see selfupdate.go
showVersion bool
checkUpdate bool
doUpdate bool
updateRefresh bool
}
var (
opt options
// Which options the user actually typed. Perl could ask "is it defined?";
// here the zero value of a flag is indistinguishable from an unset one,
// and only typed options may override a config file entry.
given = map[string]bool{}
)
func main() {
code := run()
cleanupTemps()
os.Exit(code)
}
func run() int {
fs := flag.NewFlagSet("upd", flag.ContinueOnError)
fs.SetOutput(io.Discard) // errors are reported together with the usage text
fs.StringVar(&opt.repo, "repo", "", "")
fs.StringVar(&opt.tag, "tag", "", "")
for _, n := range []string{"install", "dest", "i"} {
fs.StringVar(&opt.install, n, "", "")
}
fs.StringVar(&opt.name, "name", "", "")
fs.StringVar(&opt.asset, "asset", "", "")
fs.StringVar(&opt.pattern, "pattern", "", "")
fs.StringVar(&opt.os, "os", "", "")
fs.StringVar(&opt.arch, "arch", "", "")
fs.StringVar(&opt.token, "token", "", "")
fs.StringVar(&opt.forge, "forge", "", "")
fs.StringVar(&opt.versionFlag, "version-flag", "--version", "")
fs.IntVar(&opt.timeout, "timeout", 30, "")
fs.StringVar(&opt.config, "config", "", "")
fs.StringVar(&opt.cacert, "cacert", "", "")
fs.BoolVar(&opt.insecure, "insecure", false, "")
fs.BoolVar(&opt.insecure, "k", false, "")
fs.BoolVar(&opt.all, "all", false, "")
fs.BoolVar(&opt.check, "check", false, "")
fs.BoolVar(&opt.pre, "pre", false, "")
fs.BoolVar(&opt.force, "force", false, "")
fs.BoolVar(&opt.list, "list", false, "")
fs.BoolVar(&opt.dryRun, "dry-run", false, "")
fs.BoolVar(&opt.quiet, "quiet", false, "")
fs.BoolVar(&opt.verbose, "verbose", false, "")
fs.BoolVar(&opt.help, "help", false, "")
fs.BoolVar(&opt.help, "h", false, "")
fs.BoolVar(&opt.showVersion, "version", false, "")
fs.BoolVar(&opt.checkUpdate, "check-update", false, "")
fs.BoolVar(&opt.doUpdate, "update", false, "")
fs.BoolVar(&opt.updateRefresh, updateRefreshFlag, false, "")
if err := fs.Parse(permute(fs, os.Args[1:])); err != nil {
if errors.Is(err, flag.ErrHelp) {
usage(os.Stdout)
return exOK
}
fmt.Fprintf(os.Stderr, "Error: %v\n\n", err)
usage(os.Stderr)
return exUsage
}
if opt.help {
usage(os.Stdout)
return exOK
}
fs.Visit(func(f *flag.Flag) {
switch f.Name { // the aliases answer to the name the rest of the code knows
case "dest", "i":
given["install"] = true
default:
given[f.Name] = true
}
})
// An explicit choice about certificates switches off the automatic
// fallback: from here on it is the user's decision, not a guess.
switch {
case opt.insecure:
caCurrent = caNone
fmt.Fprintf(os.Stderr, "Warning: --insecure, the server certificate is not verified.\n")
case caCertPath() != "":
caCurrent = caFile
}
verbose("certificates: %v", caCurrent)
// Leftovers from an interrupted install must not linger next to the binary.
sig := make(chan os.Signal, 1)
signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP)
go func() {
<-sig
cleanupTemps()
os.Exit(130)
}()
// upd on itself: each of these is a whole run of its own, nothing follows.
switch {
case opt.updateRefresh: // the background look, deliberately not in the help
selfUpdate.refresh()
return exOK
case opt.showVersion:
fmt.Printf("upd %s\n", version)
return exOK
case opt.checkUpdate:
if err := selfUpdate.check(os.Stdout); err != nil {
fmt.Fprintf(os.Stderr, "%v\n", err)
return exError
}
return exOK
case opt.doUpdate:
if err := selfUpdate.install(os.Stdout); err != nil {
fmt.Fprintf(os.Stderr, "%v\n", err)
return exError
}
return exOK
}
specs, err := buildSpecs(fs.Args())
if err != nil {
if errors.Is(err, errUsage) {
fmt.Fprintf(os.Stderr, "Error: %v\n\n", err)
usage(os.Stderr)
return exUsage
}
fmt.Fprintf(os.Stderr, "%v\n", err)
return exError
}
var results []result
for _, spec := range specs {
res, err := updateOne(spec)
if err != nil {
res = result{status: "error", label: spec.label(), msg: oneLine(err.Error())}
fmt.Fprintf(os.Stderr, "%s: %v\n", res.label, err)
}
results = append(results, res)
}
if len(specs) > 1 && !opt.list {
summary(results)
}
// Costs nothing: the hint comes from the note in the cache, and the asking
// happens once a day at most, in the background.
if hint := selfUpdate.daily(); hint != "" && !opt.quiet {
fmt.Fprintln(os.Stderr, hint)
}
return worstCode(results)
}
// Go's flag package stops at the first operand, but "upd URL -i ." has to keep
// working, so operands are moved behind the flags before parsing. A flag that
// takes a value swallows the argument after it, unless it is a boolean or
// already carries "=value".
func permute(fs *flag.FlagSet, args []string) []string {
var flags, operands []string
for i := 0; i < len(args); i++ {
a := args[i]
if a == "--" {
operands = append(operands, args[i+1:]...)
break
}
if len(a) < 2 || a[0] != '-' {
operands = append(operands, a)
continue
}
name := strings.TrimLeft(a, "-")
inline := strings.Contains(name, "=")
if inline {
name, _, _ = strings.Cut(name, "=")
}
flags = append(flags, a)
if inline || isBoolFlag(fs, name) || i+1 >= len(args) {
continue
}
i++
flags = append(flags, args[i])
}
return append(flags, operands...)
}
func isBoolFlag(fs *flag.FlagSet, name string) bool {
f := fs.Lookup(name)
if f == nil {
return false // unknown: let flag.Parse produce the error
}
b, ok := f.Value.(interface{ IsBoolFlag() bool })
return ok && b.IsBoolFlag()
}
var errUsage = errors.New("missing repository URL")
func buildSpecs(args []string) ([]*Spec, error) {
if opt.all {
path := opt.config
if path == "" {
path = configPath()
}
specs, err := readConfig(path)
if err != nil {
return nil, err
}
if len(specs) == 0 {
return nil, fmt.Errorf("no entries in %s", path)
}
return specs, nil
}
url := opt.repo
if url == "" && len(args) > 0 {
url = args[0]
}
if url == "" {
url = os.Getenv("UPD_REPO")
}
if url == "" {
return nil, errUsage
}
s := &Spec{Repo: url}
s.applyGlobals()
return []*Spec{s}, nil
}
// ==============================================================================
// Reporting
// ==============================================================================
type result struct {
status string // ok, current, updated, outdated, listed, dry, error
label string
msg string
}
func summary(res []result) {
rows := res
if opt.quiet {
// --quiet is meant for cron: report only what needs attention.
rows = nil
for _, r := range res {
switch r.status {
case "error", "outdated", "updated":
rows = append(rows, r)
}
}
}
if len(rows) == 0 {
return
}
w := 0
for _, r := range rows {
if len(r.label) > w {
w = len(r.label)
}
}
fmt.Print("\nSummary:\n")
for _, r := range rows {
fmt.Printf(" %-9s %-*s %s\n", r.status, w, r.label, r.msg)
}
}
func worstCode(res []result) int {
for _, r := range res {
if r.status == "error" {
return exError
}
}
for _, r := range res {
if r.status == "outdated" {
return exOutdated
}
}
return exOK
}
func info(format string, a ...any) {
if !opt.quiet {
fmt.Printf(format+"\n", a...)
}
}
func verbose(format string, a ...any) {
if opt.verbose {
fmt.Printf(" "+format+"\n", a...)
}
}
func oneLine(s string) string {
return strings.Join(strings.Fields(strings.TrimSpace(s)), " ")
}
func usage(w io.Writer) {
fmt.Fprintf(w, `upd %s - download the matching binary from the latest Gitea/GitHub release
upd [OPTIONS] REPO-URL
upd --all [OPTIONS]
REPO-URL e.g. https://git.micw.org/mike/dns
https://github.com/sxyazi/yazi
(or use --repo / $UPD_REPO)
Options:
--install PATH, -i target directory (created if missing), e.g. ~/bin.
A path whose last segment is one of the binary names,
or an existing file, means that exact file.
Also $UPD_INSTALL. Default: the directory of a
same-named binary in $PATH, else ~/.local/bin
--name A[,B...] binary name(s) to take out of one asset, e.g.
"yazi,ya" installs both. Use "SRC:DST" to install
under a different name (default: repository name)
--all update every entry of the config file
--config PATH config file (default: %s)
--check only report whether an update is available
--tag VERSION install a specific release ("v1.2.3" or "1.2.3")
--pre consider prereleases as well
--asset NAME exact asset name instead of auto-detection
--pattern REGEX select the asset by regex
--os OS darwin|linux|windows|freebsd (default: detected)
--arch ARCH amd64|arm64|386|arm (default: detected)
--forge NAME github|gitea (default: github for github.com hosts,
gitea otherwise; needed for GitHub Enterprise)
--token TOKEN API token for private repositories. Also read from
$UPD_TOKEN, then $GITHUB_TOKEN/$GH_TOKEN (GitHub)
or $GITEA_TOKEN (Gitea)
--version-flag F flag to query the version of a binary installed
without upd (default: --version)
--cacert FILE verify against this CA bundle instead of the system
store (also $UPD_CACERT). Without it, upd uses the
system store and falls back to its own built-in
Mozilla CA list if that store does not know the issuer
--insecure, -k do not verify the server certificate at all
--force install even if it is already up to date
--list show releases and their assets
--dry-run show what would happen, write nothing
--verbose show asset scoring and cache decisions
--quiet print errors only
--timeout SEC network timeout (default: 30)
--help this help
--version print the version of upd itself
--check-update look for a newer release of upd itself
--update download and install the newest release of upd itself
Config file (one line per tool, "#" comments):
https://git.micw.org/mike/dns install=~/bin
https://github.com/sxyazi/yazi install=~/bin name=yazi,ya
Keys: %s
State is kept in %s
Exit codes: 0 ok, 1 usage, 2 error, 10 update available (--check)
upd looks for a new release of itself once a day, in the background, and says
so on stderr. UPD_NO_UPDATE_CHECK=1 turns that off.
`, version, configPath(), strings.Join(specKeys, ", "), stateDir())
}
func listReleases(rels []Release) {
for _, r := range rels {
pre := ""
if r.Prerelease {
pre = " [prerelease]"
}
published := r.PublishedAt
if len(published) > 10 {
published = published[:10]
}
fmt.Printf("%-12s %s%s\n", r.TagName, published, pre)
for _, a := range r.Assets {
fmt.Printf(" %-32s %s\n", a.Name, humanSize(a.Size))
}
}
}
// sortedKeys keeps map iteration out of the output.
func sortedKeys[V any](m map[string]V) []string {
keys := make([]string, 0, len(m))
for k := range m {
keys = append(keys, k)
}
sort.Strings(keys)
return keys
}
func homeDir() string {
if h, err := os.UserHomeDir(); err == nil && h != "" {
return h
}
return "."
}
// expandTilde handles the one form a config file realistically contains.
func expandTilde(p string) string {
if p == "~" {
return homeDir()
}
if strings.HasPrefix(p, "~/") {
return filepath.Join(homeDir(), p[2:])
}
return p
}
+328
View File
@@ -0,0 +1,328 @@
// selfupdate.go - upd updating itself.
//
// The mechanism is the one dx uses: --check-update looks, --update installs,
// and an ordinary run consults a note in the cache once a day and mentions a
// newer release on stderr - never asking in the foreground, never asking when
// nobody is watching. The work itself goes through upd's own machinery: the
// same forge layer, the same retries and CA fallback, the same checksum check
// and the same atomic install, so updating upd behaves like updating any other
// tool it installs.
//
// It assumes the layout build.sh produces: one release per version, whose tag
// is the bare number (2.0.6, a leading "v" is allowed), holding one asset
// "upd-<goos>-<goarch>" each - that is, exactly the files from ./bin.
package main
import (
"fmt"
"io"
"os"
"os/exec"
"path/filepath"
"runtime"
"strconv"
"strings"
"time"
)
// ------------------------------------------------------------ Configuration
var selfUpdate = selfUpdater{
repo: "https://git.micw.org/mike/upd",
name: "upd",
current: version, // from main.go, set by -ldflags
every: 24 * time.Hour,
quietEnv: "UPD_NO_UPDATE_CHECK",
}
type selfUpdater struct {
repo string // repo URL as in the browser: https://host/owner/repo
name string // base name of the assets, "-<goos>-<goarch>" is added
current string // the running version
every time.Duration // how often to look on its own; 0 turns that off
quietEnv string // this environment variable set: keep quiet as well
}
// updateRefreshFlag is the option upd calls itself with, in the background. It
// is deliberately absent from the help.
const updateRefreshFlag = "update-refresh"
// ------------------------------------------------------------ Looking by itself
// daily is the hook for the ordinary run. It costs nothing: in the foreground
// the network is never touched. What comes back is the line pointing at a new
// version - or "", when there is nothing to say. Should the note be older than
// `every`, daily starts a background run on the side, whose answer the next
// call will find waiting.
func (u selfUpdater) daily() string {
if u.every <= 0 || os.Getenv(u.quietEnv) != "" || !isTerminal(os.Stderr) {
return ""
}
st := u.loadState() // no file: the zero value, hence due at once
if time.Since(st.Checked) >= u.every {
// The timestamp moves on before the asking, not after: otherwise two
// simultaneous runs start two queries, and a server that is not in the
// mood would get a new one on every call. If the note does not stay
// put, nothing is asked either - else an unwritable cache directory
// would mean one process per call.
st.Checked = time.Now()
if u.saveState(st) == nil {
u.spawnRefresh()
}
}
if st.Latest == "" || compareVer(st.Latest, u.current) <= 0 {
return ""
}
return fmt.Sprintf("Note: %s %s is available, run '%s --update'.", u.name, st.Latest, u.name)
}
// refresh is the background run: ask, write it down, stay quiet. The writing
// down is done by latest; if the query fails, the old state remains.
func (u selfUpdater) refresh() {
_, _, _ = u.latest()
}
// spawnRefresh calls upd once more, only to ask, and does not wait. Without a
// Wait the child is adopted by init when this process ends - it thus outlives
// the call, and the call's output stays untouched by it.
func (u selfUpdater) spawnRefresh() {
exe, err := os.Executable()
if err != nil {
return
}
cmd := exec.Command(exe, "--"+updateRefreshFlag)
cmd.Stdin, cmd.Stdout, cmd.Stderr = nil, nil, nil // everything to /dev/null
if cmd.Start() == nil {
cmd.Process.Release()
}
}
// --------------------------------------------------------------------- Note
// updateState is what is left between two calls: when the last question was
// asked and what came of it.
type updateState struct {
Checked time.Time `json:"checked"`
Latest string `json:"latest"`
}
// The note lives in the cache directory next to the ETag cache, not in the
// state directory: if it gets lost, the only cost is asking once too early.
// The name cannot collide with a cache file, those are hex digests.
func (u selfUpdater) statePath() string {
return filepath.Join(cacheDir(), "selfupdate.json")
}
func (u selfUpdater) loadState() updateState {
var st updateState
readJSON(u.statePath(), &st) // a missing or broken file counts as none
return st
}
func (u selfUpdater) saveState(st updateState) error {
if err := os.MkdirAll(cacheDir(), 0o755); err != nil {
return err
}
return writeJSON(u.statePath(), st)
}
// ------------------------------------------------------------------ The work
// check only looks and touches nothing.
func (u selfUpdater) check(w io.Writer) error {
_, rel, err := u.latest()
if err != nil {
return err
}
if compareVer(rel.TagName, u.current) <= 0 {
fmt.Fprintf(w, "%s %s is up to date\n", u.name, u.current)
return nil
}
fmt.Fprintf(w, "%s %s is available, running %s\n", u.name, rel.TagName, u.current)
if rel.HTMLURL != "" {
fmt.Fprintf(w, " %s\n", rel.HTMLURL)
}
fmt.Fprintf(w, " run '%s --update' to install it\n", u.name)
return nil
}
// install fetches the newest release and replaces the running file with it.
func (u selfUpdater) install(w io.Writer) error {
c, rel, err := u.latest()
if err != nil {
return err
}
if compareVer(rel.TagName, u.current) <= 0 && !opt.force {
fmt.Fprintf(w, "%s %s is up to date\n", u.name, u.current)
return nil
}
want := fmt.Sprintf("%s-%s-%s", u.name, runtime.GOOS, runtime.GOARCH)
var src *Asset
for i := range rel.Assets {
if rel.Assets[i].Name == want {
src = &rel.Assets[i]
break
}
}
if src == nil {
var have []string
for _, a := range rel.Assets {
have = append(have, a.Name)
}
return fmt.Errorf("release %s has no %q (only %s)", rel.TagName, want, strings.Join(have, ", "))
}
exe, err := os.Executable()
if err != nil {
return fmt.Errorf("cannot locate the running binary: %w", err)
}
// An installed upd may be a symlink into ./bin. What should be replaced is
// the file behind it, not the link.
if real, err := filepath.EvalSymlinks(exe); err == nil {
exe = real
}
// Ask before the download, not after it: a missing write permission ought
// to show up before a few megabytes have gone down the wire.
if dir := filepath.Dir(exe); !writable(dir) {
return fmt.Errorf("no write permission in %s (use sudo, or install upd elsewhere)", dir)
}
if opt.dryRun {
fmt.Fprintf(w, "[dry-run] would install %s %s to %s\n", want, rel.TagName, exe)
return nil
}
tmpdir, err := os.MkdirTemp("", "upd-self-")
if err != nil {
return err
}
defer os.RemoveAll(tmpdir)
dl := filepath.Join(tmpdir, want)
dlURL, dlHdr := c.assetURL(src)
hdr := c.authHeaders()
for k, v := range dlHdr {
hdr[k] = v
}
fmt.Fprintf(w, "Downloading %s %s (%s) ...\n", want, rel.TagName, humanSize(src.Size))
if err := downloadTo(dl, dlURL, hdr, src.Size); err != nil {
return err
}
if st, err := os.Stat(dl); err == nil && src.Size > 0 && st.Size() < src.Size {
return fmt.Errorf("incomplete download: %d of %d bytes", st.Size(), src.Size)
}
if err := verifyDownload(c, dl, src, rel.Assets, tmpdir); err != nil {
return err
}
if err := os.Chmod(dl, 0o755); err != nil {
return err
}
if err := u.probe(dl, rel.TagName); err != nil {
return err
}
if err := installAtomic(dl, exe); err != nil {
return err
}
stripQuarantine(exe)
u.saveState(updateState{Checked: time.Now(), Latest: rel.TagName})
fmt.Fprintf(w, "%s %s -> %s, at %s\n", u.name, u.current, rel.TagName, exe)
return nil
}
// probe runs the freshly fetched binary once. That catches a file that is
// truncated, built for the wrong platform, or not executable in the first
// place, before it replaces the running one. The version flag is the built-in
// one, not spec.versionFlag(): what upd answers to is not the user's to
// configure here.
func (u selfUpdater) probe(path, tag string) error {
got := installedVersion(path, &Spec{VersionFlag: "--version"})
if got == "" {
return fmt.Errorf("the downloaded binary does not run, %s left alone", u.name)
}
if got != normVer(tag) {
return fmt.Errorf("the downloaded binary reports %s, expected %s", got, normVer(tag))
}
return nil
}
// latest asks the forge for the newest release, through the same code path an
// ordinary run uses, and notes down what came back.
func (u selfUpdater) latest() (*forgeCtx, *Release, error) {
spec := &Spec{Repo: u.repo}
c, err := newForgeCtx(spec)
if err != nil {
return nil, nil, err
}
rel, _, err := c.fetchRelease(spec)
if err != nil {
return nil, nil, err
}
if rel == nil { // --list on the command line: fetchRelease returns the list
return nil, nil, fmt.Errorf("--list cannot be combined with --update")
}
// Every question that succeeds fills the note - no matter whether it came
// from --update, from --check-update or from the background run.
u.saveState(updateState{Checked: time.Now(), Latest: rel.TagName})
return c, rel, nil
}
// ------------------------------------------------------------------ Numbers
// compareVer compares two versions component by component, numerically, so
// that 2.0.10 lands behind 2.0.9 and not in front of it. A leading "v" does not
// count, missing places count as 0 (2.1 == 2.1.0), and a suffix on the number
// makes the version older, not newer (2.0.6-rc1 < 2.0.6). The result is the one
// of strings.Compare: -1, 0, 1.
func compareVer(a, b string) int {
as := strings.Split(normVer(a), ".")
bs := strings.Split(normVer(b), ".")
for i := 0; i < len(as) || i < len(bs); i++ {
x, y := "0", "0"
if i < len(as) {
x = as[i]
}
if i < len(bs) {
y = bs[i]
}
if c := compareVerPart(x, y); c != 0 {
return c
}
}
return 0
}
func compareVerPart(a, b string) int {
na, ra := splitNum(a)
nb, rb := splitNum(b)
switch {
case na != nb:
if na < nb {
return -1
}
return 1
case ra == rb:
return 0
case ra == "": // 2.0.6 is finished, 2.0.6-rc1 is not yet
return 1
case rb == "":
return -1
}
return strings.Compare(ra, rb)
}
// splitNum separates "10-rc1" into 10 and "-rc1".
func splitNum(s string) (int, string) {
i := 0
for i < len(s) && s[i] >= '0' && s[i] <= '9' {
i++
}
n, _ := strconv.Atoi(s[:i])
return n, s[i:]
}
+223
View File
@@ -0,0 +1,223 @@
package main
import (
"bufio"
"fmt"
"os"
"path/filepath"
"strings"
)
// Spec is one repository plus the options that belong to it - either a command
// line invocation or one line of the config file.
type Spec struct {
Repo string
Install string
Name string
Tag string
Asset string
Pattern string
OS string
Arch string
Forge string
Token string
VersionFlag string
Pre bool
}
func (s *Spec) set(key, val string) error {
switch key {
case "install":
s.Install = val
case "name":
s.Name = val
case "tag":
s.Tag = val
case "asset":
s.Asset = val
case "pattern":
s.Pattern = val
case "os":
s.OS = val
case "arch":
s.Arch = val
case "forge":
s.Forge = val
case "token":
s.Token = val
case "version-flag":
s.VersionFlag = val
case "pre":
s.Pre = val != "" && val != "0"
default:
return fmt.Errorf("unknown key %q (allowed: %s)", key, strings.Join(specKeys, ", "))
}
return nil
}
// Options typed on the command line win over the config file. Unlike the Perl
// version this looks at what was actually given, so a "version-flag=" in the
// config file is no longer overwritten by its own default.
func (s *Spec) applyGlobals() {
for _, k := range specKeys {
if !given[k] {
continue
}
var v string
switch k {
case "install":
v = opt.install
case "name":
v = opt.name
case "tag":
v = opt.tag
case "asset":
v = opt.asset
case "pattern":
v = opt.pattern
case "os":
v = opt.os
case "arch":
v = opt.arch
case "forge":
v = opt.forge
case "token":
v = opt.token
case "version-flag":
v = opt.versionFlag
case "pre":
if !opt.pre {
continue // --pre only ever switches on
}
v = "1"
}
s.set(k, v)
}
}
func (s *Spec) versionFlag() string {
if s.VersionFlag != "" {
return s.VersionFlag
}
return opt.versionFlag
}
func (s *Spec) label() string {
if s.Name != "" {
return parseNames(s.Name, "?")[0].out
}
r := strings.TrimRight(s.Repo, "/")
if r == "" {
return "?"
}
return filepath.Base(r)
}
// ==============================================================================
// Config file (--all)
// ==============================================================================
func configPath() string {
base := os.Getenv("XDG_CONFIG_HOME")
if base == "" {
base = filepath.Join(homeDir(), ".config")
}
return filepath.Join(base, "upd", "tools")
}
// One entry per line: <repo-url> [key=value ...] [pre]
func readConfig(path string) ([]*Spec, error) {
fh, err := os.Open(path)
if err != nil {
return nil, fmt.Errorf("cannot read config %s: %w\n"+
"Format: one line per tool, e.g.\n"+
" https://github.com/sxyazi/yazi install=~/bin name=yazi,ya", path, err)
}
defer fh.Close()
var specs []*Spec
sc := bufio.NewScanner(fh)
for ln := 1; sc.Scan(); ln++ {
line := strings.TrimSpace(sc.Text())
if line == "" || strings.HasPrefix(line, "#") {
continue
}
tokens := splitTokens(line)
s := &Spec{Repo: tokens[0]}
for _, t := range tokens[1:] {
key, val, ok := strings.Cut(t, "=")
if !ok {
val = "1" // a bare "pre"
}
if err := s.set(strings.ToLower(key), val); err != nil {
return nil, fmt.Errorf("%s:%d: %w", path, ln, err)
}
}
s.applyGlobals()
specs = append(specs, s)
}
if err := sc.Err(); err != nil {
return nil, fmt.Errorf("cannot read config %s: %w", path, err)
}
return specs, nil
}
// Whitespace separated, with double quotes around values that contain spaces.
func splitTokens(line string) []string {
var out []string
var cur strings.Builder
quoted, escaped, started := false, false, false
for _, r := range line {
switch {
case escaped:
cur.WriteRune(r)
escaped = false
case r == '\\' && quoted:
escaped = true
case r == '"':
quoted = !quoted
started = true
case (r == ' ' || r == '\t') && !quoted:
if started {
out = append(out, cur.String())
cur.Reset()
started = false
}
default:
cur.WriteRune(r)
started = true
}
}
if started {
out = append(out, cur.String())
}
return out
}
// ==============================================================================
// Binary names
// ==============================================================================
// name holds what to look for inside the asset (src) and what to write to disk
// (out): "yazi,ya" are two binaries, "yazi:yazi-nightly" renames one.
type name struct{ src, out string }
func parseNames(list, fallback string) []name {
var names []name
for _, part := range strings.Split(list, ",") {
part = strings.TrimSpace(part)
if part == "" {
continue
}
src, out, ok := strings.Cut(part, ":")
if !ok || out == "" {
out = src
}
names = append(names, name{src: src, out: out})
}
if len(names) == 0 {
names = append(names, name{src: fallback, out: fallback})
}
return names
}
+138
View File
@@ -0,0 +1,138 @@
package main
import (
"crypto/tls"
"crypto/x509"
_ "embed"
"errors"
"fmt"
"net"
"os"
"strings"
)
// The Mozilla CA list as extracted by the curl project, refreshed with
//
// curl -o ca-bundle.pem https://curl.se/ca/cacert.pem
//
// It is not used unless the system store fails: on a machine old enough that
// its ca-certificates package predates Let's Encrypt's ISRG roots, every
// https:// forge is unreachable otherwise, and updating that package is often
// no longer possible there.
//
//go:embed ca-bundle.pem
var caBundle []byte
// How certificates are verified. Starts at the system store and falls back one
// step when that store turns out not to know the issuer.
type caMode int
const (
caSystem caMode = iota
caBundled
caFile
caNone
)
func (m caMode) String() string {
switch m {
case caBundled:
return "bundled CA list"
case caFile:
return "--cacert " + caCertPath()
case caNone:
return "no verification"
}
return "system CA store"
}
// caCurrent is the mode every request uses; the fallback in httpGet moves it
// forward at most once per run.
var caCurrent = caSystem
func bundledPool() *x509.CertPool {
pool := x509.NewCertPool()
if !pool.AppendCertsFromPEM(caBundle) {
return nil
}
return pool
}
func filePool(path string) (*x509.CertPool, error) {
pem, err := os.ReadFile(path)
if err != nil {
return nil, fmt.Errorf("cannot read --cacert %s: %w", path, err)
}
pool := x509.NewCertPool()
if !pool.AppendCertsFromPEM(pem) {
return nil, fmt.Errorf("no certificates found in %s", path)
}
return pool, nil
}
// tlsConfig builds the config for the mode currently in force. The system
// store stays the default: it is the one the administrator controls.
func tlsConfig(mode caMode) (*tls.Config, error) {
switch mode {
case caBundled:
pool := bundledPool()
if pool == nil {
return nil, errors.New("the bundled CA list could not be parsed")
}
return &tls.Config{RootCAs: pool}, nil
case caFile:
pool, err := filePool(caCertPath())
if err != nil {
return nil, err
}
return &tls.Config{RootCAs: pool}, nil
case caNone:
return &tls.Config{InsecureSkipVerify: true}, nil
}
return &tls.Config{}, nil
}
func caCertPath() string {
if opt.cacert != "" {
return expandTilde(opt.cacert)
}
return expandTilde(os.Getenv("UPD_CACERT"))
}
// A certificate the local store cannot chain up to a root it knows. This is
// the failure the bundled list exists for; no other x509 problem (expired,
// wrong host name) would be fixed by more roots.
func isUnknownAuthority(err error) bool {
var unknown x509.UnknownAuthorityError
if errors.As(err, &unknown) {
return true
}
// Some paths only carry the verifier's verdict as text.
return strings.Contains(err.Error(), "certificate signed by unknown authority") ||
strings.Contains(err.Error(), "x509: failed to load system roots")
}
// Errors that will still be errors on the next attempt: retrying a rejected
// certificate, an unresolvable host or a handshake the two sides cannot agree
// on only makes the output longer.
func isPermanent(err error) bool {
var (
hostname x509.HostnameError
invalid x509.CertificateInvalidError
verify *tls.CertificateVerificationError
record tls.RecordHeaderError
dns *net.DNSError
)
switch {
case isUnknownAuthority(err),
errors.As(err, &hostname),
errors.As(err, &invalid),
errors.As(err, &verify),
errors.As(err, &record):
return true
case errors.As(err, &dns):
return !dns.IsTemporary
}
msg := err.Error()
return strings.Contains(msg, "tls: ") || strings.Contains(msg, "x509: ")
}
+160
View File
@@ -0,0 +1,160 @@
package main
import (
"crypto/x509"
"encoding/pem"
"io"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
)
// resetTransport clears the memoised client so a test can change the
// verification mode.
func resetTransport(t *testing.T, mode caMode) {
t.Helper()
client, clientMode, caCurrent = nil, -1, mode
opt.timeout = 10
t.Cleanup(func() {
client, clientMode, caCurrent = nil, -1, caSystem
opt.cacert, opt.insecure = "", false
})
}
// tlsServer returns a server with a certificate no public CA has signed, plus
// that certificate as PEM - the stand-in for a root the system does not know.
func tlsServer(t *testing.T) (*httptest.Server, []byte) {
t.Helper()
srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
io.WriteString(w, `{"tag_name":"v1.2.3"}`)
}))
t.Cleanup(srv.Close)
certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: srv.Certificate().Raw})
return srv, certPEM
}
func TestSystemStoreRejectsUnknownIssuer(t *testing.T) {
srv, _ := tlsServer(t)
resetTransport(t, caSystem)
// Nothing to fall back to: the bundled list does not know it either.
old := caBundle
caBundle = nil
t.Cleanup(func() { caBundle = old })
_, err := httpGet(srv.URL, nil, nil)
if err == nil {
t.Fatal("expected the system store to reject this certificate")
}
if !isUnknownAuthority(err) {
t.Errorf("not recognised as an unknown authority: %v", err)
}
if !isPermanent(err) {
t.Error("a rejected certificate must not be retried")
}
if !strings.Contains(err.Error(), "--cacert") {
t.Errorf("error should point at the way out, got: %v", err)
}
}
// The failure from the old box: the system store predates the issuer's root.
// upd must notice, switch to its own list and carry on.
func TestFallbackToBundledCAs(t *testing.T) {
srv, certPEM := tlsServer(t)
resetTransport(t, caSystem)
old := caBundle
caBundle = certPEM
t.Cleanup(func() { caBundle = old })
resp, err := httpGet(srv.URL, nil, nil)
if err != nil {
t.Fatalf("fallback did not happen: %v", err)
}
if resp.status != http.StatusOK {
t.Errorf("status %d", resp.status)
}
if caCurrent != caBundled {
t.Errorf("mode is %v, want the bundled list", caCurrent)
}
}
func TestCacertFile(t *testing.T) {
srv, certPEM := tlsServer(t)
path := filepath.Join(t.TempDir(), "ca.pem")
if err := os.WriteFile(path, certPEM, 0o644); err != nil {
t.Fatal(err)
}
resetTransport(t, caFile)
opt.cacert = path
if _, err := httpGet(srv.URL, nil, nil); err != nil {
t.Fatalf("--cacert did not verify: %v", err)
}
// A bundle without the right root must still fail, and say which file.
resetTransport(t, caFile)
opt.cacert = filepath.Join(t.TempDir(), "empty.pem")
os.WriteFile(opt.cacert, []byte("not a certificate\n"), 0o644)
if _, err := httpGet(srv.URL, nil, nil); err == nil {
t.Error("expected an error for a bundle without certificates")
} else if !strings.Contains(err.Error(), "no certificates found") {
t.Errorf("unexpected error: %v", err)
}
}
func TestInsecureSkipsVerification(t *testing.T) {
srv, _ := tlsServer(t)
resetTransport(t, caNone)
if _, err := httpGet(srv.URL, nil, nil); err != nil {
t.Fatalf("--insecure should have connected anyway: %v", err)
}
}
// The bundled list has to be a usable pool and carry the roots that the old
// machines are missing - that is the whole point of embedding it.
func TestBundledCAList(t *testing.T) {
pool := bundledPool()
if pool == nil {
t.Fatal("the embedded CA bundle does not parse")
}
var subjects []string
for block, rest := pem.Decode(caBundle); block != nil; block, rest = pem.Decode(rest) {
cert, err := x509.ParseCertificate(block.Bytes)
if err != nil {
t.Fatalf("unparsable certificate in the bundle: %v", err)
}
subjects = append(subjects, cert.Subject.CommonName)
}
if len(subjects) < 100 {
t.Errorf("only %d roots in the bundle, expected the full Mozilla list", len(subjects))
}
for _, want := range []string{"ISRG Root X1", "ISRG Root X2"} {
found := false
for _, s := range subjects {
if s == want {
found = true
}
}
if !found {
t.Errorf("%s missing - Let's Encrypt hosts would still fail", want)
}
}
}
func TestPermanentClassification(t *testing.T) {
// A DNS failure for a name that cannot exist is permanent, a plain
// connection refused is not: the second one is worth another attempt.
resetTransport(t, caSystem)
if _, err := httpGet("https://no-such-host.invalid/x", nil, nil); err == nil {
t.Fatal("expected a DNS failure")
} else if !isPermanent(err) {
t.Errorf("an unresolvable host should not be retried: %v", err)
}
}
+267
View File
@@ -0,0 +1,267 @@
package main
import (
"bufio"
"fmt"
"os"
"path/filepath"
"regexp"
"runtime"
"strings"
)
// updateOne runs one repository end to end.
func updateOne(spec *Spec) (result, error) {
c, err := newForgeCtx(spec)
if err != nil {
return result{}, err
}
// Without --name the repository name is both what to look for and what to
// write; parseNames falls back to it on its own.
names := parseNames(spec.Name, c.repo)
goos, goarch := detectPlatform(spec)
info("Repo: %s/%s/%s [%s]", c.base, c.owner, c.repo, c.forge)
info("Platform: %s/%s", goos, goarch)
rel, list, err := c.fetchRelease(spec)
if err != nil {
return result{}, err
}
if opt.list {
if rel != nil {
list = []Release{*rel}
}
listReleases(list)
return result{status: "listed", label: names[0].out}, nil
}
pre := ""
if rel.Prerelease {
pre = " [prerelease]"
}
info("Release: %s (%s)%s", rel.TagName, rel.PublishedAt, pre)
if len(rel.Assets) == 0 {
return result{}, fmt.Errorf("release %s has no assets", rel.TagName)
}
asset, err := pickAsset(rel.Assets, names[0].src, goos, goarch, spec)
if err != nil {
return result{}, err
}
if asset == nil {
var have []string
for _, a := range rel.Assets {
have = append(have, a.Name)
}
return result{}, fmt.Errorf("no asset for %s/%s in release %s.\nAvailable:\n %s\n"+
"Select one explicitly with --asset <name> or --pattern <regex>",
goos, goarch, rel.TagName, strings.Join(have, "\n "))
}
info("Asset: %s (%s)", asset.Name, humanSize(asset.Size))
targets, err := resolveTargets(spec, names, goos)
if err != nil {
return result{}, err
}
var dests []string
for _, t := range targets {
dests = append(dests, t.dest)
}
info("Target: %s", strings.Join(dests, ", "))
stale := targets[:0:0]
for _, t := range targets {
if !targetCurrent(t, rel.TagName, asset, spec) {
stale = append(stale, t)
}
}
if len(stale) == 0 && !opt.force {
info("Already up to date (%s) - nothing to do.", rel.TagName)
return result{status: "current", label: names[0].out, msg: rel.TagName}, nil
}
if opt.force {
stale = targets
}
if opt.check {
var have []string
for _, t := range stale {
have = append(have, installedTag(t))
}
info("Update available: %s (installed: %s)", rel.TagName, strings.Join(have, ", "))
return result{status: "outdated", label: names[0].out,
msg: fmt.Sprintf("%s (have: %s)", rel.TagName, strings.Join(have, ", "))}, nil
}
dlURL, dlHdr := c.assetURL(asset)
if opt.dryRun {
fmt.Printf("[dry-run] would download: %s\n", dlURL)
for _, t := range stale {
fmt.Printf("[dry-run] would install to: %s\n", t.dest)
}
return result{status: "dry", label: names[0].out, msg: rel.TagName}, nil
}
// --- download -------------------------------------------------------------
tmpdir, err := os.MkdirTemp("", "upd-")
if err != nil {
return result{}, err
}
defer os.RemoveAll(tmpdir)
dl := filepath.Join(tmpdir, asset.Name)
info("Downloading %s ...", dlURL)
hdr := c.authHeaders()
for k, v := range dlHdr {
hdr[k] = v
}
if err := downloadTo(dl, dlURL, hdr, asset.Size); err != nil {
return result{}, err
}
if st, err := os.Stat(dl); err == nil && asset.Size > 0 && st.Size() < asset.Size {
return result{}, fmt.Errorf("incomplete download: %d of %d bytes", st.Size(), asset.Size)
}
if err := verifyDownload(c, dl, asset, rel.Assets, tmpdir); err != nil {
return result{}, err
}
// --- extract and install --------------------------------------------------
root, err := extractIfArchive(dl, tmpdir)
if err != nil {
return result{}, err
}
for _, t := range stale {
src := dl
if root != "" {
src = findInTree(root, t.name)
if src == "" {
return result{}, fmt.Errorf("binary %q not found inside %s", t.name, asset.Name)
}
}
if !looksExecutable(src) {
fmt.Fprintf(os.Stderr, "Warning: %s does not look like an executable.\n", t.name)
}
if err := installAtomic(src, t.dest); err != nil {
return result{}, err
}
stripQuarantine(t.dest)
if err := writeState(t, c, rel.TagName, asset); err != nil {
return result{}, err
}
info("Installed: %s (%s)", t.dest, rel.TagName)
if v := installedVersion(t.dest, spec); v != "" {
info("Version: %s", v)
}
}
seen := map[string]bool{}
for _, t := range stale {
dir := filepath.Dir(t.dest)
if !seen[dir] && !inPath(dir) {
fmt.Fprintf(os.Stderr, "Note: %s is not in $PATH.\n", dir)
}
seen[dir] = true
}
return result{status: "updated", label: names[0].out, msg: rel.TagName}, nil
}
func detectPlatform(spec *Spec) (string, string) {
goos, goarch := runtime.GOOS, runtime.GOARCH
if spec.OS != "" {
goos = spec.OS
}
if spec.Arch != "" {
goarch = spec.Arch
}
return goos, goarch
}
// ==============================================================================
// Checksums
// ==============================================================================
var (
sha256Re = regexp.MustCompile(`(?i)^sha256:([0-9a-f]{64})$`)
sumLineRe = regexp.MustCompile(`^([0-9a-fA-F]{64})(\s+\*?(\S+))?`)
sumsFileRe = regexp.MustCompile(`(?i)^(sha256sums?(\.txt)?|checksums?\.txt)$`)
)
func verifyDownload(c *forgeCtx, file string, asset *Asset, assets []Asset, tmpdir string) error {
var want, src string
if m := sha256Re.FindStringSubmatch(asset.Digest); m != nil {
want, src = strings.ToLower(m[1]), "asset digest"
} else {
var sum *Asset
for i := range assets {
if assets[i].Name == asset.Name+".sha256" {
sum = &assets[i]
}
}
if sum == nil {
for i := range assets {
if sumsFileRe.MatchString(assets[i].Name) {
sum = &assets[i]
break
}
}
}
if sum == nil {
return nil // nothing published to check against
}
sf := filepath.Join(tmpdir, "sums")
u, h := c.assetURL(sum)
hdr := c.authHeaders()
for k, v := range h {
hdr[k] = v
}
if err := downloadTo(sf, u, hdr, 0); err != nil {
fmt.Fprintf(os.Stderr, "Checksum file could not be downloaded, skipping verification.\n")
return nil
}
fh, err := os.Open(sf)
if err != nil {
return nil
}
defer fh.Close()
sc := bufio.NewScanner(fh)
for sc.Scan() {
// "<hash> <name>", or just "<hash>" in a single-file .sha256
m := sumLineRe.FindStringSubmatch(sc.Text())
if m == nil {
continue
}
if m[3] != "" && filepath.Base(m[3]) != asset.Name {
continue
}
want, src = strings.ToLower(m[1]), sum.Name
break
}
if want == "" {
// An unreadable or unrelated checksum file is not a failed
// verification, only a missing one.
if err := sc.Err(); err != nil {
verbose("could not read %s: %v", sum.Name, err)
}
return nil
}
}
got := fileSHA256(file)
if got != want {
return fmt.Errorf("SHA256 mismatch (%s)!\n expected: %s\n got: %s", src, want, got)
}
info("SHA256: ok (%s)", src)
return nil
}
+46
View File
@@ -0,0 +1,46 @@
package main
import (
"encoding/json"
"fmt"
"os"
)
func readJSON(path string, v any) error {
body, err := os.ReadFile(path)
if err != nil {
return err
}
return json.Unmarshal(body, v)
}
// writeJSON replaces the file atomically: a half-written state file would look
// like a corrupt install on the next run.
func writeJSON(path string, v any) error {
body, err := json.MarshalIndent(v, "", " ")
if err != nil {
return err
}
tmp := fmt.Sprintf("%s.%d", path, os.Getpid())
addTemp(tmp)
defer dropTemp(tmp)
if err := os.WriteFile(tmp, append(body, '\n'), 0o644); err != nil {
return fmt.Errorf("cannot write %s: %w", tmp, err)
}
if err := os.Rename(tmp, path); err != nil {
os.Remove(tmp)
return fmt.Errorf("cannot update %s: %w", path, err)
}
return nil
}
func humanSize(n int64) string {
switch {
case n >= 1024*1024:
return fmt.Sprintf("%.1f MB", float64(n)/1024/1024)
case n >= 1024:
return fmt.Sprintf("%.1f kB", float64(n)/1024)
}
return fmt.Sprintf("%d B", n)
}
+1
View File
@@ -0,0 +1 @@
2.0.3