mirror/yay
0
0
Fork 0
mirror of https://github.com/Jguer/yay.git synced 2024-11-06 09:07:21 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity

feat(download): download PKGBUILD repos interface

Browse Source
This commit is contained in:
jguer 2021-08-02 18:00:50 +02:00 committed by J Guerreiro
parent 50c0ece9a7
commit 79b44fd544
11 changed files with 258 additions and 220 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.golangci.yml
View File

@ -54,7 +54,6 @@ linters:
- gocritic
- gofmt
- goimports
- golint
- goprintffuncname
- gosec
- gosimple
@ -63,6 +62,8 @@ linters:
- lll
- misspell
- nakedret
- prealloc
- revive
- rowserrcheck
- staticcheck
- structcheck
@ -73,7 +74,6 @@ linters:
- unused
- varcheck
- whitespace
- prealloc
# disabled want to fix
#- scopelint

4
cmd.go
View File

@ -277,7 +277,9 @@ func handleGetpkgbuild(cmdArgs *settings.Arguments, dbExecutor db.Executor) erro
if cmdArgs.ExistsArg("p", "print") {
return printPkgbuilds(dbExecutor, config.Runtime.HTTPClient, cmdArgs.Targets)
}
return getPkgbuilds(cmdArgs.Targets, dbExecutor, cmdArgs.ExistsArg("f", "force"))
return getPkgbuilds(dbExecutor,
config.Runtime.CmdRunner, config.Runtime.CmdBuilder,
cmdArgs.Targets, config.Runtime.Mode, config.AURURL, cmdArgs.ExistsArg("f", "force"))
}
func handleYogurt(cmdArgs *settings.Arguments, dbExecutor db.Executor) error {

209
download.go
View File

@ -3,20 +3,15 @@ package main
import (
"fmt"
"os"
"os/exec"
"path/filepath"
"strings"
"sync"
"github.com/leonelquinteros/gotext"
"github.com/pkg/errors"
alpm "github.com/Jguer/go-alpm/v2"
"github.com/Jguer/yay/v10/pkg/db"
"github.com/Jguer/yay/v10/pkg/dep"
"github.com/Jguer/yay/v10/pkg/multierror"
"github.com/Jguer/yay/v10/pkg/query"
"github.com/Jguer/yay/v10/pkg/download"
"github.com/Jguer/yay/v10/pkg/settings"
"github.com/Jguer/yay/v10/pkg/settings/exe"
"github.com/Jguer/yay/v10/pkg/text"
)
@ -80,34 +75,6 @@ func gitHasDiff(path, name string) (bool, error) {
return true, nil
}
// TODO: yay-next passes args through the header, use that to unify ABS and AUR
func gitDownloadABS(url, path, name string) (bool, error) {
if err := os.MkdirAll(path, 0o755); err != nil {
return false, err
}
if _, errExist := os.Stat(filepath.Join(path, name)); os.IsNotExist(errExist) {
cmd := config.Runtime.CmdBuilder.BuildGitCmd(path, "clone", "--no-progress", "--single-branch",
"-b", "packages/"+name, url, name)
_, stderr, err := config.Runtime.CmdRunner.Capture(cmd, 0)
if err != nil {
return false, fmt.Errorf(gotext.Get("error cloning %s: %s", name, stderr))
}
return true, nil
} else if errExist != nil {
return false, fmt.Errorf(gotext.Get("error reading %s", filepath.Join(path, name, ".git")))
}
cmd := config.Runtime.CmdBuilder.BuildGitCmd(filepath.Join(path, name), "pull", "--ff-only")
_, stderr, err := config.Runtime.CmdRunner.Capture(cmd, 0)
if err != nil {
return false, fmt.Errorf(gotext.Get("error fetching %s: %s", name, stderr))
}
return true, nil
}
func gitDownload(url, path, name string) (bool, error) {
_, err := os.Stat(filepath.Join(path, name, ".git"))
if os.IsNotExist(err) {
@ -149,173 +116,33 @@ func gitMerge(path, name string) error {
return nil
}
func getPkgbuilds(pkgs []string, dbExecutor db.Executor, force bool) error {
missing := false
func getPkgbuilds(dbExecutor db.Executor,
cmdRunner exe.Runner,
cmdBuilder exe.GitCmdBuilder, targets []string,
mode settings.TargetMode,
aurURL string,
force bool) error {
wd, err := os.Getwd()
if err != nil {
return err
}
pkgs = query.RemoveInvalidTargets(pkgs, config.Runtime.Mode)
aur, repo := packageSlices(pkgs, dbExecutor)
for n := range aur {
_, pkg := text.SplitDBFromName(aur[n])
aur[n] = pkg
cloned, errD := download.PKGBUILDRepos(dbExecutor, cmdRunner, cmdBuilder, targets, mode, aurURL, wd, force)
if errD != nil {
text.Errorln(errD)
}
info, err := query.AURInfoPrint(config.Runtime.AURClient, aur, config.RequestSplitN)
if err != nil {
return err
}
if len(repo) > 0 {
missing, err = getPkgbuildsfromABS(repo, wd, dbExecutor, force)
if err != nil {
return err
}
}
if len(aur) > 0 {
allBases := dep.GetBases(info)
bases := make([]dep.Base, 0)
for _, base := range allBases {
name := base.Pkgbase()
pkgDest := filepath.Join(wd, name)
_, err = os.Stat(pkgDest)
if os.IsNotExist(err) {
bases = append(bases, base)
} else if err != nil {
text.Errorln(err)
continue
} else {
if force {
if err = os.RemoveAll(pkgDest); err != nil {
text.Errorln(err)
continue
}
bases = append(bases, base)
} else {
text.Warnln(gotext.Get("%s already exists. Use -f/--force to overwrite", pkgDest))
continue
}
if len(targets) != len(cloned) {
missing := []string{}
for _, target := range targets {
if _, ok := cloned[target]; !ok {
missing = append(missing, target)
}
}
text.Warnln(gotext.Get("Unable to find the following packages:"), strings.Join(missing, ", "))
if _, err = downloadPkgbuilds(bases, nil, wd); err != nil {
return err
}
missing = missing || len(aur) != len(info)
}
if missing {
err = fmt.Errorf("")
}
return err
}
// GetPkgbuild downloads pkgbuild from the ABS.
func getPkgbuildsfromABS(pkgs []string, path string, dbExecutor db.Executor, force bool) (bool, error) {
var wg sync.WaitGroup
var mux sync.Mutex
var errs multierror.MultiError
names := make(map[string]string)
missing := make([]string, 0)
downloaded := 0
for _, pkgN := range pkgs {
var pkg alpm.IPackage
var err error
var url string
pkgDB, name := text.SplitDBFromName(pkgN)
if pkgDB != "" {
pkg = dbExecutor.SatisfierFromDB(name, pkgDB)
} else {
pkg = dbExecutor.SyncSatisfier(name)
}
if pkg == nil {
missing = append(missing, name)
continue
}
name = pkg.Base()
if name == "" {
name = pkg.Name()
}
// TODO: Check existence with ls-remote
// https://git.archlinux.org/svntogit/packages.git
switch pkg.DB().Name() {
case "core", "extra", "testing":
url = "https://github.com/archlinux/svntogit-packages.git"
case "community", "multilib", "community-testing", "multilib-testing":
url = "https://github.com/archlinux/svntogit-community.git"
default:
missing = append(missing, name)
continue
}
_, err = os.Stat(filepath.Join(path, name))
switch {
case err != nil && !os.IsNotExist(err):
text.Errorln(err)
continue
case os.IsNotExist(err), force:
if err = os.RemoveAll(filepath.Join(path, name)); err != nil {
text.Errorln(err)
continue
}
default:
text.Warn(gotext.Get("%s already downloaded -- use -f to overwrite", text.Cyan(name)))
continue
}
names[name] = url
}
if len(missing) != 0 {
text.Warnln(gotext.Get("Missing ABS packages:"),
text.Cyan(strings.Join(missing, ", ")))
}
download := func(pkg string, url string) {
defer wg.Done()
if _, err := gitDownloadABS(url, config.ABSDir, pkg); err != nil {
errs.Add(errors.New(gotext.Get("failed to get pkgbuild: %s: %s", text.Cyan(pkg), err.Error())))
return
}
_, stderr, err := config.Runtime.CmdRunner.Capture(
exec.Command(
"cp", "-r",
filepath.Join(config.ABSDir, pkg, "trunk"),
filepath.Join(path, pkg)), 0)
mux.Lock()
downloaded++
if err != nil {
errs.Add(errors.New(gotext.Get("failed to link %s: %s", text.Cyan(pkg), stderr)))
} else {
fmt.Fprintln(os.Stdout, gotext.Get("(%d/%d) Downloaded PKGBUILD from ABS: %s", downloaded, len(names), text.Cyan(pkg)))
}
mux.Unlock()
}
count := 0
for name, url := range names {
wg.Add(1)
go download(name, url)
count++
if count%25 == 0 {
wg.Wait()
}
}
wg.Wait()
return len(missing) != 0, errs.Return()
}

48
pkg/download/abs.go
View File

@ -7,6 +7,8 @@ import (
"net/http"
"github.com/leonelquinteros/gotext"
"github.com/Jguer/yay/v10/pkg/settings/exe"
)
const (
@ -21,22 +23,37 @@ var (
ABSCommunityURL = "https://github.com/archlinux/svntogit-community"
)
func getRepoURL(db string) (string, error) {
switch db {
case "core", "extra", "testing":
return ABSPackageURL, nil
case "community", "multilib", "community-testing", "multilib-testing":
return ABSCommunityURL, nil
}
return "", ErrInvalidRepository
}
// Return format for pkgbuild
// https://github.com/archlinux/svntogit-community/raw/packages/neovim/trunk/PKGBUILD
func getPackageURL(db, pkgName string) (string, error) {
repoURL := ""
switch db {
case "core", "extra", "testing":
repoURL = ABSPackageURL
case "community", "multilib", "community-testing", "multilib-testing":
repoURL = ABSCommunityURL
default:
return "", ErrInvalidRepository
repoURL, err := getRepoURL(db)
if err != nil {
return "", err
}
return fmt.Sprintf(_urlPackagePath, repoURL, pkgName), nil
return fmt.Sprintf(_urlPackagePath, repoURL, pkgName), err
}
// Return format for pkgbuild repo
// https://github.com/archlinux/svntogit-community.git
func getPackageRepoURL(db string) (string, error) {
repoURL, err := getRepoURL(db)
return repoURL + ".git", err
}
// GetABSPkgbuild retrieves the PKGBUILD file to a dest directory
func GetABSPkgbuild(httpClient *http.Client, dbName, pkgName string) ([]byte, error) {
packageURL, err := getPackageURL(dbName, pkgName)
if err != nil {
@ -48,7 +65,7 @@ func GetABSPkgbuild(httpClient *http.Client, dbName, pkgName string) ([]byte, er
return nil, err
}
if resp.StatusCode != 200 {
if resp.StatusCode != http.StatusOK {
return nil, ErrABSPackageNotFound
}
@ -61,3 +78,14 @@ func GetABSPkgbuild(httpClient *http.Client, dbName, pkgName string) ([]byte, er
return pkgBuild, nil
}
// ABSPkgbuildRepo retrieves the PKGBUILD repository to a dest directory
func ABSPkgbuildRepo(cmdRunner exe.Runner, cmdBuilder exe.GitCmdBuilder, dbName, pkgName, dest string, force bool) error {
pkgURL, err := getPackageRepoURL(dbName)
if err != nil {
return err
}
return downloadGitRepo(cmdRunner, cmdBuilder, pkgURL,
pkgName, dest, force, "--single-branch", "-b", "packages/"+pkgName)
}

2
pkg/download/abs_test.go
View File

@ -81,13 +81,11 @@ func Test_getPackageURL(t *testing.T) {
func TestGetABSPkgbuild(t *testing.T) {
pkgBuildHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(200)
w.Write([]byte(gitExtrasPKGBUILD))
})
notFoundHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(404)
})

18
pkg/download/aur.go
View File

@ -1,18 +1,16 @@
package download
import (
"errors"
"fmt"
"io/ioutil"
"net/http"
"net/url"
"github.com/leonelquinteros/gotext"
"github.com/Jguer/yay/v10/pkg/settings/exe"
)
var AURPackageURL = "https://aur.archlinux.org/cgit/aur.git"
var ErrAURPackageNotFound = errors.New(gotext.Get("package not found in AUR"))
func GetAURPkgbuild(httpClient *http.Client, pkgName string) ([]byte, error) {
values := url.Values{}
values.Set("h", pkgName)
@ -22,8 +20,9 @@ func GetAURPkgbuild(httpClient *http.Client, pkgName string) ([]byte, error) {
if err != nil {
return nil, err
}
if resp.StatusCode != 200 {
return nil, ErrAURPackageNotFound
if resp.StatusCode != http.StatusOK {
return nil, ErrAURPackageNotFound{pkgName: pkgName}
}
defer resp.Body.Close()
@ -35,3 +34,10 @@ func GetAURPkgbuild(httpClient *http.Client, pkgName string) ([]byte, error) {
return pkgBuild, nil
}
// AURPkgbuildRepo retrieves the PKGBUILD repository to a dest directory.
func AURPkgbuildRepo(cmdRunner exe.Runner, cmdBuilder exe.GitCmdBuilder, aurURL, pkgName, dest string, force bool) error {
pkgURL := fmt.Sprintf("%s/%s.git", aurURL, pkgName)
return downloadGitRepo(cmdRunner, cmdBuilder, pkgURL, pkgName, dest, force)
}

2
pkg/download/aur_test.go
View File

@ -10,13 +10,11 @@ import (
func TestGetAURPkgbuild(t *testing.T) {
pkgBuildHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(200)
w.Write([]byte(gitExtrasPKGBUILD))
})
notFoundHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(404)
})

31
pkg/download/errors.go Normal file
View File

@ -0,0 +1,31 @@
package download
import (
"fmt"
"github.com/leonelquinteros/gotext"
)
// ErrAURPackageNotFound means that package was not found in AUR.
type ErrAURPackageNotFound struct {
pkgName string
}
func (e ErrAURPackageNotFound) Error() string {
return fmt.Sprintln(gotext.Get("package not found in AUR"), ":", e.pkgName)
}
type ErrGetPKGBUILDRepo struct {
inner error
pkgName string
errOut string
}
func (e ErrGetPKGBUILDRepo) Error() string {
return fmt.Sprintln(gotext.Get("error fetching %s: %s", e.pkgName, e.errOut),
"\n\t context:", e.inner.Error())
}
func (e *ErrGetPKGBUILDRepo) Unwrap() error {
return e.inner
}

152
pkg/download/unified.go
View File

@ -2,14 +2,61 @@ package download
import (
"net/http"
"os"
"path/filepath"
"sync"
"github.com/leonelquinteros/gotext"
"github.com/Jguer/go-alpm/v2"
"github.com/Jguer/yay/v10/pkg/db"
"github.com/Jguer/yay/v10/pkg/multierror"
"github.com/Jguer/yay/v10/pkg/settings"
"github.com/Jguer/yay/v10/pkg/settings/exe"
"github.com/Jguer/yay/v10/pkg/text"
)
func downloadGitRepo(cmdRunner exe.Runner,
cmdBuilder exe.GitCmdBuilder, pkgURL, pkgName, dest string, force bool, gitArgs ...string) error {
gitArgs = append(gitArgs, pkgURL, pkgName)
cloneArgs := make([]string, 0, len(gitArgs)+4)
cloneArgs = append(cloneArgs, "clone", "--no-progress")
cloneArgs = append(cloneArgs, gitArgs...)
finalDir := filepath.Join(dest, pkgName)
if _, err := os.Stat(filepath.Join(finalDir, ".git")); os.IsNotExist(err) {
if _, errD := os.Stat(finalDir); force && errD == nil {
if errR := os.RemoveAll(finalDir); errR != nil {
return ErrGetPKGBUILDRepo{inner: errR, pkgName: pkgName}
}
}
cmd := cmdBuilder.BuildGitCmd(dest, cloneArgs...)
_, stderr, errCapture := cmdRunner.Capture(cmd, 0)
if errCapture != nil {
return ErrGetPKGBUILDRepo{inner: errCapture, pkgName: pkgName, errOut: stderr}
}
} else if err != nil {
return ErrGetPKGBUILDRepo{
inner: err,
pkgName: pkgName,
errOut: gotext.Get("error reading %s", filepath.Join(dest, pkgName, ".git")),
}
} else {
cmd := cmdBuilder.BuildGitCmd(filepath.Join(dest, pkgName), "pull", "--ff-only")
_, stderr, errCmd := cmdRunner.Capture(cmd, 0)
if errCmd != nil {
return ErrGetPKGBUILDRepo{inner: errCmd, pkgName: pkgName, errOut: stderr}
}
}
return nil
}
func getURLName(pkg db.IPackage) string {
name := pkg.Base()
if name == "" {
@ -21,13 +68,18 @@ func getURLName(pkg db.IPackage) string {
func GetPkgbuilds(dbExecutor db.Executor, httpClient *http.Client, targets []string, mode settings.TargetMode) (map[string][]byte, error) {
pkgbuilds := make(map[string][]byte, len(targets))
var mux sync.Mutex
var errs multierror.MultiError
var wg sync.WaitGroup
var (
mux sync.Mutex
errs multierror.MultiError
wg sync.WaitGroup
)
sem := make(chan uint8, MaxConcurrentFetch)
for _, target := range targets {
aur := true
dbName, name := text.SplitDBFromName(target)
if dbName != "aur" && (mode == settings.ModeAny || mode == settings.ModeRepo) {
if pkg := dbExecutor.SyncPackage(name); pkg != nil {
@ -43,11 +95,14 @@ func GetPkgbuilds(dbExecutor db.Executor, httpClient *http.Client, targets []str
}
sem <- 1
wg.Add(1)
go func(target, dbName, pkgName string, aur bool) {
var err error
var pkgbuild []byte
var (
err error
pkgbuild []byte
)
if aur {
pkgbuild, err = GetAURPkgbuild(httpClient, pkgName)
@ -72,3 +127,90 @@ func GetPkgbuilds(dbExecutor db.Executor, httpClient *http.Client, targets []str
return pkgbuilds, errs.Return()
}
func PKGBUILDRepos(dbExecutor db.Executor,
cmdRunner exe.Runner,
cmdBuilder exe.GitCmdBuilder,
targets []string, mode settings.TargetMode, aurURL, dest string, force bool) (map[string]bool, error) {
cloned := make(map[string]bool, len(targets))
var (
mux sync.Mutex
errs multierror.MultiError
wg sync.WaitGroup
)
sem := make(chan uint8, MaxConcurrentFetch)
for _, target := range targets {
aur := true
dbName, name := text.SplitDBFromName(target)
if dbName != "aur" && (mode == settings.ModeAny || mode == settings.ModeRepo) {
var pkg alpm.IPackage
if dbName != "" {
pkg = dbExecutor.SatisfierFromDB(name, dbName)
if pkg == nil {
// if the user precised a db but the package is not in the db
// then it is missing
continue
}
} else {
pkg = dbExecutor.SyncPackage(name)
}
if pkg != nil {
aur = false
name = getURLName(pkg)
dbName = pkg.DB().Name()
}
}
if aur && mode == settings.ModeRepo {
// Mode does not allow AUR packages
continue
}
sem <- 1
wg.Add(1)
go func(target, dbName, pkgName string, aur bool) {
var (
err error
)
if aur {
err = AURPkgbuildRepo(cmdRunner, cmdBuilder, aurURL, pkgName, dest, force)
} else {
err = ABSPkgbuildRepo(cmdRunner, cmdBuilder, dbName, pkgName, dest, force)
}
success := err == nil
if success {
mux.Lock()
cloned[target] = success
mux.Unlock()
}
if !success {
errs.Add(err)
}
if aur {
text.OperationInfoln(
gotext.Get("(%d/%d) Downloaded PKGBUILD: %s",
len(cloned), len(targets), text.Cyan(pkgName)))
} else {
text.OperationInfoln(
gotext.Get("(%d/%d) Downloaded PKGBUILD from ABS: %s",
len(cloned), len(targets), text.Cyan(pkgName)))
}
<-sem
wg.Done()
}(target, dbName, name, aur)
}
wg.Wait()
return cloned, errs.Return()
}

4
pkg/settings/exe/passers.go
View File

@ -5,6 +5,10 @@ import (
"os/exec"
)
type GitCmdBuilder interface {
BuildGitCmd(dir string, extraArgs ...string) *exec.Cmd
}
type CmdBuilder struct {
GitBin string
GitFlags []string

4
print.go
View File

@ -291,7 +291,9 @@ func printPkgbuilds(dbExecutor db.Executor, httpClient *http.Client, targets []s
missing = append(missing, target)
}
}
text.Warnln("Unable to find the following packages:", strings.Join(missing, ", "))
text.Warnln(gotext.Get("Unable to find the following packages:"), strings.Join(missing, ", "))
return fmt.Errorf("")
}
return nil