mirror of
https://github.com/Jguer/yay.git
synced 2024-11-06 09:07:21 +01:00
feat(aur): add option to limit concurrent downloads (#1768)
* feat(aur): add option to limit concurrent downloads Adds to config file option 'maxconcurrentdownloads' which if set to value != 0, limits number of concurrent AUR downloads to specified. Fixes #1763. * fix lint issue
This commit is contained in:
vqrkxo
GitHub
parent
18fbc24e68
commit
e0006ec272
@ -65,7 +65,7 @@ func downloadPKGBUILDSourceWorker(ctx context.Context, wg *sync.WaitGroup, dest
|
||||
}
|
||||
|
||||
func downloadPKGBUILDSourceFanout(ctx context.Context, cmdBuilder exe.ICmdBuilder, dest string,
|
||||
bases []dep.Base, incompatible stringset.StringSet) error {
|
||||
bases []dep.Base, incompatible stringset.StringSet, maxConcurrentDownloads int) error {
|
||||
if len(bases) == 1 {
|
||||
return downloadPKGBUILDSource(ctx, cmdBuilder, dest, bases[0].Pkgbase(), incompatible)
|
||||
}
|
||||
@ -78,6 +78,10 @@ func downloadPKGBUILDSourceFanout(ctx context.Context, cmdBuilder exe.ICmdBuilde
|
||||
fanInChanErrors = make(chan error)
|
||||
)
|
||||
|
||||
if maxConcurrentDownloads != 0 {
|
||||
numOfWorkers = maxConcurrentDownloads
|
||||
}
|
||||
|
||||
go func() {
|
||||
for _, base := range bases {
|
||||
c <- base.Pkgbase()
|
||||
|
||||
@ -2,6 +2,7 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os/exec"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
@ -82,13 +83,6 @@ func Test_downloadPKGBUILDSourceError(t *testing.T) {
|
||||
// THEN 5 calls should be made to makepkg
|
||||
func Test_downloadPKGBUILDSourceFanout(t *testing.T) {
|
||||
t.Parallel()
|
||||
cmdBuilder := &TestMakepkgBuilder{
|
||||
parentBuilder: &exe.CmdBuilder{
|
||||
MakepkgConfPath: "/etc/not.conf",
|
||||
MakepkgFlags: []string{"--nocheck"}, MakepkgBin: "makepkg",
|
||||
},
|
||||
test: t,
|
||||
}
|
||||
|
||||
bases := []dep.Base{
|
||||
{&aur.Pkg{PackageBase: "yay"}},
|
||||
@ -98,9 +92,21 @@ func Test_downloadPKGBUILDSourceFanout(t *testing.T) {
|
||||
{&aur.Pkg{PackageBase: "yay-v12"}},
|
||||
}
|
||||
|
||||
err := downloadPKGBUILDSourceFanout(context.TODO(), cmdBuilder, "/tmp", bases, stringset.Make())
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, 5, int(cmdBuilder.passes))
|
||||
for _, maxConcurrentDownloads := range []int{0, 3} {
|
||||
t.Run(fmt.Sprintf("maxconcurrentdownloads set to %d", maxConcurrentDownloads), func(t *testing.T) {
|
||||
cmdBuilder := &TestMakepkgBuilder{
|
||||
parentBuilder: &exe.CmdBuilder{
|
||||
MakepkgConfPath: "/etc/not.conf",
|
||||
MakepkgFlags: []string{"--nocheck"}, MakepkgBin: "makepkg",
|
||||
},
|
||||
test: t,
|
||||
}
|
||||
|
||||
err := downloadPKGBUILDSourceFanout(context.TODO(), cmdBuilder, "/tmp", bases, stringset.Make(), maxConcurrentDownloads)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, 5, int(cmdBuilder.passes))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// GIVEN 1 package
|
||||
@ -120,7 +126,7 @@ func Test_downloadPKGBUILDSourceFanoutNoCC(t *testing.T) {
|
||||
{&aur.Pkg{PackageBase: "yay"}},
|
||||
}
|
||||
|
||||
err := downloadPKGBUILDSourceFanout(context.TODO(), cmdBuilder, "/tmp", bases, stringset.Make())
|
||||
err := downloadPKGBUILDSourceFanout(context.TODO(), cmdBuilder, "/tmp", bases, stringset.Make(), 0)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, 1, int(cmdBuilder.passes))
|
||||
}
|
||||
@ -147,7 +153,7 @@ func Test_downloadPKGBUILDSourceFanoutError(t *testing.T) {
|
||||
{&aur.Pkg{PackageBase: "yay-v12"}},
|
||||
}
|
||||
|
||||
err := downloadPKGBUILDSourceFanout(context.TODO(), cmdBuilder, "/tmp", bases, stringset.Make())
|
||||
err := downloadPKGBUILDSourceFanout(context.TODO(), cmdBuilder, "/tmp", bases, stringset.Make(), 0)
|
||||
assert.Error(t, err)
|
||||
assert.Equal(t, 5, int(cmdBuilder.passes))
|
||||
assert.Len(t, err.(*multierror.MultiError).Errors, 5)
|
||||
|
||||
@ -330,7 +330,8 @@ func install(ctx context.Context, cmdArgs *parser.Arguments, dbExecutor db.Execu
|
||||
config.AURURL, config.Runtime.CompletionPath, config.CompletionInterval, false)
|
||||
}()
|
||||
|
||||
if errP := downloadPKGBUILDSourceFanout(ctx, config.Runtime.CmdBuilder, config.BuildDir, do.Aur, incompatible); errP != nil {
|
||||
if errP := downloadPKGBUILDSourceFanout(ctx, config.Runtime.CmdBuilder, config.BuildDir,
|
||||
do.Aur, incompatible, config.MaxConcurrentDownloads); errP != nil {
|
||||
text.Errorln(errP)
|
||||
}
|
||||
|
||||
|
||||
@ -30,50 +30,51 @@ var NoConfirm = false
|
||||
|
||||
// Configuration stores yay's config.
|
||||
type Configuration struct {
|
||||
AURURL string `json:"aururl"`
|
||||
BuildDir string `json:"buildDir"`
|
||||
Editor string `json:"editor"`
|
||||
EditorFlags string `json:"editorflags"`
|
||||
MakepkgBin string `json:"makepkgbin"`
|
||||
MakepkgConf string `json:"makepkgconf"`
|
||||
PacmanBin string `json:"pacmanbin"`
|
||||
PacmanConf string `json:"pacmanconf"`
|
||||
ReDownload string `json:"redownload"`
|
||||
ReBuild string `json:"rebuild"`
|
||||
AnswerClean string `json:"answerclean"`
|
||||
AnswerDiff string `json:"answerdiff"`
|
||||
AnswerEdit string `json:"answeredit"`
|
||||
AnswerUpgrade string `json:"answerupgrade"`
|
||||
GitBin string `json:"gitbin"`
|
||||
GpgBin string `json:"gpgbin"`
|
||||
GpgFlags string `json:"gpgflags"`
|
||||
MFlags string `json:"mflags"`
|
||||
SortBy string `json:"sortby"`
|
||||
SearchBy string `json:"searchby"`
|
||||
GitFlags string `json:"gitflags"`
|
||||
RemoveMake string `json:"removemake"`
|
||||
SudoBin string `json:"sudobin"`
|
||||
SudoFlags string `json:"sudoflags"`
|
||||
RequestSplitN int `json:"requestsplitn"`
|
||||
CompletionInterval int `json:"completionrefreshtime"`
|
||||
BottomUp bool `json:"bottomup"`
|
||||
SudoLoop bool `json:"sudoloop"`
|
||||
TimeUpdate bool `json:"timeupdate"`
|
||||
Devel bool `json:"devel"`
|
||||
CleanAfter bool `json:"cleanAfter"`
|
||||
Provides bool `json:"provides"`
|
||||
PGPFetch bool `json:"pgpfetch"`
|
||||
UpgradeMenu bool `json:"upgrademenu"`
|
||||
CleanMenu bool `json:"cleanmenu"`
|
||||
DiffMenu bool `json:"diffmenu"`
|
||||
EditMenu bool `json:"editmenu"`
|
||||
CombinedUpgrade bool `json:"combinedupgrade"`
|
||||
UseAsk bool `json:"useask"`
|
||||
BatchInstall bool `json:"batchinstall"`
|
||||
SingleLineResults bool `json:"singlelineresults"`
|
||||
SeparateSources bool `json:"separatesources"`
|
||||
Runtime *Runtime `json:"-"`
|
||||
Version string `json:"version"`
|
||||
AURURL string `json:"aururl"`
|
||||
BuildDir string `json:"buildDir"`
|
||||
Editor string `json:"editor"`
|
||||
EditorFlags string `json:"editorflags"`
|
||||
MakepkgBin string `json:"makepkgbin"`
|
||||
MakepkgConf string `json:"makepkgconf"`
|
||||
PacmanBin string `json:"pacmanbin"`
|
||||
PacmanConf string `json:"pacmanconf"`
|
||||
ReDownload string `json:"redownload"`
|
||||
ReBuild string `json:"rebuild"`
|
||||
AnswerClean string `json:"answerclean"`
|
||||
AnswerDiff string `json:"answerdiff"`
|
||||
AnswerEdit string `json:"answeredit"`
|
||||
AnswerUpgrade string `json:"answerupgrade"`
|
||||
GitBin string `json:"gitbin"`
|
||||
GpgBin string `json:"gpgbin"`
|
||||
GpgFlags string `json:"gpgflags"`
|
||||
MFlags string `json:"mflags"`
|
||||
SortBy string `json:"sortby"`
|
||||
SearchBy string `json:"searchby"`
|
||||
GitFlags string `json:"gitflags"`
|
||||
RemoveMake string `json:"removemake"`
|
||||
SudoBin string `json:"sudobin"`
|
||||
SudoFlags string `json:"sudoflags"`
|
||||
RequestSplitN int `json:"requestsplitn"`
|
||||
CompletionInterval int `json:"completionrefreshtime"`
|
||||
MaxConcurrentDownloads int `json:"maxconcurrentdownloads"`
|
||||
BottomUp bool `json:"bottomup"`
|
||||
SudoLoop bool `json:"sudoloop"`
|
||||
TimeUpdate bool `json:"timeupdate"`
|
||||
Devel bool `json:"devel"`
|
||||
CleanAfter bool `json:"cleanAfter"`
|
||||
Provides bool `json:"provides"`
|
||||
PGPFetch bool `json:"pgpfetch"`
|
||||
UpgradeMenu bool `json:"upgrademenu"`
|
||||
CleanMenu bool `json:"cleanmenu"`
|
||||
DiffMenu bool `json:"diffmenu"`
|
||||
EditMenu bool `json:"editmenu"`
|
||||
CombinedUpgrade bool `json:"combinedupgrade"`
|
||||
UseAsk bool `json:"useask"`
|
||||
BatchInstall bool `json:"batchinstall"`
|
||||
SingleLineResults bool `json:"singlelineresults"`
|
||||
SeparateSources bool `json:"separatesources"`
|
||||
Runtime *Runtime `json:"-"`
|
||||
Version string `json:"version"`
|
||||
}
|
||||
|
||||
// SaveConfig writes yay config to file.
|
||||
@ -178,48 +179,49 @@ func (c *Configuration) setPrivilegeElevator() error {
|
||||
|
||||
func DefaultConfig(version string) *Configuration {
|
||||
return &Configuration{
|
||||
AURURL: "https://aur.archlinux.org",
|
||||
BuildDir: os.ExpandEnv("$HOME/.cache/yay"),
|
||||
CleanAfter: false,
|
||||
Editor: "",
|
||||
EditorFlags: "",
|
||||
Devel: false,
|
||||
MakepkgBin: "makepkg",
|
||||
MakepkgConf: "",
|
||||
PacmanBin: "pacman",
|
||||
PGPFetch: true,
|
||||
PacmanConf: "/etc/pacman.conf",
|
||||
GpgFlags: "",
|
||||
MFlags: "",
|
||||
GitFlags: "",
|
||||
BottomUp: true,
|
||||
CompletionInterval: 7,
|
||||
SortBy: "votes",
|
||||
SearchBy: "name-desc",
|
||||
SudoLoop: false,
|
||||
GitBin: "git",
|
||||
GpgBin: "gpg",
|
||||
SudoBin: "sudo",
|
||||
SudoFlags: "",
|
||||
TimeUpdate: false,
|
||||
RequestSplitN: 150,
|
||||
ReDownload: "no",
|
||||
ReBuild: "no",
|
||||
BatchInstall: false,
|
||||
AnswerClean: "",
|
||||
AnswerDiff: "",
|
||||
AnswerEdit: "",
|
||||
AnswerUpgrade: "",
|
||||
RemoveMake: "ask",
|
||||
Provides: true,
|
||||
UpgradeMenu: true,
|
||||
CleanMenu: true,
|
||||
DiffMenu: true,
|
||||
EditMenu: false,
|
||||
UseAsk: false,
|
||||
CombinedUpgrade: false,
|
||||
SeparateSources: true,
|
||||
Version: version,
|
||||
AURURL: "https://aur.archlinux.org",
|
||||
BuildDir: os.ExpandEnv("$HOME/.cache/yay"),
|
||||
CleanAfter: false,
|
||||
Editor: "",
|
||||
EditorFlags: "",
|
||||
Devel: false,
|
||||
MakepkgBin: "makepkg",
|
||||
MakepkgConf: "",
|
||||
PacmanBin: "pacman",
|
||||
PGPFetch: true,
|
||||
PacmanConf: "/etc/pacman.conf",
|
||||
GpgFlags: "",
|
||||
MFlags: "",
|
||||
GitFlags: "",
|
||||
BottomUp: true,
|
||||
CompletionInterval: 7,
|
||||
MaxConcurrentDownloads: 0,
|
||||
SortBy: "votes",
|
||||
SearchBy: "name-desc",
|
||||
SudoLoop: false,
|
||||
GitBin: "git",
|
||||
GpgBin: "gpg",
|
||||
SudoBin: "sudo",
|
||||
SudoFlags: "",
|
||||
TimeUpdate: false,
|
||||
RequestSplitN: 150,
|
||||
ReDownload: "no",
|
||||
ReBuild: "no",
|
||||
BatchInstall: false,
|
||||
AnswerClean: "",
|
||||
AnswerDiff: "",
|
||||
AnswerEdit: "",
|
||||
AnswerUpgrade: "",
|
||||
RemoveMake: "ask",
|
||||
Provides: true,
|
||||
UpgradeMenu: true,
|
||||
CleanMenu: true,
|
||||
DiffMenu: true,
|
||||
EditMenu: false,
|
||||
UseAsk: false,
|
||||
CombinedUpgrade: false,
|
||||
SeparateSources: true,
|
||||
Version: version,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user