feat(sources): concurrent makepkg downloading

This commit is contained in:
jguer 2021-08-13 23:37:12 +02:00 committed by J Guerreiro
parent 5d9e424942
commit aedbcffc80
2 changed files with 123 additions and 22 deletions

121
aur_source.go Normal file
View File

@ -0,0 +1,121 @@
package main
import (
"context"
"fmt"
"path/filepath"
"runtime"
"sync"
"github.com/leonelquinteros/gotext"
"github.com/Jguer/yay/v10/pkg/dep"
"github.com/Jguer/yay/v10/pkg/multierror"
"github.com/Jguer/yay/v10/pkg/settings/exe"
"github.com/Jguer/yay/v10/pkg/stringset"
"github.com/Jguer/yay/v10/pkg/text"
)
type ErrDownloadSource struct {
inner error
pkgName string
errOut string
}
func (e ErrDownloadSource) Error() string {
return fmt.Sprintln(gotext.Get("error downloading sources: %s", text.Cyan(e.pkgName)),
"\n\t context:", e.inner.Error(), "\n\t", e.errOut)
}
func (e *ErrDownloadSource) Unwrap() error {
return e.inner
}
func downloadPKGBUILDSource(ctx context.Context, cmdBuilder exe.ICmdBuilder,
base string, incompatible stringset.StringSet) (err error) {
dir := filepath.Join(config.BuildDir, base)
args := []string{"--verifysource", "-Ccf"}
if incompatible.Get(base) {
args = append(args, "--ignorearch")
}
err = cmdBuilder.Show(
cmdBuilder.BuildMakepkgCmd(ctx, dir, args...))
if err != nil {
return ErrDownloadSource{inner: err, pkgName: base, errOut: ""}
}
return nil
}
func downloadPKGBUILDSourceWorker(ctx context.Context, wg *sync.WaitGroup,
cBase <-chan string, valOut chan<- string, errOut chan<- error,
cmdBuilder exe.ICmdBuilder, incompatible stringset.StringSet) {
for base := range cBase {
err := downloadPKGBUILDSource(ctx, cmdBuilder, base, incompatible)
if err != nil {
errOut <- ErrDownloadSource{inner: err, pkgName: base, errOut: ""}
} else {
valOut <- base
}
}
wg.Done()
}
func downloadPKGBUILDSourceFanout(ctx context.Context, cmdBuilder exe.ICmdBuilder,
bases []dep.Base, incompatible stringset.StringSet) error {
if len(bases) == 1 {
return downloadPKGBUILDSource(ctx, cmdBuilder, bases[0].Pkgbase(), incompatible)
}
var (
numOfWorkers = runtime.NumCPU()
wg = &sync.WaitGroup{}
c = make(chan string)
fanInChanValues = make(chan string)
fanInChanErrors = make(chan error)
)
go func() {
for _, base := range bases {
c <- base.Pkgbase()
}
close(c)
}()
// Launch Workers
wg.Add(numOfWorkers)
for s := 0; s < numOfWorkers; s++ {
go downloadPKGBUILDSourceWorker(ctx, wg, c,
fanInChanValues, fanInChanErrors, cmdBuilder, incompatible)
}
go func() {
wg.Wait()
close(fanInChanValues)
close(fanInChanErrors)
}()
returnErr := multierror.MultiError{}
receiver:
for {
select {
case _, ok := <-fanInChanValues:
if !ok {
break receiver
}
case err, ok := <-fanInChanErrors:
if !ok {
break receiver
}
returnErr.Add(err)
}
}
return returnErr.Return()
}

View File

@ -396,9 +396,9 @@ func install(ctx context.Context, cmdArgs *parser.Arguments, dbExecutor db.Execu
config.AURURL, config.Runtime.CompletionPath, config.CompletionInterval, false)
}()
err = downloadPkgbuildsSources(ctx, do.Aur, incompatible)
err = downloadPKGBUILDSourceFanout(ctx, config.Runtime.CmdBuilder, do.Aur, incompatible)
if err != nil {
return err
text.Errorln(err)
}
err = buildInstallPkgbuilds(ctx, cmdArgs, dbExecutor, dp, do, srcinfos, incompatible, conflicts, noDeps, noCheck)
@ -876,26 +876,6 @@ func mergePkgbuilds(ctx context.Context, bases []dep.Base) error {
return nil
}
func downloadPkgbuildsSources(ctx context.Context, bases []dep.Base, incompatible stringset.StringSet) (err error) {
for _, base := range bases {
pkg := base.Pkgbase()
dir := filepath.Join(config.BuildDir, pkg)
args := []string{"--verifysource", "-Ccf"}
if incompatible.Get(pkg) {
args = append(args, "--ignorearch")
}
err = config.Runtime.CmdBuilder.Show(
config.Runtime.CmdBuilder.BuildMakepkgCmd(ctx, dir, args...))
if err != nil {
return errors.New(gotext.Get("error downloading sources: %s", text.Cyan(base.String())))
}
}
return
}
func buildInstallPkgbuilds(
ctx context.Context,
cmdArgs *parser.Arguments,