From 57a8048cb89d9267ec8e12f1b24fa2a51b37d00c Mon Sep 17 00:00:00 2001 From: morganamilo Date: Thu, 8 Mar 2018 21:34:12 +0000 Subject: [PATCH 1/2] Use goroutinuies for devel updates The update process was already partly parallel: repo, aur and devel updates where each given their own goroutine. This gives two further layers of parallelization. Firstly the `needsUpdate()` function is now run in parallel for each package checked. One package may have many vcs sources so `needsUpdate()` also checks all of its sources in parallel. This gives an aproxamte 3x speedup for `yay -Su --devel` timing from when the command starts to when the number menu apears. unfortunately git://bitbucket.org/volumesoffun/polyvox.git never resolves on my machine for some reason so it always hits the 5 second timout period. It then moves on to http:/bitbucket.org/volumesoffun/polyvox.git/ which does resolve as expected. I have not looked into it but I fear this applies to all gitbucket repos. Luckly they are few and far between. --- query.go | 5 +---- upgrade.go | 58 +++++++++++++++++++++++++++++++++++++----------------- vcs.go | 18 ++++++++++++++--- 3 files changed, 56 insertions(+), 25 deletions(-) diff --git a/query.go b/query.go index d5f7c901..2d767bb0 100644 --- a/query.go +++ b/query.go @@ -338,21 +338,18 @@ func aurInfo(names []string) ([]rpc.Pkg, error) { outOfDate := make([]string, 0, len(names)) makeRequest := func(n, max int) { + defer wg.Done() tempInfo, requestErr := rpc.Info(names[n:max]) if err != nil { - wg.Done() return } if requestErr != nil { - //return info, err err = requestErr - wg.Done() return } mux.Lock() info = append(info, tempInfo...) mux.Unlock() - wg.Done() } for n := 0; n < len(names); n += config.RequestSplitN { diff --git a/upgrade.go b/upgrade.go index 2b1f08fd..cf5188f7 100644 --- a/upgrade.go +++ b/upgrade.go @@ -5,6 +5,7 @@ import ( "fmt" "os" "sort" + "sync" "unicode" alpm "github.com/jguer/go-alpm" @@ -128,29 +129,50 @@ loop: } func upDevel(remote []alpm.Package, packageC chan upgrade, done chan bool) { - for vcsName, e := range savedInfo { + toUpdate := make([]alpm.Package, 0, 0) + toRemove := make([]string, 0, 0) + + var mux1 sync.Mutex + var mux2 sync.Mutex + var wg sync.WaitGroup + + checkUpdate := func(vcsName string, e shaInfos) { + defer wg.Done() + if e.needsUpdate() { - found := false - var pkg alpm.Package - for _, r := range remote { - if r.Name() == vcsName { - found = true - pkg = r + for _, pkg := range remote { + if pkg.Name() == vcsName { + mux1.Lock() + toUpdate = append(toUpdate, pkg) + mux1.Unlock() + return } } - if found { - if pkg.ShouldIgnore() { - left, right := getVersionDiff(pkg.Version(), "latest-commit") - fmt.Print(magenta("Warning: ")) - fmt.Printf("%s ignoring package upgrade (%s => %s)\n", cyan(pkg.Name()), left, right) - } else { - packageC <- upgrade{pkg.Name(), "devel", pkg.Version(), "latest-commit"} - } - } else { - removeVCSPackage([]string{vcsName}) - } + + mux2.Lock() + toRemove = append(toRemove, vcsName) + mux2.Unlock() } } + + for vcsName, e := range savedInfo { + wg.Add(1) + go checkUpdate(vcsName, e) + } + + wg.Wait() + + for _, pkg := range toUpdate { + if pkg.ShouldIgnore() { + left, right := getVersionDiff(pkg.Version(), "latest-commit") + fmt.Print(magenta("Warning: ")) + fmt.Printf("%s ignoring package upgrade (%s => %s)\n", cyan(pkg.Name()), left, right) + } else { + packageC <- upgrade{pkg.Name(), "devel", pkg.Version(), "latest-commit"} + } + } + + removeVCSPackage(toRemove) done <- true } diff --git a/vcs.go b/vcs.go index b54963b2..e673de03 100644 --- a/vcs.go +++ b/vcs.go @@ -6,6 +6,7 @@ import ( "os" "os/exec" "strings" + "sync" "time" ) @@ -142,14 +143,25 @@ func getCommit(url string, branch string, protocols []string) string { } func (infos shaInfos) needsUpdate() bool { - for url, info := range infos { + var wg sync.WaitGroup + hasUpdate := false + + checkHash := func(url string, info shaInfo) { + defer wg.Done() hash := getCommit(url, info.Brach, info.Protocols) if hash != "" && hash != info.SHA { - return true + hasUpdate = true } } - return false + for url, info := range infos { + wg.Add(1) + go checkHash(url, info) + } + + wg.Wait() + + return hasUpdate } func inStore(pkgName string) shaInfos { From 1d2b07fa84774d2ddeb34dc1d0688183ae64d983 Mon Sep 17 00:00:00 2001 From: morganamilo Date: Thu, 8 Mar 2018 22:20:47 +0000 Subject: [PATCH 2/2] Use channels over WaitGroup in needsUpdate() Using a WaitGroup forced the code to wait for every goroutine to finish. Using channels allows us to exit early if any of a packages sources need an update. No point in waiting for the other requests to finish if we know there is an update. --- vcs.go | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/vcs.go b/vcs.go index e673de03..dc517319 100644 --- a/vcs.go +++ b/vcs.go @@ -6,7 +6,6 @@ import ( "os" "os/exec" "strings" - "sync" "time" ) @@ -143,25 +142,38 @@ func getCommit(url string, branch string, protocols []string) string { } func (infos shaInfos) needsUpdate() bool { - var wg sync.WaitGroup - hasUpdate := false + //used to signal we have gone through all sources and found nothing + finished := make(chan struct{}) + alive := 0 + + //if we find an update we use this to exit early and return true + hasUpdate := make(chan struct{}) checkHash := func(url string, info shaInfo) { - defer wg.Done() hash := getCommit(url, info.Brach, info.Protocols) if hash != "" && hash != info.SHA { - hasUpdate = true + hasUpdate <- struct{}{} + } else { + finished <- struct{}{} } } for url, info := range infos { - wg.Add(1) + alive++ go checkHash(url, info) } - wg.Wait() - - return hasUpdate + for { + select { + case <- hasUpdate: + return true + case <- finished: + alive-- + if alive == 0 { + return false + } + } + } } func inStore(pkgName string) shaInfos {