yay/vcs.go

302 lines
9.3 KiB
Go
Raw Normal View History

package main
2017-05-01 03:23:03 +02:00
2017-05-01 03:34:40 +02:00
import (
"encoding/json"
"fmt"
2017-11-21 01:27:01 +01:00
"io/ioutil"
2017-05-01 03:34:40 +02:00
"net/http"
"os"
2017-05-01 03:34:40 +02:00
"strings"
"time"
2017-05-01 03:34:40 +02:00
)
2017-05-01 03:23:03 +02:00
// branch contains the information of a repository branch
type branch struct {
Name string `json:"name"`
Commit struct {
SHA string `json:"sha"`
URL string `json:"url"`
} `json:"commit"`
}
type branches []branch
// Info contains the last commit sha of a repo
type Info struct {
2017-05-02 12:50:11 +02:00
Package string `json:"pkgname"`
URL string `json:"url"`
SHA string `json:"sha"`
}
type infos []Info
// Repo contains information about the repository
type repo struct {
ID int `json:"id"`
Name string `json:"name"`
FullName string `json:"full_name"`
Owner struct {
Login string `json:"login"`
ID int `json:"id"`
AvatarURL string `json:"avatar_url"`
GravatarID string `json:"gravatar_id"`
URL string `json:"url"`
HTMLURL string `json:"html_url"`
FollowersURL string `json:"followers_url"`
FollowingURL string `json:"following_url"`
GistsURL string `json:"gists_url"`
StarredURL string `json:"starred_url"`
SubscriptionsURL string `json:"subscriptions_url"`
OrganizationsURL string `json:"organizations_url"`
ReposURL string `json:"repos_url"`
EventsURL string `json:"events_url"`
ReceivedEventsURL string `json:"received_events_url"`
Type string `json:"type"`
SiteAdmin bool `json:"site_admin"`
} `json:"owner"`
Private bool `json:"private"`
HTMLURL string `json:"html_url"`
Description string `json:"description"`
Fork bool `json:"fork"`
URL string `json:"url"`
ForksURL string `json:"forks_url"`
KeysURL string `json:"keys_url"`
CollaboratorsURL string `json:"collaborators_url"`
TeamsURL string `json:"teams_url"`
HooksURL string `json:"hooks_url"`
IssueEventsURL string `json:"issue_events_url"`
EventsURL string `json:"events_url"`
AssigneesURL string `json:"assignees_url"`
BranchesURL string `json:"branches_url"`
TagsURL string `json:"tags_url"`
BlobsURL string `json:"blobs_url"`
GitTagsURL string `json:"git_tags_url"`
GitRefsURL string `json:"git_refs_url"`
TreesURL string `json:"trees_url"`
StatusesURL string `json:"statuses_url"`
LanguagesURL string `json:"languages_url"`
StargazersURL string `json:"stargazers_url"`
ContributorsURL string `json:"contributors_url"`
SubscribersURL string `json:"subscribers_url"`
SubscriptionURL string `json:"subscription_url"`
CommitsURL string `json:"commits_url"`
GitCommitsURL string `json:"git_commits_url"`
CommentsURL string `json:"comments_url"`
IssueCommentURL string `json:"issue_comment_url"`
ContentsURL string `json:"contents_url"`
CompareURL string `json:"compare_url"`
MergesURL string `json:"merges_url"`
ArchiveURL string `json:"archive_url"`
DownloadsURL string `json:"downloads_url"`
IssuesURL string `json:"issues_url"`
PullsURL string `json:"pulls_url"`
MilestonesURL string `json:"milestones_url"`
NotificationsURL string `json:"notifications_url"`
LabelsURL string `json:"labels_url"`
ReleasesURL string `json:"releases_url"`
DeploymentsURL string `json:"deployments_url"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
PushedAt time.Time `json:"pushed_at"`
GitURL string `json:"git_url"`
SSHURL string `json:"ssh_url"`
CloneURL string `json:"clone_url"`
SvnURL string `json:"svn_url"`
Homepage string `json:"homepage"`
Size int `json:"size"`
StargazersCount int `json:"stargazers_count"`
WatchersCount int `json:"watchers_count"`
Language string `json:"language"`
HasIssues bool `json:"has_issues"`
HasProjects bool `json:"has_projects"`
HasDownloads bool `json:"has_downloads"`
HasWiki bool `json:"has_wiki"`
HasPages bool `json:"has_pages"`
ForksCount int `json:"forks_count"`
MirrorURL interface{} `json:"mirror_url"`
Archived bool `json:"archived"`
OpenIssuesCount int `json:"open_issues_count"`
License struct {
Key string `json:"key"`
Name string `json:"name"`
SpdxID interface{} `json:"spdx_id"`
URL interface{} `json:"url"`
} `json:"license"`
Forks int `json:"forks"`
OpenIssues int `json:"open_issues"`
Watchers int `json:"watchers"`
DefaultBranch string `json:"default_branch"`
Organization struct {
Login string `json:"login"`
ID int `json:"id"`
AvatarURL string `json:"avatar_url"`
GravatarID string `json:"gravatar_id"`
URL string `json:"url"`
HTMLURL string `json:"html_url"`
FollowersURL string `json:"followers_url"`
FollowingURL string `json:"following_url"`
GistsURL string `json:"gists_url"`
StarredURL string `json:"starred_url"`
SubscriptionsURL string `json:"subscriptions_url"`
OrganizationsURL string `json:"organizations_url"`
ReposURL string `json:"repos_url"`
EventsURL string `json:"events_url"`
ReceivedEventsURL string `json:"received_events_url"`
Type string `json:"type"`
SiteAdmin bool `json:"site_admin"`
} `json:"organization"`
NetworkCount int `json:"network_count"`
SubscribersCount int `json:"subscribers_count"`
}
2017-10-14 18:11:47 +02:00
// createDevelDB forces yay to create a DB of the existing development packages
func createDevelDB() error {
_, _, _, remoteNames, err := filterPackages()
if err != nil {
return err
}
config.NoConfirm = true
New install algorithm I have replaced the old install and dependancy algorithms with a new design that attemps to be more pacaur like. Mostly in minimizing user input. Ask every thing first then do everything with no need for more user input. It is not yet fully complete but is finished enough so that it works, should not fail in most cases and provides a base for more contributors to help address the existing problems. The new install chain is as follows: Source info about the provided targets Fetch a list of all dependancies needed to install targets I put alot of effort into fetching the dependancy tree while making the least amount of aur requests as possible. I'm actually very happy with how it turned out and yay wil now resolve dependancies noticably faster than pacaur when there are many aur dependancies. Install repo targets by passing to pacman Print dependancy tree and ask to confirm Ask to clean build if directory already exists Download all pkgbuilds Ask to edit all pkgbuilds Ask to continue with the install Download the sources for each packagebuild Build and install every package using -s to get repo deps and -i to install Ask to remove make dependancies There are still a lot of things that need to be done for a fully working system. Here are the problems I found with this system, either new or existing: Formating I am not so good at formatting myself, I thought best to leave it until last so I could get feedback on how it should look and help implementing it. Dependancy tree The dependancy tree is usually correct although I have noticed times where it doesnt detect all the dependancies that it should. I have only noticed this when there are circular dependancies so i think this might be the cause. It's not a big deal currently because makepkg -i installed repo deps for us which handles the repo deps for us and will get the correct ones. So yay might not list all the dependancies. but they will get installed so I consider this a visual bug. I have yet to see any circular dependancies in the AUR so I can not say what will happend but I#m guessing that it will break. Versioned packages/dependencies Targets and dependancies with version constriants such as 'linux>=4.1' will not be checked on the aur side of things but will be checked on the repo side. Ignorepkg/Ignoregroup Currently I do not handle this in any way but it shouldn't be too hard to implement. Conflict checking This is not currently implemented either Split Paclages Split packages are not Handles properly. If we only specify one package so install from a split package makepkg -i ends up installing them all anyway. If we specify more than one (n) package it will actually build the package base n times and reinstall every split package n times. Makepkg To get things working I decided to keep using the makepkg -i method. I plan to eventually replace this with a pacman -U based method. This should allow passing args such as --dbpath and --config to aur packages aswell as help solve some problems such as the split packages. Clean build I plan to improve the clean build choice to be a little more smart and instead of check if the directory exists, check if the package is already build and if so skip the build all together.
2018-01-17 22:48:23 +01:00
arguments := makeArguments()
arguments.addArg("gendb")
New install algorithm I have replaced the old install and dependancy algorithms with a new design that attemps to be more pacaur like. Mostly in minimizing user input. Ask every thing first then do everything with no need for more user input. It is not yet fully complete but is finished enough so that it works, should not fail in most cases and provides a base for more contributors to help address the existing problems. The new install chain is as follows: Source info about the provided targets Fetch a list of all dependancies needed to install targets I put alot of effort into fetching the dependancy tree while making the least amount of aur requests as possible. I'm actually very happy with how it turned out and yay wil now resolve dependancies noticably faster than pacaur when there are many aur dependancies. Install repo targets by passing to pacman Print dependancy tree and ask to confirm Ask to clean build if directory already exists Download all pkgbuilds Ask to edit all pkgbuilds Ask to continue with the install Download the sources for each packagebuild Build and install every package using -s to get repo deps and -i to install Ask to remove make dependancies There are still a lot of things that need to be done for a fully working system. Here are the problems I found with this system, either new or existing: Formating I am not so good at formatting myself, I thought best to leave it until last so I could get feedback on how it should look and help implementing it. Dependancy tree The dependancy tree is usually correct although I have noticed times where it doesnt detect all the dependancies that it should. I have only noticed this when there are circular dependancies so i think this might be the cause. It's not a big deal currently because makepkg -i installed repo deps for us which handles the repo deps for us and will get the correct ones. So yay might not list all the dependancies. but they will get installed so I consider this a visual bug. I have yet to see any circular dependancies in the AUR so I can not say what will happend but I#m guessing that it will break. Versioned packages/dependencies Targets and dependancies with version constriants such as 'linux>=4.1' will not be checked on the aur side of things but will be checked on the repo side. Ignorepkg/Ignoregroup Currently I do not handle this in any way but it shouldn't be too hard to implement. Conflict checking This is not currently implemented either Split Paclages Split packages are not Handles properly. If we only specify one package so install from a split package makepkg -i ends up installing them all anyway. If we specify more than one (n) package it will actually build the package base n times and reinstall every split package n times. Makepkg To get things working I decided to keep using the makepkg -i method. I plan to eventually replace this with a pacman -U based method. This should allow passing args such as --dbpath and --config to aur packages aswell as help solve some problems such as the split packages. Clean build I plan to improve the clean build choice to be a little more smart and instead of check if the directory exists, check if the package is already build and if so skip the build all together.
2018-01-17 22:48:23 +01:00
arguments.addTarget(remoteNames...)
err = install(arguments)
return err
2017-05-01 03:23:03 +02:00
}
2017-10-14 18:11:47 +02:00
// parseSource returns owner and repo from source
func parseSource(source string) (owner string, repo string) {
if !(strings.Contains(source, "git://") ||
strings.Contains(source, ".git") ||
strings.Contains(source, "git+https://")) {
return
}
2017-05-01 03:23:03 +02:00
split := strings.Split(source, "github.com/")
if len(split) > 1 {
secondSplit := strings.Split(split[1], "/")
if len(secondSplit) > 1 {
owner = secondSplit[0]
thirdSplit := strings.Split(secondSplit[1], ".git")
if len(thirdSplit) > 0 {
repo = thirdSplit[0]
}
}
}
return
}
2017-05-01 03:34:40 +02:00
2017-05-02 12:50:11 +02:00
func (info *Info) needsUpdate() bool {
var newRepo repo
var newBranches branches
if strings.HasSuffix(info.URL, "/branches") {
info.URL = info.URL[:len(info.URL)-9]
}
infoResp, infoErr := http.Get(info.URL)
if infoErr != nil {
fmt.Println(infoErr)
return false
}
defer infoResp.Body.Close()
infoBody, _ := ioutil.ReadAll(infoResp.Body)
var err = json.Unmarshal(infoBody, &newRepo)
2017-05-02 12:50:11 +02:00
if err != nil {
fmt.Printf("Cannot update '%v'\nError: %v\nStatus code: %v\nBody: %v\n",
info.Package, err, infoResp.StatusCode, string(infoBody))
return false
}
defaultBranch := newRepo.DefaultBranch
branchesURL := info.URL + "/branches"
branchResp, branchErr := http.Get(branchesURL)
if branchErr != nil {
fmt.Println(branchErr)
2017-05-02 12:50:11 +02:00
return false
}
defer branchResp.Body.Close()
branchBody, _ := ioutil.ReadAll(branchResp.Body)
err = json.Unmarshal(branchBody, &newBranches)
2017-05-02 12:50:11 +02:00
if err != nil {
2017-11-21 01:27:01 +01:00
fmt.Printf("Cannot update '%v'\nError: %v\nStatus code: %v\nBody: %v\n",
info.Package, err, branchResp.StatusCode, string(branchBody))
2017-05-02 12:50:11 +02:00
return false
}
for _, e := range newBranches {
if e.Name == defaultBranch {
return e.Commit.SHA != info.SHA
2017-05-02 12:50:11 +02:00
}
}
return false
}
func inStore(pkgName string) *Info {
for i, e := range savedInfo {
if pkgName == e.Package {
return &savedInfo[i]
}
}
return nil
}
2017-10-14 18:11:47 +02:00
// branchInfo updates saved information
func branchInfo(pkgName string, owner string, repoName string) (err error) {
2017-08-04 11:26:53 +02:00
updated = true
var newRepo repo
var newBranches branches
url := "https://api.github.com/repos/" + owner + "/" + repoName
repoResp, err := http.Get(url)
if err != nil {
return
}
defer repoResp.Body.Close()
_ = json.NewDecoder(repoResp.Body).Decode(&newRepo)
defaultBranch := newRepo.DefaultBranch
branchesUrl := url + "/branches"
branchResp, err := http.Get(branchesUrl)
2017-05-01 03:34:40 +02:00
if err != nil {
return
}
defer branchResp.Body.Close()
2017-05-01 03:34:40 +02:00
_ = json.NewDecoder(branchResp.Body).Decode(&newBranches)
2017-05-01 03:34:40 +02:00
packinfo := inStore(pkgName)
for _, e := range newBranches {
if e.Name == defaultBranch {
if packinfo != nil {
packinfo.Package = pkgName
2017-05-02 12:50:11 +02:00
packinfo.URL = url
packinfo.SHA = e.Commit.SHA
} else {
savedInfo = append(savedInfo, Info{Package: pkgName, URL: url, SHA: e.Commit.SHA})
}
}
}
2017-05-01 03:34:40 +02:00
return
}
2017-05-02 12:50:11 +02:00
func saveVCSInfo() error {
2017-08-04 11:26:53 +02:00
marshalledinfo, err := json.MarshalIndent(savedInfo, "", "\t")
2017-05-09 15:44:34 +02:00
if err != nil || string(marshalledinfo) == "null" {
return err
}
in, err := os.OpenFile(vcsFile, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
2017-05-02 12:50:11 +02:00
if err != nil {
return err
}
defer in.Close()
_, err = in.Write(marshalledinfo)
if err != nil {
return err
}
err = in.Sync()
return err
}