Commit 1dcceac6 authored by Mustafa Gezen's avatar Mustafa Gezen
Browse files

abstract modes and blob storage. support import from git.c.o

parent 777015bb
package main
import (
"cloud.google.com/go/storage"
"context"
"fmt"
"github.com/go-git/go-git/v5/plumbing/transport/ssh"
"github.com/mstg/srpmproc/internal/blob"
"github.com/mstg/srpmproc/internal/blob/gcs"
"github.com/mstg/srpmproc/internal/blob/s3"
"log"
"os/user"
"path/filepath"
"strings"
"github.com/mstg/srpmproc/internal"
......@@ -15,10 +20,12 @@ var (
sshKeyLocation string
sshUser string
upstreamPrefix string
branch string
gcsBucket string
version int
storageAddr string
gitCommitterName string
gitCommitterEmail string
modulePrefix string
rpmPrefix string
)
var root = &cobra.Command{
......@@ -27,28 +34,59 @@ var root = &cobra.Command{
}
func mn(_ *cobra.Command, _ []string) {
ctx := context.Background()
client, err := storage.NewClient(ctx)
if err != nil {
log.Fatalf("could not create gcloud client: %v", err)
switch version {
case 8:
break
default:
log.Fatalf("unsupported upstream version %d", version)
}
var importer internal.ImportMode
var blobStorage blob.Storage
if strings.HasPrefix(storageAddr, "gs://") {
blobStorage = gcs.New(strings.Replace(storageAddr, "gs://", "", 1))
} else if strings.HasPrefix(storageAddr, "s3://") {
blobStorage = s3.New(strings.Replace(storageAddr, "s3://", "", 1))
} else {
log.Fatalf("invalid blob storage")
}
sourceRpmLocation := ""
if strings.HasPrefix(sourceRpm, "file://") {
sourceRpmLocation = strings.TrimPrefix(sourceRpm, "file://")
importer = &internal.SrpmMode{}
} else {
log.Fatal("non-local SRPMs are currently not supported")
sourceRpmLocation = fmt.Sprintf("%s/%s", rpmPrefix, sourceRpm)
importer = &internal.GitMode{}
}
lastKeyLocation := sshKeyLocation
if lastKeyLocation == "" {
usr, err := user.Current()
if err != nil {
log.Fatalf("could not get user: %v", err)
}
lastKeyLocation = filepath.Join(usr.HomeDir, ".ssh/id_rsa")
}
// create ssh key authenticator
authenticator, err := ssh.NewPublicKeysFromFile(sshUser, lastKeyLocation, "")
if err != nil {
log.Fatalf("could not get git authenticator: %v", err)
}
internal.ProcessRPM(&internal.ProcessData{
Importer: importer,
RpmLocation: sourceRpmLocation,
UpstreamPrefix: upstreamPrefix,
SshKeyLocation: sshKeyLocation,
SshUser: sshUser,
Branch: branch,
Bucket: client.Bucket(gcsBucket),
Version: version,
BlobStorage: blobStorage,
GitCommitterName: gitCommitterName,
GitCommitterEmail: gitCommitterEmail,
ModulePrefix: modulePrefix,
Authenticator: authenticator,
})
}
......@@ -57,15 +95,17 @@ func main() {
_ = root.MarkFlagRequired("source-rpm")
root.Flags().StringVar(&upstreamPrefix, "upstream-prefix", "", "Upstream git repository prefix")
_ = root.MarkFlagRequired("upstream-prefix")
root.Flags().StringVar(&branch, "branch", "", "Upstream branch")
_ = root.MarkFlagRequired("branch")
root.Flags().StringVar(&gcsBucket, "gcs-bucket", "", "Bucket to use as blob storage")
_ = root.MarkFlagRequired("gcs-bucket")
root.Flags().StringVar(&gitCommitterName, "git-committer-name", "distrobuild-bot", "Name of committer")
root.Flags().StringVar(&gitCommitterEmail, "git-committer-email", "mustafa+distrobuild@bycrates.com", "Email of committer")
root.Flags().IntVar(&version, "version", 0, "Upstream version")
_ = root.MarkFlagRequired("version")
root.Flags().StringVar(&storageAddr, "storage-addr", "", "Bucket to use as blob storage")
_ = root.MarkFlagRequired("storage-addr")
root.Flags().StringVar(&sshKeyLocation, "ssh-key-location", "", "Location of the SSH key to use to authenticate against upstream")
root.Flags().StringVar(&sshUser, "ssh-user", "git", "SSH User")
root.Flags().StringVar(&gitCommitterName, "git-committer-name", "distrobuild-bot", "Name of committer")
root.Flags().StringVar(&gitCommitterEmail, "git-committer-email", "mustafa+distrobuild@bycrates.com", "Email of committer")
root.Flags().StringVar(&modulePrefix, "module-prefix", "https://git.centos.org/modules", "Where to retrieve modules if exists. Only used when source-rpm is a git repo")
root.Flags().StringVar(&rpmPrefix, "rpm-prefix", "https://git.centos.org/rpms", "Where to retrieve SRPM content. Only used when source-rpm is not a local file")
if err := root.Execute(); err != nil {
log.Fatal(err)
......
......@@ -47,6 +47,8 @@ github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hC
github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
github.com/aws/aws-sdk-go v1.36.12 h1:YJpKFEMbqEoo+incs5qMe61n1JH3o4O1IMkMexLzJG8=
github.com/aws/aws-sdk-go v1.36.12/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
......@@ -188,6 +190,9 @@ github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANyt
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A=
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo=
github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
......@@ -228,6 +233,7 @@ github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FI
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
......@@ -369,6 +375,8 @@ golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81R
golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20200904194848-62affa334b73 h1:MXfv8rhZWmFeqX3GNZRsd6vOLoaCHjYEX3qkRo3YBUA=
golang.org/x/net v0.0.0-20200904194848-62affa334b73/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20201110031124-69a78807bb2b h1:uwuIcX0g4Yl1NC5XAz37xsr2lTtcqevgzYNVt49waME=
golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
......@@ -419,6 +427,8 @@ golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20200828194041-157a740278f4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f h1:Fqb3ao1hUmOR3GkUOg/Y+BadLwykBIzs5q8Ez2SbHyc=
golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f h1:+Nyd8tzPX9R7BWHguqsrbFdRx3WQ/1ib8I44HXV5yTA=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
......
package blob
type Storage interface {
Write(path string, content []byte)
}
package gcs
import (
"cloud.google.com/go/storage"
"context"
"log"
)
type GCS struct {
bucket *storage.BucketHandle
}
func New(name string) *GCS {
ctx := context.Background()
client, err := storage.NewClient(ctx)
if err != nil {
log.Fatalf("could not create gcloud client: %v", err)
}
return &GCS{
bucket: client.Bucket(name),
}
}
func (g *GCS) Write(path string, content []byte) {
ctx := context.Background()
obj := g.bucket.Object(path)
w := obj.NewWriter(ctx)
_, err := w.Write(content)
if err != nil {
log.Fatalf("could not write file to gcs: %v", err)
}
// Close, just like writing a file.
if err := w.Close(); err != nil {
log.Fatalf("could not close gcs writer to source: %v", err)
}
}
package s3
import (
"bytes"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3/s3manager"
"log"
)
type S3 struct {
bucket string
uploader *s3manager.Uploader
}
func New(name string) *S3 {
sess := session.Must(session.NewSession())
uploader := s3manager.NewUploader(sess)
return &S3{
bucket: name,
uploader: uploader,
}
}
func (s *S3) Write(path string, content []byte) {
buf := bytes.NewBuffer(content)
_, err := s.uploader.Upload(&s3manager.UploadInput{
Bucket: aws.String(s.bucket),
Key: aws.String(path),
Body: buf,
})
if err != nil {
log.Fatalf("failed to upload file to s3, %v", err)
}
}
package internal
import (
"fmt"
"github.com/go-git/go-billy/v5/memfs"
"github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/config"
"github.com/go-git/go-git/v5/plumbing"
"github.com/go-git/go-git/v5/storage/memory"
"io/ioutil"
"log"
"net/http"
"path/filepath"
"sort"
"strings"
)
type GitMode struct{}
func (g *GitMode) RetrieveSource(pd *ProcessData) *modeData {
repo, err := git.Init(memory.NewStorage(), memfs.New())
if err != nil {
log.Fatalf("could not init git repo: %v", err)
}
w, err := repo.Worktree()
if err != nil {
log.Fatalf("could not get worktree: %v", err)
}
refspec := config.RefSpec("+refs/heads/*:refs/remotes/*")
remote, err := repo.CreateRemote(&config.RemoteConfig{
Name: "upstream",
URLs: []string{pd.RpmLocation},
Fetch: []config.RefSpec{refspec},
})
if err != nil {
log.Fatalf("could not create remote: %v", err)
}
list, err := remote.List(&git.ListOptions{})
if err != nil {
log.Fatalf("could not list remote: %v", err)
}
var branches []string
for _, ref := range list {
log.Println(ref.String())
name := string(ref.Name())
if strings.HasPrefix(name, fmt.Sprintf("refs/tags/imports/c%d", pd.Version)) {
branches = append(branches, name)
}
}
sort.Strings(branches)
return &modeData{
repo: repo,
worktree: w,
rpmFile: createPackageFile(filepath.Base(pd.RpmLocation)),
fileWrites: nil,
branches: branches,
}
}
func (g *GitMode) WriteSource(md *modeData) {
remote, err := md.repo.Remote("upstream")
if err != nil {
log.Fatalf("could not get upstream remote: %v", err)
}
match := tagImportRegex.FindStringSubmatch(md.tagBranch)
refspec := config.RefSpec(fmt.Sprintf("+refs/heads/%s:%s", match[2], md.tagBranch))
err = remote.Fetch(&git.FetchOptions{
RemoteName: "upstream",
RefSpecs: []config.RefSpec{refspec},
Tags: git.AllTags,
Force: true,
})
if err != nil {
log.Fatalf("could not fetch upstream: %v", err)
}
err = md.worktree.Checkout(&git.CheckoutOptions{
Branch: plumbing.ReferenceName(md.tagBranch),
Force: true,
})
if err != nil {
log.Fatalf("could not checkout source from git: %v", err)
}
_, err = md.worktree.Add(".")
if err != nil {
log.Fatalf("could not add worktree: %v", err)
}
metadataFile, err := md.worktree.Filesystem.Open(fmt.Sprintf(".%s.metadata", md.rpmFile.Name()))
if err != nil {
log.Fatalf("could not open metadata file: %v", err)
}
fileBytes, err := ioutil.ReadAll(metadataFile)
if err != nil {
log.Fatalf("could not read metadata file: %v", err)
}
client := &http.Client{
Transport: &http.Transport{
DisableCompression: false,
},
}
fileContent := strings.Split(string(fileBytes), "\n")
for _, line := range fileContent {
if strings.TrimSpace(line) == "" {
continue
}
lineInfo := strings.Split(line, " ")
hash := lineInfo[0]
path := lineInfo[1]
url := fmt.Sprintf("https://git.centos.org/sources/%s/%s/%s", md.rpmFile.Name(), match[2], hash)
log.Printf("downloading %s", url)
req, err := http.NewRequest("GET", url, nil)
if err != nil {
log.Fatalf("could not create new http request: %v", err)
}
req.Header.Set("Accept-Encoding", "*")
resp, err := client.Do(req)
if err != nil {
log.Fatalf("could not download dist-git file: %v", err)
}
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Fatalf("could not read the whole dist-git file: %v", err)
}
err = resp.Body.Close()
if err != nil {
log.Fatalf("could not close body handle: %v", err)
}
f, err := md.worktree.Filesystem.Create(path)
if err != nil {
log.Fatalf("could not open file pointer: %v", err)
}
hasher := compareHash(body, hash)
if hasher == nil {
log.Fatal("checksum in metadata does not match dist-git file")
}
md.sourcesToIgnore = append(md.sourcesToIgnore, &ignoredSource{
name: filepath.Base(path),
hashFunction: hasher,
})
_, err = f.Write(body)
if err != nil {
log.Fatalf("could not copy dist-git file to in-tree: %v", err)
}
_ = f.Close()
}
}
func (g *GitMode) PostProcess(md *modeData) {
for _, source := range md.sourcesToIgnore {
err := md.worktree.Filesystem.Remove(filepath.Join("SOURCES", source.name))
if err != nil {
log.Fatalf("could not remove dist-git file: %v", err)
}
}
_, err := md.worktree.Add(".")
if err != nil {
log.Fatalf("could not add git sources: %v", err)
}
}
func (g *GitMode) ImportName(_ *ProcessData, md *modeData) string {
match := tagImportRegex.FindStringSubmatch(md.tagBranch)
return match[3]
}
package internal
import (
"github.com/cavaliercoder/go-rpm"
)
// TODO: ugly hack, should create an interface
// since GitMode does not parse an RPM file, we just mimick
// the headers of an actual file to reuse rpmFile.Name()
func createPackageFile(name string) *rpm.PackageFile {
return &rpm.PackageFile{
Lead: rpm.Lead{},
Headers: []rpm.Header{
{},
{
Version: 0,
IndexCount: 1,
Length: 1,
Indexes: []rpm.IndexEntry{
{
Tag: 1000,
Type: rpm.IndexDataTypeStringArray,
Offset: 0,
ItemCount: 1,
Value: []string{name},
},
},
},
},
}
}
package internal
type ImportMode interface {
RetrieveSource(pd *ProcessData) *modeData
WriteSource(md *modeData)
PostProcess(md *modeData)
ImportName(pd *ProcessData, md *modeData) string
}
package internal
import (
"bytes"
"github.com/bluekeyes/go-gitdiff/gitdiff"
"github.com/go-git/go-git/v5"
"log"
"path/filepath"
"strings"
)
func srpmPatches(w *git.Worktree) {
// check SRPM patches
_, err := w.Filesystem.Stat("ROCKY/SRPM")
if err == nil {
// iterate through patches
infos, err := w.Filesystem.ReadDir("ROCKY/SRPM")
if err != nil {
log.Fatalf("could not walk patches: %v", err)
}
for _, info := range infos {
// can only process .patch files
if !strings.HasSuffix(info.Name(), ".patch") {
continue
}
log.Printf("applying %s", info.Name())
filePath := filepath.Join("ROCKY/SRPM", info.Name())
patch, err := w.Filesystem.Open(filePath)
if err != nil {
log.Fatalf("could not open patch file %s: %v", info.Name(), err)
}
files, _, err := gitdiff.Parse(patch)
if err != nil {
log.Fatalf("could not parse patch file: %v", err)
}
for _, patchedFile := range files {
srcPath := patchedFile.NewName
if !strings.HasPrefix(srcPath, "SPECS") {
srcPath = filepath.Join("SOURCES", patchedFile.NewName)
}
var output bytes.Buffer
if !patchedFile.IsDelete && !patchedFile.IsNew {
patchSubjectFile, err := w.Filesystem.Open(srcPath)
if err != nil {
log.Fatalf("could not open patch subject: %v", err)
}
err = gitdiff.NewApplier(patchSubjectFile).ApplyFile(&output, patchedFile)
if err != nil {
log.Fatalf("could not apply patch: %v", err)
}
}
oldName := filepath.Join("SOURCES", patchedFile.OldName)
_ = w.Filesystem.Remove(oldName)
_ = w.Filesystem.Remove(srcPath)
if patchedFile.IsNew {
newFile, err := w.Filesystem.Create(srcPath)
if err != nil {
log.Fatalf("could not create new file: %v", err)
}
err = gitdiff.NewApplier(newFile).ApplyFile(&output, patchedFile)
if err != nil {
log.Fatalf("could not apply patch: %v", err)
}
_, err = newFile.Write(output.Bytes())
if err != nil {
log.Fatalf("could not write post-patch file: %v", err)
}
_, err = w.Add(srcPath)
if err != nil {
log.Fatalf("could not add file %s to git: %v", srcPath, err)
}
log.Printf("git add %s", srcPath)
} else if !patchedFile.IsDelete {
newFile, err := w.Filesystem.Create(srcPath)
if err != nil {
log.Fatalf("could not create post-patch file: %v", err)
}
_, err = newFile.Write(output.Bytes())
if err != nil {
log.Fatalf("could not write post-patch file: %v", err)
}
_, err = w.Add(srcPath)
if err != nil {
log.Fatalf("could not add file %s to git: %v", srcPath, err)
}
log.Printf("git add %s", srcPath)
} else {
_, err = w.Remove(oldName)
if err != nil {
log.Fatalf("could not remove file %s to git: %v", oldName, err)
}
log.Printf("git rm %s", oldName)
}
}
_, err = w.Add(filePath)
if err != nil {
log.Fatalf("could not add file %s to git: %v", filePath, err)
}
log.Printf("git add %s", filePath)
}
}
}
func executePatches(w *git.Worktree) {
// check if patches exist
_, err := w.Filesystem.Stat("ROCKY")
if err == nil {
srpmPatches(w)
}