Skip to content

GitLab

Commit 1dcceac6 authored by Mustafa Gezen's avatar Mustafa Gezen
Browse files

abstract modes and blob storage. support import from git.c.o

parent 777015bb
Hide whitespace changes
Inline Side-by-side
cmd/srpmproc/main.go
View file @ 1dcceac6
package main
import (
"cloud.google.com/go/storage"
"context"
"fmt"
"github.com/go-git/go-git/v5/plumbing/transport/ssh"
"github.com/mstg/srpmproc/internal/blob"
"github.com/mstg/srpmproc/internal/blob/gcs"
"github.com/mstg/srpmproc/internal/blob/s3"
"log"
"os/user"
"path/filepath"
"strings"
"github.com/mstg/srpmproc/internal"
......@@ -15,10 +20,12 @@ var (
sshKeyLocation string
sshUser string
upstreamPrefix string
branch string
gcsBucket string
version int
storageAddr string
gitCommitterName string
gitCommitterEmail string
modulePrefix string
rpmPrefix string
)
var root = &cobra.Command{
......@@ -27,28 +34,59 @@ var root = &cobra.Command{
}
func mn(_ *cobra.Command, _ []string) {
ctx := context.Background()
client, err := storage.NewClient(ctx)
if err != nil {
log.Fatalf("could not create gcloud client: %v", err)
switch version {
case 8:
break
default:
log.Fatalf("unsupported upstream version %d", version)
}
var importer internal.ImportMode
var blobStorage blob.Storage
if strings.HasPrefix(storageAddr, "gs://") {
blobStorage = gcs.New(strings.Replace(storageAddr, "gs://", "", 1))
} else if strings.HasPrefix(storageAddr, "s3://") {
blobStorage = s3.New(strings.Replace(storageAddr, "s3://", "", 1))
} else {
log.Fatalf("invalid blob storage")
}
sourceRpmLocation := ""
if strings.HasPrefix(sourceRpm, "file://") {
sourceRpmLocation = strings.TrimPrefix(sourceRpm, "file://")
importer = &internal.SrpmMode{}
} else {
log.Fatal("non-local SRPMs are currently not supported")
sourceRpmLocation = fmt.Sprintf("%s/%s", rpmPrefix, sourceRpm)
importer = &internal.GitMode{}
}
lastKeyLocation := sshKeyLocation
if lastKeyLocation == "" {
usr, err := user.Current()
if err != nil {
log.Fatalf("could not get user: %v", err)
}
lastKeyLocation = filepath.Join(usr.HomeDir, ".ssh/id_rsa")
}
// create ssh key authenticator
authenticator, err := ssh.NewPublicKeysFromFile(sshUser, lastKeyLocation, "")
if err != nil {
log.Fatalf("could not get git authenticator: %v", err)
}
internal.ProcessRPM(&internal.ProcessData{
Importer: importer,
RpmLocation: sourceRpmLocation,
UpstreamPrefix: upstreamPrefix,
SshKeyLocation: sshKeyLocation,
SshUser: sshUser,
Branch: branch,
Bucket: client.Bucket(gcsBucket),
Version: version,
BlobStorage: blobStorage,
GitCommitterName: gitCommitterName,
GitCommitterEmail: gitCommitterEmail,
ModulePrefix: modulePrefix,
Authenticator: authenticator,
})
}
......@@ -57,15 +95,17 @@ func main() {
_ = root.MarkFlagRequired("source-rpm")
root.Flags().StringVar(&upstreamPrefix, "upstream-prefix", "", "Upstream git repository prefix")
_ = root.MarkFlagRequired("upstream-prefix")
root.Flags().StringVar(&branch, "branch", "", "Upstream branch")
_ = root.MarkFlagRequired("branch")
root.Flags().StringVar(&gcsBucket, "gcs-bucket", "", "Bucket to use as blob storage")
_ = root.MarkFlagRequired("gcs-bucket")
root.Flags().StringVar(&gitCommitterName, "git-committer-name", "distrobuild-bot", "Name of committer")
root.Flags().StringVar(&gitCommitterEmail, "git-committer-email", "mustafa+distrobuild@bycrates.com", "Email of committer")
root.Flags().IntVar(&version, "version", 0, "Upstream version")
_ = root.MarkFlagRequired("version")
root.Flags().StringVar(&storageAddr, "storage-addr", "", "Bucket to use as blob storage")
_ = root.MarkFlagRequired("storage-addr")
root.Flags().StringVar(&sshKeyLocation, "ssh-key-location", "", "Location of the SSH key to use to authenticate against upstream")
root.Flags().StringVar(&sshUser, "ssh-user", "git", "SSH User")
root.Flags().StringVar(&gitCommitterName, "git-committer-name", "distrobuild-bot", "Name of committer")
root.Flags().StringVar(&gitCommitterEmail, "git-committer-email", "mustafa+distrobuild@bycrates.com", "Email of committer")
root.Flags().StringVar(&modulePrefix, "module-prefix", "https://git.centos.org/modules", "Where to retrieve modules if exists. Only used when source-rpm is a git repo")
root.Flags().StringVar(&rpmPrefix, "rpm-prefix", "https://git.centos.org/rpms", "Where to retrieve SRPM content. Only used when source-rpm is not a local file")
if err := root.Execute(); err != nil {
log.Fatal(err)
......
go.sum
View file @ 1dcceac6
......@@ -47,6 +47,8 @@ github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hC
github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
github.com/aws/aws-sdk-go v1.36.12 h1:YJpKFEMbqEoo+incs5qMe61n1JH3o4O1IMkMexLzJG8=
github.com/aws/aws-sdk-go v1.36.12/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
......@@ -188,6 +190,9 @@ github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANyt
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A=
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo=
github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
......@@ -228,6 +233,7 @@ github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FI
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
......@@ -369,6 +375,8 @@ golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81R
golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20200904194848-62affa334b73 h1:MXfv8rhZWmFeqX3GNZRsd6vOLoaCHjYEX3qkRo3YBUA=
golang.org/x/net v0.0.0-20200904194848-62affa334b73/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20201110031124-69a78807bb2b h1:uwuIcX0g4Yl1NC5XAz37xsr2lTtcqevgzYNVt49waME=
golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
......@@ -419,6 +427,8 @@ golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20200828194041-157a740278f4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f h1:Fqb3ao1hUmOR3GkUOg/Y+BadLwykBIzs5q8Ez2SbHyc=
golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f h1:+Nyd8tzPX9R7BWHguqsrbFdRx3WQ/1ib8I44HXV5yTA=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
......
internal/blob/blob.go 0 → 100644
View file @ 1dcceac6
package blob
type Storage interface {
Write(path string, content []byte)
}
internal/blob/gcs/gcs.go 0 → 100644
View file @ 1dcceac6
package gcs
import (
"cloud.google.com/go/storage"
"context"
"log"
)
type GCS struct {
bucket *storage.BucketHandle
}
func New(name string) *GCS {
ctx := context.Background()
client, err := storage.NewClient(ctx)
if err != nil {
log.Fatalf("could not create gcloud client: %v", err)
}
return &GCS{
bucket: client.Bucket(name),
}
}
func (g *GCS) Write(path string, content []byte) {
ctx := context.Background()
obj := g.bucket.Object(path)
w := obj.NewWriter(ctx)
_, err := w.Write(content)
if err != nil {
log.Fatalf("could not write file to gcs: %v", err)
}
// Close, just like writing a file.
if err := w.Close(); err != nil {
log.Fatalf("could not close gcs writer to source: %v", err)
}
}
internal/blob/s3/s3.go 0 → 100644
View file @ 1dcceac6
package s3
import (
"bytes"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3/s3manager"
"log"
)
type S3 struct {
bucket string
uploader *s3manager.Uploader
}
func New(name string) *S3 {
sess := session.Must(session.NewSession())
uploader := s3manager.NewUploader(sess)
return &S3{
bucket: name,
uploader: uploader,
}
}
func (s *S3) Write(path string, content []byte) {
buf := bytes.NewBuffer(content)
_, err := s.uploader.Upload(&s3manager.UploadInput{
Bucket: aws.String(s.bucket),
Key: aws.String(path),
Body: buf,
})
if err != nil {
log.Fatalf("failed to upload file to s3, %v", err)
}
}
internal/git.go 0 → 100644
View file @ 1dcceac6
package internal
import (
"fmt"
"github.com/go-git/go-billy/v5/memfs"
"github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/config"
"github.com/go-git/go-git/v5/plumbing"
"github.com/go-git/go-git/v5/storage/memory"
"io/ioutil"
"log"
"net/http"
"path/filepath"
"sort"
"strings"
)
type GitMode struct{}
func (g *GitMode) RetrieveSource(pd *ProcessData) *modeData {
repo, err := git.Init(memory.NewStorage(), memfs.New())
if err != nil {
log.Fatalf("could not init git repo: %v", err)
}
w, err := repo.Worktree()
if err != nil {
log.Fatalf("could not get worktree: %v", err)
}
refspec := config.RefSpec("+refs/heads/*:refs/remotes/*")
remote, err := repo.CreateRemote(&config.RemoteConfig{
Name: "upstream",
URLs: []string{pd.RpmLocation},
Fetch: []config.RefSpec{refspec},
})
if err != nil {
log.Fatalf("could not create remote: %v", err)
}
list, err := remote.List(&git.ListOptions{})
if err != nil {
log.Fatalf("could not list remote: %v", err)
}
var branches []string
for _, ref := range list {
log.Println(ref.String())
name := string(ref.Name())
if strings.HasPrefix(name, fmt.Sprintf("refs/tags/imports/c%d", pd.Version)) {
branches = append(branches, name)
}
}
sort.Strings(branches)
return &modeData{
repo: repo,
worktree: w,
rpmFile: createPackageFile(filepath.Base(pd.RpmLocation)),
fileWrites: nil,
branches: branches,
}
}
func (g *GitMode) WriteSource(md *modeData) {
remote, err := md.repo.Remote("upstream")
if err != nil {
log.Fatalf("could not get upstream remote: %v", err)
}
match := tagImportRegex.FindStringSubmatch(md.tagBranch)
refspec := config.RefSpec(fmt.Sprintf("+refs/heads/%s:%s", match[2], md.tagBranch))
err = remote.Fetch(&git.FetchOptions{
RemoteName: "upstream",
RefSpecs: []config.RefSpec{refspec},
Tags: git.AllTags,
Force: true,
})
if err != nil {
log.Fatalf("could not fetch upstream: %v", err)
}
err = md.worktree.Checkout(&git.CheckoutOptions{
Branch: plumbing.ReferenceName(md.tagBranch),
Force: true,
})
if err != nil {
log.Fatalf("could not checkout source from git: %v", err)
}
_, err = md.worktree.Add(".")
if err != nil {
log.Fatalf("could not add worktree: %v", err)
}
metadataFile, err := md.worktree.Filesystem.Open(fmt.Sprintf(".%s.metadata", md.rpmFile.Name()))
if err != nil {
log.Fatalf("could not open metadata file: %v", err)
}
fileBytes, err := ioutil.ReadAll(metadataFile)
if err != nil {
log.Fatalf("could not read metadata file: %v", err)
}
client := &http.Client{
Transport: &http.Transport{
DisableCompression: false,
},
}
fileContent := strings.Split(string(fileBytes), "\n")
for _, line := range fileContent {
if strings.TrimSpace(line) == "" {
continue
}
lineInfo := strings.Split(line, " ")
hash := lineInfo[0]
path := lineInfo[1]
url := fmt.Sprintf("https://git.centos.org/sources/%s/%s/%s", md.rpmFile.Name(), match[2], hash)
log.Printf("downloading %s", url)
req, err := http.NewRequest("GET", url, nil)
if err != nil {
log.Fatalf("could not create new http request: %v", err)
}
req.Header.Set("Accept-Encoding", "*")
resp, err := client.Do(req)
if err != nil {
log.Fatalf("could not download dist-git file: %v", err)
}
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Fatalf("could not read the whole dist-git file: %v", err)
}
err = resp.Body.Close()
if err != nil {
log.Fatalf("could not close body handle: %v", err)
}
f, err := md.worktree.Filesystem.Create(path)
if err != nil {
log.Fatalf("could not open file pointer: %v", err)
}
hasher := compareHash(body, hash)
if hasher == nil {
log.Fatal("checksum in metadata does not match dist-git file")
}
md.sourcesToIgnore = append(md.sourcesToIgnore, &ignoredSource{
name: filepath.Base(path),
hashFunction: hasher,
})
_, err = f.Write(body)
if err != nil {
log.Fatalf("could not copy dist-git file to in-tree: %v", err)
}
_ = f.Close()
}
}
func (g *GitMode) PostProcess(md *modeData) {
for _, source := range md.sourcesToIgnore {
err := md.worktree.Filesystem.Remove(filepath.Join("SOURCES", source.name))
if err != nil {
log.Fatalf("could not remove dist-git file: %v", err)
}
}
_, err := md.worktree.Add(".")
if err != nil {
log.Fatalf("could not add git sources: %v", err)
}
}
func (g *GitMode) ImportName(_ *ProcessData, md *modeData) string {
match := tagImportRegex.FindStringSubmatch(md.tagBranch)
return match[3]
}
internal/gitrpm.go 0 → 100644
View file @ 1dcceac6
package internal
import (
"github.com/cavaliercoder/go-rpm"
)
// TODO: ugly hack, should create an interface
// since GitMode does not parse an RPM file, we just mimick
// the headers of an actual file to reuse rpmFile.Name()
func createPackageFile(name string) *rpm.PackageFile {
return &rpm.PackageFile{
Lead: rpm.Lead{},
Headers: []rpm.Header{
{},
{
Version: 0,
IndexCount: 1,
Length: 1,
Indexes: []rpm.IndexEntry{
{
Tag: 1000,
Type: rpm.IndexDataTypeStringArray,
Offset: 0,
ItemCount: 1,
Value: []string{name},
},
},
},
},
}
}
internal/mode.go 0 → 100644
View file @ 1dcceac6
package internal
type ImportMode interface {
RetrieveSource(pd *ProcessData) *modeData
WriteSource(md *modeData)
PostProcess(md *modeData)
ImportName(pd *ProcessData, md *modeData) string
}
internal/patch.go 0 → 100644
View file @ 1dcceac6
package internal
import (
"bytes"
"github.com/bluekeyes/go-gitdiff/gitdiff"
"github.com/go-git/go-git/v5"
"log"
"path/filepath"
"strings"
)
func srpmPatches(w *git.Worktree) {
// check SRPM patches
_, err := w.Filesystem.Stat("ROCKY/SRPM")
if err == nil {
// iterate through patches
infos, err := w.Filesystem.ReadDir("ROCKY/SRPM")
if err != nil {
log.Fatalf("could not walk patches: %v", err)
}
for _, info := range infos {
// can only process .patch files
if !strings.HasSuffix(info.Name(), ".patch") {
continue
}
log.Printf("applying %s", info.Name())
filePath := filepath.Join("ROCKY/SRPM", info.Name())
patch, err := w.Filesystem.Open(filePath)
if err != nil {
log.Fatalf("could not open patch file %s: %v", info.Name(), err)
}
files, _, err := gitdiff.Parse(patch)
if err != nil {
log.Fatalf("could not parse patch file: %v", err)
}
for _, patchedFile := range files {
srcPath := patchedFile.NewName
if !strings.HasPrefix(srcPath, "SPECS") {
srcPath = filepath.Join("SOURCES", patchedFile.NewName)
}
var output bytes.Buffer
if !patchedFile.IsDelete && !patchedFile.IsNew {
patchSubjectFile, err := w.Filesystem.Open(srcPath)
if err != nil {
log.Fatalf("could not open patch subject: %v", err)
}
err = gitdiff.NewApplier(patchSubjectFile).ApplyFile(&output, patchedFile)
if err != nil {
log.Fatalf("could not apply patch: %v", err)
}
}
oldName := filepath.Join("SOURCES", patchedFile.OldName)
_ = w.Filesystem.Remove(oldName)
_ = w.Filesystem.Remove(srcPath)
if patchedFile.IsNew {
newFile, err := w.Filesystem.Create(srcPath)
if err != nil {
log.Fatalf("could not create new file: %v", err)
}
err = gitdiff.NewApplier(newFile).ApplyFile(&output, patchedFile)
if err != nil {
log.Fatalf("could not apply patch: %v", err)
}
_, err = newFile.Write(output.Bytes())
if err != nil {
log.Fatalf("could not write post-patch file: %v", err)
}
_, err = w.Add(srcPath)
if err != nil {
log.Fatalf("could not add file %s to git: %v", srcPath, err)
}
log.Printf("git add %s", srcPath)
} else if !patchedFile.IsDelete {
newFile, err := w.Filesystem.Create(srcPath)
if err != nil {
log.Fatalf("could not create post-patch file: %v", err)
}
_, err = newFile.Write(output.Bytes())
if err != nil {
log.Fatalf("could not write post-patch file: %v", err)
}
_, err = w.Add(srcPath)
if err != nil {
log.Fatalf("could not add file %s to git: %v", srcPath, err)
}
log.Printf("git add %s", srcPath)
} else {
_, err = w.Remove(oldName)
if err != nil {
log.Fatalf("could not remove file %s to git: %v", oldName, err)
}
log.Printf("git rm %s", oldName)
}
}
_, err = w.Add(filePath)
if err != nil {
log.Fatalf("could not add file %s to git: %v", filePath, err)
}
log.Printf("git add %s", filePath)
}
}
}
func executePatches(w *git.Worktree) {
// check if patches exist
_, err := w.Filesystem.Stat("ROCKY")
if err == nil {
srpmPatches(w)
}