Commit 2782a668 authored by Mustafa Gezen's avatar Mustafa Gezen
Browse files

change module package, add gitlab tag workaround and add new directives

parent 0c150ed1
......@@ -3,7 +3,7 @@ package main
import (
"errors"
"fmt"
"github.com/mstg/srpmproc/internal"
"git.rockylinux.org/release-engineering/public/srpmproc/internal"
"github.com/spf13/cobra"
"io/ioutil"
"log"
......
......@@ -2,16 +2,16 @@ package main
import (
"fmt"
"git.rockylinux.org/release-engineering/public/srpmproc/internal/blob"
"git.rockylinux.org/release-engineering/public/srpmproc/internal/blob/gcs"
"git.rockylinux.org/release-engineering/public/srpmproc/internal/blob/s3"
"github.com/go-git/go-git/v5/plumbing/transport/ssh"
"github.com/mstg/srpmproc/internal/blob"
"github.com/mstg/srpmproc/internal/blob/gcs"
"github.com/mstg/srpmproc/internal/blob/s3"
"log"
"os/user"
"path/filepath"
"strings"
"github.com/mstg/srpmproc/internal"
"git.rockylinux.org/release-engineering/public/srpmproc/internal"
"github.com/spf13/cobra"
)
......
......@@ -40,7 +40,9 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/ghQa61ZWa/C2Aw3RkjiTBOix7dkqa1VLIs=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5VpdgMhJosfJnn5/FoN2SRZ4p7fJNX58YPaU=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZqLG4oE62mJzwPIB8+Tee4RNCL9ulrY=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
......@@ -242,6 +244,7 @@ github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:
github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
github.com/prometheus/common v0.4.0 h1:7etb9YClo3a6HjLzfl6rIQaU+FDfi0VSX39io3aQ+DM=
github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
......@@ -577,6 +580,7 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD
google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c=
google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc=
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
......
package directives
import (
"errors"
"fmt"
srpmprocpb "git.rockylinux.org/release-engineering/public/srpmproc/pb"
"github.com/go-git/go-git/v5"
"io/ioutil"
"os"
"path/filepath"
)
func add(cfg *srpmprocpb.Cfg, patchTree *git.Worktree, pushTree *git.Worktree) error {
for _, add := range cfg.Add {
filePath := checkAddPrefix(filepath.Base(add.File))
f, err := pushTree.Filesystem.OpenFile(filePath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
if err != nil {
return errors.New(fmt.Sprintf("COULD_NOT_OPEN_DESTINATION:%s", filePath))
}
fPatch, err := patchTree.Filesystem.OpenFile(add.File, os.O_RDONLY, 0644)
if err != nil {
return errors.New(fmt.Sprintf("COULD_NOT_OPEN_FROM:%s", add.File))
}
replacingBytes, err := ioutil.ReadAll(fPatch)
if err != nil {
return errors.New(fmt.Sprintf("COULD_NOT_READ_FROM:%s", add.File))
}
_, err = f.Write(replacingBytes)
if err != nil {
return errors.New(fmt.Sprintf("COULD_NOT_WRITE_DESTIONATION:%s", filePath))
}
}
return nil
}
package directives
import (
"errors"
"fmt"
srpmprocpb "git.rockylinux.org/release-engineering/public/srpmproc/pb"
"github.com/go-git/go-git/v5"
)
func del(cfg *srpmprocpb.Cfg, _ *git.Worktree, pushTree *git.Worktree) error {
for _, del := range cfg.Delete {
filePath := del.File
_, err := pushTree.Filesystem.Stat(filePath)
if err != nil {
return errors.New(fmt.Sprintf("FILE_DOES_NOT_EXIST:%s", filePath))
}
err = pushTree.Filesystem.Remove(filePath)
if err != nil {
return errors.New(fmt.Sprintf("COULD_NOT_DELETE_FILE:%s", filePath))
}
}
return nil
}
package directives
import (
"encoding/json"
srpmprocpb "git.rockylinux.org/release-engineering/public/srpmproc/pb"
"github.com/go-git/go-git/v5"
srpmprocpb "github.com/mstg/srpmproc/pb"
"log"
"os"
"path/filepath"
"strings"
)
......@@ -17,5 +20,34 @@ func checkAddPrefix(file string) string {
}
func Apply(cfg *srpmprocpb.Cfg, patchTree *git.Worktree, pushTree *git.Worktree) {
replace(cfg, patchTree, pushTree)
var errs []string
err := replace(cfg, patchTree, pushTree)
if err != nil {
errs = append(errs, err.Error())
}
err = del(cfg, patchTree, pushTree)
if err != nil {
errs = append(errs, err.Error())
}
err = add(cfg, patchTree, pushTree)
if err != nil {
errs = append(errs, err.Error())
}
err = patch(cfg, patchTree, pushTree)
if err != nil {
errs = append(errs, err.Error())
}
if len(errs) > 0 {
err := json.NewEncoder(os.Stdout).Encode(errs)
if err != nil {
log.Fatal(errs)
}
os.Exit(1)
}
}
package directives
import (
"bytes"
"errors"
"fmt"
srpmprocpb "git.rockylinux.org/release-engineering/public/srpmproc/pb"
"github.com/bluekeyes/go-gitdiff/gitdiff"
"github.com/go-git/go-git/v5"
"log"
)
func patch(cfg *srpmprocpb.Cfg, patchTree *git.Worktree, pushTree *git.Worktree) error {
for _, patch := range cfg.Patch {
patchFile, err := patchTree.Filesystem.Open(patch.File)
if err != nil {
return errors.New(fmt.Sprintf("COULD_NOT_OPEN_PATCH_FILE:%s", patch.File))
}
files, _, err := gitdiff.Parse(patchFile)
if err != nil {
log.Printf("could not parse patch file: %v", err)
return errors.New(fmt.Sprintf("COULD_NOT_PARSE_PATCH_FILE:%s", patch.File))
}
for _, patchedFile := range files {
srcPath := patchedFile.NewName
if !patch.Strict {
srcPath = checkAddPrefix(patchedFile.NewName)
}
var output bytes.Buffer
if !patchedFile.IsDelete && !patchedFile.IsNew {
patchSubjectFile, err := pushTree.Filesystem.Open(srcPath)
if err != nil {
return errors.New(fmt.Sprintf("COULD_NOT_OPEN_PATCH_SUBJECT:%s", srcPath))
}
err = gitdiff.NewApplier(patchSubjectFile).ApplyFile(&output, patchedFile)
if err != nil {
log.Printf("could not apply patch: %v", err)
return errors.New(fmt.Sprintf("COULD_NOT_APPLY_PATCH_WITH_SUBJECT:%s", srcPath))
}
}
oldName := patchedFile.OldName
if !patch.Strict {
oldName = checkAddPrefix(patchedFile.OldName)
}
_ = pushTree.Filesystem.Remove(oldName)
_ = pushTree.Filesystem.Remove(srcPath)
if patchedFile.IsNew {
newFile, err := pushTree.Filesystem.Create(srcPath)
if err != nil {
return errors.New(fmt.Sprintf("COULD_NOT_CREATE_NEW_FILE:%s", srcPath))
}
err = gitdiff.NewApplier(newFile).ApplyFile(&output, patchedFile)
if err != nil {
return errors.New(fmt.Sprintf("COULD_NOT_APPLY_PATCH_TO_NEW_FILE:%s", srcPath))
}
_, err = newFile.Write(output.Bytes())
if err != nil {
return errors.New(fmt.Sprintf("COULD_NOT_WRITE_TO_NEW_FILE:%s", srcPath))
}
_, err = pushTree.Add(srcPath)
if err != nil {
return errors.New(fmt.Sprintf("COULD_NOT_ADD_NEW_FILE_TO_GIT:%s", srcPath))
}
} else if !patchedFile.IsDelete {
newFile, err := pushTree.Filesystem.Create(srcPath)
if err != nil {
return errors.New(fmt.Sprintf("COULD_NOT_CREATE_POST_PATCH_FILE:%s", srcPath))
}
_, err = newFile.Write(output.Bytes())
if err != nil {
return errors.New(fmt.Sprintf("COULD_NOT_WRITE_POST_PATCH_FILE:%s", srcPath))
}
_, err = pushTree.Add(srcPath)
if err != nil {
return errors.New(fmt.Sprintf("COULD_NOT_ADD_POST_PATCH_FILE_TO_GIT:%s", srcPath))
}
} else {
_, err = pushTree.Remove(oldName)
if err != nil {
return errors.New(fmt.Sprintf("COULD_NOT_REMOVE_FILE_FROM_GIT:%s", oldName))
}
}
}
}
return nil
}
package directives
import (
"errors"
"fmt"
srpmprocpb "git.rockylinux.org/release-engineering/public/srpmproc/pb"
"github.com/go-git/go-git/v5"
srpmprocpb "github.com/mstg/srpmproc/pb"
"io/ioutil"
"log"
"os"
)
func replace(cfg *srpmprocpb.Cfg, patchTree *git.Worktree, pushTree *git.Worktree) {
func replace(cfg *srpmprocpb.Cfg, patchTree *git.Worktree, pushTree *git.Worktree) error {
for _, replace := range cfg.Replace {
filePath := checkAddPrefix(replace.File)
stat, err := pushTree.Filesystem.Stat(filePath)
if replace.File == "" || err != nil {
log.Fatalf("file to replace is invalid")
return errors.New(fmt.Sprintf("INVALID_FILE:%s", filePath))
}
err = pushTree.Filesystem.Remove(filePath)
if err != nil {
log.Fatalf("could not remove old file: %v", err)
return errors.New(fmt.Sprintf("COULD_NOT_REMOVE_OLD_FILE:%s", filePath))
}
f, err := pushTree.Filesystem.OpenFile(filePath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, stat.Mode())
if err != nil {
log.Fatalf("could not open replacement file: %v", err)
return errors.New(fmt.Sprintf("COULD_NOT_OPEN_REPLACEMENT:%s", filePath))
}
switch replacing := replace.Replacing.(type) {
case *srpmprocpb.Replace_WithFile:
fPatch, err := patchTree.Filesystem.OpenFile(replacing.WithFile, os.O_RDONLY, 0644)
if err != nil {
log.Fatalf("could not open replacing file: %v", err)
return errors.New(fmt.Sprintf("COULD_NOT_OPEN_REPLACING:%s", replacing.WithFile))
}
replacingBytes, err := ioutil.ReadAll(fPatch)
if err != nil {
log.Fatalf("could not read replacing file: %v", err)
return errors.New(fmt.Sprintf("COULD_NOT_READ_REPLACING:%s", replacing.WithFile))
}
_, err = f.Write(replacingBytes)
if err != nil {
log.Fatalf("could not write replacing file: %v", err)
return errors.New(fmt.Sprintf("COULD_NOT_WRITE_REPLACING:%s", replacing.WithFile))
}
break
case *srpmprocpb.Replace_WithInline:
_, err := f.Write([]byte(replacing.WithInline))
if err != nil {
log.Fatalf("could not write inline replacement: %v", err)
return errors.New(fmt.Sprintf("COULD_NOT_WRITE_INLINE:%s", filePath))
}
break
}
}
return nil
}
......@@ -60,10 +60,9 @@ func (g *GitMode) RetrieveSource(pd *ProcessData) *modeData {
}
err = remote.Fetch(&git.FetchOptions{
RemoteName: "upstream",
RefSpecs: []config.RefSpec{refspec},
Tags: git.AllTags,
Force: true,
RefSpecs: []config.RefSpec{refspec},
Tags: git.AllTags,
Force: true,
})
if err != nil {
log.Fatalf("could not fetch upstream: %v", err)
......@@ -71,20 +70,45 @@ func (g *GitMode) RetrieveSource(pd *ProcessData) *modeData {
var branches remoteTargetSlice
tagIter, err := repo.TagObjects()
if err != nil {
log.Fatalf("could not get tag objects: %v", err)
}
_ = tagIter.ForEach(func(tag *object.Tag) error {
log.Printf("tag: %s", tag.Name)
if strings.HasPrefix(tag.Name, fmt.Sprintf("imports/c%d", pd.Version)) {
tagAdd := func(tag *object.Tag) error {
if strings.HasPrefix(tag.Name, fmt.Sprintf("imports/%s%d", pd.ImportBranchPrefix, pd.Version)) {
log.Printf("tag: %s", tag.Name)
branches = append(branches, remoteTarget{
remote: fmt.Sprintf("refs/tags/%s", tag.Name),
when: tag.Tagger.When,
})
}
return nil
})
}
tagIter, err := repo.TagObjects()
if err != nil {
log.Fatalf("could not get tag objects: %v", err)
}
_ = tagIter.ForEach(tagAdd)
if len(branches) == 0 {
list, err := remote.List(&git.ListOptions{})
if err != nil {
log.Fatalf("could not list upstream: %v", err)
}
for _, ref := range list {
if ref.Hash().IsZero() {
continue
}
commit, err := repo.CommitObject(ref.Hash())
if err != nil {
log.Fatalf("could not get commit object: %v", err)
}
_ = tagAdd(&object.Tag{
Name: strings.TrimPrefix(string(ref.Name()), "refs/tags/"),
Tagger: commit.Committer,
})
}
}
sort.Sort(branches)
var sortedBranches []string
......@@ -125,7 +149,7 @@ func (g *GitMode) WriteSource(pd *ProcessData, md *modeData) {
Tags: git.AllTags,
Force: true,
})
if err != nil {
if err != nil && err != git.NoErrAlreadyUpToDate {
log.Fatalf("could not fetch upstream: %v", err)
}
......
......@@ -3,15 +3,15 @@ package internal
import (
"bytes"
"fmt"
"git.rockylinux.org/release-engineering/public/srpmproc/internal/directives"
"git.rockylinux.org/release-engineering/public/srpmproc/modulemd"
srpmprocpb "git.rockylinux.org/release-engineering/public/srpmproc/pb"
"github.com/bluekeyes/go-gitdiff/gitdiff"
"github.com/go-git/go-billy/v5/memfs"
"github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/config"
"github.com/go-git/go-git/v5/plumbing"
"github.com/go-git/go-git/v5/storage/memory"
"github.com/mstg/srpmproc/internal/directives"
"github.com/mstg/srpmproc/modulemd"
srpmprocpb "github.com/mstg/srpmproc/pb"
"google.golang.org/protobuf/encoding/prototext"
"io/ioutil"
"log"
......
......@@ -4,6 +4,7 @@ import (
"encoding/hex"
"encoding/json"
"fmt"
"git.rockylinux.org/release-engineering/public/srpmproc/internal/blob"
"github.com/cavaliercoder/go-rpm"
"github.com/go-git/go-billy/v5/memfs"
"github.com/go-git/go-git/v5"
......@@ -12,7 +13,6 @@ import (
"github.com/go-git/go-git/v5/plumbing/object"
"github.com/go-git/go-git/v5/plumbing/transport/ssh"
"github.com/go-git/go-git/v5/storage/memory"
"github.com/mstg/srpmproc/internal/blob"
"hash"
"io/ioutil"
"log"
......@@ -162,8 +162,8 @@ func ProcessRPM(pd *ProcessData) {
}
match := tagImportRegex.FindStringSubmatch(matchString)
md.pushBranch = pd.BranchPrefix + strings.TrimPrefix(match[2], "c")
newTag := "imports/" + pd.BranchPrefix + strings.TrimPrefix(match[1], "imports/c")
md.pushBranch = pd.BranchPrefix + strings.TrimPrefix(match[2], pd.ImportBranchPrefix)
newTag := "imports/" + pd.BranchPrefix + strings.TrimPrefix(match[1], "imports/"+pd.ImportBranchPrefix)
shouldContinue := true
for _, ignoredTag := range tagIgnoreList {
......
This diff is collapsed.
syntax = "proto3";
option go_package = "github.com/mstg/srpmproc/pb;srpmprocpb";
option go_package = "git.rockylinux.org/release-engineering/public/srpmproc/pb;srpmprocpb";
package srpmproc;
// Replace directive replaces literal files with other files.
// Replace directive replaces a file from the rpm repository
// with a file from the patch repository.
// Replacing content can either be inline or in the same patch-tree.
message Replace {
// required - replaced file
// Required - Replaced file
string file = 1;
oneof replacing {
// replace with in-tree file
// Replace with in-tree file
string with_file = 2;
// replace with inline content
// Replace with inline content
string with_inline = 3;
}
}
// Delete directive deletes literal files from the rpm repository.
// Won't delete from spec automatically.
// Use the `SpecChange` directive for that
message Delete {
// Required
string file = 1;
}
// Add directive adds a file from the patch repository to the rpm repository.
// The file is added in the `SOURCES` directory
// Won't add to spec automatically.
// Use the `SpecChange` directive for that
message Add {
// Required - file to add
string file = 1;
// Overrides file name if specified
string name = 2;
}
// Lookaside directive puts patched files in blob storage.
// If tar is true, the files will be put into a tarball and gzipped
message Lookaside {
// Required - List of files that should be stored in blob storage
repeated string files = 1;
// Whether files should be put into a tarball and gzipped
bool tar = 2;
}
// SpecChange directive makes it possible to execute certain
// plans against the package spec
message SpecChange {
// The FileOperation plan allows patchers to add or delete
// a file from the spec.
message FileOperation {
enum Type {
Unknown = 0;
Source = 1;
Patch = 2;
}
// File name
string file = 1;
// File type
Type type = 2;
oneof mode {
// Add won't add the file to the tree.
// Use the `Add` directive for that
bool add = 3;
// Delete won't delete the file from the tree.
// Use the `Delete` directive for that
bool delete = 4;
}
}
oneof operation {
FileOperation file_operation = 1;
}
}
message Patch {
// Path to patch file from repo root
string file = 1;
// Srpmproc adds `SOURCES/` to files in a diff
// without a prefix if strict is false.
// If strict is true, then that is disabled.
bool strict = 2;
}
message Cfg {
repeated Replace replace = 1;
repeated Delete delete = 2;
repeated Add add = 3;
repeated Lookaside lookaside = 4;
repeated SpecChange spec_change = 5;
repeated Patch patch = 6;
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment