Commit 01380b9b authored by Mustafa Gezen's avatar Mustafa Gezen
Browse files

messy code ahead. implement new directives (spec_change) and restructure code...

messy code ahead. implement new directives (spec_change) and restructure code to prevent import cycles
parent 2782a668
......@@ -3,10 +3,17 @@ package main
import (
"fmt"
"git.rockylinux.org/release-engineering/public/srpmproc/internal/blob"
"git.rockylinux.org/release-engineering/public/srpmproc/internal/blob/file"
"git.rockylinux.org/release-engineering/public/srpmproc/internal/blob/gcs"
"git.rockylinux.org/release-engineering/public/srpmproc/internal/blob/s3"
"git.rockylinux.org/release-engineering/public/srpmproc/internal/data"
"github.com/go-git/go-billy/v5"
"github.com/go-git/go-billy/v5/memfs"
"github.com/go-git/go-billy/v5/osfs"
"github.com/go-git/go-git/v5/plumbing/transport/ssh"
"io/ioutil"
"log"
"os"
"os/user"
"path/filepath"
"strings"
......@@ -28,8 +35,12 @@ var (
rpmPrefix string
importBranchPrefix string
branchPrefix string
singleTag string
noDupMode bool
moduleMode bool
tmpFsMode bool
noStorageDownload bool
noStorageUpload bool
)
var root = &cobra.Command{
......@@ -45,13 +56,15 @@ func mn(_ *cobra.Command, _ []string) {
log.Fatalf("unsupported upstream version %d", version)
}
var importer internal.ImportMode
var importer data.ImportMode
var blobStorage blob.Storage
if strings.HasPrefix(storageAddr, "gs://") {
blobStorage = gcs.New(strings.Replace(storageAddr, "gs://", "", 1))
} else if strings.HasPrefix(storageAddr, "s3://") {
blobStorage = s3.New(strings.Replace(storageAddr, "s3://", "", 1))
} else if strings.HasPrefix(storageAddr, "file://") {
blobStorage = file.New(strings.Replace(storageAddr, "file://", "", 1))
} else {
log.Fatalf("invalid blob storage")
}
......@@ -83,7 +96,22 @@ func mn(_ *cobra.Command, _ []string) {
log.Fatalf("could not get git authenticator: %v", err)
}
internal.ProcessRPM(&internal.ProcessData{
fsCreator := func() billy.Filesystem {
return memfs.New()
}
if tmpFsMode {
tmpDir, err := ioutil.TempDir(os.TempDir(), "srpmproc-*")
if err != nil {
log.Fatalf("could not create temp dir: %v", err)
}
log.Printf("using temp dir: %s", tmpDir)
fsCreator = func() billy.Filesystem {
return osfs.New(tmpDir)
}
}
internal.ProcessRPM(&data.ProcessData{
Importer: importer,
RpmLocation: sourceRpmLocation,
UpstreamPrefix: upstreamPrefix,
......@@ -96,9 +124,14 @@ func mn(_ *cobra.Command, _ []string) {
ModulePrefix: modulePrefix,
ImportBranchPrefix: importBranchPrefix,
BranchPrefix: branchPrefix,
SingleTag: singleTag,
Authenticator: authenticator,
NoDupMode: noDupMode,
ModuleMode: moduleMode,
TmpFsMode: tmpFsMode,
NoStorageDownload: noStorageDownload,
NoStorageUpload: noStorageUpload,
FsCreator: fsCreator,
})
}
......@@ -120,8 +153,12 @@ func main() {
root.Flags().StringVar(&rpmPrefix, "rpm-prefix", "https://git.centos.org/rpms", "Where to retrieve SRPM content. Only used when source-rpm is not a local file")
root.Flags().StringVar(&importBranchPrefix, "import-branch-prefix", "c", "Import branch prefix")
root.Flags().StringVar(&branchPrefix, "branch-prefix", "r", "Branch prefix (replaces import-branch-prefix)")
root.Flags().StringVar(&singleTag, "single-tag", "", "If set, only this tag is imported")
root.Flags().BoolVar(&noDupMode, "no-dup-mode", false, "If enabled, skips already imported tags")
root.Flags().BoolVar(&moduleMode, "module-mode", false, "If enabled, imports a module instead of a package")
root.Flags().BoolVar(&tmpFsMode, "tmpfs-mode", false, "If enabled, packages are imported and patched but not pushed")
root.Flags().BoolVar(&noStorageDownload, "no-storage-download", false, "If enabled, blobs are always downloaded from upstream")
root.Flags().BoolVar(&noStorageUpload, "no-storage-upload", false, "If enabled, blobs are not uploaded to blob storage")
if err := root.Execute(); err != nil {
log.Fatal(err)
......
package file
import (
"io/ioutil"
"log"
"os"
"path/filepath"
)
type File struct {
path string
}
func New(path string) *File {
return &File{
path: path,
}
}
func (f *File) Write(path string, content []byte) {
w, err := os.OpenFile(filepath.Join(f.path, path), os.O_CREATE|os.O_TRUNC|os.O_RDWR, 0644)
if err != nil {
log.Fatalf("could not open file: %v", err)
}
_, err = w.Write(content)
if err != nil {
log.Fatalf("could not write file to file: %v", err)
}
// Close, just like writing a file.
if err := w.Close(); err != nil {
log.Fatalf("could not close file writer to source: %v", err)
}
}
func (f *File) Read(path string) []byte {
r, err := os.OpenFile(filepath.Join(f.path, path), os.O_RDONLY, 0644)
if err != nil {
return nil
}
body, err := ioutil.ReadAll(r)
if err != nil {
return nil
}
return body
}
func (f *File) Exists(path string) bool {
_, err := os.Stat(filepath.Join(f.path, path))
return !os.IsNotExist(err)
}
package data
import (
"github.com/cavaliercoder/go-rpm"
"github.com/go-git/go-git/v5"
"hash"
)
type ImportMode interface {
RetrieveSource(pd *ProcessData) *ModeData
WriteSource(pd *ProcessData, md *ModeData)
PostProcess(md *ModeData)
ImportName(pd *ProcessData, md *ModeData) string
}
type ModeData struct {
Repo *git.Repository
Worktree *git.Worktree
RpmFile *rpm.PackageFile
FileWrites map[string][]byte
TagBranch string
PushBranch string
Branches []string
SourcesToIgnore []*IgnoredSource
BlobCache map[string][]byte
}
type IgnoredSource struct {
Name string
HashFunction hash.Hash
Expired bool
}
package data
import (
"git.rockylinux.org/release-engineering/public/srpmproc/internal/blob"
"github.com/go-git/go-billy/v5"
"github.com/go-git/go-git/v5/plumbing/transport/ssh"
)
type ProcessData struct {
RpmLocation string
UpstreamPrefix string
SshKeyLocation string
SshUser string
Version int
GitCommitterName string
GitCommitterEmail string
Mode int
ModulePrefix string
ImportBranchPrefix string
BranchPrefix string
SingleTag string
Authenticator *ssh.PublicKeys
Importer ImportMode
BlobStorage blob.Storage
NoDupMode bool
ModuleMode bool
TmpFsMode bool
NoStorageDownload bool
NoStorageUpload bool
FsCreator func() billy.Filesystem
}
......@@ -3,6 +3,7 @@ package directives
import (
"errors"
"fmt"
"git.rockylinux.org/release-engineering/public/srpmproc/internal/data"
srpmprocpb "git.rockylinux.org/release-engineering/public/srpmproc/pb"
"github.com/go-git/go-git/v5"
"io/ioutil"
......@@ -10,7 +11,7 @@ import (
"path/filepath"
)
func add(cfg *srpmprocpb.Cfg, patchTree *git.Worktree, pushTree *git.Worktree) error {
func add(cfg *srpmprocpb.Cfg, _ *data.ProcessData, _ *data.ModeData, patchTree *git.Worktree, pushTree *git.Worktree) error {
for _, add := range cfg.Add {
filePath := checkAddPrefix(filepath.Base(add.File))
......
......@@ -3,11 +3,12 @@ package directives
import (
"errors"
"fmt"
"git.rockylinux.org/release-engineering/public/srpmproc/internal/data"
srpmprocpb "git.rockylinux.org/release-engineering/public/srpmproc/pb"
"github.com/go-git/go-git/v5"
)
func del(cfg *srpmprocpb.Cfg, _ *git.Worktree, pushTree *git.Worktree) error {
func del(cfg *srpmprocpb.Cfg, _ *data.ProcessData, _ *data.ModeData, _ *git.Worktree, pushTree *git.Worktree) error {
for _, del := range cfg.Delete {
filePath := del.File
_, err := pushTree.Filesystem.Stat(filePath)
......
......@@ -2,6 +2,7 @@ package directives
import (
"encoding/json"
"git.rockylinux.org/release-engineering/public/srpmproc/internal/data"
srpmprocpb "git.rockylinux.org/release-engineering/public/srpmproc/pb"
"github.com/go-git/go-git/v5"
"log"
......@@ -19,27 +20,23 @@ func checkAddPrefix(file string) string {
return filepath.Join("SOURCES", file)
}
func Apply(cfg *srpmprocpb.Cfg, patchTree *git.Worktree, pushTree *git.Worktree) {
func Apply(cfg *srpmprocpb.Cfg, pd *data.ProcessData, md *data.ModeData, patchTree *git.Worktree, pushTree *git.Worktree) {
var errs []string
err := replace(cfg, patchTree, pushTree)
if err != nil {
errs = append(errs, err.Error())
directives := []func(*srpmprocpb.Cfg, *data.ProcessData, *data.ModeData, *git.Worktree, *git.Worktree) error{
replace,
del,
add,
patch,
lookaside,
specChange,
}
err = del(cfg, patchTree, pushTree)
if err != nil {
errs = append(errs, err.Error())
}
err = add(cfg, patchTree, pushTree)
if err != nil {
errs = append(errs, err.Error())
}
err = patch(cfg, patchTree, pushTree)
if err != nil {
errs = append(errs, err.Error())
for _, directive := range directives {
err := directive(cfg, pd, md, patchTree, pushTree)
if err != nil {
errs = append(errs, err.Error())
}
}
if len(errs) > 0 {
......
package directives
import (
"git.rockylinux.org/release-engineering/public/srpmproc/internal/data"
srpmprocpb "git.rockylinux.org/release-engineering/public/srpmproc/pb"
"github.com/go-git/go-git/v5"
)
func lookaside(cfg *srpmprocpb.Cfg, _ *data.ProcessData, _ *data.ModeData, _ *git.Worktree, pushTree *git.Worktree) error {
return nil
}
......@@ -4,13 +4,14 @@ import (
"bytes"
"errors"
"fmt"
"git.rockylinux.org/release-engineering/public/srpmproc/internal/data"
srpmprocpb "git.rockylinux.org/release-engineering/public/srpmproc/pb"
"github.com/bluekeyes/go-gitdiff/gitdiff"
"github.com/go-git/go-git/v5"
"log"
)
func patch(cfg *srpmprocpb.Cfg, patchTree *git.Worktree, pushTree *git.Worktree) error {
func patch(cfg *srpmprocpb.Cfg, _ *data.ProcessData, _ *data.ModeData, patchTree *git.Worktree, pushTree *git.Worktree) error {
for _, patch := range cfg.Patch {
patchFile, err := patchTree.Filesystem.Open(patch.File)
if err != nil {
......
......@@ -3,13 +3,14 @@ package directives
import (
"errors"
"fmt"
"git.rockylinux.org/release-engineering/public/srpmproc/internal/data"
srpmprocpb "git.rockylinux.org/release-engineering/public/srpmproc/pb"
"github.com/go-git/go-git/v5"
"io/ioutil"
"os"
)
func replace(cfg *srpmprocpb.Cfg, patchTree *git.Worktree, pushTree *git.Worktree) error {
func replace(cfg *srpmprocpb.Cfg, _ *data.ProcessData, _ *data.ModeData, patchTree *git.Worktree, pushTree *git.Worktree) error {
for _, replace := range cfg.Replace {
filePath := checkAddPrefix(replace.File)
stat, err := pushTree.Filesystem.Stat(filePath)
......
package directives
import (
"errors"
"fmt"
"git.rockylinux.org/release-engineering/public/srpmproc/internal/data"
srpmprocpb "git.rockylinux.org/release-engineering/public/srpmproc/pb"
"github.com/go-git/go-git/v5"
"io/ioutil"
"math"
"os"
"path/filepath"
"regexp"
"strconv"
"strings"
"time"
)
type sourcePatchOperationInLoopRequest struct {
cfg *srpmprocpb.Cfg
field string
value *string
longestField int
lastNum *int
in *bool
expectedField string
}
type sourcePatchOperationAfterLoopRequest struct {
cfg *srpmprocpb.Cfg
inLoopNum int
lastNum *int
longestField int
newLines *[]string
in *bool
expectedField string
operation srpmprocpb.SpecChange_FileOperation_Type
}
func sourcePatchOperationInLoop(req *sourcePatchOperationInLoopRequest) error {
if strings.HasPrefix(req.field, req.expectedField) {
for _, file := range req.cfg.SpecChange.File {
if file.Type != srpmprocpb.SpecChange_FileOperation_Source {
continue
}
switch file.Mode.(type) {
case *srpmprocpb.SpecChange_FileOperation_Delete:
if file.Name == *req.value {
*req.value = ""
}
break
}
}
sourceNum, err := strconv.Atoi(strings.Split(req.field, req.expectedField)[1])
if err != nil {
return errors.New(fmt.Sprintf("INVALID_%s_NUM", strings.ToUpper(req.expectedField)))
}
*req.lastNum = sourceNum
}
return nil
}
func sourcePatchOperationAfterLoop(req *sourcePatchOperationAfterLoopRequest) (bool, error) {
if req.inLoopNum == *req.lastNum && *req.in {
for _, file := range req.cfg.SpecChange.File {
if file.Type != req.operation {
continue
}
switch file.Mode.(type) {
case *srpmprocpb.SpecChange_FileOperation_Add:
field := fmt.Sprintf("%s%d", req.expectedField, *req.lastNum+1)
spaces := calculateSpaces(req.longestField, len(field))
*req.newLines = append(*req.newLines, fmt.Sprintf("%s:%s%s", field, spaces, file.Name))
*req.lastNum++
break
}
}
*req.in = false
return true, nil
}
return false, nil
}
func calculateSpaces(longestField int, fieldLength int) string {
return strings.Repeat(" ", longestField+8-fieldLength)
}
func searchAndReplaceLine(line string, sar []*srpmprocpb.SpecChange_SearchAndReplaceOperation) string {
for _, searchAndReplace := range sar {
switch searchAndReplace.Identifier.(type) {
case *srpmprocpb.SpecChange_SearchAndReplaceOperation_Any:
line = strings.Replace(line, searchAndReplace.Find, searchAndReplace.Replace, int(searchAndReplace.N))
break
case *srpmprocpb.SpecChange_SearchAndReplaceOperation_StartsWith:
if strings.HasPrefix(line, searchAndReplace.Find) {
line = strings.Replace(line, searchAndReplace.Find, searchAndReplace.Replace, int(searchAndReplace.N))
}
break
case *srpmprocpb.SpecChange_SearchAndReplaceOperation_EndsWith:
if strings.HasSuffix(line, searchAndReplace.Find) {
line = strings.Replace(line, searchAndReplace.Find, searchAndReplace.Replace, int(searchAndReplace.N))
}
break
}
}
return line
}
func specChange(cfg *srpmprocpb.Cfg, pd *data.ProcessData, md *data.ModeData, _ *git.Worktree, pushTree *git.Worktree) error {
// no spec change operations present
// skip parsing spec
if cfg.SpecChange == nil {
return nil
}
specFiles, err := pushTree.Filesystem.ReadDir("SPECS")
if err != nil {
return errors.New("COULD_NOT_READ_SPECS_DIR")
}
if len(specFiles) != 1 {
return errors.New("ONLY_ONE_SPEC_FILE_IS_SUPPORTED")
}
filePath := filepath.Join("SPECS", specFiles[0].Name())
stat, err := pushTree.Filesystem.Stat(filePath)
if err != nil {
return errors.New("COULD_NOT_STAT_SPEC_FILE")
}
specFile, err := pushTree.Filesystem.OpenFile(filePath, os.O_RDONLY, 0644)
if err != nil {
return errors.New("COULD_NOT_READ_SPEC_FILE")
}
specBts, err := ioutil.ReadAll(specFile)
if err != nil {
return errors.New("COULD_NOT_READ_ALL_BYTES")
}
specStr := string(specBts)
lines := strings.Split(specStr, "\n")
var newLines []string
lastSourceNum := 0
lastPatchNum := 0
inSources := false
inPatches := false
inChangelog := false
lastSource := ""
lastPatch := ""
version := ""
importNameSplit := strings.SplitN(pd.Importer.ImportName(pd, md), "-", 2)
if len(importNameSplit) == 2 {
versionSplit := strings.SplitN(importNameSplit[1], ".el", 2)
if len(versionSplit) == 2 {
version = versionSplit[0]
}
}
fieldValueRegex := regexp.MustCompile("^[A-Z].+:")
longestField := 0
for _, line := range lines {
if fieldValueRegex.MatchString(line) {
fieldValue := strings.SplitN(line, ":", 2)
field := strings.TrimSpace(fieldValue[0])
longestField = int(math.Max(float64(len(field)), float64(longestField)))
if strings.HasPrefix(field, "Source") {
lastSource = field
}
if strings.HasPrefix(field, "Patch") {
lastPatch = field
}
}
}
for _, line := range lines {
inLoopSourceNum := lastSourceNum
inLoopPatchNum := lastPatchNum
prefixLine := strings.TrimSpace(line)
if fieldValueRegex.MatchString(line) {
line = searchAndReplaceLine(line, cfg.SpecChange.SearchAndReplace)
fieldValue := strings.SplitN(line, ":", 2)
field := strings.TrimSpace(fieldValue[0])
value := strings.TrimSpace(fieldValue[1])
if field == lastSource {
inSources = true
} else if field == lastPatch {
inPatches = true
}
if field == "Version" && version == "" {
version = value
}
for _, searchAndReplace := range cfg.SpecChange.SearchAndReplace {
switch identifier := searchAndReplace.Identifier.(type) {
case *srpmprocpb.SpecChange_SearchAndReplaceOperation_Field:
if field == identifier.Field {
value = strings.Replace(value, searchAndReplace.Find, searchAndReplace.Replace, int(searchAndReplace.N))
}
break
}
}
spaces := calculateSpaces(longestField, len(field))
err := sourcePatchOperationInLoop(&sourcePatchOperationInLoopRequest{
cfg: cfg,
field: field,
value: &value,
lastNum: &lastSourceNum,
longestField: longestField,
expectedField: "Source",
in: &inSources,
})
if err != nil {
return err
}
err = sourcePatchOperationInLoop(&sourcePatchOperationInLoopRequest{
cfg: cfg,
field: field,
value: &value,
longestField: longestField,
lastNum: &lastPatchNum,
in: &inPatches,
expectedField: "Patch",
})
if err != nil {
return err
}
if value != "" {
newLines = append(newLines, fmt.Sprintf("%s:%s%s", field, spaces, value))
}
} else {
executed, err := sourcePatchOperationAfterLoop(&sourcePatchOperationAfterLoopRequest{
cfg: cfg,
inLoopNum: inLoopSourceNum,
lastNum: &lastSourceNum,
longestField: longestField,
newLines: &newLines,
expectedField: "Source",
in: &inSources,