messy code ahead. implement new directives (spec_change) and restructure code...

messy code ahead. implement new directives (spec_change) and restructure code to prevent import cycles

cmd/srpmproc/main.go

@@ -3,10 +3,17 @@ package main
 import (
 	"fmt"
 	"git.rockylinux.org/release-engineering/public/srpmproc/internal/blob"
+	"git.rockylinux.org/release-engineering/public/srpmproc/internal/blob/file"
 	"git.rockylinux.org/release-engineering/public/srpmproc/internal/blob/gcs"
 	"git.rockylinux.org/release-engineering/public/srpmproc/internal/blob/s3"
+	"git.rockylinux.org/release-engineering/public/srpmproc/internal/data"
+	"github.com/go-git/go-billy/v5"
+	"github.com/go-git/go-billy/v5/memfs"
+	"github.com/go-git/go-billy/v5/osfs"
 	"github.com/go-git/go-git/v5/plumbing/transport/ssh"
+	"io/ioutil"
 	"log"
+	"os"
 	"os/user"
 	"path/filepath"
 	"strings"
@@ -28,8 +35,12 @@ var (
 	rpmPrefix          string
 	importBranchPrefix string
 	branchPrefix       string
+	singleTag          string
 	noDupMode          bool
 	moduleMode         bool
+	tmpFsMode          bool
+	noStorageDownload  bool
+	noStorageUpload    bool
 )
 
 var root = &cobra.Command{
@@ -45,13 +56,15 @@ func mn(_ *cobra.Command, _ []string) {
 		log.Fatalf("unsupported upstream version %d", version)
 	}
 
-	var importer internal.ImportMode
+	var importer data.ImportMode
 	var blobStorage blob.Storage
 
 	if strings.HasPrefix(storageAddr, "gs://") {
 		blobStorage = gcs.New(strings.Replace(storageAddr, "gs://", "", 1))
 	} else if strings.HasPrefix(storageAddr, "s3://") {
 		blobStorage = s3.New(strings.Replace(storageAddr, "s3://", "", 1))
+	} else if strings.HasPrefix(storageAddr, "file://") {
+		blobStorage = file.New(strings.Replace(storageAddr, "file://", "", 1))
 	} else {
 		log.Fatalf("invalid blob storage")
 	}
@@ -83,7 +96,22 @@ func mn(_ *cobra.Command, _ []string) {
 		log.Fatalf("could not get git authenticator: %v", err)
 	}
 
-	internal.ProcessRPM(&internal.ProcessData{
+	fsCreator := func() billy.Filesystem {
+		return memfs.New()
+	}
+
+	if tmpFsMode {
+		tmpDir, err := ioutil.TempDir(os.TempDir(), "srpmproc-*")
+		if err != nil {
+			log.Fatalf("could not create temp dir: %v", err)
+		}
+		log.Printf("using temp dir: %s", tmpDir)
+		fsCreator = func() billy.Filesystem {
+			return osfs.New(tmpDir)
+		}
+	}
+
+	internal.ProcessRPM(&data.ProcessData{
 		Importer:           importer,
 		RpmLocation:        sourceRpmLocation,
 		UpstreamPrefix:     upstreamPrefix,
@@ -96,9 +124,14 @@ func mn(_ *cobra.Command, _ []string) {
 		ModulePrefix:       modulePrefix,
 		ImportBranchPrefix: importBranchPrefix,
 		BranchPrefix:       branchPrefix,
+		SingleTag:          singleTag,
 		Authenticator:      authenticator,
 		NoDupMode:          noDupMode,
 		ModuleMode:         moduleMode,
+		TmpFsMode:          tmpFsMode,
+		NoStorageDownload:  noStorageDownload,
+		NoStorageUpload:    noStorageUpload,
+		FsCreator:          fsCreator,
 	})
 }
 
@@ -120,8 +153,12 @@ func main() {
 	root.Flags().StringVar(&rpmPrefix, "rpm-prefix", "https://git.centos.org/rpms", "Where to retrieve SRPM content. Only used when source-rpm is not a local file")
 	root.Flags().StringVar(&importBranchPrefix, "import-branch-prefix", "c", "Import branch prefix")
 	root.Flags().StringVar(&branchPrefix, "branch-prefix", "r", "Branch prefix (replaces import-branch-prefix)")
+	root.Flags().StringVar(&singleTag, "single-tag", "", "If set, only this tag is imported")
 	root.Flags().BoolVar(&noDupMode, "no-dup-mode", false, "If enabled, skips already imported tags")
 	root.Flags().BoolVar(&moduleMode, "module-mode", false, "If enabled, imports a module instead of a package")
+	root.Flags().BoolVar(&tmpFsMode, "tmpfs-mode", false, "If enabled, packages are imported and patched but not pushed")
+	root.Flags().BoolVar(&noStorageDownload, "no-storage-download", false, "If enabled, blobs are always downloaded from upstream")
+	root.Flags().BoolVar(&noStorageUpload, "no-storage-upload", false, "If enabled, blobs are not uploaded to blob storage")
 
 	if err := root.Execute(); err != nil {
 		log.Fatal(err)

internal/blob/file/file.go

+package file
+
+import (
+	"io/ioutil"
+	"log"
+	"os"
+	"path/filepath"
+)
+
+type File struct {
+	path string
+}
+
+func New(path string) *File {
+	return &File{
+		path: path,
+	}
+}
+
+func (f *File) Write(path string, content []byte) {
+	w, err := os.OpenFile(filepath.Join(f.path, path), os.O_CREATE|os.O_TRUNC|os.O_RDWR, 0644)
+	if err != nil {
+		log.Fatalf("could not open file: %v", err)
+	}
+
+	_, err = w.Write(content)
+	if err != nil {
+		log.Fatalf("could not write file to file: %v", err)
+	}
+
+	// Close, just like writing a file.
+	if err := w.Close(); err != nil {
+		log.Fatalf("could not close file writer to source: %v", err)
+	}
+}
+
+func (f *File) Read(path string) []byte {
+	r, err := os.OpenFile(filepath.Join(f.path, path), os.O_RDONLY, 0644)
+	if err != nil {
+		return nil
+	}
+
+	body, err := ioutil.ReadAll(r)
+	if err != nil {
+		return nil
+	}
+
+	return body
+}
+
+func (f *File) Exists(path string) bool {
+	_, err := os.Stat(filepath.Join(f.path, path))
+	return !os.IsNotExist(err)
+}

internal/data/import.go

+package data
+
+import (
+	"github.com/cavaliercoder/go-rpm"
+	"github.com/go-git/go-git/v5"
+	"hash"
+)
+
+type ImportMode interface {
+	RetrieveSource(pd *ProcessData) *ModeData
+	WriteSource(pd *ProcessData, md *ModeData)
+	PostProcess(md *ModeData)
+	ImportName(pd *ProcessData, md *ModeData) string
+}
+
+type ModeData struct {
+	Repo            *git.Repository
+	Worktree        *git.Worktree
+	RpmFile         *rpm.PackageFile
+	FileWrites      map[string][]byte
+	TagBranch       string
+	PushBranch      string
+	Branches        []string
+	SourcesToIgnore []*IgnoredSource
+	BlobCache       map[string][]byte
+}
+
+type IgnoredSource struct {
+	Name         string
+	HashFunction hash.Hash
+	Expired      bool
+}

internal/data/process.go

+package data
+
+import (
+	"git.rockylinux.org/release-engineering/public/srpmproc/internal/blob"
+	"github.com/go-git/go-billy/v5"
+	"github.com/go-git/go-git/v5/plumbing/transport/ssh"
+)
+
+type ProcessData struct {
+	RpmLocation        string
+	UpstreamPrefix     string
+	SshKeyLocation     string
+	SshUser            string
+	Version            int
+	GitCommitterName   string
+	GitCommitterEmail  string
+	Mode               int
+	ModulePrefix       string
+	ImportBranchPrefix string
+	BranchPrefix       string
+	SingleTag          string
+	Authenticator      *ssh.PublicKeys
+	Importer           ImportMode
+	BlobStorage        blob.Storage
+	NoDupMode          bool
+	ModuleMode         bool
+	TmpFsMode          bool
+	NoStorageDownload  bool
+	NoStorageUpload    bool
+	FsCreator          func() billy.Filesystem
+}

internal/directives/add.go

@@ -3,6 +3,7 @@ package directives
 import (
 	"errors"
 	"fmt"
+	"git.rockylinux.org/release-engineering/public/srpmproc/internal/data"
 	srpmprocpb "git.rockylinux.org/release-engineering/public/srpmproc/pb"
 	"github.com/go-git/go-git/v5"
 	"io/ioutil"
@@ -10,7 +11,7 @@ import (
 	"path/filepath"
 )
 
-func add(cfg *srpmprocpb.Cfg, patchTree *git.Worktree, pushTree *git.Worktree) error {
+func add(cfg *srpmprocpb.Cfg, _ *data.ProcessData, _ *data.ModeData, patchTree *git.Worktree, pushTree *git.Worktree) error {
 	for _, add := range cfg.Add {
 		filePath := checkAddPrefix(filepath.Base(add.File))
 

internal/directives/delete.go

@@ -3,11 +3,12 @@ package directives
 import (
 	"errors"
 	"fmt"
+	"git.rockylinux.org/release-engineering/public/srpmproc/internal/data"
 	srpmprocpb "git.rockylinux.org/release-engineering/public/srpmproc/pb"
 	"github.com/go-git/go-git/v5"
 )
 
-func del(cfg *srpmprocpb.Cfg, _ *git.Worktree, pushTree *git.Worktree) error {
+func del(cfg *srpmprocpb.Cfg, _ *data.ProcessData, _ *data.ModeData, _ *git.Worktree, pushTree *git.Worktree) error {
 	for _, del := range cfg.Delete {
 		filePath := del.File
 		_, err := pushTree.Filesystem.Stat(filePath)

internal/directives/directives.go

@@ -2,6 +2,7 @@ package directives
 
 import (
 	"encoding/json"
+	"git.rockylinux.org/release-engineering/public/srpmproc/internal/data"
 	srpmprocpb "git.rockylinux.org/release-engineering/public/srpmproc/pb"
 	"github.com/go-git/go-git/v5"
 	"log"
@@ -19,27 +20,23 @@ func checkAddPrefix(file string) string {
 	return filepath.Join("SOURCES", file)
 }
 
-func Apply(cfg *srpmprocpb.Cfg, patchTree *git.Worktree, pushTree *git.Worktree) {
+func Apply(cfg *srpmprocpb.Cfg, pd *data.ProcessData, md *data.ModeData, patchTree *git.Worktree, pushTree *git.Worktree) {
 	var errs []string
 
-	err := replace(cfg, patchTree, pushTree)
-	if err != nil {
-		errs = append(errs, err.Error())
+	directives := []func(*srpmprocpb.Cfg, *data.ProcessData, *data.ModeData, *git.Worktree, *git.Worktree) error{
+		replace,
+		del,
+		add,
+		patch,
+		lookaside,
+		specChange,
 	}
 
-	err = del(cfg, patchTree, pushTree)
-	if err != nil {
-		errs = append(errs, err.Error())
-	}
-
-	err = add(cfg, patchTree, pushTree)
-	if err != nil {
-		errs = append(errs, err.Error())
-	}
-
-	err = patch(cfg, patchTree, pushTree)
-	if err != nil {
-		errs = append(errs, err.Error())
+	for _, directive := range directives {
+		err := directive(cfg, pd, md, patchTree, pushTree)
+		if err != nil {
+			errs = append(errs, err.Error())
+		}
 	}
 
 	if len(errs) > 0 {

internal/directives/lookaside.go

+package directives
+
+import (
+	"git.rockylinux.org/release-engineering/public/srpmproc/internal/data"
+	srpmprocpb "git.rockylinux.org/release-engineering/public/srpmproc/pb"
+	"github.com/go-git/go-git/v5"
+)
+
+func lookaside(cfg *srpmprocpb.Cfg, _ *data.ProcessData, _ *data.ModeData, _ *git.Worktree, pushTree *git.Worktree) error {
+	return nil
+}

internal/directives/patch.go

@@ -4,13 +4,14 @@ import (
 	"bytes"
 	"errors"
 	"fmt"
+	"git.rockylinux.org/release-engineering/public/srpmproc/internal/data"
 	srpmprocpb "git.rockylinux.org/release-engineering/public/srpmproc/pb"
 	"github.com/bluekeyes/go-gitdiff/gitdiff"
 	"github.com/go-git/go-git/v5"
 	"log"
 )
 
-func patch(cfg *srpmprocpb.Cfg, patchTree *git.Worktree, pushTree *git.Worktree) error {
+func patch(cfg *srpmprocpb.Cfg, _ *data.ProcessData, _ *data.ModeData, patchTree *git.Worktree, pushTree *git.Worktree) error {
 	for _, patch := range cfg.Patch {
 		patchFile, err := patchTree.Filesystem.Open(patch.File)
 		if err != nil {

internal/directives/replace.go

@@ -3,13 +3,14 @@ package directives
 import (
 	"errors"
 	"fmt"
+	"git.rockylinux.org/release-engineering/public/srpmproc/internal/data"
 	srpmprocpb "git.rockylinux.org/release-engineering/public/srpmproc/pb"
 	"github.com/go-git/go-git/v5"
 	"io/ioutil"
 	"os"
 )
 
-func replace(cfg *srpmprocpb.Cfg, patchTree *git.Worktree, pushTree *git.Worktree) error {
+func replace(cfg *srpmprocpb.Cfg, _ *data.ProcessData, _ *data.ModeData, patchTree *git.Worktree, pushTree *git.Worktree) error {
 	for _, replace := range cfg.Replace {
 		filePath := checkAddPrefix(replace.File)
 		stat, err := pushTree.Filesystem.Stat(filePath)

internal/directives/spec_change.go

+package directives
+
+import (
+	"errors"
+	"fmt"
+	"git.rockylinux.org/release-engineering/public/srpmproc/internal/data"
+	srpmprocpb "git.rockylinux.org/release-engineering/public/srpmproc/pb"
+	"github.com/go-git/go-git/v5"
+	"io/ioutil"
+	"math"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strconv"
+	"strings"
+	"time"
+)
+
+type sourcePatchOperationInLoopRequest struct {
+	cfg           *srpmprocpb.Cfg
+	field         string
+	value         *string
+	longestField  int
+	lastNum       *int
+	in            *bool
+	expectedField string
+}
+
+type sourcePatchOperationAfterLoopRequest struct {
+	cfg           *srpmprocpb.Cfg
+	inLoopNum     int
+	lastNum       *int
+	longestField  int
+	newLines      *[]string
+	in            *bool
+	expectedField string
+	operation     srpmprocpb.SpecChange_FileOperation_Type
+}
+
+func sourcePatchOperationInLoop(req *sourcePatchOperationInLoopRequest) error {
+	if strings.HasPrefix(req.field, req.expectedField) {
+		for _, file := range req.cfg.SpecChange.File {
+			if file.Type != srpmprocpb.SpecChange_FileOperation_Source {
+				continue
+			}
+
+			switch file.Mode.(type) {
+			case *srpmprocpb.SpecChange_FileOperation_Delete:
+				if file.Name == *req.value {
+					*req.value = ""
+				}
+				break
+			}
+		}
+		sourceNum, err := strconv.Atoi(strings.Split(req.field, req.expectedField)[1])
+		if err != nil {
+			return errors.New(fmt.Sprintf("INVALID_%s_NUM", strings.ToUpper(req.expectedField)))
+		}
+		*req.lastNum = sourceNum
+	}
+
+	return nil
+}
+
+func sourcePatchOperationAfterLoop(req *sourcePatchOperationAfterLoopRequest) (bool, error) {
+	if req.inLoopNum == *req.lastNum && *req.in {
+		for _, file := range req.cfg.SpecChange.File {
+			if file.Type != req.operation {
+				continue
+			}
+
+			switch file.Mode.(type) {
+			case *srpmprocpb.SpecChange_FileOperation_Add:
+				field := fmt.Sprintf("%s%d", req.expectedField, *req.lastNum+1)
+				spaces := calculateSpaces(req.longestField, len(field))
+				*req.newLines = append(*req.newLines, fmt.Sprintf("%s:%s%s", field, spaces, file.Name))
+				*req.lastNum++
+				break
+			}
+		}
+		*req.in = false
+
+		return true, nil
+	}
+
+	return false, nil
+}
+
+func calculateSpaces(longestField int, fieldLength int) string {
+	return strings.Repeat(" ", longestField+8-fieldLength)
+}
+
+func searchAndReplaceLine(line string, sar []*srpmprocpb.SpecChange_SearchAndReplaceOperation) string {
+	for _, searchAndReplace := range sar {
+		switch searchAndReplace.Identifier.(type) {
+		case *srpmprocpb.SpecChange_SearchAndReplaceOperation_Any:
+			line = strings.Replace(line, searchAndReplace.Find, searchAndReplace.Replace, int(searchAndReplace.N))
+			break
+		case *srpmprocpb.SpecChange_SearchAndReplaceOperation_StartsWith:
+			if strings.HasPrefix(line, searchAndReplace.Find) {
+				line = strings.Replace(line, searchAndReplace.Find, searchAndReplace.Replace, int(searchAndReplace.N))
+			}
+			break
+		case *srpmprocpb.SpecChange_SearchAndReplaceOperation_EndsWith:
+			if strings.HasSuffix(line, searchAndReplace.Find) {
+				line = strings.Replace(line, searchAndReplace.Find, searchAndReplace.Replace, int(searchAndReplace.N))
+			}
+			break
+		}
+	}
+
+	return line
+}
+
+func specChange(cfg *srpmprocpb.Cfg, pd *data.ProcessData, md *data.ModeData, _ *git.Worktree, pushTree *git.Worktree) error {
+	// no spec change operations present
+	// skip parsing spec
+	if cfg.SpecChange == nil {
+		return nil
+	}
+
+	specFiles, err := pushTree.Filesystem.ReadDir("SPECS")
+	if err != nil {
+		return errors.New("COULD_NOT_READ_SPECS_DIR")
+	}
+
+	if len(specFiles) != 1 {
+		return errors.New("ONLY_ONE_SPEC_FILE_IS_SUPPORTED")
+	}
+
+	filePath := filepath.Join("SPECS", specFiles[0].Name())
+	stat, err := pushTree.Filesystem.Stat(filePath)
+	if err != nil {
+		return errors.New("COULD_NOT_STAT_SPEC_FILE")
+	}
+
+	specFile, err := pushTree.Filesystem.OpenFile(filePath, os.O_RDONLY, 0644)
+	if err != nil {
+		return errors.New("COULD_NOT_READ_SPEC_FILE")
+	}
+
+	specBts, err := ioutil.ReadAll(specFile)
+	if err != nil {
+		return errors.New("COULD_NOT_READ_ALL_BYTES")
+	}
+
+	specStr := string(specBts)
+	lines := strings.Split(specStr, "\n")
+
+	var newLines []string
+	lastSourceNum := 0
+	lastPatchNum := 0
+	inSources := false
+	inPatches := false
+	inChangelog := false
+	lastSource := ""
+	lastPatch := ""
+
+	version := ""
+	importNameSplit := strings.SplitN(pd.Importer.ImportName(pd, md), "-", 2)
+	if len(importNameSplit) == 2 {
+		versionSplit := strings.SplitN(importNameSplit[1], ".el", 2)
+		if len(versionSplit) == 2 {
+			version = versionSplit[0]
+		}
+	}
+
+	fieldValueRegex := regexp.MustCompile("^[A-Z].+:")
+
+	longestField := 0
+	for _, line := range lines {
+		if fieldValueRegex.MatchString(line) {
+			fieldValue := strings.SplitN(line, ":", 2)
+			field := strings.TrimSpace(fieldValue[0])
+			longestField = int(math.Max(float64(len(field)), float64(longestField)))
+
+			if strings.HasPrefix(field, "Source") {
+				lastSource = field
+			}
+
+			if strings.HasPrefix(field, "Patch") {
+				lastPatch = field
+			}
+		}
+	}
+
+	for _, line := range lines {
+		inLoopSourceNum := lastSourceNum
+		inLoopPatchNum := lastPatchNum
+		prefixLine := strings.TrimSpace(line)
+
+		if fieldValueRegex.MatchString(line) {
+			line = searchAndReplaceLine(line, cfg.SpecChange.SearchAndReplace)
+			fieldValue := strings.SplitN(line, ":", 2)
+			field := strings.TrimSpace(fieldValue[0])
+			value := strings.TrimSpace(fieldValue[1])
+
+			if field == lastSource {
+				inSources = true
+			} else if field == lastPatch {
+				inPatches = true
+			}
+
+			if field == "Version" && version == "" {
+				version = value
+			}
+
+			for _, searchAndReplace := range cfg.SpecChange.SearchAndReplace {
+				switch identifier := searchAndReplace.Identifier.(type) {
+				case *srpmprocpb.SpecChange_SearchAndReplaceOperation_Field:
+					if field == identifier.Field {
+						value = strings.Replace(value, searchAndReplace.Find, searchAndReplace.Replace, int(searchAndReplace.N))
+					}
+					break
+				}
+			}
+
+			spaces := calculateSpaces(longestField, len(field))
+
+			err := sourcePatchOperationInLoop(&sourcePatchOperationInLoopRequest{
+				cfg:           cfg,
+				field:         field,
+				value:         &value,
+				lastNum:       &lastSourceNum,
+				longestField:  longestField,
+				expectedField: "Source",
+				in:            &inSources,
+			})
+			if err != nil {
+				return err
+			}
+
+			err = sourcePatchOperationInLoop(&sourcePatchOperationInLoopRequest{
+				cfg:           cfg,
+				field:         field,
+				value:         &value,
+				longestField:  longestField,
+				lastNum:       &lastPatchNum,
+				in:            &inPatches,
+				expectedField: "Patch",
+			})
+			if err != nil {
+				return err
+			}
+
+			if value != "" {
+				newLines = append(newLines, fmt.Sprintf("%s:%s%s", field, spaces, value))
+			}
+		} else {
+			executed, err := sourcePatchOperationAfterLoop(&sourcePatchOperationAfterLoopRequest{
+				cfg:           cfg,
+				inLoopNum:     inLoopSourceNum,
+				lastNum:       &lastSourceNum,
+				longestField:  longestField,
+				newLines:      &newLines,
+				expectedField: "Source",
+				in:            &inSources,