fix empty spec files and automatically upload tarball to gcs

cmd/srpmproc/main.go

 package main
 
 import (
-    "log"
-    "strings"
+	"cloud.google.com/go/storage"
+	"context"
+	"log"
+	"strings"
 
-    "github.com/mstg/srpmproc/internal"
-    "github.com/spf13/cobra"
+	"github.com/mstg/srpmproc/internal"
+	"github.com/spf13/cobra"
 )
 
 var (
-    sourceRpm         string
-    sshKeyLocation    string
-    sshUser           string
-    upstreamPrefix    string
-    branch            string
-    gcsBucket         string
-    debrandedTarballs []string
+	sourceRpm         string
+	sshKeyLocation    string
+	sshUser           string
+	upstreamPrefix    string
+	branch            string
+	gcsBucket         string
+	debrandedTarballs []string
 )
 
 var root = &cobra.Command{
-    Use: "srpmproc",
-    Run: mn,
+	Use: "srpmproc",
+	Run: mn,
 }
 
 func mn(_ *cobra.Command, _ []string) {
-    sourceRpmLocation := ""
-    if strings.HasPrefix(sourceRpm, "file://") {
-        sourceRpmLocation = strings.TrimPrefix(sourceRpm, "file://")
-    } else {
-        log.Fatal("non-local SRPMs are currently not supported")
-    }
-
-    internal.ProcessRPM(&internal.ProcessData{
-        RpmLocation:    sourceRpmLocation,
-        UpstreamPrefix: upstreamPrefix,
-        SshKeyLocation: sshKeyLocation,
-        SshUser:        sshUser,
-        Branch:         branch,
-    })
+	ctx := context.Background()
+	client, err := storage.NewClient(ctx)
+	if err != nil {
+		log.Fatalf("could not create gcloud client: %v", err)
+	}
+
+	sourceRpmLocation := ""
+	if strings.HasPrefix(sourceRpm, "file://") {
+		sourceRpmLocation = strings.TrimPrefix(sourceRpm, "file://")
+	} else {
+		log.Fatal("non-local SRPMs are currently not supported")
+	}
+
+	internal.ProcessRPM(&internal.ProcessData{
+		RpmLocation:    sourceRpmLocation,
+		UpstreamPrefix: upstreamPrefix,
+		SshKeyLocation: sshKeyLocation,
+		SshUser:        sshUser,
+		Branch:         branch,
+		Bucket:         client.Bucket(gcsBucket),
+	})
 }
 
 func main() {
-    root.Flags().StringVar(&sourceRpm, "source-rpm", "", "Location of RPM to process")
-    _ = root.MarkFlagRequired("source-rpm")
-    root.Flags().StringVar(&upstreamPrefix, "upstream-prefix", "", "Upstream git repository prefix")
-    _ = root.MarkFlagRequired("upstream-prefix")
-    root.Flags().StringVar(&branch, "branch", "", "Upstream branch")
-    _ = root.MarkFlagRequired("branch")
-    root.Flags().StringVar(&gcsBucket, "gcs-bucket", "", "Bucket to use as blob storage")
-    _ = root.MarkFlagRequired("gcs-bucket")
-
-    root.Flags().StringVar(&sshKeyLocation, "ssh-key-location", "", "Location of the SSH key to use to authenticate against upstream (Optional)")
-    root.Flags().StringVar(&sshUser, "ssh-user", "git", "SSH User (Optional, default git)")
-    root.Flags().StringArrayVar(&debrandedTarballs, "debranded-tarball", []string{}, "GCS urls to debranded tarballs (stage 2) (Optional)")
-
-    if err := root.Execute(); err != nil {
-        log.Fatal(err)
-    }
+	root.Flags().StringVar(&sourceRpm, "source-rpm", "", "Location of RPM to process")
+	_ = root.MarkFlagRequired("source-rpm")
+	root.Flags().StringVar(&upstreamPrefix, "upstream-prefix", "", "Upstream git repository prefix")
+	_ = root.MarkFlagRequired("upstream-prefix")
+	root.Flags().StringVar(&branch, "branch", "", "Upstream branch")
+	_ = root.MarkFlagRequired("branch")
+	root.Flags().StringVar(&gcsBucket, "gcs-bucket", "", "Bucket to use as blob storage")
+	_ = root.MarkFlagRequired("gcs-bucket")
+
+	root.Flags().StringVar(&sshKeyLocation, "ssh-key-location", "", "Location of the SSH key to use to authenticate against upstream (Optional)")
+	root.Flags().StringVar(&sshUser, "ssh-user", "git", "SSH User (Optional, default git)")
+	root.Flags().StringArrayVar(&debrandedTarballs, "debranded-tarball", []string{}, "GCS urls to debranded tarballs (stage 2) (Optional)")
+
+	if err := root.Execute(); err != nil {
+		log.Fatal(err)
+	}
 }

go.mod

@@ -3,7 +3,7 @@ module github.com/mstg/srpmproc
 go 1.15
 
 require (
-	cloud.google.com/go/storage v1.12.0 // indirect
+	cloud.google.com/go/storage v1.12.0
 	github.com/bluekeyes/go-gitdiff v0.5.0
 	github.com/cavaliercoder/go-cpio v0.0.0-20180626203310-925f9528c45e
 	github.com/cavaliercoder/go-rpm v0.0.0-20200122174316-8cb9fd9c31a8
@@ -11,4 +11,5 @@ require (
 	github.com/go-git/go-git/v5 v5.2.0
 	github.com/sirupsen/logrus v1.2.0
 	github.com/spf13/cobra v1.1.1
+	google.golang.org/api v0.32.0
 )

internal/process.go

@@ -2,6 +2,8 @@ package internal
 
 import (
 	"bytes"
+	"cloud.google.com/go/storage"
+	"context"
 	"crypto/sha1"
 	"encoding/hex"
 	"fmt"
@@ -33,6 +35,7 @@ type ProcessData struct {
 	SshKeyLocation string
 	SshUser        string
 	Branch         string
+	Bucket         *storage.BucketHandle
 }
 
 type specFileData struct {
@@ -118,20 +121,14 @@ func ProcessRPM(pd *ProcessData) {
 			log.Fatalln(err)
 		}
 
-		// set spec file name
-		if filepath.Ext(hdr.Name) == ".spec" {
-			var err error
-			specBts, err = ioutil.ReadAll(r)
-			if err != nil {
-				log.Fatalf("could not read spec: %v", err)
-			}
-		}
-
 		bts, err := ioutil.ReadAll(r)
 		if err != nil {
 			log.Fatalf("could not copy file to virtual filesystem: %v", err)
 		}
 		fileWrites[hdr.Name] = bts
+		if filepath.Ext(hdr.Name) == ".spec" {
+			specBts = bts
+		}
 	}
 
 	w, err := repo.Worktree()
@@ -298,7 +295,10 @@ func ProcessRPM(pd *ProcessData) {
 				}
 
 				for _, patchedFile := range files {
-					srcPath := filepath.Join("SOURCES", patchedFile.NewName)
+					srcPath := patchedFile.NewName
+					if !strings.HasPrefix(srcPath, "SPECS") {
+						srcPath = filepath.Join("SOURCES", patchedFile.NewName)
+					}
 					var output bytes.Buffer
 					if !patchedFile.IsDelete {
 						patchSubjectFile, err := w.Filesystem.Open(srcPath)
@@ -365,6 +365,20 @@ func ProcessRPM(pd *ProcessData) {
 			log.Fatalf("could not read the whole of ignored source file: %v", err)
 		}
 
+		ctx := context.Background()
+		gcsPath := fmt.Sprintf("%s-%s/%s", rpmFile.Name(), pd.Branch, source)
+		obj := pd.Bucket.Object(gcsPath)
+		w := obj.NewWriter(ctx)
+		_, err = w.Write(sourceFileBts)
+		if err != nil {
+			log.Fatalf("could not write tarball to gcs: %v", err)
+		}
+		// Close, just like writing a file.
+		if err := w.Close(); err != nil {
+			log.Fatalf("could not close gcs writer to source %s: %v", source, err)
+		}
+		log.Printf("wrote %s to gcs", gcsPath)
+
 		checksum := sha1.Sum(sourceFileBts)
 		checksumLine := fmt.Sprintf("%s %s\n", hex.EncodeToString(checksum[:]), sourcePath)
 		_, err = metadata.Write([]byte(checksumLine))