process.go 9.36 KB
Newer Older
Mustafa Gezen's avatar
Mustafa Gezen committed
1
2
3
package internal

import (
Mustafa Gezen's avatar
update    
Mustafa Gezen committed
4
5
6
7
8
9
10
11
12
13
	"encoding/hex"
	"fmt"
	"github.com/cavaliercoder/go-rpm"
	"github.com/go-git/go-billy/v5/memfs"
	"github.com/go-git/go-git/v5"
	"github.com/go-git/go-git/v5/config"
	"github.com/go-git/go-git/v5/plumbing"
	"github.com/go-git/go-git/v5/plumbing/object"
	"github.com/go-git/go-git/v5/plumbing/transport/ssh"
	"github.com/go-git/go-git/v5/storage/memory"
14
15
	"github.com/mstg/srpmproc/internal/blob"
	"hash"
Mustafa Gezen's avatar
update    
Mustafa Gezen committed
16
17
	"io/ioutil"
	"log"
Mustafa Gezen's avatar
Mustafa Gezen committed
18
	"path/filepath"
19
	"regexp"
Mustafa Gezen's avatar
update    
Mustafa Gezen committed
20
21
	"strings"
	"time"
Mustafa Gezen's avatar
Mustafa Gezen committed
22
23
)

24
var tagImportRegex *regexp.Regexp
25

Mustafa Gezen's avatar
Mustafa Gezen committed
26
type ProcessData struct {
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
	RpmLocation        string
	UpstreamPrefix     string
	SshKeyLocation     string
	SshUser            string
	Version            int
	GitCommitterName   string
	GitCommitterEmail  string
	Mode               int
	ModulePrefix       string
	ImportBranchPrefix string
	BranchPrefix       string
	Authenticator      *ssh.PublicKeys
	Importer           ImportMode
	BlobStorage        blob.Storage
	NoDupMode          bool
	ModuleMode         bool
Mustafa Gezen's avatar
Mustafa Gezen committed
43
44
}

45
46
47
type ignoredSource struct {
	name         string
	hashFunction hash.Hash
48
	expired      bool
49
}
Mustafa Gezen's avatar
Mustafa Gezen committed
50

51
52
53
54
55
56
57
58
59
type modeData struct {
	repo            *git.Repository
	worktree        *git.Worktree
	rpmFile         *rpm.PackageFile
	fileWrites      map[string][]byte
	tagBranch       string
	pushBranch      string
	branches        []string
	sourcesToIgnore []*ignoredSource
Mustafa Gezen's avatar
Mustafa Gezen committed
60
61
62
63
64
65
66
67
68
69
}

// ProcessRPM checks the RPM specs and discards any remote files
// This functions also sorts files into directories
// .spec files goes into -> SPECS
// metadata files goes to root
// source files goes into -> SOURCES
// all files that are remote goes into .gitignore
// all ignored files' hash goes into .{name}.metadata
func ProcessRPM(pd *ProcessData) {
70
	tagImportRegex = regexp.MustCompile(fmt.Sprintf("refs/tags/(imports/(%s.|%s.-.+)/(.*))", pd.ImportBranchPrefix, pd.ImportBranchPrefix))
71
	md := pd.Importer.RetrieveSource(pd)
Mustafa Gezen's avatar
update    
Mustafa Gezen committed
72

Mustafa Gezen's avatar
Mustafa Gezen committed
73
	remotePrefix := "rpms"
Mustafa Gezen's avatar
Mustafa Gezen committed
74
75
76
77
	if pd.ModuleMode {
		remotePrefix = "modules"
	}

78
79
80
81
82
83
84
	// if no-dup-mode is enabled then skip already imported versions
	var tagIgnoreList []string
	if pd.NoDupMode {
		repo, err := git.Init(memory.NewStorage(), memfs.New())
		if err != nil {
			log.Fatalf("could not init git repo: %v", err)
		}
Mustafa Gezen's avatar
Mustafa Gezen committed
85
		remoteUrl := fmt.Sprintf("%s/%s/%s.git", pd.UpstreamPrefix, remotePrefix, md.rpmFile.Name())
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
		refspec := config.RefSpec("+refs/heads/*:refs/remotes/origin/*")

		remote, err := repo.CreateRemote(&config.RemoteConfig{
			Name:  "origin",
			URLs:  []string{remoteUrl},
			Fetch: []config.RefSpec{refspec},
		})
		if err != nil {
			log.Fatalf("could not create remote: %v", err)
		}

		list, err := remote.List(&git.ListOptions{
			Auth: pd.Authenticator,
		})
		if err != nil {
Mustafa Gezen's avatar
Mustafa Gezen committed
101
			log.Println("ignoring no-dup-mode")
102
103
104
105
106
107
		} else {
			for _, ref := range list {
				if !strings.HasPrefix(string(ref.Name()), "refs/tags/imports") {
					continue
				}
				tagIgnoreList = append(tagIgnoreList, string(ref.Name()))
108
109
110
111
			}
		}
	}

112
113
	sourceRepo := *md.repo
	sourceWorktree := *md.worktree
Mustafa Gezen's avatar
update    
Mustafa Gezen committed
114

115
116
117
118
	for _, branch := range md.branches {
		md.repo = &sourceRepo
		md.worktree = &sourceWorktree
		md.tagBranch = branch
119
120
121
		for _, source := range md.sourcesToIgnore {
			source.expired = true
		}
122
123
124
125

		rpmFile := md.rpmFile
		// create new repo for final dist
		repo, err := git.Init(memory.NewStorage(), memfs.New())
Mustafa Gezen's avatar
update    
Mustafa Gezen committed
126
		if err != nil {
127
			log.Fatalf("could not create new dist repo: %v", err)
Mustafa Gezen's avatar
update    
Mustafa Gezen committed
128
		}
129
		w, err := repo.Worktree()
Mustafa Gezen's avatar
update    
Mustafa Gezen committed
130
		if err != nil {
131
			log.Fatalf("could not get dist worktree: %v", err)
Mustafa Gezen's avatar
update    
Mustafa Gezen committed
132
133
		}

Mustafa Gezen's avatar
Mustafa Gezen committed
134
		var matchString string
135
		if !tagImportRegex.MatchString(md.tagBranch) {
Mustafa Gezen's avatar
Mustafa Gezen committed
136
			if pd.ModuleMode {
137
				prefix := fmt.Sprintf("refs/heads/%s%d", pd.ImportBranchPrefix, pd.Version)
Mustafa Gezen's avatar
Mustafa Gezen committed
138
139
140
141
142
143
144
				if strings.HasPrefix(md.tagBranch, prefix) {
					replace := strings.Replace(md.tagBranch, "refs/heads/", "", 1)
					matchString = fmt.Sprintf("refs/tags/imports/%s/%s", replace, filepath.Base(pd.RpmLocation))
					log.Printf("using match string: %s", matchString)
				}
			}
			if !tagImportRegex.MatchString(matchString) {
145
				continue
Mustafa Gezen's avatar
Mustafa Gezen committed
146
147
148
			}
		} else {
			matchString = md.tagBranch
149
		}
Mustafa Gezen's avatar
update    
Mustafa Gezen committed
150

Mustafa Gezen's avatar
Mustafa Gezen committed
151
		match := tagImportRegex.FindStringSubmatch(matchString)
152
153
		md.pushBranch = pd.BranchPrefix + strings.TrimPrefix(match[2], "c")
		newTag := "imports/" + pd.BranchPrefix + strings.TrimPrefix(match[1], "imports/c")
154
155
156
157
158
159
160
161
162
163
164

		shouldContinue := true
		for _, ignoredTag := range tagIgnoreList {
			if ignoredTag == "refs/tags/"+newTag {
				log.Printf("skipping %s", ignoredTag)
				shouldContinue = false
			}
		}
		if !shouldContinue {
			continue
		}
Mustafa Gezen's avatar
update    
Mustafa Gezen committed
165

166
		// create a new remote
Mustafa Gezen's avatar
Mustafa Gezen committed
167
		remoteUrl := fmt.Sprintf("%s/%s/%s.git", pd.UpstreamPrefix, remotePrefix, rpmFile.Name())
168
169
170
171
172
173
174
175
		log.Printf("using remote: %s", remoteUrl)
		refspec := config.RefSpec(fmt.Sprintf("+refs/heads/%s:refs/remotes/origin/%s", md.pushBranch, md.pushBranch))
		log.Printf("using refspec: %s", refspec)

		_, err = repo.CreateRemote(&config.RemoteConfig{
			Name:  "origin",
			URLs:  []string{remoteUrl},
			Fetch: []config.RefSpec{refspec},
Mustafa Gezen's avatar
update    
Mustafa Gezen committed
176
177
		})
		if err != nil {
178
			log.Fatalf("could not create remote: %v", err)
Mustafa Gezen's avatar
update    
Mustafa Gezen committed
179
180
		}

181
182
183
184
185
		err = repo.Fetch(&git.FetchOptions{
			RemoteName: "origin",
			RefSpecs:   []config.RefSpec{refspec},
			Auth:       pd.Authenticator,
		})
Mustafa Gezen's avatar
update    
Mustafa Gezen committed
186

187
188
		refName := plumbing.NewBranchReferenceName(md.pushBranch)
		log.Printf("set reference to ref: %s", refName)
189

Mustafa Gezen's avatar
update    
Mustafa Gezen committed
190
		if err != nil {
191
192
193
194
195
196
197
198
199
200
201
202
			h := plumbing.NewSymbolicReference(plumbing.HEAD, refName)
			if err := repo.Storer.CheckAndSetReference(h, nil); err != nil {
				log.Fatalf("could not set reference: %v", err)
			}
		} else {
			err = w.Checkout(&git.CheckoutOptions{
				Branch: plumbing.NewRemoteReferenceName("origin", md.pushBranch),
				Force:  true,
			})
			if err != nil {
				log.Fatalf("could not checkout: %v", err)
			}
Mustafa Gezen's avatar
update    
Mustafa Gezen committed
203
204
		}

205
		pd.Importer.WriteSource(md)
Mustafa Gezen's avatar
update    
Mustafa Gezen committed
206

207
208
209
		copyFromFs(md.worktree.Filesystem, w.Filesystem, ".")
		md.repo = repo
		md.worktree = w
Mustafa Gezen's avatar
update    
Mustafa Gezen committed
210

Mustafa Gezen's avatar
Mustafa Gezen committed
211
212
213
214
215
		if pd.ModuleMode {
			patchModuleYaml(pd, md)
		} else {
			executePatchesRpm(pd, md)
		}
Mustafa Gezen's avatar
update    
Mustafa Gezen committed
216

217
218
		// already uploaded blobs are skipped
		var alreadyUploadedBlobs []string
219
220
221
		// get ignored files hash and add to .{name}.metadata
		metadataFile := fmt.Sprintf(".%s.metadata", rpmFile.Name())
		metadata, err := w.Filesystem.Create(metadataFile)
Mustafa Gezen's avatar
update    
Mustafa Gezen committed
222
		if err != nil {
223
			log.Fatalf("could not create metadata file: %v", err)
Mustafa Gezen's avatar
update    
Mustafa Gezen committed
224
		}
225
		for _, source := range md.sourcesToIgnore {
226
227
228
229
			sourcePath := source.name

			_, err := w.Filesystem.Stat(sourcePath)
			if source.expired || err != nil {
230
231
232
				continue
			}

233
234
235
236
237
238
239
240
			sourceFile, err := w.Filesystem.Open(sourcePath)
			if err != nil {
				log.Fatalf("could not open ignored source file %s: %v", sourcePath, err)
			}
			sourceFileBts, err := ioutil.ReadAll(sourceFile)
			if err != nil {
				log.Fatalf("could not read the whole of ignored source file: %v", err)
			}
Mustafa Gezen's avatar
update    
Mustafa Gezen committed
241

242
243
			source.hashFunction.Reset()
			_, err = source.hashFunction.Write(sourceFileBts)
Mustafa Gezen's avatar
update    
Mustafa Gezen committed
244
			if err != nil {
245
				log.Fatalf("could not write bytes to hash function: %v", err)
Mustafa Gezen's avatar
update    
Mustafa Gezen committed
246
			}
Mustafa Gezen's avatar
Mustafa Gezen committed
247
248
			checksum := hex.EncodeToString(source.hashFunction.Sum(nil))
			checksumLine := fmt.Sprintf("%s %s\n", checksum, sourcePath)
249
250
251
			_, err = metadata.Write([]byte(checksumLine))
			if err != nil {
				log.Fatalf("could not write to metadata file: %v", err)
Mustafa Gezen's avatar
update    
Mustafa Gezen committed
252
			}
Mustafa Gezen's avatar
Mustafa Gezen committed
253

254
			path := checksum
255
256
257
			if strContains(alreadyUploadedBlobs, path) {
				continue
			}
Mustafa Gezen's avatar
Mustafa Gezen committed
258
259
			pd.BlobStorage.Write(path, sourceFileBts)
			log.Printf("wrote %s to blob storage", path)
260
			alreadyUploadedBlobs = append(alreadyUploadedBlobs, path)
Mustafa Gezen's avatar
update    
Mustafa Gezen committed
261
262
		}

263
		_, err = w.Add(metadataFile)
Mustafa Gezen's avatar
update    
Mustafa Gezen committed
264
		if err != nil {
265
			log.Fatalf("could not add metadata file: %v", err)
Mustafa Gezen's avatar
update    
Mustafa Gezen committed
266
267
		}

268
269
		lastFilesToAdd := []string{".gitignore", "SPECS"}
		for _, f := range lastFilesToAdd {
Mustafa Gezen's avatar
Mustafa Gezen committed
270
271
272
273
274
275
			_, err := w.Filesystem.Stat(f)
			if err == nil {
				_, err := w.Add(f)
				if err != nil {
					log.Fatalf("could not add %s: %v", f, err)
				}
276
			}
277
278
		}

279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
		pd.Importer.PostProcess(md)

		// show status
		status, _ := w.Status()
		log.Printf("successfully processed:\n%s", status)

		var hashes []plumbing.Hash
		var pushRefspecs []config.RefSpec

		head, err := repo.Head()
		if err != nil {
			hashes = nil
			pushRefspecs = append(pushRefspecs, "*:*")
		} else {
			log.Printf("tip %s", head.String())
			hashes = append(hashes, head.Hash())
			refOrigin := "refs/heads/" + md.pushBranch
			pushRefspecs = append(pushRefspecs, config.RefSpec(fmt.Sprintf("HEAD:%s", refOrigin)))
		}

		// we are now finished with the tree and are going to push it to the src repo
		// create import commit
		commit, err := w.Commit("import "+pd.Importer.ImportName(pd, md), &git.CommitOptions{
			Author: &object.Signature{
				Name:  pd.GitCommitterName,
				Email: pd.GitCommitterEmail,
				When:  time.Now(),
			},
			Parents: hashes,
		})
Mustafa Gezen's avatar
update    
Mustafa Gezen committed
309
		if err != nil {
310
			log.Fatalf("could not commit object: %v", err)
Mustafa Gezen's avatar
update    
Mustafa Gezen committed
311
312
		}

313
		obj, err := repo.CommitObject(commit)
Mustafa Gezen's avatar
update    
Mustafa Gezen committed
314
		if err != nil {
315
			log.Fatalf("could not get commit object: %v", err)
Mustafa Gezen's avatar
update    
Mustafa Gezen committed
316
317
		}

318
		log.Printf("committed:\n%s", obj.String())
Mustafa Gezen's avatar
update    
Mustafa Gezen committed
319

320
321
322
323
324
325
326
327
328
329
330
331
		_, err = repo.CreateTag(newTag, commit, &git.CreateTagOptions{
			Tagger: &object.Signature{
				Name:  pd.GitCommitterName,
				Email: pd.GitCommitterEmail,
				When:  time.Now(),
			},
			Message: "import " + md.tagBranch + " from " + pd.RpmLocation,
			SignKey: nil,
		})
		if err != nil {
			log.Fatalf("could not create tag: %v", err)
		}
Mustafa Gezen's avatar
update    
Mustafa Gezen committed
332

333
		pushRefspecs = append(pushRefspecs, config.RefSpec(fmt.Sprintf("HEAD:%s", plumbing.NewTagReferenceName(newTag))))
Mustafa Gezen's avatar
update    
Mustafa Gezen committed
334

335
336
337
338
339
340
341
342
343
		err = repo.Push(&git.PushOptions{
			RemoteName: "origin",
			Auth:       pd.Authenticator,
			RefSpecs:   pushRefspecs,
			Force:      true,
		})
		if err != nil {
			log.Fatalf("could not push to remote: %v", err)
		}
Mustafa Gezen's avatar
update    
Mustafa Gezen committed
344
	}
Mustafa Gezen's avatar
Mustafa Gezen committed
345
}