git.go 7.27 KB
Newer Older
1
2
3
4
package internal

import (
	"fmt"
5
	"git.rockylinux.org/release-engineering/public/srpmproc/internal/data"
6
7
8
9
	"github.com/go-git/go-billy/v5/memfs"
	"github.com/go-git/go-git/v5"
	"github.com/go-git/go-git/v5/config"
	"github.com/go-git/go-git/v5/plumbing"
10
	"github.com/go-git/go-git/v5/plumbing/object"
11
12
13
14
15
16
17
	"github.com/go-git/go-git/v5/storage/memory"
	"io/ioutil"
	"log"
	"net/http"
	"path/filepath"
	"sort"
	"strings"
18
	"time"
19
20
)

21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
type remoteTarget struct {
	remote string
	when   time.Time
}

type remoteTargetSlice []remoteTarget

func (p remoteTargetSlice) Len() int {
	return len(p)
}

func (p remoteTargetSlice) Less(i, j int) bool {
	return p[i].when.Before(p[j].when)
}

func (p remoteTargetSlice) Swap(i, j int) {
	p[i], p[j] = p[j], p[i]
}

40
41
type GitMode struct{}

42
func (g *GitMode) RetrieveSource(pd *data.ProcessData) *data.ModeData {
43
44
	repo, err := git.Init(memory.NewStorage(), memfs.New())
	if err != nil {
45
		log.Fatalf("could not init git Repo: %v", err)
46
47
48
49
	}

	w, err := repo.Worktree()
	if err != nil {
50
		log.Fatalf("could not get Worktree: %v", err)
51
52
53
54
55
56
57
58
59
60
61
62
	}

	refspec := config.RefSpec("+refs/heads/*:refs/remotes/*")
	remote, err := repo.CreateRemote(&config.RemoteConfig{
		Name:  "upstream",
		URLs:  []string{pd.RpmLocation},
		Fetch: []config.RefSpec{refspec},
	})
	if err != nil {
		log.Fatalf("could not create remote: %v", err)
	}

63
	err = remote.Fetch(&git.FetchOptions{
64
65
66
		RefSpecs: []config.RefSpec{refspec},
		Tags:     git.AllTags,
		Force:    true,
67
	})
68
	if err != nil {
69
		log.Fatalf("could not fetch upstream: %v", err)
70
71
	}

72
	var branches remoteTargetSlice
73

74
75
	latestTags := map[string]*remoteTarget{}

76
77
	tagAdd := func(tag *object.Tag) error {
		if strings.HasPrefix(tag.Name, fmt.Sprintf("imports/%s%d", pd.ImportBranchPrefix, pd.Version)) {
78
79
80
81
82
83
84
85
86
87
88
89
90
91
			refSpec := fmt.Sprintf("refs/tags/%s", tag.Name)
			if tagImportRegex.MatchString(refSpec) {
				match := tagImportRegex.FindStringSubmatch(refSpec)

				exists := latestTags[match[2]]
				if exists != nil && exists.when.After(tag.Tagger.When) {
					return nil
				}

				latestTags[match[2]] = &remoteTarget{
					remote: refSpec,
					when:   tag.Tagger.When,
				}
			}
Mustafa Gezen's avatar
Mustafa Gezen committed
92
		}
93
		return nil
94
95
96
97
98
99
100
101
	}

	tagIter, err := repo.TagObjects()
	if err != nil {
		log.Fatalf("could not get tag objects: %v", err)
	}
	_ = tagIter.ForEach(tagAdd)

102
	if len(latestTags) == 0 {
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
		list, err := remote.List(&git.ListOptions{})
		if err != nil {
			log.Fatalf("could not list upstream: %v", err)
		}

		for _, ref := range list {
			if ref.Hash().IsZero() {
				continue
			}

			commit, err := repo.CommitObject(ref.Hash())
			if err != nil {
				log.Fatalf("could not get commit object: %v", err)
			}
			_ = tagAdd(&object.Tag{
				Name:   strings.TrimPrefix(string(ref.Name()), "refs/tags/"),
				Tagger: commit.Committer,
			})
		}
	}

124
125
126
127
128
	for _, branch := range latestTags {
		log.Printf("tag: %s", strings.TrimPrefix(branch.remote, "refs/tags/"))
		branches = append(branches, *branch)
	}

129
130
131
132
133
	sort.Sort(branches)

	var sortedBranches []string
	for _, branch := range branches {
		sortedBranches = append(sortedBranches, branch.remote)
Mustafa Gezen's avatar
Mustafa Gezen committed
134
	}
135

136
137
138
139
140
141
	return &data.ModeData{
		Repo:       repo,
		Worktree:   w,
		RpmFile:    createPackageFile(filepath.Base(pd.RpmLocation)),
		FileWrites: nil,
		Branches:   sortedBranches,
142
143
144
	}
}

145
146
func (g *GitMode) WriteSource(pd *data.ProcessData, md *data.ModeData) {
	remote, err := md.Repo.Remote("upstream")
147
148
149
150
	if err != nil {
		log.Fatalf("could not get upstream remote: %v", err)
	}

Mustafa Gezen's avatar
Mustafa Gezen committed
151
152
153
	var refspec config.RefSpec
	var branchName string

154
155
156
	if strings.HasPrefix(md.TagBranch, "refs/heads") {
		refspec = config.RefSpec(fmt.Sprintf("+%s:%s", md.TagBranch, md.TagBranch))
		branchName = strings.TrimPrefix(md.TagBranch, "refs/heads/")
Mustafa Gezen's avatar
Mustafa Gezen committed
157
	} else {
158
		match := tagImportRegex.FindStringSubmatch(md.TagBranch)
Mustafa Gezen's avatar
Mustafa Gezen committed
159
		branchName = match[2]
160
		refspec = config.RefSpec(fmt.Sprintf("+refs/heads/%s:%s", branchName, md.TagBranch))
Mustafa Gezen's avatar
Mustafa Gezen committed
161
	}
162
	log.Printf("checking out upstream refspec %s", refspec)
163
164
165
166
167
168
	err = remote.Fetch(&git.FetchOptions{
		RemoteName: "upstream",
		RefSpecs:   []config.RefSpec{refspec},
		Tags:       git.AllTags,
		Force:      true,
	})
169
	if err != nil && err != git.NoErrAlreadyUpToDate {
170
171
172
		log.Fatalf("could not fetch upstream: %v", err)
	}

173
174
	err = md.Worktree.Checkout(&git.CheckoutOptions{
		Branch: plumbing.ReferenceName(md.TagBranch),
175
176
177
178
179
180
		Force:  true,
	})
	if err != nil {
		log.Fatalf("could not checkout source from git: %v", err)
	}

181
	_, err = md.Worktree.Add(".")
182
	if err != nil {
183
		log.Fatalf("could not add Worktree: %v", err)
184
185
	}

186
	metadataFile, err := md.Worktree.Filesystem.Open(fmt.Sprintf(".%s.metadata", md.RpmFile.Name()))
187
	if err != nil {
Mustafa Gezen's avatar
Mustafa Gezen committed
188
189
		log.Printf("warn: could not open metadata file, so skipping: %v", err)
		return
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
	}

	fileBytes, err := ioutil.ReadAll(metadataFile)
	if err != nil {
		log.Fatalf("could not read metadata file: %v", err)
	}

	client := &http.Client{
		Transport: &http.Transport{
			DisableCompression: false,
		},
	}
	fileContent := strings.Split(string(fileBytes), "\n")
	for _, line := range fileContent {
		if strings.TrimSpace(line) == "" {
			continue
		}

208
209
210
		lineInfo := strings.SplitN(line, " ", 2)
		hash := strings.TrimSpace(lineInfo[0])
		path := strings.TrimSpace(lineInfo[1])
211

212
213
		var body []byte

214
215
		if md.BlobCache[hash] != nil {
			body = md.BlobCache[hash]
216
217
218
			log.Printf("retrieving %s from cache", hash)
		} else {
			fromBlobStorage := pd.BlobStorage.Read(hash)
219
			if fromBlobStorage != nil && !pd.NoStorageDownload {
220
221
222
				body = fromBlobStorage
				log.Printf("downloading %s from blob storage", hash)
			} else {
223
				url := fmt.Sprintf("https://git.centos.org/sources/%s/%s/%s", md.RpmFile.Name(), branchName, hash)
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
				log.Printf("downloading %s", url)

				req, err := http.NewRequest("GET", url, nil)
				if err != nil {
					log.Fatalf("could not create new http request: %v", err)
				}
				req.Header.Set("Accept-Encoding", "*")

				resp, err := client.Do(req)
				if err != nil {
					log.Fatalf("could not download dist-git file: %v", err)
				}

				body, err = ioutil.ReadAll(resp.Body)
				if err != nil {
					log.Fatalf("could not read the whole dist-git file: %v", err)
				}
				err = resp.Body.Close()
				if err != nil {
					log.Fatalf("could not close body handle: %v", err)
				}
			}

247
			md.BlobCache[hash] = body
248
249
		}

250
		f, err := md.Worktree.Filesystem.Create(path)
251
252
253
254
		if err != nil {
			log.Fatalf("could not open file pointer: %v", err)
		}

Mustafa Gezen's avatar
Mustafa Gezen committed
255
		hasher := CompareHash(body, hash)
256
257
258
259
		if hasher == nil {
			log.Fatal("checksum in metadata does not match dist-git file")
		}

260
261
262
		md.SourcesToIgnore = append(md.SourcesToIgnore, &data.IgnoredSource{
			Name:         path,
			HashFunction: hasher,
263
264
265
266
267
268
269
270
271
272
		})

		_, err = f.Write(body)
		if err != nil {
			log.Fatalf("could not copy dist-git file to in-tree: %v", err)
		}
		_ = f.Close()
	}
}

273
274
275
func (g *GitMode) PostProcess(md *data.ModeData) {
	for _, source := range md.SourcesToIgnore {
		_, err := md.Worktree.Filesystem.Stat(source.Name)
276
		if err == nil {
277
			err := md.Worktree.Filesystem.Remove(source.Name)
278
279
280
			if err != nil {
				log.Fatalf("could not remove dist-git file: %v", err)
			}
281
282
283
		}
	}

284
	_, err := md.Worktree.Add(".")
285
286
287
288
289
	if err != nil {
		log.Fatalf("could not add git sources: %v", err)
	}
}

290
291
292
func (g *GitMode) ImportName(_ *data.ProcessData, md *data.ModeData) string {
	if tagImportRegex.MatchString(md.TagBranch) {
		match := tagImportRegex.FindStringSubmatch(md.TagBranch)
Mustafa Gezen's avatar
Mustafa Gezen committed
293
294
295
		return match[3]
	}

296
	return strings.TrimPrefix(md.TagBranch, "refs/heads/")
297
}