git.go 6.87 KB
Newer Older
1
2
3
4
package internal

import (
	"fmt"
5
	"git.rockylinux.org/release-engineering/public/srpmproc/internal/data"
6
7
8
9
	"github.com/go-git/go-billy/v5/memfs"
	"github.com/go-git/go-git/v5"
	"github.com/go-git/go-git/v5/config"
	"github.com/go-git/go-git/v5/plumbing"
10
	"github.com/go-git/go-git/v5/plumbing/object"
11
12
13
14
15
16
17
	"github.com/go-git/go-git/v5/storage/memory"
	"io/ioutil"
	"log"
	"net/http"
	"path/filepath"
	"sort"
	"strings"
18
	"time"
19
20
)

21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
type remoteTarget struct {
	remote string
	when   time.Time
}

type remoteTargetSlice []remoteTarget

func (p remoteTargetSlice) Len() int {
	return len(p)
}

func (p remoteTargetSlice) Less(i, j int) bool {
	return p[i].when.Before(p[j].when)
}

func (p remoteTargetSlice) Swap(i, j int) {
	p[i], p[j] = p[j], p[i]
}

40
41
type GitMode struct{}

42
func (g *GitMode) RetrieveSource(pd *data.ProcessData) *data.ModeData {
43
44
	repo, err := git.Init(memory.NewStorage(), memfs.New())
	if err != nil {
45
		log.Fatalf("could not init git Repo: %v", err)
46
47
48
49
	}

	w, err := repo.Worktree()
	if err != nil {
50
		log.Fatalf("could not get Worktree: %v", err)
51
52
53
54
55
56
57
58
59
60
61
62
	}

	refspec := config.RefSpec("+refs/heads/*:refs/remotes/*")
	remote, err := repo.CreateRemote(&config.RemoteConfig{
		Name:  "upstream",
		URLs:  []string{pd.RpmLocation},
		Fetch: []config.RefSpec{refspec},
	})
	if err != nil {
		log.Fatalf("could not create remote: %v", err)
	}

63
	err = remote.Fetch(&git.FetchOptions{
64
65
66
		RefSpecs: []config.RefSpec{refspec},
		Tags:     git.AllTags,
		Force:    true,
67
	})
68
	if err != nil {
69
		log.Fatalf("could not fetch upstream: %v", err)
70
71
	}

72
	var branches remoteTargetSlice
73

74
75
76
	tagAdd := func(tag *object.Tag) error {
		if strings.HasPrefix(tag.Name, fmt.Sprintf("imports/%s%d", pd.ImportBranchPrefix, pd.Version)) {
			log.Printf("tag: %s", tag.Name)
77
78
79
80
			branches = append(branches, remoteTarget{
				remote: fmt.Sprintf("refs/tags/%s", tag.Name),
				when:   tag.Tagger.When,
			})
Mustafa Gezen's avatar
Mustafa Gezen committed
81
		}
82
		return nil
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
	}

	tagIter, err := repo.TagObjects()
	if err != nil {
		log.Fatalf("could not get tag objects: %v", err)
	}
	_ = tagIter.ForEach(tagAdd)

	if len(branches) == 0 {
		list, err := remote.List(&git.ListOptions{})
		if err != nil {
			log.Fatalf("could not list upstream: %v", err)
		}

		for _, ref := range list {
			if ref.Hash().IsZero() {
				continue
			}

			commit, err := repo.CommitObject(ref.Hash())
			if err != nil {
				log.Fatalf("could not get commit object: %v", err)
			}
			_ = tagAdd(&object.Tag{
				Name:   strings.TrimPrefix(string(ref.Name()), "refs/tags/"),
				Tagger: commit.Committer,
			})
		}
	}

113
114
115
116
117
	sort.Sort(branches)

	var sortedBranches []string
	for _, branch := range branches {
		sortedBranches = append(sortedBranches, branch.remote)
Mustafa Gezen's avatar
Mustafa Gezen committed
118
	}
119

120
121
122
123
124
125
	return &data.ModeData{
		Repo:       repo,
		Worktree:   w,
		RpmFile:    createPackageFile(filepath.Base(pd.RpmLocation)),
		FileWrites: nil,
		Branches:   sortedBranches,
126
127
128
	}
}

129
130
func (g *GitMode) WriteSource(pd *data.ProcessData, md *data.ModeData) {
	remote, err := md.Repo.Remote("upstream")
131
132
133
134
	if err != nil {
		log.Fatalf("could not get upstream remote: %v", err)
	}

Mustafa Gezen's avatar
Mustafa Gezen committed
135
136
137
	var refspec config.RefSpec
	var branchName string

138
139
140
	if strings.HasPrefix(md.TagBranch, "refs/heads") {
		refspec = config.RefSpec(fmt.Sprintf("+%s:%s", md.TagBranch, md.TagBranch))
		branchName = strings.TrimPrefix(md.TagBranch, "refs/heads/")
Mustafa Gezen's avatar
Mustafa Gezen committed
141
	} else {
142
		match := tagImportRegex.FindStringSubmatch(md.TagBranch)
Mustafa Gezen's avatar
Mustafa Gezen committed
143
		branchName = match[2]
144
		refspec = config.RefSpec(fmt.Sprintf("+refs/heads/%s:%s", branchName, md.TagBranch))
Mustafa Gezen's avatar
Mustafa Gezen committed
145
	}
146
	log.Printf("checking out upstream refspec %s", refspec)
147
148
149
150
151
152
	err = remote.Fetch(&git.FetchOptions{
		RemoteName: "upstream",
		RefSpecs:   []config.RefSpec{refspec},
		Tags:       git.AllTags,
		Force:      true,
	})
153
	if err != nil && err != git.NoErrAlreadyUpToDate {
154
155
156
		log.Fatalf("could not fetch upstream: %v", err)
	}

157
158
	err = md.Worktree.Checkout(&git.CheckoutOptions{
		Branch: plumbing.ReferenceName(md.TagBranch),
159
160
161
162
163
164
		Force:  true,
	})
	if err != nil {
		log.Fatalf("could not checkout source from git: %v", err)
	}

165
	_, err = md.Worktree.Add(".")
166
	if err != nil {
167
		log.Fatalf("could not add Worktree: %v", err)
168
169
	}

170
	metadataFile, err := md.Worktree.Filesystem.Open(fmt.Sprintf(".%s.metadata", md.RpmFile.Name()))
171
	if err != nil {
Mustafa Gezen's avatar
Mustafa Gezen committed
172
173
		log.Printf("warn: could not open metadata file, so skipping: %v", err)
		return
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
	}

	fileBytes, err := ioutil.ReadAll(metadataFile)
	if err != nil {
		log.Fatalf("could not read metadata file: %v", err)
	}

	client := &http.Client{
		Transport: &http.Transport{
			DisableCompression: false,
		},
	}
	fileContent := strings.Split(string(fileBytes), "\n")
	for _, line := range fileContent {
		if strings.TrimSpace(line) == "" {
			continue
		}

192
193
194
		lineInfo := strings.SplitN(line, " ", 2)
		hash := strings.TrimSpace(lineInfo[0])
		path := strings.TrimSpace(lineInfo[1])
195

196
197
		var body []byte

198
199
		if md.BlobCache[hash] != nil {
			body = md.BlobCache[hash]
200
201
202
			log.Printf("retrieving %s from cache", hash)
		} else {
			fromBlobStorage := pd.BlobStorage.Read(hash)
203
			if fromBlobStorage != nil && !pd.NoStorageDownload {
204
205
206
				body = fromBlobStorage
				log.Printf("downloading %s from blob storage", hash)
			} else {
207
				url := fmt.Sprintf("https://git.centos.org/sources/%s/%s/%s", md.RpmFile.Name(), branchName, hash)
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
				log.Printf("downloading %s", url)

				req, err := http.NewRequest("GET", url, nil)
				if err != nil {
					log.Fatalf("could not create new http request: %v", err)
				}
				req.Header.Set("Accept-Encoding", "*")

				resp, err := client.Do(req)
				if err != nil {
					log.Fatalf("could not download dist-git file: %v", err)
				}

				body, err = ioutil.ReadAll(resp.Body)
				if err != nil {
					log.Fatalf("could not read the whole dist-git file: %v", err)
				}
				err = resp.Body.Close()
				if err != nil {
					log.Fatalf("could not close body handle: %v", err)
				}
			}

231
			md.BlobCache[hash] = body
232
233
		}

234
		f, err := md.Worktree.Filesystem.Create(path)
235
236
237
238
		if err != nil {
			log.Fatalf("could not open file pointer: %v", err)
		}

Mustafa Gezen's avatar
Mustafa Gezen committed
239
		hasher := CompareHash(body, hash)
240
241
242
243
		if hasher == nil {
			log.Fatal("checksum in metadata does not match dist-git file")
		}

244
245
246
		md.SourcesToIgnore = append(md.SourcesToIgnore, &data.IgnoredSource{
			Name:         path,
			HashFunction: hasher,
247
248
249
250
251
252
253
254
255
256
		})

		_, err = f.Write(body)
		if err != nil {
			log.Fatalf("could not copy dist-git file to in-tree: %v", err)
		}
		_ = f.Close()
	}
}

257
258
259
func (g *GitMode) PostProcess(md *data.ModeData) {
	for _, source := range md.SourcesToIgnore {
		_, err := md.Worktree.Filesystem.Stat(source.Name)
260
		if err == nil {
261
			err := md.Worktree.Filesystem.Remove(source.Name)
262
263
264
			if err != nil {
				log.Fatalf("could not remove dist-git file: %v", err)
			}
265
266
267
		}
	}

268
	_, err := md.Worktree.Add(".")
269
270
271
272
273
	if err != nil {
		log.Fatalf("could not add git sources: %v", err)
	}
}

274
275
276
func (g *GitMode) ImportName(_ *data.ProcessData, md *data.ModeData) string {
	if tagImportRegex.MatchString(md.TagBranch) {
		match := tagImportRegex.FindStringSubmatch(md.TagBranch)
Mustafa Gezen's avatar
Mustafa Gezen committed
277
278
279
		return match[3]
	}

280
	return strings.TrimPrefix(md.TagBranch, "refs/heads/")
281
}