Commit c57785b6 authored by Mustafa Gezen's avatar Mustafa Gezen
Browse files

Add support for scratch and bootstrapping from koji. Click for more info.

* Add support for scratch+merge builds to rebuild for a different architecture
* Add support for importing successful builds from Koji (so it's not registered as not built in Distrobuild)
* Fix package link in Imports and Builds page.
* Add executor to Imports, Builds and Batch pages.
* Correctly expire the redis session
* Add distinct status for scratch builds
* Check that an import exists for a batch build requested package
* Add a new merge_scratch task to accommodate the first change
* Add a /merge_scratch endpoint for batch builds (single merges not supported right now)
* Auto refresh page instead of a success modal for package actions
* Add support for arch_override (Builds)
* Show all batches (including batches with one package only)
parent f1865e70
......@@ -69,7 +69,7 @@ async def batch_list_check(packages):
elif package.get("package_name"):
filters["name"] = package["package_name"]
db_package = await Package.filter(**filters).first()
db_package = await Package.filter(**filters).prefetch_related("imports").first()
if not db_package:
detail = ""
if package.get("package_id"):
......@@ -77,6 +77,13 @@ async def batch_list_check(packages):
elif package.get("package_name"):
detail = f"Package with name {package['package_name']} not found"
raise HTTPException(412, detail=detail)
elif not db_package.imports or len(db_package.imports) == 0:
detail = ""
if package.get("package_id"):
detail = f"Package with id {package['package_id']} not imported"
elif package.get("package_name"):
detail = f"Package with name {package['package_name']} not imported"
raise HTTPException(412, detail=detail)
def get_user(request: Request) -> dict:
......
......@@ -37,5 +37,5 @@ def testing() -> str:
return f"{base()}-testing"
def scratch() -> str:
return "__scratch"
def modular_updates_candidate() -> str:
return "modular-updates-candidate"
......@@ -92,12 +92,15 @@ class RedisSessionMiddleware(BaseHTTPMiddleware):
redis_val = b64encode(json.dumps(request.scope["session"]).encode("utf-8"))
if redis_key == "":
redis_key = secrets.token_hex(32)
await self.redis.set(redis_key, redis_val)
await self.redis.expire(redis_key, 3000)
await self.redis.set(redis_key, redis_val)
await self.redis.expire(redis_key, 3000)
else:
await self.redis.set(redis_key, redis_val)
data = self.signer.sign(redis_key)
response.set_cookie(self.session_cookie, data.decode("utf-8"), self.max_age, httponly=True,
samesite=self.same_site, path="/", secure=self.https_only)
elif not initial_session_was_empty:
response.delete_cookie(self.session_cookie, path="/")
await self.redis.delete(redis_key)
return response
......@@ -67,11 +67,14 @@ class Build(Model):
status = fields.CharEnumField(BuildStatus)
mbs = fields.BooleanField(default=False)
signed = fields.BooleanField(default=False)
scratch = fields.BooleanField(default=False)
scratch_merged = fields.BooleanField(default=False)
koji_id = fields.BigIntField(null=True)
mbs_id = fields.BigIntField(null=True)
import_commit = fields.ForeignKeyField("distrobuild.ImportCommit", on_delete="RESTRICT", related_name="builds")
executor_username = fields.CharField(max_length=255)
force_tag = fields.CharField(max_length=255, null=True)
arch_override = fields.TextField(null=True)
exclude_compose = fields.BooleanField(default=False)
point_release = fields.CharField(max_length=255)
......
......@@ -18,19 +18,19 @@
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
from typing import List
from typing import List, Optional
from fastapi import APIRouter, Depends, Request, HTTPException
from fastapi_pagination import Page, pagination_params
from fastapi_pagination.ext.tortoise import paginate
from pydantic import BaseModel
from tortoise.functions import Count
from distrobuild.common import batch_list_check, get_user
from distrobuild.models import BatchImport, BatchBuild, ImportStatus, BuildStatus
from distrobuild.routes.builds import BuildRequest, queue_build
from distrobuild.routes.imports import ImportRequest, import_package_route
from distrobuild.serialize import BatchImport_Pydantic, BatchBuild_Pydantic
from distrobuild_scheduler import merge_scratch_task
router = APIRouter(prefix="/batches")
......@@ -43,6 +43,9 @@ class BatchImportRequest(BaseModel):
class BatchBuildRequest(BaseModel):
should_precheck: bool = True
ignore_modules: bool = False
scratch: bool = False
arch_override: Optional[str]
force_tag: Optional[str]
packages: List[BuildRequest]
......@@ -53,7 +56,7 @@ class NewBatchResponse(BaseModel):
@router.get("/imports/", response_model=Page[BatchImport_Pydantic], dependencies=[Depends(pagination_params)])
async def list_batch_imports():
return await paginate(
BatchImport.annotate(imports_count=Count('imports')).filter(imports_count__gte=2).all().prefetch_related(
BatchImport.all().prefetch_related(
"imports", "imports__package", "imports__commits").order_by("-created_at"))
......@@ -110,7 +113,7 @@ async def retry_failed_batch_imports(request: Request, batch_import_id: int):
@router.get("/builds/", response_model=Page[BatchBuild_Pydantic], dependencies=[Depends(pagination_params)])
async def list_batch_builds():
return await paginate(
BatchBuild.annotate(builds_count=Count('builds')).filter(builds_count__gte=2).all().prefetch_related(
BatchBuild.all().prefetch_related(
"builds", "builds__package", "builds__import_commit").order_by(
"-created_at"))
......@@ -125,7 +128,8 @@ async def batch_queue_build(request: Request, body: BatchBuildRequest):
batch = await BatchBuild.create()
for build_request in body.packages:
await queue_build(request, dict(**dict(build_request), ignore_modules=body.ignore_modules), batch.id)
await queue_build(request, dict(**dict(build_request), ignore_modules=body.ignore_modules, scratch=body.scratch,
arch_override=body.arch_override, force_tag=body.force_tag), batch.id)
return NewBatchResponse(id=batch.id)
......@@ -150,6 +154,21 @@ async def cancel_batch_build(request: Request, batch_build_id: int):
return {}
@router.post("/builds/{batch_build_id}/merge_scratch", status_code=202)
async def merge_batch_build(request: Request, batch_build_id: int):
get_user(request)
batch_build_obj = await BatchBuild.filter(id=batch_build_id).prefetch_related("builds").get_or_none()
if not batch_build_obj:
raise HTTPException(404, detail="batch build does not exist")
for build in batch_build_obj.builds:
if build.scratch and not build.scratch_merged:
await merge_scratch_task(build.id)
return {}
@router.post("/builds/{batch_build_id}/retry_failed", response_model=NewBatchResponse)
async def retry_failed_batch_builds(request: Request, batch_build_id: int):
get_user(request)
......
......@@ -17,17 +17,40 @@
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
from datetime import datetime
from fastapi import APIRouter, Request
from fastapi.responses import JSONResponse
from tortoise.transactions import atomic
from distrobuild.bootstrap import process_repo_dump, process_module_dump
from distrobuild.common import get_user
from distrobuild.models import Repo
from distrobuild.models import Repo, Package, Build, Import, ImportStatus, ImportCommit, BuildStatus
from distrobuild.session import koji_session
from distrobuild.settings import settings
router = APIRouter(prefix="/bootstrap")
@atomic()
async def import_build_from_koji(username: str, package: Package, koji_build):
new_import = await Import.create(package_id=package.id, status=ImportStatus.SUCCEEDED,
executor_username=username, version=settings.version)
commit = koji_build["source"].split("#")[1]
import_commit = await ImportCommit.create(branch=f"{settings.original_import_branch_prefix}{settings.version}",
commit=commit, import__id=new_import.id)
package.last_import = datetime.now()
package.last_build = datetime.now()
await Build.create(package_id=package.id, status=BuildStatus.SUCCEEDED,
executor_username=username,
point_release=f"{settings.version}_{settings.default_point_release}",
import_commit_id=import_commit.id, koji_id=koji_build["task_id"])
await package.save()
@router.post("/modules")
async def bootstrap_modules(request: Request):
user = get_user(request)
......@@ -35,6 +58,22 @@ async def bootstrap_modules(request: Request):
return JSONResponse(content={})
@router.post("/import_from_koji", status_code=202)
async def import_from_koji(request: Request):
user = get_user(request)
all_koji_builds = koji_session.listBuilds()
packages_without_builds = await Package.filter(last_build__isnull=True).all()
for package in packages_without_builds:
for koji_build in all_koji_builds:
if package.name == koji_build["name"] and not package.is_module and not package.part_of_module and \
koji_build["state"] == 1:
await import_build_from_koji(user["preferred_username"], package, koji_build)
return {}
@router.post("/{repo}")
async def bootstrap_repo(request: Request, repo: Repo):
user = get_user(request)
......
......@@ -25,20 +25,22 @@ from fastapi_pagination import Page, pagination_params
from fastapi_pagination.ext.tortoise import paginate
from pydantic import BaseModel, validator
from distrobuild.common import gen_body_filters, get_user, tags
from distrobuild.common import gen_body_filters, get_user
from distrobuild.models import Build, Import, ImportCommit, Package, PackageModule, BuildStatus, Repo, BatchBuildPackage
from distrobuild.serialize import Build_Pydantic, BuildGeneral_Pydantic
from distrobuild.session import message_cipher
from distrobuild.settings import settings
from distrobuild_scheduler import build_package_task
from distrobuild_scheduler import build_package_task, merge_scratch_task
router = APIRouter(prefix="/builds")
class BuildRequest(BaseModel):
scratch: bool = False
testing: bool = False
ignore_modules: bool = False
arch_override: Optional[str]
force_tag: Optional[str]
only_branch: Optional[str]
package_id: Optional[int]
package_name: Optional[str]
......@@ -75,6 +77,19 @@ async def cancel_build(request: Request, build_id: int):
return {}
@router.post("/{build_id}/merge_scratch", status_code=202)
async def merge_build(request: Request, build_id: int):
get_user(request)
build_obj = await Build.filter(id=build_id, scratch_merged=False, scratch=True).get_or_none()
if not build_obj:
raise HTTPException(404, detail="build does not exist, is already merged or not a scratch build")
await merge_scratch_task(build_obj.id)
return {}
@router.post("/", status_code=202)
async def queue_build(request: Request, body: Dict[str, BuildRequest], batch_build_id: Optional[int] = None):
user = get_user(request)
......@@ -93,16 +108,29 @@ async def queue_build(request: Request, body: Dict[str, BuildRequest], batch_bui
if len(package_modules) > 0 or package.is_module:
token = message_cipher.encrypt(request.session.get("token").encode()).decode()
if body.get("testing") and not body.get("scratch"):
extras["force_tag"] = tags.testing()
if body.get("force_tag"):
extras["force_tag"] = body.get("force_tag")
if body.get("scratch"):
extras["force_tag"] = tags.scratch()
extras["scratch"] = True
latest_build = await Build.filter(package_id=package.id, status=BuildStatus.SUCCEEDED).prefetch_related(
"import_commit").order_by(
"-created_at").first()
import_commits = [latest_build.import_commit]
else:
latest_import = await Import.filter(package_id=package.id).order_by("-created_at").first()
import_commits = await ImportCommit.filter(import__id=latest_import.id).all()
latest_import = await Import.filter(package_id=package.id).order_by("-created_at").first()
import_commits = await ImportCommit.filter(import__id=latest_import.id).all()
if body.get("arch_override"):
extras["arch_override"] = body.get("arch_override")
only_branch = body.get("only_branch")
for import_commit in import_commits:
if "-beta" not in import_commit.branch:
if only_branch and import_commit.branch != only_branch:
continue
stream_branch_prefix = f"{settings.original_import_branch_prefix}{settings.version}-stream"
if import_commit.branch.startswith(stream_branch_prefix):
if body.get("ignore_modules"):
......
......@@ -80,3 +80,18 @@ async def build_package_task(package_id: int, build_id: int, token: Optional[str
),
routing_key=settings.routing_key,
)
async def merge_scratch_task(build_id: int):
msg_body = {
"message": "merge_scratch",
"build_id": build_id,
}
encoded = json.dumps(msg_body).encode()
await channel.default_exchange.publish(
aio_pika.Message(
body=encoded,
),
routing_key=settings.routing_key,
)
......@@ -22,6 +22,7 @@ import datetime
from typing import Optional
import koji
from tortoise.transactions import atomic
from distrobuild.common import tags
......@@ -47,7 +48,14 @@ async def do(package: Package, build: Build, token: Optional[str]):
host = f"git+https://{settings.gitlab_host}{settings.repo_prefix}"
source = f"{host}/rpms/{gitlabify(package.name)}.git?#{build.import_commit.commit}"
task_id = koji_session.build(source, target)
opts = {}
if build.scratch:
opts["scratch"] = True
if build.arch_override:
opts["arch_override"] = koji.parse_arches(build.arch_override)
task_id = koji_session.build(source, target, opts)
build.koji_id = task_id
build.status = BuildStatus.BUILDING
......
......@@ -32,7 +32,7 @@ from distrobuild.settings import TORTOISE_ORM, settings
from distrobuild.session import message_cipher
# noinspection PyUnresolvedReferences
from distrobuild_scheduler import init_channel, build_package, import_package, logger, periodic_tasks
from distrobuild_scheduler import init_channel, build_package, import_package, logger, periodic_tasks, merge_scratch
async def consume_messages(i: int):
......@@ -53,6 +53,8 @@ async def consume_messages(i: int):
if token:
token = message_cipher.decrypt(token.encode()).decode()
await build_package.task(body["package_id"], body["build_id"], token)
elif msg == "merge_scratch":
await merge_scratch.task(body["build_id"])
else:
logger.error("[*] Received unknown message")
......
# Copyright (c) 2021 The Distrobuild Authors
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
from tortoise.transactions import atomic
from distrobuild.models import Build, BuildStatus
from distrobuild.session import koji_session
@atomic()
async def do(build: Build):
if build.koji_id and build.scratch and not build.scratch_merged and build.status == BuildStatus.SUCCEEDED:
koji_session.mergeScratch(build.koji_id)
build.scratch_merged = True
await build.save()
# noinspection DuplicatedCode
async def task(build_id: int):
build = await Build.filter(id=build_id, scratch=True).get()
if not build:
return
if build.status == BuildStatus.CANCELLED:
return
try:
await do(build)
except Exception as e:
print(e)
/*
* Copyright (c) 2021 The Distrobuild Authors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
-- upgrade --
alter table builds
add column scratch bool default false not null;
alter table builds
add column scratch_merged bool default false not null;
alter table builds
add column arch_override text;
-- downgrade --
alter table builds
drop column scratch;
alter table builds
drop column scratch_merged;
alter table builds
drop column arch_override;
......@@ -69,6 +69,8 @@ export interface IBuild {
commit: string;
import_commit: Commit;
package: IPackage;
scratch: boolean;
scratch_merged: boolean;
}
export interface IBatchBuild {
......
......@@ -39,6 +39,7 @@ import {
TableRow,
TableToolbar,
TextArea,
TextInput,
} from 'carbon-components-react';
import { IPaginated, Axios } from '../api';
......@@ -55,6 +56,9 @@ export const BatchList = <T extends DataTableRow>(props: BatchListProps) => {
const [ignoreModules, setIgnoreModules] = React.useState<boolean | null>(
null
);
const [scratch, setScratch] = React.useState<boolean | null>(null);
const [archOverride, setArchOverride] = React.useState<string | null>(null);
const [forceTag, setForceTag] = React.useState<string | null>(null);
const [disable, setDisable] = React.useState(false);
const [page, setPage] = React.useState(Number(getQueryParam('page') || 1));
......@@ -99,7 +103,10 @@ export const BatchList = <T extends DataTableRow>(props: BatchListProps) => {
const [err, res] = await to(
Axios.post(`/batches/${props.name}/`, {
packages,
scratch,
ignore_modules: ignoreModules,
arch_override: archOverride,
force_tag: forceTag,
})
);
if (err) {
......@@ -133,6 +140,7 @@ export const BatchList = <T extends DataTableRow>(props: BatchListProps) => {
{ header: 'Package count', key: 'package_count' },
{ header: 'Failed', key: 'failed' },
{ header: 'Succeeded', key: 'succeeded' },
{ header: 'Executor', key: 'executor_username' },
];
return batchRes.items ? (
......@@ -152,12 +160,28 @@ export const BatchList = <T extends DataTableRow>(props: BatchListProps) => {
labelText="Package list"
/>
{props.name === 'builds' && (
<Checkbox
className="mt-4"
id="ignore_modules"
labelText="Ignore modules"
onChange={onIgnoreModulesChange}
/>
<div className="mt-4">
<Checkbox
id="ignore_modules"
labelText="Ignore modules"
onChange={onIgnoreModulesChange}
/>
<Checkbox
id="scratch"
labelText="Scratch build"
onChange={(checked) => setScratch(checked)}
/>
<TextInput
id="arch_override"
labelText="Arch override"
onChange={(e) => setArchOverride(e.currentTarget.value)}
/>
<TextInput
id="force_tag"
labelText="Force target"
onChange={(e) => setForceTag(e.currentTarget.value)}
/>
</div>
)}
</>
</Modal>
......@@ -248,6 +272,20 @@ export const BatchList = <T extends DataTableRow>(props: BatchListProps) => {
}
</TableCell>
)}
{pkg && cell.info.header === 'executor_username' && (
<TableCell key={`${cell.id}-succeeded`}>
{pkg[props.name].length > 0 ? (
<a
target="_blank"
href={`${window.SETTINGS.gitlabUrl}/${cell.value}`}
>
{pkg[props.name][0]['executor_username']}
</a>
) : (
<span>System</span>
)}
</TableCell>
)}
</>
))}
</TableRow>
......
......@@ -45,6 +45,7 @@ export const BatchShow = <T extends unknown>(props: BatchShowProps) => {
const [showCancelModal, setShowCancelModal] = React.useState(false);
const [showSuccessModal, setShowSuccessModal] = React.useState(false);
const [showRetryModal, setShowRetryModal] = React.useState(false);
const [showScratchModal, setShowScratchModal] = React.useState(false);
const [disable, setDisable] = React.useState(false);
const params = useParams<BatchShowParams>();
......@@ -102,6 +103,25 @@ export const BatchShow = <T extends unknown>(props: BatchShowProps) => {
})().then();
};
const mergeScratch = () => {
setDisable(true);
(async () => {
const [err] = await to(
Axios.post(`/batches/${props.name}/${params.id}/merge_scratch`)
);
if (err) {
alert('API Error');
setDisable(false);
return;
}
setShowScratchModal(false);
setDisable(false);
window.location.reload();
})().then();
};
const failedItems =
res &&
res[props.name]
......@@ -165,6 +185,18 @@ export const BatchShow = <T extends unknown>(props: BatchShowProps) => {
>
Are you sure you want to retry failed {props.name}?
</Modal>
{props.name === 'builds' && (
<Modal
open={showScratchModal}
primaryButtonText="Merge"
secondaryButtonText="Go back"
primaryButtonDisabled={disable}
onRequestClose={() => setShowScratchModal(false)}
onRequestSubmit={() => mergeScratch()}
>
Are you sure you want merge all scratch builds?
</Modal>
)}
<Tile className="flex items-center justify-between">
<h3>Batch #{params.id}</h3>
......@@ -177,6 +209,15 @@ export const BatchShow = <T extends unknown>(props: BatchShowProps) => {
Retry failed
</Button>