Commit 66751759 authored by Mustafa Gezen's avatar Mustafa Gezen 🏗
Browse files

add mbs support, add package ownership and some other changes

parent 7ff1257d
......@@ -19,22 +19,32 @@ from distrobuild import session
from distrobuild_scheduler import init_channel
app = FastAPI()
app.add_middleware(SessionMiddleware, secret_key=settings.session_secret)
app.add_middleware(SessionMiddleware, secret_key=settings.session_secret, max_age=3500)
app.mount("/static/files", StaticFiles(directory="ui/dist/files"), name="static")
register_routes(app)
templates = Jinja2Templates(directory="ui/dist/templates")
static_templates = Jinja2Templates(directory="distrobuild/templates")
@app.get("/{full_path:path}", response_class=HTMLResponse, include_in_schema=False)
async def serve_frontend(request: Request):
not_authorized_message = request.session.get("not_authorized")
if not_authorized_message:
request.session.pop("not_authorized")
return static_templates.TemplateResponse("not_authorized.html.j2", {
"request": request,
"message": not_authorized_message,
})
return templates.TemplateResponse("index.html", {
"request": request,
"distribution": settings.distribution,
"authenticated": "true" if request.session.get("user") else "false",
"full_name": request.session["user"]["name"] if request.session.get("user") else "",
"full_name": request.session.get("user").get("name") if request.session.get("user") else "",
"koji_weburl": session.koji_config.get("weburl"),
"gitlab_url": f"https://{settings.gitlab_host}{settings.repo_prefix}"
"gitlab_url": f"https://{settings.gitlab_host}",
"repo_prefix": settings.repo_prefix
})
......
......@@ -9,7 +9,7 @@ oauth.register(
client_secret=settings.oidc_client_secret,
server_metadata_url=f"{settings.oidc_issuer}/.well-known/openid-configuration",
client_kwargs={
"scope": f"openid profile {settings.oidc_scopes}"
"scope": f"openid profile groups {settings.oidc_scopes}"
}
)
......
import json
from distrobuild.models import Package, PackageModule, Repo
from distrobuild.settings import settings
async def process_repo_dump(repo: Repo) -> None:
async def _internal_create_package(**kwargs) -> Package:
return await Package.create(
el8=True if settings.version == 8 else False,
el9=True if settings.version == 9 else False,
**kwargs,
)
async def process_repo_dump(repo: Repo, responsible_username: str) -> None:
with open(f"/tmp/{repo}_ALL.txt", "r") as f:
lines = f.readlines()
for line in lines:
package_name = line.strip()
if await Package.filter(name=package_name).get_or_none():
continue
await Package.create(name=package_name, el8=True, is_package=True, repo=repo)
await _internal_create_package(name=package_name,
is_package=True,
repo=repo,
responsible_username=responsible_username)
async def process_module_dump() -> None:
async def process_module_dump(responsible_username: str) -> None:
f = open("/tmp/modules.json")
module_list = json.loads(f.read())
f.close()
......@@ -22,19 +34,26 @@ async def process_module_dump() -> None:
package_name = module.strip()
existing_package = await Package.filter(name=package_name).get_or_none()
if existing_package:
if not existing_package.is_module:
if not existing_package.is_module and existing_package.repo != Repo.MODULAR_CANDIDATE:
existing_package.is_module = True
await existing_package.save()
else:
existing_package = await Package.create(name=package_name, el8=True, is_module=True)
existing_package = await _internal_create_package(name=package_name,
is_module=True,
responsible_username=responsible_username)
subpackages = [x.strip() for x in module_list[module].split(",")]
for module_package in subpackages:
m_package_name = module_package.strip()
m_package = await Package.filter(name=m_package_name).get_or_none()
if m_package and m_package.repo != Repo.MODULAR_CANDIDATE:
m_package.part_of_module = True
if not m_package:
continue
m_package = await _internal_create_package(name=m_package_name,
repo=Repo.MODULAR_CANDIDATE,
responsible_username=responsible_username)
m_package.part_of_module = True
await m_package.save()
await PackageModule.create(package_id=m_package.id, module_parent_package_id=existing_package.id)
from typing import List, Tuple, Optional
from typing import List, Tuple
from pydantic import BaseModel, validator
from fastapi import Request, HTTPException
from distrobuild.models import Import, Package, PackageModule, BuildStatus
from distrobuild.models import Import, ImportStatus, Package, PackageModule
from distrobuild.settings import settings
class BuildRequest(BaseModel):
package_id: Optional[int]
package_name: Optional[str]
def gen_body_filters(body: dict) -> dict:
if body["package_id"]:
return {"id": body["package_id"]}
if body["package_name"]:
return {"name": body["package_name"]}
@validator('package_name')
def validate(cls, package_name, values):
if (not values.get('package_id') and not package_name) or (values.get('package_id') and package_name):
raise ValueError('either package_id or package_name is required')
return package_name
class BatchBuildRequest(BaseModel):
packages: List[BuildRequest]
def gen_body_filters(body_in) -> dict:
body = BuildRequest(**body_in)
if body.package_id:
return {"id": body.package_id}
if body.package_name:
return {"name": body.package_name}
async def create_build_order(package: Package) -> List[Tuple[int, int]]:
async def create_import_order(package: Package, username: str) -> List[Tuple[int, int]]:
pkg_list = []
if package.is_package:
package_import = await Import.create(package_id=package.id, status=BuildStatus.QUEUED, version=8)
package_import = await Import.create(package_id=package.id, status=ImportStatus.QUEUED,
executor_username=username, version=settings.version)
pkg_list.append((package.id, package_import.id))
if package.is_module:
......@@ -41,10 +27,23 @@ async def create_build_order(package: Package) -> List[Tuple[int, int]]:
imports = await Import.filter(package_id=subpackage.package_id).all()
if not imports or len(imports) == 0:
subpackage_package = await Package.filter(id=subpackage.package_id).get()
pkg_list += await create_build_order(subpackage_package)
pkg_list += await create_import_order(subpackage_package, username)
package_module_import = await Import.create(package_id=package.id, status=BuildStatus.QUEUED, version=8,
module=True)
package_module_import = await Import.create(package_id=package.id, status=ImportStatus.QUEUED,
module=True, executor_username=username, version=settings.version)
pkg_list.append((package.id, package_module_import.id))
return pkg_list
def get_user(request: Request) -> dict:
user = request.session.get("user")
token = request.session.get("token")
if not user or not token:
if request.session.get("token"):
request.session.pop("token")
if request.session.get("user"):
request.session.pop("user")
raise HTTPException(401, "not authenticated")
return user
from distrobuild.settings import settings
def base() -> str:
return f"dist-{settings.tag_prefix}{settings.version}"
def compose() -> str:
return f"{base()}-compose"
def testing() -> str:
return f"{base()}-testing"
def scratch() -> str:
return "__scratch"
from dataclasses import dataclass
import httpx
from distrobuild.settings import settings
class MBSConflictException(BaseException):
pass
class MBSBuildNotFound(BaseException):
pass
@dataclass
class MBSClient:
mbs_url: str
async def get_build(self, mbs_id: int):
client = httpx.AsyncClient()
async with client:
r = await client.get(f"{self.mbs_url}/1/module-builds/{mbs_id}")
if r.status_code == 404:
raise MBSBuildNotFound()
return r.json()
async def build(self, token: str, name: str, branch: str, commit: str) -> int:
client = httpx.AsyncClient()
async with client:
r = await client.post(
f"{self.mbs_url}/1/module-builds/",
headers={
"Authorization": f"Bearer {token}"
},
json={
"scmurl": f"https://{settings.gitlab_host}{settings.repo_prefix}/modules/{name}?#{commit}",
"branch": branch,
}
)
if r.status_code == 409:
raise MBSConflictException()
data = r.json()
return data["id"]
from distrobuild.models.enums import BuildStatus, Repo
from distrobuild.models.enums import ImportStatus, BuildStatus, Repo
from distrobuild.models.package import Package, PackageModule
from distrobuild.models.build import Build, Import, ImportCommit
from tortoise import Model, fields
from distrobuild.models.enums import BuildStatus
class Build(Model):
id = fields.BigIntField(pk=True)
created_at = fields.DatetimeField(auto_now_add=True)
updated_at = fields.DatetimeField(auto_add=True, null=True)
package = fields.ForeignKeyField("distrobuild.Package", on_delete="CASCADE")
status = fields.CharEnumField(BuildStatus)
mbs = fields.BooleanField(default=False)
koji_id = fields.BigIntField(null=True)
mbs_id = fields.BigIntField(null=True)
commit = fields.CharField(max_length=255)
branch = fields.CharField(max_length=255)
class Meta:
table = "builds"
from distrobuild.models.enums import BuildStatus, ImportStatus
class Import(Model):
id = fields.BigIntField(pk=True)
created_at = fields.DatetimeField(auto_now_add=True)
updated_at = fields.DatetimeField(auto_add=True, null=True)
package = fields.ForeignKeyField("distrobuild.Package", on_delete="CASCADE")
status = fields.CharEnumField(BuildStatus)
version = fields.IntField()
package = fields.ForeignKeyField("distrobuild.Package", on_delete="RESTRICT")
status = fields.CharEnumField(ImportStatus)
module = fields.BooleanField(default=False)
version = fields.IntField()
executor_username = fields.CharField(max_length=255)
commits: fields.ReverseRelation["ImportCommit"] = fields.ReverseRelation
......@@ -40,7 +25,35 @@ class ImportCommit(Model):
id = fields.BigIntField(pk=True)
commit = fields.CharField(max_length=255)
branch = fields.CharField(max_length=255)
import_ = fields.ForeignKeyField("distrobuild.Import", on_delete="CASCADE")
import_ = fields.ForeignKeyField("distrobuild.Import", on_delete="RESTRICT")
class Meta:
table = "import_commits"
class Build(Model):
id = fields.BigIntField(pk=True)
created_at = fields.DatetimeField(auto_now_add=True)
updated_at = fields.DatetimeField(auto_add=True, null=True)
package = fields.ForeignKeyField("distrobuild.Package", on_delete="RESTRICT")
status = fields.CharEnumField(BuildStatus)
mbs = fields.BooleanField(default=False)
koji_id = fields.BigIntField(null=True)
mbs_id = fields.BigIntField(null=True)
import_commit = fields.ForeignKeyField("distrobuild.ImportCommit", on_delete="RESTRICT")
executor_username = fields.CharField(max_length=255)
force_tag = fields.CharField(max_length=255, null=True)
exclude_compose = fields.BooleanField(default=False)
point_release = fields.CharField(max_length=255)
class Meta:
table = "builds"
class Logs(Model):
id = fields.BigIntField(pk=True)
created_at = fields.DatetimeField(auto_now_add=True)
content = fields.TextField()
class Meta:
table = "logs"
......@@ -5,6 +5,10 @@ class Repo(str, Enum):
BASEOS = "BASEOS"
APPSTREAM = "APPSTREAM"
POWERTOOLS = "POWERTOOLS"
EXTERNAL = "EXTERNAL"
MODULAR_CANDIDATE = "MODULAR_CANDIDATE"
ORIGINAL = "ORIGINAL"
INFRA = "INFRA"
class BuildStatus(str, Enum):
......@@ -13,3 +17,11 @@ class BuildStatus(str, Enum):
FAILED = "FAILED"
SUCCEEDED = "SUCCEEDED"
CANCELLED = "CANCELLED"
class ImportStatus(str, Enum):
QUEUED = "QUEUED"
IN_PROGRESS = "IN_PROGRESS"
FAILED = "FAILED"
SUCCEEDED = "SUCCEEDED"
CANCELLED = "CANCELLED"
......@@ -7,7 +7,7 @@ class Package(Model):
created_at = fields.DatetimeField(auto_now_add=True)
updated_at = fields.DatetimeField(auto_add=True, null=True)
name = fields.CharField(max_length=255)
responsible_user_id = fields.CharField(max_length=255, null=True)
responsible_username = fields.CharField(max_length=255)
is_module = fields.BooleanField(default=False)
is_package = fields.BooleanField(default=False)
part_of_module = fields.BooleanField(default=False)
......@@ -15,6 +15,7 @@ class Package(Model):
last_build = fields.DatetimeField(null=True)
el8 = fields.BooleanField(default=False)
el9 = fields.BooleanField(default=False)
repo = fields.CharEnumField(Repo, null=True)
class Meta:
......@@ -32,5 +33,8 @@ class PackageModule(Model):
module_parent_package = fields.ForeignKeyField("distrobuild.Package", on_delete="RESTRICT",
related_name="m_module_parent_pacakges")
class Meta:
table = "package_modules"
class PydanticMeta:
backward_relations = False
from fastapi import APIRouter
from fastapi import APIRouter, Request
from fastapi.responses import JSONResponse
from distrobuild.bootstrap import process_repo_dump, process_module_dump
from distrobuild.common import get_user
from distrobuild.models import Repo
router = APIRouter(prefix="/bootstrap")
@router.post("/modules")
async def bootstrap_modules():
await process_module_dump()
async def bootstrap_modules(request: Request):
user = get_user(request)
await process_module_dump(user["preferred_username"])
return JSONResponse(content={})
@router.post("/{repo}")
async def bootstrap_repo(repo: Repo):
await process_repo_dump(repo)
async def bootstrap_repo(request: Request, repo: Repo):
user = get_user(request)
await process_repo_dump(repo, user["preferred_username"])
return JSONResponse(content={})
from fastapi import APIRouter, Depends, HTTPException
from typing import Optional, List, Dict
from fastapi import APIRouter, Depends, HTTPException, Request
from fastapi_pagination import Page, pagination_params
from fastapi_pagination.ext.tortoise import paginate
from pydantic import BaseModel, validator
from distrobuild.common import BuildRequest, gen_body_filters, BatchBuildRequest
from distrobuild.models import Build, Import, ImportCommit, Package, PackageModule, BuildStatus
from distrobuild.common import gen_body_filters, get_user, tags
from distrobuild.models import Build, Import, ImportCommit, Package, PackageModule, BuildStatus, Repo
from distrobuild.serialize import Build_Pydantic
from distrobuild.session import message_cipher
from distrobuild_scheduler import build_package_task
router = APIRouter(prefix="/builds")
class BuildRequest(BaseModel):
scratch: bool = False
testing: bool = False
package_id: Optional[int]
package_name: Optional[str]
@validator("package_name")
def validate(cls, package_name, values):
if (not values.get("package_id") and not package_name) or (values.get("package_id") and package_name):
raise ValueError("either package_id or package_name is required")
return package_name
class BatchBuildRequest(BaseModel):
packages: List[BuildRequest]
@router.get("/", response_model=Page[Build_Pydantic], dependencies=[Depends(pagination_params)])
async def list_builds():
return await paginate(Build.all().order_by('-created_at').prefetch_related("package"))
return await paginate(Build.all().order_by("-created_at").prefetch_related("package", "import_commit"))
@router.get("/{build_id}", response_model=Build_Pydantic)
async def get_build(build_id: int):
return await Build_Pydantic.from_queryset_single(
Build.filter(id=build_id).prefetch_related("package", "import_commit").first()
)
@router.post("/", status_code=202)
async def queue_build(body: BuildRequest):
async def queue_build(request: Request, body: Dict[str, BuildRequest]):
user = get_user(request)
filters = gen_body_filters(body)
package = await Package.filter(**filters).get_or_none()
if not package:
raise HTTPException(404, detail="package does not exist")
mbs = package.is_module
if package.repo == Repo.MODULAR_CANDIDATE:
raise HTTPException(401, detail="modular subpackages cannot be built, build the main module")
if package.part_of_module:
raise HTTPException(401, detail="this package is part of a module. build the main module")
extras = {
"mbs": package.is_module
}
token = None
package_modules = await PackageModule.filter(package_id=package.id)
if len(package_modules) > 0:
mbs = True
extras["mbs"] = True
token = message_cipher.encrypt(request.session.get("token").encode()).decode()
if body.get("testing") and not body.get("scratch"):
extras["force_tag"] = tags.testing()
if body.get("scratch"):
extras["force_tag"] = tags.scratch()
latest_import = await Import.filter(package_id=package.id).order_by("-created_at").first()
import_commits = await ImportCommit.filter(import__id=latest_import.id).all()
for import_commit in import_commits:
if "-beta" not in import_commit.branch:
build = await Build.create(package_id=package.id, status=BuildStatus.QUEUED, mbs=mbs,
commit=import_commit.commit, branch=import_commit.branch)
await build_package_task(package.id, build.id)
build = await Build.create(package_id=package.id, status=BuildStatus.QUEUED,
executor_username=user["preferred_username"], point_release="8_4",
import_commit_id=import_commit.id, **extras)
await build_package_task(package.id, build.id, token)
return {}
@router.post("/batch", status_code=202)
async def batch_queue_build(body: BatchBuildRequest):
async def batch_queue_build(request: Request, body: BatchBuildRequest):
for build_request in body.packages:
await queue_build(build_request)
await queue_build(request, dict(build_request))
return {}
from fastapi import APIRouter, Depends, HTTPException
from fastapi_pagination import pagination_params
from typing import Optional, List, Dict
from fastapi import APIRouter, Depends, HTTPException, Request
from fastapi_pagination import pagination_params, Page
from fastapi_pagination.ext.tortoise import paginate
from pydantic import BaseModel, validator
from starlette.responses import PlainTextResponse
from distrobuild.common import BuildRequest, gen_body_filters, BatchBuildRequest, create_build_order
from distrobuild.models import Import, Package
from distrobuild.common import gen_body_filters, create_import_order, get_user
from distrobuild.models import Import, Package, Repo
from distrobuild.serialize import Import_Pydantic
from distrobuild_scheduler import import_package_task
router = APIRouter(prefix="/imports")
# response_model causes some weird errors with Import. why?
# TODO: find out (removing response_model for now)
@router.get("/", dependencies=[Depends(pagination_params)])
class ImportRequest(BaseModel):
full_history: bool = False
single_tag: Optional[str]
package_id: Optional[int]
package_name: Optional[str]
@validator("package_name")
def validate(cls, package_name, values):
if (not values.get("package_id") and not package_name) or (values.get("package_id") and package_name):
raise ValueError("either package_id or package_name is required")
return package_name
class BatchImportRequest(BaseModel):
packages: List[ImportRequest]
@router.get("/", response_model=Page[Import_Pydantic], dependencies=[Depends(pagination_params)])
async def list_imports():
return await paginate(Import.all().order_by('-created_at').prefetch_related("package"))
return await paginate(Import.all().order_by("-created_at").prefetch_related("package"))
@router.get("/{import_id}", response_model=Import_Pydantic)
async def get_import(import_id: int):
return await Import_Pydantic.from_queryset_single(Import.filter(id=import_id).prefetch_related("package").first())
@router.get("/{import_id}/logs", response_class=PlainTextResponse)
......@@ -30,21 +54,26 @@ async def get_import_logs(import_id: int):
@router.post("/", status_code=202)
async def import_package_route(body: BuildRequest):
async def import_package_route(request: Request, body: Dict[str, ImportRequest]):
user = get_user(request)
filters = gen_body_filters(body)
package = await Package.filter(**filters).get_or_none()
if not package:
raise HTTPException(404, detail="package does not exist")
build_order = await create_build_order(package)
if package.repo == Repo.MODULAR_CANDIDATE:
raise HTTPException(401, detail="modular subpackages cannot be imported")