add mbs support, add package ownership and some other changes

distrobuild/app.py

@@ -19,22 +19,32 @@ from distrobuild import session
 from distrobuild_scheduler import init_channel
 
 app = FastAPI()
-app.add_middleware(SessionMiddleware, secret_key=settings.session_secret)
+app.add_middleware(SessionMiddleware, secret_key=settings.session_secret, max_age=3500)
 app.mount("/static/files", StaticFiles(directory="ui/dist/files"), name="static")
 register_routes(app)
 
 templates = Jinja2Templates(directory="ui/dist/templates")
+static_templates = Jinja2Templates(directory="distrobuild/templates")
 
 
 @app.get("/{full_path:path}", response_class=HTMLResponse, include_in_schema=False)
 async def serve_frontend(request: Request):
+    not_authorized_message = request.session.get("not_authorized")
+    if not_authorized_message:
+        request.session.pop("not_authorized")
+        return static_templates.TemplateResponse("not_authorized.html.j2", {
+            "request": request,
+            "message": not_authorized_message,
+        })
+
     return templates.TemplateResponse("index.html", {
         "request": request,
         "distribution": settings.distribution,
         "authenticated": "true" if request.session.get("user") else "false",
-        "full_name": request.session["user"]["name"] if request.session.get("user") else "",
+        "full_name": request.session.get("user").get("name") if request.session.get("user") else "",
         "koji_weburl": session.koji_config.get("weburl"),
-        "gitlab_url": f"https://{settings.gitlab_host}{settings.repo_prefix}"
+        "gitlab_url": f"https://{settings.gitlab_host}",
+        "repo_prefix": settings.repo_prefix
     })
 
 

distrobuild/auth.py

@@ -9,7 +9,7 @@ oauth.register(
     client_secret=settings.oidc_client_secret,
     server_metadata_url=f"{settings.oidc_issuer}/.well-known/openid-configuration",
     client_kwargs={
-        "scope": f"openid profile {settings.oidc_scopes}"
+        "scope": f"openid profile groups {settings.oidc_scopes}"
     }
 )
 

distrobuild/bootstrap/__init__.py

 import json
 
 from distrobuild.models import Package, PackageModule, Repo
+from distrobuild.settings import settings
 
 
-async def process_repo_dump(repo: Repo) -> None:
+async def _internal_create_package(**kwargs) -> Package:
+    return await Package.create(
+        el8=True if settings.version == 8 else False,
+        el9=True if settings.version == 9 else False,
+        **kwargs,
+    )
+
+
+async def process_repo_dump(repo: Repo, responsible_username: str) -> None:
     with open(f"/tmp/{repo}_ALL.txt", "r") as f:
         lines = f.readlines()
         for line in lines:
             package_name = line.strip()
             if await Package.filter(name=package_name).get_or_none():
                 continue
-            await Package.create(name=package_name, el8=True, is_package=True, repo=repo)
+            await _internal_create_package(name=package_name,
+                                           is_package=True,
+                                           repo=repo,
+                                           responsible_username=responsible_username)
 
 
-async def process_module_dump() -> None:
+async def process_module_dump(responsible_username: str) -> None:
     f = open("/tmp/modules.json")
     module_list = json.loads(f.read())
     f.close()
@@ -22,19 +34,26 @@ async def process_module_dump() -> None:
         package_name = module.strip()
         existing_package = await Package.filter(name=package_name).get_or_none()
         if existing_package:
-            if not existing_package.is_module:
+            if not existing_package.is_module and existing_package.repo != Repo.MODULAR_CANDIDATE:
                 existing_package.is_module = True
                 await existing_package.save()
         else:
-            existing_package = await Package.create(name=package_name, el8=True, is_module=True)
+            existing_package = await _internal_create_package(name=package_name,
+                                                              is_module=True,
+                                                              responsible_username=responsible_username)
 
         subpackages = [x.strip() for x in module_list[module].split(",")]
         for module_package in subpackages:
             m_package_name = module_package.strip()
             m_package = await Package.filter(name=m_package_name).get_or_none()
+
+            if m_package and m_package.repo != Repo.MODULAR_CANDIDATE:
+                m_package.part_of_module = True
+
             if not m_package:
-                continue
+                m_package = await _internal_create_package(name=m_package_name,
+                                                           repo=Repo.MODULAR_CANDIDATE,
+                                                           responsible_username=responsible_username)
 
-            m_package.part_of_module = True
             await m_package.save()
             await PackageModule.create(package_id=m_package.id, module_parent_package_id=existing_package.id)

distrobuild/common/__init__.py

-from typing import List, Tuple, Optional
+from typing import List, Tuple
 
-from pydantic import BaseModel, validator
+from fastapi import Request, HTTPException
 
-from distrobuild.models import Import, Package, PackageModule, BuildStatus
+from distrobuild.models import Import, ImportStatus, Package, PackageModule
+from distrobuild.settings import settings
 
 
-class BuildRequest(BaseModel):
-    package_id: Optional[int]
-    package_name: Optional[str]
+def gen_body_filters(body: dict) -> dict:
+    if body["package_id"]:
+        return {"id": body["package_id"]}
+    if body["package_name"]:
+        return {"name": body["package_name"]}
 
-    @validator('package_name')
-    def validate(cls, package_name, values):
-        if (not values.get('package_id') and not package_name) or (values.get('package_id') and package_name):
-            raise ValueError('either package_id or package_name is required')
-        return package_name
 
-
-class BatchBuildRequest(BaseModel):
-    packages: List[BuildRequest]
-
-
-def gen_body_filters(body_in) -> dict:
-    body = BuildRequest(**body_in)
-    if body.package_id:
-        return {"id": body.package_id}
-    if body.package_name:
-        return {"name": body.package_name}
-
-
-async def create_build_order(package: Package) -> List[Tuple[int, int]]:
+async def create_import_order(package: Package, username: str) -> List[Tuple[int, int]]:
     pkg_list = []
 
     if package.is_package:
-        package_import = await Import.create(package_id=package.id, status=BuildStatus.QUEUED, version=8)
+        package_import = await Import.create(package_id=package.id, status=ImportStatus.QUEUED,
+                                             executor_username=username, version=settings.version)
         pkg_list.append((package.id, package_import.id))
 
     if package.is_module:
@@ -41,10 +27,23 @@ async def create_build_order(package: Package) -> List[Tuple[int, int]]:
             imports = await Import.filter(package_id=subpackage.package_id).all()
             if not imports or len(imports) == 0:
                 subpackage_package = await Package.filter(id=subpackage.package_id).get()
-                pkg_list += await create_build_order(subpackage_package)
+                pkg_list += await create_import_order(subpackage_package, username)
 
-        package_module_import = await Import.create(package_id=package.id, status=BuildStatus.QUEUED, version=8,
-                                                    module=True)
+        package_module_import = await Import.create(package_id=package.id, status=ImportStatus.QUEUED,
+                                                    module=True, executor_username=username, version=settings.version)
         pkg_list.append((package.id, package_module_import.id))
 
     return pkg_list
+
+
+def get_user(request: Request) -> dict:
+    user = request.session.get("user")
+    token = request.session.get("token")
+    if not user or not token:
+        if request.session.get("token"):
+            request.session.pop("token")
+        if request.session.get("user"):
+            request.session.pop("user")
+        raise HTTPException(401, "not authenticated")
+
+    return user

distrobuild/common/tags.py

+from distrobuild.settings import settings
+
+
+def base() -> str:
+    return f"dist-{settings.tag_prefix}{settings.version}"
+
+
+def compose() -> str:
+    return f"{base()}-compose"
+
+
+def testing() -> str:
+    return f"{base()}-testing"
+
+
+def scratch() -> str:
+    return "__scratch"

distrobuild/mbs/__init__.py

+from dataclasses import dataclass
+
+import httpx
+
+from distrobuild.settings import settings
+
+
+class MBSConflictException(BaseException):
+    pass
+
+
+class MBSBuildNotFound(BaseException):
+    pass
+
+
+@dataclass
+class MBSClient:
+    mbs_url: str
+
+    async def get_build(self, mbs_id: int):
+        client = httpx.AsyncClient()
+        async with client:
+            r = await client.get(f"{self.mbs_url}/1/module-builds/{mbs_id}")
+
+            if r.status_code == 404:
+                raise MBSBuildNotFound()
+
+            return r.json()
+
+    async def build(self, token: str, name: str, branch: str, commit: str) -> int:
+        client = httpx.AsyncClient()
+        async with client:
+            r = await client.post(
+                f"{self.mbs_url}/1/module-builds/",
+                headers={
+                    "Authorization": f"Bearer {token}"
+                },
+                json={
+                    "scmurl": f"https://{settings.gitlab_host}{settings.repo_prefix}/modules/{name}?#{commit}",
+                    "branch": branch,
+                }
+            )
+
+            if r.status_code == 409:
+                raise MBSConflictException()
+
+            data = r.json()
+            return data["id"]

distrobuild/models/__init__.py

-from distrobuild.models.enums import BuildStatus, Repo
+from distrobuild.models.enums import ImportStatus, BuildStatus, Repo
 from distrobuild.models.package import Package, PackageModule
 from distrobuild.models.build import Build, Import, ImportCommit
 

distrobuild/models/build.py

 from tortoise import Model, fields
-from distrobuild.models.enums import BuildStatus
-
-
-class Build(Model):
-    id = fields.BigIntField(pk=True)
-    created_at = fields.DatetimeField(auto_now_add=True)
-    updated_at = fields.DatetimeField(auto_add=True, null=True)
-    package = fields.ForeignKeyField("distrobuild.Package", on_delete="CASCADE")
-    status = fields.CharEnumField(BuildStatus)
-    mbs = fields.BooleanField(default=False)
-    koji_id = fields.BigIntField(null=True)
-    mbs_id = fields.BigIntField(null=True)
-    commit = fields.CharField(max_length=255)
-    branch = fields.CharField(max_length=255)
-
-    class Meta:
-        table = "builds"
+from distrobuild.models.enums import BuildStatus, ImportStatus
 
 
 class Import(Model):
     id = fields.BigIntField(pk=True)
     created_at = fields.DatetimeField(auto_now_add=True)
     updated_at = fields.DatetimeField(auto_add=True, null=True)
-    package = fields.ForeignKeyField("distrobuild.Package", on_delete="CASCADE")
-    status = fields.CharEnumField(BuildStatus)
-    version = fields.IntField()
+    package = fields.ForeignKeyField("distrobuild.Package", on_delete="RESTRICT")
+    status = fields.CharEnumField(ImportStatus)
     module = fields.BooleanField(default=False)
+    version = fields.IntField()
+    executor_username = fields.CharField(max_length=255)
 
     commits: fields.ReverseRelation["ImportCommit"] = fields.ReverseRelation
 
@@ -40,7 +25,35 @@ class ImportCommit(Model):
     id = fields.BigIntField(pk=True)
     commit = fields.CharField(max_length=255)
     branch = fields.CharField(max_length=255)
-    import_ = fields.ForeignKeyField("distrobuild.Import", on_delete="CASCADE")
+    import_ = fields.ForeignKeyField("distrobuild.Import", on_delete="RESTRICT")
 
     class Meta:
         table = "import_commits"
+
+
+class Build(Model):
+    id = fields.BigIntField(pk=True)
+    created_at = fields.DatetimeField(auto_now_add=True)
+    updated_at = fields.DatetimeField(auto_add=True, null=True)
+    package = fields.ForeignKeyField("distrobuild.Package", on_delete="RESTRICT")
+    status = fields.CharEnumField(BuildStatus)
+    mbs = fields.BooleanField(default=False)
+    koji_id = fields.BigIntField(null=True)
+    mbs_id = fields.BigIntField(null=True)
+    import_commit = fields.ForeignKeyField("distrobuild.ImportCommit", on_delete="RESTRICT")
+    executor_username = fields.CharField(max_length=255)
+    force_tag = fields.CharField(max_length=255, null=True)
+    exclude_compose = fields.BooleanField(default=False)
+    point_release = fields.CharField(max_length=255)
+
+    class Meta:
+        table = "builds"
+
+
+class Logs(Model):
+    id = fields.BigIntField(pk=True)
+    created_at = fields.DatetimeField(auto_now_add=True)
+    content = fields.TextField()
+
+    class Meta:
+        table = "logs"

distrobuild/models/enums.py

@@ -5,6 +5,10 @@ class Repo(str, Enum):
     BASEOS = "BASEOS"
     APPSTREAM = "APPSTREAM"
     POWERTOOLS = "POWERTOOLS"
+    EXTERNAL = "EXTERNAL"
+    MODULAR_CANDIDATE = "MODULAR_CANDIDATE"
+    ORIGINAL = "ORIGINAL"
+    INFRA = "INFRA"
 
 
 class BuildStatus(str, Enum):
@@ -13,3 +17,11 @@ class BuildStatus(str, Enum):
     FAILED = "FAILED"
     SUCCEEDED = "SUCCEEDED"
     CANCELLED = "CANCELLED"
+
+
+class ImportStatus(str, Enum):
+    QUEUED = "QUEUED"
+    IN_PROGRESS = "IN_PROGRESS"
+    FAILED = "FAILED"
+    SUCCEEDED = "SUCCEEDED"
+    CANCELLED = "CANCELLED"

distrobuild/models/package.py

@@ -7,7 +7,7 @@ class Package(Model):
     created_at = fields.DatetimeField(auto_now_add=True)
     updated_at = fields.DatetimeField(auto_add=True, null=True)
     name = fields.CharField(max_length=255)
-    responsible_user_id = fields.CharField(max_length=255, null=True)
+    responsible_username = fields.CharField(max_length=255)
     is_module = fields.BooleanField(default=False)
     is_package = fields.BooleanField(default=False)
     part_of_module = fields.BooleanField(default=False)
@@ -15,6 +15,7 @@ class Package(Model):
     last_build = fields.DatetimeField(null=True)
 
     el8 = fields.BooleanField(default=False)
+    el9 = fields.BooleanField(default=False)
     repo = fields.CharEnumField(Repo, null=True)
 
     class Meta:
@@ -32,5 +33,8 @@ class PackageModule(Model):
     module_parent_package = fields.ForeignKeyField("distrobuild.Package", on_delete="RESTRICT",
                                                    related_name="m_module_parent_pacakges")
 
+    class Meta:
+        table = "package_modules"
+
     class PydanticMeta:
         backward_relations = False

distrobuild/routes/bootstrap.py

-from fastapi import APIRouter
+from fastapi import APIRouter, Request
 from fastapi.responses import JSONResponse
 
 from distrobuild.bootstrap import process_repo_dump, process_module_dump
+from distrobuild.common import get_user
 from distrobuild.models import Repo
 
 router = APIRouter(prefix="/bootstrap")
 
 
 @router.post("/modules")
-async def bootstrap_modules():
-    await process_module_dump()
+async def bootstrap_modules(request: Request):
+    user = get_user(request)
+    await process_module_dump(user["preferred_username"])
     return JSONResponse(content={})
 
 
 @router.post("/{repo}")
-async def bootstrap_repo(repo: Repo):
-    await process_repo_dump(repo)
+async def bootstrap_repo(request: Request, repo: Repo):
+    user = get_user(request)
+    await process_repo_dump(repo, user["preferred_username"])
     return JSONResponse(content={})

distrobuild/routes/builds.py

-from fastapi import APIRouter, Depends, HTTPException
+from typing import Optional, List, Dict
+
+from fastapi import APIRouter, Depends, HTTPException, Request
 from fastapi_pagination import Page, pagination_params
 from fastapi_pagination.ext.tortoise import paginate
+from pydantic import BaseModel, validator
 
-from distrobuild.common import BuildRequest, gen_body_filters, BatchBuildRequest
-from distrobuild.models import Build, Import, ImportCommit, Package, PackageModule, BuildStatus
+from distrobuild.common import gen_body_filters, get_user, tags
+from distrobuild.models import Build, Import, ImportCommit, Package, PackageModule, BuildStatus, Repo
 from distrobuild.serialize import Build_Pydantic
+from distrobuild.session import message_cipher
 from distrobuild_scheduler import build_package_task
 
 router = APIRouter(prefix="/builds")
 
 
+class BuildRequest(BaseModel):
+    scratch: bool = False
+    testing: bool = False
+    package_id: Optional[int]
+    package_name: Optional[str]
+
+    @validator("package_name")
+    def validate(cls, package_name, values):
+        if (not values.get("package_id") and not package_name) or (values.get("package_id") and package_name):
+            raise ValueError("either package_id or package_name is required")
+        return package_name
+
+
+class BatchBuildRequest(BaseModel):
+    packages: List[BuildRequest]
+
+
 @router.get("/", response_model=Page[Build_Pydantic], dependencies=[Depends(pagination_params)])
 async def list_builds():
-    return await paginate(Build.all().order_by('-created_at').prefetch_related("package"))
+    return await paginate(Build.all().order_by("-created_at").prefetch_related("package", "import_commit"))
+
+
+@router.get("/{build_id}", response_model=Build_Pydantic)
+async def get_build(build_id: int):
+    return await Build_Pydantic.from_queryset_single(
+        Build.filter(id=build_id).prefetch_related("package", "import_commit").first()
+    )
 
 
 @router.post("/", status_code=202)
-async def queue_build(body: BuildRequest):
+async def queue_build(request: Request, body: Dict[str, BuildRequest]):
+    user = get_user(request)
+
     filters = gen_body_filters(body)
     package = await Package.filter(**filters).get_or_none()
     if not package:
         raise HTTPException(404, detail="package does not exist")
 
-    mbs = package.is_module
+    if package.repo == Repo.MODULAR_CANDIDATE:
+        raise HTTPException(401, detail="modular subpackages cannot be built, build the main module")
+
+    if package.part_of_module:
+        raise HTTPException(401, detail="this package is part of a module. build the main module")
+
+    extras = {
+        "mbs": package.is_module
+    }
+    token = None
     package_modules = await PackageModule.filter(package_id=package.id)
     if len(package_modules) > 0:
-        mbs = True
+        extras["mbs"] = True
+        token = message_cipher.encrypt(request.session.get("token").encode()).decode()
+
+    if body.get("testing") and not body.get("scratch"):
+        extras["force_tag"] = tags.testing()
+
+    if body.get("scratch"):
+        extras["force_tag"] = tags.scratch()
 
     latest_import = await Import.filter(package_id=package.id).order_by("-created_at").first()
     import_commits = await ImportCommit.filter(import__id=latest_import.id).all()
     for import_commit in import_commits:
         if "-beta" not in import_commit.branch:
-            build = await Build.create(package_id=package.id, status=BuildStatus.QUEUED, mbs=mbs,
-                                       commit=import_commit.commit, branch=import_commit.branch)
-            await build_package_task(package.id, build.id)
+            build = await Build.create(package_id=package.id, status=BuildStatus.QUEUED,
+                                       executor_username=user["preferred_username"], point_release="8_4",
+                                       import_commit_id=import_commit.id, **extras)
+            await build_package_task(package.id, build.id, token)
 
     return {}
 
 
 @router.post("/batch", status_code=202)
-async def batch_queue_build(body: BatchBuildRequest):
+async def batch_queue_build(request: Request, body: BatchBuildRequest):
     for build_request in body.packages:
-        await queue_build(build_request)
+        await queue_build(request, dict(build_request))
 
     return {}

distrobuild/routes/imports.py

-from fastapi import APIRouter, Depends, HTTPException
-from fastapi_pagination import pagination_params
+from typing import Optional, List, Dict
+
+from fastapi import APIRouter, Depends, HTTPException, Request
+from fastapi_pagination import pagination_params, Page
 from fastapi_pagination.ext.tortoise import paginate
+from pydantic import BaseModel, validator
 from starlette.responses import PlainTextResponse
 
-from distrobuild.common import BuildRequest, gen_body_filters, BatchBuildRequest, create_build_order
-from distrobuild.models import Import, Package
+from distrobuild.common import gen_body_filters, create_import_order, get_user
+from distrobuild.models import Import, Package, Repo
+from distrobuild.serialize import Import_Pydantic
 from distrobuild_scheduler import import_package_task
 
 router = APIRouter(prefix="/imports")
 
 
-# response_model causes some weird errors with Import. why?
-# TODO: find out (removing response_model for now)
-@router.get("/", dependencies=[Depends(pagination_params)])
+class ImportRequest(BaseModel):
+    full_history: bool = False
+    single_tag: Optional[str]
+    package_id: Optional[int]
+    package_name: Optional[str]
+
+    @validator("package_name")
+    def validate(cls, package_name, values):
+        if (not values.get("package_id") and not package_name) or (values.get("package_id") and package_name):
+            raise ValueError("either package_id or package_name is required")
+        return package_name
+
+
+class BatchImportRequest(BaseModel):
+    packages: List[ImportRequest]
+
+
+@router.get("/", response_model=Page[Import_Pydantic], dependencies=[Depends(pagination_params)])
 async def list_imports():
-    return await paginate(Import.all().order_by('-created_at').prefetch_related("package"))
+    return await paginate(Import.all().order_by("-created_at").prefetch_related("package"))
+
+
+@router.get("/{import_id}", response_model=Import_Pydantic)
+async def get_import(import_id: int):
+    return await Import_Pydantic.from_queryset_single(Import.filter(id=import_id).prefetch_related("package").first())
 
 
 @router.get("/{import_id}/logs", response_class=PlainTextResponse)
@@ -30,21 +54,26 @@ async def get_import_logs(import_id: int):
 
 
 @router.post("/", status_code=202)
-async def import_package_route(body: BuildRequest):
+async def import_package_route(request: Request, body: Dict[str, ImportRequest]):
+    user = get_user(request)
+
     filters = gen_body_filters(body)
     package = await Package.filter(**filters).get_or_none()
     if not package:
         raise HTTPException(404, detail="package does not exist")
 
-    build_order = await create_build_order(package)
+    if package.repo == Repo.MODULAR_CANDIDATE:
+        raise HTTPException(401, detail="modular subpackages cannot be imported")
+