Commit eab5d95d authored by Mustafa Gezen's avatar Mustafa Gezen 🏗
Browse files

This is a very big update. Click to read more

* Move batches into another namespace (`/batches`)
* Add UI for listing and showing batches
* Add Redis backed sessions to not leak JWT
* Add Lookaside API
* Add Lookaside UI
* Ability to skip modules during batch builds
* Ability to cancel batch builds/imports
* Ability to retry failed builds/imports in batches
* Add logout link to UI
* Add imports list
* Add builds list
* Add commit info to imports in the UI
* Add commit, Koji and MBS info to builds in the UI
* Fix IN_PROGRESS status for imports
parent 256b010e
......@@ -28,18 +28,20 @@ from fastapi import FastAPI, Request
from fastapi.responses import HTMLResponse
from fastapi.templating import Jinja2Templates
from fastapi.staticfiles import StaticFiles
from starlette.middleware.sessions import SessionMiddleware
from tortoise.contrib.fastapi import register_tortoise
from distrobuild.middleware.redis_session import RedisSessionMiddleware
from distrobuild.settings import TORTOISE_ORM, settings
from distrobuild.routes import register_routes
# init sessions
from distrobuild import session
from distrobuild_scheduler import init_channel
app = FastAPI()
app.add_middleware(SessionMiddleware, secret_key=settings.session_secret, max_age=3500)
app.add_middleware(RedisSessionMiddleware, secret_key=settings.session_secret, max_age=3000, fapi=app,
https_only=settings.production)
app.mount("/static/files", StaticFiles(directory="ui/dist/files"), name="static")
register_routes(app)
......
......@@ -68,6 +68,8 @@ async def process_module_dump(responsible_username: str) -> None:
subpackages = [x.strip() for x in module_list[module].split(",")]
for module_package in subpackages:
if len(module_package.strip()) == 0:
continue
m_package_name = module_package.strip()
m_package = await Package.filter(name=m_package_name).get_or_none()
......
......@@ -18,11 +18,11 @@
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
from typing import List, Tuple
from typing import List, Tuple, Optional
from fastapi import Request, HTTPException
from distrobuild.models import Import, ImportStatus, Package, PackageModule
from distrobuild.models import Import, ImportStatus, Package, PackageModule, BatchImportPackage
from distrobuild.settings import settings
......@@ -33,12 +33,15 @@ def gen_body_filters(body: dict) -> dict:
return {"id": body["package_id"]}
async def create_import_order(package: Package, username: str) -> List[Tuple[int, int]]:
async def create_import_order(package: Package, username: str, batch_import_id: Optional[int] = None) -> List[
Tuple[int, int]]:
pkg_list = []
if package.is_package:
package_import = await Import.create(package_id=package.id, status=ImportStatus.QUEUED,
executor_username=username, version=settings.version)
if batch_import_id:
await BatchImportPackage.create(import_id=package_import.id, batch_import_id=batch_import_id)
pkg_list.append((package.id, package_import.id))
if package.is_module:
......@@ -47,15 +50,35 @@ async def create_import_order(package: Package, username: str) -> List[Tuple[int
imports = await Import.filter(package_id=subpackage.package_id).all()
if not imports or len(imports) == 0:
subpackage_package = await Package.filter(id=subpackage.package_id).get()
pkg_list += await create_import_order(subpackage_package, username)
pkg_list += await create_import_order(subpackage_package, username, batch_import_id)
package_module_import = await Import.create(package_id=package.id, status=ImportStatus.QUEUED,
module=True, executor_username=username, version=settings.version)
if batch_import_id:
await BatchImportPackage.create(import_id=package_module_import.id, batch_import_id=batch_import_id)
pkg_list.append((package.id, package_module_import.id))
return pkg_list
async def batch_list_check(packages):
for package in packages:
filters = {}
if package.get("package_id"):
filters["id"] = package["package_id"]
elif package.get("package_name"):
filters["name"] = package["package_name"]
db_package = await Package.filter(**filters).first()
if not db_package:
detail = ""
if package.get("package_id"):
detail = f"Package with id {package['package_id']} not found"
elif package.get("package_name"):
detail = f"Package with name {package['package_name']} not found"
raise HTTPException(412, detail=detail)
def get_user(request: Request) -> dict:
user = request.session.get("user")
token = request.session.get("token")
......
# Copyright (c) 2021 The Distrobuild Authors
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
from enum import Enum
from typing import BinaryIO
import boto3
from botocore.exceptions import ClientError
from google.cloud import storage
class LookasideUploadException(Exception):
pass
class LookasideBackend(str, Enum):
FILE = "file"
S3 = "s3"
GCS = "gcs"
class Lookaside:
backend: LookasideBackend
def __init__(self, url: str):
if url.startswith("file://"):
self.backend = LookasideBackend.FILE
self.dir = url.removeprefix("file://")
elif url.startswith("s3://"):
self.backend = LookasideBackend.S3
self.s3 = boto3.client("s3")
self.bucket = url.removeprefix("s3://")
elif url.startswith("gs://"):
self.backend = LookasideBackend.GCS
self.gcs = storage.Client().bucket(url.removeprefix("gs://"))
def upload(self, f: BinaryIO, name: str):
if self.backend == LookasideBackend.FILE:
file_content = f.read()
with open(f"{self.dir}/{name}", "wb") as f2:
f2.write(file_content)
elif self.backend == LookasideBackend.S3:
try:
self.s3.upload_fileobj(f, self.bucket, name)
except ClientError as e:
raise LookasideUploadException(e)
elif self.backend == LookasideBackend.GCS:
blob = self.gcs.blob(name)
blob.upload_from_file(f)
......@@ -25,11 +25,15 @@ import httpx
from distrobuild.settings import settings
class MBSConflictException(BaseException):
class MBSConflictException(Exception):
pass
class MBSBuildNotFound(BaseException):
class MBSUnauthorizedException(Exception):
pass
class MBSBuildNotFound(Exception):
pass
......@@ -49,7 +53,6 @@ class MBSClient:
async def build(self, token: str, name: str, branch: str, commit: str) -> int:
scmurl = f"https://{settings.gitlab_host}{settings.repo_prefix}/modules/{name}?#{commit}"
print(scmurl)
client = httpx.AsyncClient()
async with client:
......@@ -67,6 +70,8 @@ class MBSClient:
if r.status_code == 409:
raise MBSConflictException()
if r.status_code == 401:
raise MBSUnauthorizedException()
data = r.json()
print(data)
return data["id"]
# Copyright (c) 2021 The Distrobuild Authors
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
from base64 import b64decode, b64encode
import typing
import secrets
import itsdangerous
import json
import aioredis
from fastapi import FastAPI
from itsdangerous import SignatureExpired, BadTimeSignature
from starlette.datastructures import Secret
from starlette.middleware.base import BaseHTTPMiddleware, RequestResponseEndpoint, DispatchFunction
from starlette.requests import Request
from starlette.responses import Response
from starlette.types import ASGIApp
from distrobuild.settings import settings
class RedisSessionMiddleware(BaseHTTPMiddleware):
def __init__(
self,
app: ASGIApp,
fapi: FastAPI,
secret_key: typing.Union[str, Secret],
session_cookie: str = "session",
max_age: int = 3000,
same_site: str = "lax",
https_only: bool = False,
dispatch: DispatchFunction = None
) -> None:
super().__init__(app, dispatch)
self.redis = aioredis.create_redis_pool(settings.redis_url)
self.redis_inited = False
self.app = app
self.signer = itsdangerous.TimestampSigner(str(secret_key))
self.session_cookie = session_cookie
self.max_age = max_age
self.same_site = same_site
self.https_only = https_only
@fapi.on_event("shutdown")
async def shutdown():
self.redis.close()
await self.redis.wait_closed()
async def dispatch(self, request: Request, call_next: RequestResponseEndpoint) -> Response:
if not self.redis_inited:
self.redis = await self.redis
self.redis_inited = True
initial_session_was_empty = True
redis_key = ""
if self.session_cookie in request.cookies:
data = request.cookies[self.session_cookie].encode("utf-8")
try:
data = self.signer.unsign(data, max_age=self.max_age)
redis_key = data
redis_val = await self.redis.get(data, encoding="utf-8")
request.scope["session"] = json.loads(b64decode(redis_val))
initial_session_was_empty = False
except (BadTimeSignature, SignatureExpired):
request.scope["session"] = {}
else:
request.scope["session"] = {}
response = await call_next(request)
if request.scope["session"]:
redis_val = b64encode(json.dumps(request.scope["session"]).encode("utf-8"))
if redis_key == "":
redis_key = secrets.token_hex(32)
await self.redis.set(redis_key, redis_val)
await self.redis.expire(redis_key, 3000)
data = self.signer.sign(redis_key)
response.set_cookie(self.session_cookie, data.decode("utf-8"), self.max_age, httponly=True,
samesite=self.same_site, path="/", secure=self.https_only)
elif not initial_session_was_empty:
response.delete_cookie(self.session_cookie, path="/")
return response
......@@ -21,4 +21,6 @@
from distrobuild.models.enums import ImportStatus, BuildStatus, Repo
from distrobuild.models.package import Package, PackageModule
from distrobuild.models.build import Build, Import, ImportCommit
from distrobuild.models.batch import BatchImport, BatchBuild, BatchImportPackage, BatchBuildPackage
from distrobuild.models.lookaside import LookasideBlob
# Copyright (c) 2021 The Distrobuild Authors
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
from tortoise import Model, fields
from distrobuild.models.build import Import, Build
class BatchImport(Model):
id = fields.BigIntField(pk=True)
created_at = fields.DatetimeField(auto_now_add=True)
updated_at = fields.DatetimeField(auto_add=True, null=True)
imports: fields.ManyToManyRelation[Import] = fields.ManyToManyField("distrobuild.Import",
related_name="batch_imports",
backward_key="batch_import_id",
through="batch_import_packages")
class Meta:
table = "batch_imports"
class BatchBuild(Model):
id = fields.BigIntField(pk=True)
created_at = fields.DatetimeField(auto_now_add=True)
updated_at = fields.DatetimeField(auto_add=True, null=True)
builds: fields.ManyToManyRelation[Build] = fields.ManyToManyField("distrobuild.Build",
related_name="batch_builds",
backward_key="batch_build_id",
through="batch_build_packages")
class Meta:
table = "batch_builds"
class BatchImportPackage(Model):
id = fields.BigIntField(pk=True)
import_id = fields.BigIntField()
batch_import_id = fields.BigIntField()
class Meta:
table = "batch_import_packages"
class BatchBuildPackage(Model):
id = fields.BigIntField(pk=True)
build_id = fields.BigIntField()
batch_build_id = fields.BigIntField()
class Meta:
table = "batch_build_packages"
......@@ -26,41 +26,50 @@ class Import(Model):
id = fields.BigIntField(pk=True)
created_at = fields.DatetimeField(auto_now_add=True)
updated_at = fields.DatetimeField(auto_add=True, null=True)
package = fields.ForeignKeyField("distrobuild.Package", on_delete="RESTRICT")
package = fields.ForeignKeyField("distrobuild.Package", on_delete="RESTRICT", related_name="imports")
status = fields.CharEnumField(ImportStatus)
module = fields.BooleanField(default=False)
version = fields.IntField()
executor_username = fields.CharField(max_length=255)
commits: fields.ReverseRelation["ImportCommit"] = fields.ReverseRelation
commits: fields.ManyToManyRelation["ImportCommit"] = fields.ManyToManyField("distrobuild.ImportCommit",
related_name="imports",
forward_key="id",
backward_key="import__id",
through="import_commits")
class Meta:
table = "imports"
ordering = ["-created_at"]
class PydanticMeta:
backward_relations = False
exclude = ("batch_imports",)
class ImportCommit(Model):
id = fields.BigIntField(pk=True)
commit = fields.CharField(max_length=255)
branch = fields.CharField(max_length=255)
import_ = fields.ForeignKeyField("distrobuild.Import", on_delete="RESTRICT")
import__id = fields.BigIntField()
class Meta:
table = "import_commits"
class PydanticMeta:
exclude = ("import_", "imports", "builds")
class Build(Model):
id = fields.BigIntField(pk=True)
created_at = fields.DatetimeField(auto_now_add=True)
updated_at = fields.DatetimeField(auto_add=True, null=True)
package = fields.ForeignKeyField("distrobuild.Package", on_delete="RESTRICT")
package = fields.ForeignKeyField("distrobuild.Package", on_delete="RESTRICT", related_name="builds")
status = fields.CharEnumField(BuildStatus)
mbs = fields.BooleanField(default=False)
signed = fields.BooleanField(default=False)
koji_id = fields.BigIntField(null=True)
mbs_id = fields.BigIntField(null=True)
import_commit = fields.ForeignKeyField("distrobuild.ImportCommit", on_delete="RESTRICT")
import_commit = fields.ForeignKeyField("distrobuild.ImportCommit", on_delete="RESTRICT", related_name="builds")
executor_username = fields.CharField(max_length=255)
force_tag = fields.CharField(max_length=255, null=True)
exclude_compose = fields.BooleanField(default=False)
......@@ -68,6 +77,10 @@ class Build(Model):
class Meta:
table = "builds"
ordering = ["-created_at"]
class PydanticMeta:
exclude = ("batch_builds",)
class Logs(Model):
......
# Copyright (c) 2021 The Distrobuild Authors
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
from tortoise import Model, fields
class LookasideBlob(Model):
id = fields.BigIntField(pk=True)
created_at = fields.DatetimeField(auto_now_add=True)
sum = fields.TextField()
executor_username = fields.TextField()
class Meta:
table = "lookaside_blobs"
......@@ -17,8 +17,11 @@
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
from typing import Optional
from tortoise import Model, fields
from distrobuild.models.build import Build, Import
from distrobuild.models.enums import Repo
......@@ -30,7 +33,9 @@ class Package(Model):
responsible_username = fields.CharField(max_length=255)
is_module = fields.BooleanField(default=False)
is_package = fields.BooleanField(default=False)
is_published = fields.BooleanField(default=False)
part_of_module = fields.BooleanField(default=False)
signed = fields.BooleanField(default=False)
last_import = fields.DatetimeField(null=True)
last_build = fields.DatetimeField(null=True)
......@@ -38,11 +43,14 @@ class Package(Model):
el9 = fields.BooleanField(default=False)
repo = fields.CharEnumField(Repo, null=True)
builds: Optional[fields.ReverseRelation[Build]]
imports: Optional[fields.ReverseRelation[Import]]
class Meta:
table = "packages"
class PydanticMeta:
backward_relations = False
exclude = ("m_module_parent_packages", "m_subpackages")
class PackageModule(Model):
......@@ -51,7 +59,7 @@ class PackageModule(Model):
updated_at = fields.DatetimeField(auto_add=True, null=True)
package = fields.ForeignKeyField("distrobuild.Package", on_delete="RESTRICT", related_name="m_subpackages")
module_parent_package = fields.ForeignKeyField("distrobuild.Package", on_delete="RESTRICT",
related_name="m_module_parent_pacakges")
related_name="m_module_parent_packages")
class Meta:
table = "package_modules"
......
......@@ -20,7 +20,7 @@
from fastapi import APIRouter
from distrobuild.routes import builds, imports, packages, bootstrap, oidc
from distrobuild.routes import builds, imports, packages, bootstrap, oidc, batches, lookaside
_base_router = APIRouter(prefix="/api")
......@@ -31,5 +31,7 @@ def register_routes(app):
_base_router.include_router(bootstrap.router)
_base_router.include_router(builds.router)
_base_router.include_router(imports.router)
_base_router.include_router(batches.router)
_base_router.include_router(lookaside.router)
app.include_router(_base_router)
# Copyright (c) 2021 The Distrobuild Authors
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
from typing import List
from fastapi import APIRouter, Depends, Request, HTTPException
from fastapi_pagination import Page, pagination_params
from fastapi_pagination.ext.tortoise import paginate
from pydantic import BaseModel
from tortoise.functions import Count
from distrobuild.common import batch_list_check
from distrobuild.models import BatchImport, BatchBuild, ImportStatus, BuildStatus
from distrobuild.routes.builds import BuildRequest, queue_build
from distrobuild.routes.imports import ImportRequest, import_package_route
from distrobuild.serialize import BatchImport_Pydantic, BatchBuild_Pydantic
router = APIRouter(prefix="/batches")
class BatchImportRequest(BaseModel):
should_precheck: bool = True
packages: List[ImportRequest]
class BatchBuildRequest(BaseModel):
should_precheck: bool = True
ignore_modules: bool = False
packages: List[BuildRequest]
class NewBatchResponse(BaseModel):