Commit 47b47f85 authored by Mustafa Gezen's avatar Mustafa Gezen 🏗
Browse files

Initial commit

parents
File added
.env
ui/node_modules
ui/dist
__pycache__
*.pyc
.venv
FROM quay.io/centos/centos@sha256:dbbacecc49b088458781c16f3775f2a2ec7521079034a7ba499c8b0bb7f86875 # centos8.3.2011
[aerich]
tortoise_orm = distrobuild.settings.TORTOISE_ORM
location = ./migrations
from tortoise import Tortoise
Tortoise.init_models(["distrobuild.models"], "distrobuild")
from fastapi import FastAPI, Request
from fastapi.responses import HTMLResponse
from fastapi.templating import Jinja2Templates
from fastapi.staticfiles import StaticFiles
from tortoise.contrib.fastapi import register_tortoise
from distrobuild import settings
from distrobuild.routes import register_routes
# init sessions
import distrobuild.session
app = FastAPI()
app.mount("/static/files", StaticFiles(directory="ui/dist/files"), name="static")
register_routes(app)
templates = Jinja2Templates(directory="ui/dist/templates")
@app.get("/{full_path:path}", response_class=HTMLResponse, include_in_schema=False)
async def serve_frontend(request: Request):
return templates.TemplateResponse("index.html", {"request": request})
register_tortoise(
app,
config=settings.TORTOISE_ORM
)
import json
import re
from distrobuild.models import Package, PackageModule, Repo
async def process_repo_dump(repo: Repo) -> None:
with open(f"/tmp/{repo}_ALL.txt", "r") as f:
lines = f.readlines()
for line in lines:
package_name = line.strip()
if await Package.filter(name=package_name).get_or_none():
continue
await Package.create(name=package_name, el8=True, is_package=True, repo=repo)
async def process_module_dump() -> None:
f = open("/tmp/modules.json")
module_list = json.loads(f.read())
f.close()
for module in module_list.keys():
package_name = module.strip()
existing_package = await Package.filter(name=package_name).get_or_none()
if existing_package:
if not existing_package.is_module:
existing_package.is_module = True
await existing_package.save()
else:
existing_package = await Package.create(name=package_name, el8=True, is_module=True)
subpackages = [x.strip() for x in module_list[module].split(",")]
for module_package in subpackages:
m_package_name = module_package.strip()
m_package = await Package.filter(name=m_package_name).get_or_none()
if not m_package:
continue
m_package.part_of_module = True
await m_package.save()
await PackageModule.create(package_id=m_package.id, module_parent_package_id=existing_package.id)
import datetime
from typing import List
from enum import Enum
from tortoise import Model, Tortoise, fields
class Repo(str, Enum):
BASEOS = "BASEOS"
APPSTREAM = "APPSTREAM"
POWERTOOLS = "POWERTOOLS"
class BuildStatus(str, Enum):
QUEUED = "QUEUED"
BUILDING = "BUILDING"
FAILED = "FAILED"
SUCCEEDED = "SUCCEEDED"
CANCELLED = "CANCELLED"
class Package(Model):
id = fields.BigIntField(pk=True)
created_at = fields.DatetimeField(auto_now_add=True)
updated_at = fields.DatetimeField(auto_add=True, null=True)
name = fields.CharField(max_length=255)
responsible_user_id = fields.CharField(max_length=255, null=True)
is_module = fields.BooleanField(default=False)
is_package = fields.BooleanField(default=False)
part_of_module = fields.BooleanField(default=False)
last_import = fields.DatetimeField(null=True)
el8 = fields.BooleanField(default=False)
repo = fields.CharEnumField(Repo, null=True)
class Meta:
table = "packages"
class PydanticMeta:
backward_relations = False
class PackageModule(Model):
id = fields.BigIntField(pk=True)
created_at = fields.DatetimeField(auto_now_add=True)
updated_at = fields.DatetimeField(auto_add=True, null=True)
package = fields.ForeignKeyField("distrobuild.Package", on_delete="RESTRICT", related_name="m_subpackages")
module_parent_package = fields.ForeignKeyField("distrobuild.Package", on_delete="RESTRICT", related_name="m_module_parent_pacakges")
class PydanticMeta:
backward_relations = False
class Build(Model):
id = fields.BigIntField(pk=True)
created_at = fields.DatetimeField(auto_now_add=True)
updated_at = fields.DatetimeField(auto_add=True, null=True)
package = fields.ForeignKeyField("distrobuild.Package", on_delete="CASCADE")
status = fields.CharEnumField(BuildStatus)
mbs = fields.BooleanField(default=False)
koji_id = fields.BigIntField(null=True)
mbs_id = fields.BigIntField(null=True)
class Meta:
table = "builds"
class Import(Model):
id = fields.BigIntField(pk=True)
created_at = fields.DatetimeField(auto_now_add=True)
updated_at = fields.DatetimeField(auto_add=True, null=True)
package = fields.ForeignKeyField("distrobuild.Package", on_delete="CASCADE")
status = fields.CharEnumField(BuildStatus)
version = fields.IntField()
commit = fields.CharField(max_length=255, null=True)
class Meta:
table = "imports"
from tortoise.contrib.pydantic import pydantic_model_creator
from distrobuild import models
Package_Pydantic = pydantic_model_creator(models.Package, name="Package")
PackageModule_Pydantic = pydantic_model_creator(models.PackageModule, name="PackageModule")
Build_Pydantic = pydantic_model_creator(models.Build, name="Build")
Import_Pydantic = pydantic_model_creator(models.Import, name="Import")
mustafa@Mustafas-MacBook-Pro.local.55439
\ No newline at end of file
from fastapi import APIRouter
from distrobuild.routes import build, packages, bootstrap
_base_router = APIRouter(prefix="/api")
def register_routes(app):
_base_router.include_router(packages.router)
_base_router.include_router(bootstrap.router)
_base_router.include_router(build.router)
app.include_router(_base_router)
from fastapi import APIRouter
from fastapi.responses import JSONResponse
from distrobuild.bootstrap import process_repo_dump, process_module_dump
from distrobuild.models import Repo
router = APIRouter(prefix="/bootstrap")
@router.post("/modules")
async def bootstrap_modules():
await process_module_dump()
return JSONResponse(content={})
@router.post("/{repo}")
async def bootstrap_repo(repo: Repo):
await process_repo_dump(repo)
return JSONResponse(content={})
import datetime
from typing import Optional, List
from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException
from fastapi.responses import PlainTextResponse
from fastapi_pagination import Page, pagination_params
from fastapi_pagination.ext.tortoise import paginate
from tortoise.transactions import atomic
from pydantic import BaseModel, validator
from distrobuild.models import Build, Import, Package, PackageModule, BuildStatus
from distrobuild.pydantic import Build_Pydantic, Import_Pydantic
from distrobuild.session import gl, koji_session
from distrobuild.settings import settings
from distrobuild import srpmproc
router = APIRouter(prefix="/build")
class BuildRequest(BaseModel):
package_id: Optional[int]
package_name: Optional[str]
@validator('package_name')
def validate(cls, package_name, values):
if (not values.get('package_id') and not package_name) or (values.get('package_id') and package_name):
raise ValueError('either package_id or package_name is required')
return package_name
class BatchBuildRequest(BaseModel):
packages: List[BuildRequest]
def gen_body_filters(body: BuildRequest) -> dict:
if body.get("package_id"):
return {"id": body["package_id"]}
if body.get("package_name"):
return {"name": body["package_name"]}
@atomic()
async def do_build_task(package: Package, build: Build):
if build.mbs:
pass
else:
scratch = False
target = "dist-rocky8"
latest_import = await Import.filter(package_id=package.id).order_by("-created_at").first()
source = f"git+https://{settings.gitlab_host}{settings.repo_prefix}/rpms/{package.name}.git?#{latest_import.commit}"
task_id = koji_session.build(source, target)
build.koji_id = task_id
build.status = BuildStatus.BUILDING
await build.save()
async def build_task(package: Package, build: Build):
try:
await do_build_task(package, build)
except Exception as e:
print(e)
build.status = BuildStatus.FAILED
await build.save()
@atomic()
async def do_import_task(package: Package, import_obj: Import):
koji_session.packageListAdd("dist-rocky8", package.name, "distrobuild")
import_obj.status = BuildStatus.BUILDING
await import_obj.save()
await srpmproc.import_project(import_obj.id, package.name)
package.last_import = datetime.datetime.now()
await package.save()
project = gl.projects.get(f"{settings.repo_prefix.removeprefix('/')}/rpms/{package.name}")
project.visibility = "public"
project.save()
latest_commit = project.commits.list(ref_name=f"r{import_obj.version}")[0].id
import_obj.status = BuildStatus.SUCCEEDED
import_obj.commit = latest_commit
await import_obj.save()
async def import_task(package: Package, import_obj: Import):
try:
await do_import_task(package, import_obj)
except Exception as e:
print(e)
import_obj.status = BuildStatus.FAILED
await import_obj.save()
@router.get("/", response_model=Page[Build_Pydantic], dependencies=[Depends(pagination_params)])
async def list_builds():
return await paginate(Build.all().order_by('-created_at').prefetch_related("package"))
# response_model causes some weird errors with Import. why?
# TODO: find out (removing response_model for now)
@router.get("/imports/", dependencies=[Depends(pagination_params)])
async def list_imports():
return await paginate(Import.all().order_by('-created_at').prefetch_related("package"))
@router.get("/imports/{id}/logs", response_class=PlainTextResponse)
async def get_import_logs(id: int):
import_obj = await Import.filter(id=id).get()
with open(f"/tmp/import-{import_obj.id}.log") as f:
return f.read()
@router.post("/", status_code=202)
async def queue_build(body: BuildRequest, background_tasks: BackgroundTasks):
filters = gen_body_filters(body)
package = await Package.filter(**filters).get_or_none()
if not package:
raise HTTPException(404, detail="package does not exist")
mbs = package.is_module
package_modules = await PackageModule.filter(package_id=package.id)
if len(package_modules) > 0:
mbs = True
build = await Build.create(package_id=package.id, status=BuildStatus.QUEUED, mbs=mbs)
background_tasks.add_task(build_task, package, build)
return {}
@router.post("/batch", status_code=202)
async def batch_queue_build(body: BatchBuildRequest, background_tasks: BackgroundTasks):
for build_request in body.packages:
await queue_build(build_request, background_tasks)
return {}
@router.post("/imports/", status_code=202)
async def import_package(body: BuildRequest, background_tasks: BackgroundTasks):
filters = gen_body_filters(body)
package = await Package.filter(**filters).get_or_none()
if not package:
raise HTTPException(404, detail="package does not exist")
import_obj = await Import.create(package_id=package.id, status=BuildStatus.QUEUED, version=8)
background_tasks.add_task(import_task, package, import_obj)
return {}
@router.post("/imports/batch", status_code=202)
async def batch_import_package(body: BatchBuildRequest, background_tasks: BackgroundTasks):
for build_request in body.packages:
await import_package(build_request, background_tasks)
return {}
from typing import List, Optional
from fastapi import APIRouter, Depends
from pydantic import BaseModel
from fastapi_pagination import Page, pagination_params
from fastapi_pagination.ext.tortoise import paginate
from distrobuild.models import Package
from distrobuild.pydantic import Package_Pydantic
router = APIRouter(prefix="/packages")
@router.get("/", response_model=Page[Package_Pydantic], dependencies=[Depends(pagination_params)])
async def list_packages(name: Optional[str] = None, modules_only: bool = False, non_modules_only: bool = False):
filters = {}
if name:
filters["name__icontains"] = name
if modules_only:
filters["is_module"] = True
if non_modules_only:
filters["is_module"] = False
filters["part_of_module"] = False
return await paginate(Package.all().order_by('updated_at', 'name').filter(**filters))
import os
import socket
import optparse
import gitlab
import koji
import requests
from distrobuild.settings import settings
gl = gitlab.Gitlab(f"https://{settings.gitlab_host}", private_token=settings.gitlab_api_key)
# from https://pagure.io/koji/blob/master/f/cli/koji_cli/lib.py
def ensure_connection(session):
try:
ret = session.getAPIVersion()
except requests.exceptions.ConnectionError:
raise Exception("Error: Unable to connect to server")
if ret != koji.API_VERSION:
print("WARNING: The server is at API version %d and "
"the client is at %d" % (ret, koji.API_VERSION))
def activate_session(session, options):
"""Test and login the session is applicable"""
if isinstance(options, dict):
options = optparse.Values(options)
noauth = options.authtype == "noauth" or getattr(options, 'noauth', False)
runas = getattr(options, 'runas', None)
if noauth:
# skip authentication
pass
elif options.authtype == "ssl" or os.path.isfile(options.cert) and options.authtype is None:
# authenticate using SSL client cert
session.ssl_login(options.cert, None, options.serverca, proxyuser=runas)
elif options.authtype == "password" \
or getattr(options, 'user', None) \
and options.authtype is None:
# authenticate using user/password
session.login()
elif options.authtype == "kerberos" or options.authtype is None:
try:
if getattr(options, 'keytab', None) and getattr(options, 'principal', None):
session.gssapi_login(principal=options.principal, keytab=options.keytab,
proxyuser=runas)
else:
session.gssapi_login(proxyuser=runas)
except socket.error as e:
print("Could not connect to Kerberos authentication service: %s" % e.args[1])
if not noauth and not session.logged_in:
raise Exception("Unable to log in, no authentication methods available")
ensure_connection(session)
if getattr(options, 'debug', None):
print("successfully connected to hub")
# end
koji_config = koji.read_config("koji")
koji_session = koji.ClientSession(settings.koji_hub_url, koji_config)
activate_session(koji_session, koji_config)
from typing import Optional
from enum import Enum
from pydantic import BaseSettings
class Settings(BaseSettings):
gitlab_host: str
koji_hub_url: str
repo_prefix: str
storage_addr: str
ssh_user: str = "git"
ssh_port: int = 22
ssh_key_location: Optional[str]
version: int = 8
bugs_api_key: str
gitlab_api_key: str
class Config:
env_file = "/etc/distrobuild/settings"
settings = Settings()
TORTOISE_ORM = {
"connections": {"default": "postgres://postgres:postgres@localhost/dbuild"},
"apps": {
"distrobuild": {
"models": [
"aerich.models",
"distrobuild.models",
],
"default_connection": "default",
},
},
}
import asyncio
from distrobuild.settings import settings
async def import_project(import_id: int, source_rpm: str):
upstream_prefix = f"ssh://{settings.ssh_user}@{settings.gitlab_host}:{settings.ssh_port}{settings.repo_prefix}"
args = [
"--version",
str(settings.version),
"--source-rpm",
source_rpm,
"--upstream-prefix",
upstream_prefix,
"--storage-addr",
settings.storage_addr,
"--ssh-user",
settings.ssh_user
]
if settings.ssh_key_location:
args.append("--ssh-key-location")
args.append(settings.ssh_key_location)
f = open(f"/tmp/import-{import_id}.log", "w")
proc = await asyncio.create_subprocess_exec("srpmproc", *args, stdout=f, stderr=f)
await proc.wait()
f.close()
if proc.returncode != 0:
raise Exception("srpmproc failed")
-- upgrade --
CREATE TABLE IF NOT EXISTS "aerich" (
"id" SERIAL NOT NULL PRIMARY KEY,
"version" VARCHAR(255) NOT NULL,
"app" VARCHAR(20) NOT NULL,
"content" TEXT NOT NULL
);
-- upgrade --
CREATE TABLE IF NOT EXISTS "packages" (
"id" BIGSERIAL NOT NULL PRIMARY KEY,
"created_at" TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
"updated_at" TIMESTAMPTZ,
"name" VARCHAR(255) NOT NULL,
"responsible_user_id" VARCHAR(255),
"is_module" BOOL NOT NULL DEFAULT False,
"is_package" BOOL NOT NULL DEFAULT False,
"part_of_module" BOOL NOT NULL DEFAULT False,
"latest_dist_commit" TEXT,
"el8" BOOL NOT NULL DEFAULT False,
"repo" VARCHAR(10)
);
COMMENT ON COLUMN "packages"."repo" IS 'BASEOS: BASEOS\nAPPSTREAM: APPSTREAM\nPOWERTOOLS: POWERTOOLS';;
CREATE TABLE IF NOT EXISTS "packagemodule" (
"id" BIGSERIAL NOT NULL PRIMARY KEY,
"created_at" TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
"updated_at" TIMESTAMPTZ,
"package_id" BIGINT NOT NULL REFERENCES "packages" ("id") ON DELETE RESTRICT,
"module_parent_package_id" BIGINT NOT NULL REFERENCES "packages" ("id") ON DELETE RESTRICT
);;
-- downgrade --
DROP TABLE IF EXISTS "packages";
DROP TABLE IF EXISTS "packagemodule";
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment