Commit f132a6d3 authored by Mustafa Gezen's avatar Mustafa Gezen 🏗
Browse files

add build order support and last_build column. some restructuring too

parent f1478ca0
from enum import Enum
from distrobuild.models.enums import BuildStatus, Repo
from distrobuild.models.package import Package, PackageModule
from distrobuild.models.build import Build, Import, ImportCommit
from tortoise import Model, fields
class Repo(str, Enum):
BASEOS = "BASEOS"
APPSTREAM = "APPSTREAM"
POWERTOOLS = "POWERTOOLS"
class BuildStatus(str, Enum):
QUEUED = "QUEUED"
BUILDING = "BUILDING"
FAILED = "FAILED"
SUCCEEDED = "SUCCEEDED"
CANCELLED = "CANCELLED"
class Package(Model):
id = fields.BigIntField(pk=True)
created_at = fields.DatetimeField(auto_now_add=True)
updated_at = fields.DatetimeField(auto_add=True, null=True)
name = fields.CharField(max_length=255)
responsible_user_id = fields.CharField(max_length=255, null=True)
is_module = fields.BooleanField(default=False)
is_package = fields.BooleanField(default=False)
part_of_module = fields.BooleanField(default=False)
last_import = fields.DatetimeField(null=True)
el8 = fields.BooleanField(default=False)
repo = fields.CharEnumField(Repo, null=True)
class Meta:
table = "packages"
class PydanticMeta:
backward_relations = False
class PackageModule(Model):
id = fields.BigIntField(pk=True)
created_at = fields.DatetimeField(auto_now_add=True)
updated_at = fields.DatetimeField(auto_add=True, null=True)
package = fields.ForeignKeyField("distrobuild.Package", on_delete="RESTRICT", related_name="m_subpackages")
module_parent_package = fields.ForeignKeyField("distrobuild.Package", on_delete="RESTRICT",
related_name="m_module_parent_pacakges")
class PydanticMeta:
backward_relations = False
class Build(Model):
id = fields.BigIntField(pk=True)
created_at = fields.DatetimeField(auto_now_add=True)
updated_at = fields.DatetimeField(auto_add=True, null=True)
package = fields.ForeignKeyField("distrobuild.Package", on_delete="CASCADE")
status = fields.CharEnumField(BuildStatus)
mbs = fields.BooleanField(default=False)
koji_id = fields.BigIntField(null=True)
mbs_id = fields.BigIntField(null=True)
commit = fields.CharField(max_length=255)
branch = fields.CharField(max_length=255)
class Meta:
table = "builds"
class Import(Model):
id = fields.BigIntField(pk=True)
created_at = fields.DatetimeField(auto_now_add=True)
updated_at = fields.DatetimeField(auto_add=True, null=True)
package = fields.ForeignKeyField("distrobuild.Package", on_delete="CASCADE")
status = fields.CharEnumField(BuildStatus)
version = fields.IntField()
module = fields.BooleanField(default=False)
commits: fields.ReverseRelation["ImportCommit"] = fields.ReverseRelation
class Meta:
table = "imports"
class PydanticMeta:
backward_relations = False
class ImportCommit(Model):
id = fields.BigIntField(pk=True)
commit = fields.CharField(max_length=255)
branch = fields.CharField(max_length=255)
import_ = fields.ForeignKeyField("distrobuild.Import", on_delete="CASCADE")
class Meta:
table = "import_commits"
from tortoise import Model, fields
from distrobuild.models.enums import BuildStatus
class Build(Model):
id = fields.BigIntField(pk=True)
created_at = fields.DatetimeField(auto_now_add=True)
updated_at = fields.DatetimeField(auto_add=True, null=True)
package = fields.ForeignKeyField("distrobuild.Package", on_delete="CASCADE")
status = fields.CharEnumField(BuildStatus)
mbs = fields.BooleanField(default=False)
koji_id = fields.BigIntField(null=True)
mbs_id = fields.BigIntField(null=True)
commit = fields.CharField(max_length=255)
branch = fields.CharField(max_length=255)
class Meta:
table = "builds"
class Import(Model):
id = fields.BigIntField(pk=True)
created_at = fields.DatetimeField(auto_now_add=True)
updated_at = fields.DatetimeField(auto_add=True, null=True)
package = fields.ForeignKeyField("distrobuild.Package", on_delete="CASCADE")
status = fields.CharEnumField(BuildStatus)
version = fields.IntField()
module = fields.BooleanField(default=False)
commits: fields.ReverseRelation["ImportCommit"] = fields.ReverseRelation
class Meta:
table = "imports"
class PydanticMeta:
backward_relations = False
class ImportCommit(Model):
id = fields.BigIntField(pk=True)
commit = fields.CharField(max_length=255)
branch = fields.CharField(max_length=255)
import_ = fields.ForeignKeyField("distrobuild.Import", on_delete="CASCADE")
class Meta:
table = "import_commits"
from enum import Enum
class Repo(str, Enum):
BASEOS = "BASEOS"
APPSTREAM = "APPSTREAM"
POWERTOOLS = "POWERTOOLS"
class BuildStatus(str, Enum):
QUEUED = "QUEUED"
BUILDING = "BUILDING"
FAILED = "FAILED"
SUCCEEDED = "SUCCEEDED"
CANCELLED = "CANCELLED"
from tortoise import Model, fields
from distrobuild.models.enums import Repo
class Package(Model):
id = fields.BigIntField(pk=True)
created_at = fields.DatetimeField(auto_now_add=True)
updated_at = fields.DatetimeField(auto_add=True, null=True)
name = fields.CharField(max_length=255)
responsible_user_id = fields.CharField(max_length=255, null=True)
is_module = fields.BooleanField(default=False)
is_package = fields.BooleanField(default=False)
part_of_module = fields.BooleanField(default=False)
last_import = fields.DatetimeField(null=True)
last_build = fields.DatetimeField(null=True)
el8 = fields.BooleanField(default=False)
repo = fields.CharEnumField(Repo, null=True)
class Meta:
table = "packages"
class PydanticMeta:
backward_relations = False
class PackageModule(Model):
id = fields.BigIntField(pk=True)
created_at = fields.DatetimeField(auto_now_add=True)
updated_at = fields.DatetimeField(auto_add=True, null=True)
package = fields.ForeignKeyField("distrobuild.Package", on_delete="RESTRICT", related_name="m_subpackages")
module_parent_package = fields.ForeignKeyField("distrobuild.Package", on_delete="RESTRICT",
related_name="m_module_parent_pacakges")
class PydanticMeta:
backward_relations = False
......@@ -6,11 +6,13 @@ from distrobuild.models import Repo
router = APIRouter(prefix="/bootstrap")
@router.post("/modules")
async def bootstrap_modules():
await process_module_dump()
return JSONResponse(content={})
@router.post("/{repo}")
async def bootstrap_repo(repo: Repo):
await process_repo_dump(repo)
......
from typing import Optional, List
from typing import Optional, List, Tuple
from fastapi import APIRouter, Depends, HTTPException
from fastapi.responses import PlainTextResponse
......@@ -92,30 +92,37 @@ async def batch_queue_build(body: BatchBuildRequest):
return {}
@router.post("/imports/", status_code=202)
async def import_package_route(body: BuildRequest):
filters = gen_body_filters(body)
package = await Package.filter(**filters).get_or_none()
if not package:
raise HTTPException(404, detail="package does not exist")
async def create_build_order(package: Package) -> List[Tuple[int, int]]:
pkg_list = []
if package.is_package:
package_import = await Import.create(package_id=package.id, status=BuildStatus.QUEUED, version=8)
await import_package_task(package.id, package_import.id)
pkg_list.append((package.id, package_import.id))
if package.is_module:
subpackages = await PackageModule.filter(module_parent_package_id=package.id).all()
all_packages_imported = True
for subpackage in subpackages:
imports = await Import.filter(package_id=subpackage.package_id).all()
if not imports or len(imports) == 0:
all_packages_imported = False
await import_package_route(BuildRequest(package_id=subpackage.package_id))
subpackage_package = await Package.filter(id=subpackage.package_id).get()
pkg_list += await create_build_order(subpackage_package)
package_module_import = await Import.create(package_id=package.id, status=BuildStatus.QUEUED, version=8,
module=True)
pkg_list.append((package.id, package_module_import.id))
return pkg_list
@router.post("/imports/", status_code=202)
async def import_package_route(body: BuildRequest):
filters = gen_body_filters(body)
package = await Package.filter(**filters).get_or_none()
if not package:
raise HTTPException(404, detail="package does not exist")
if all_packages_imported:
package_import = await Import.create(package_id=package.id, status=BuildStatus.QUEUED, version=8,
module=True)
await import_package_task(package.id, package_import.id)
build_order = await create_build_order(package)
await import_package_task(build_order[0][0], build_order[0][1], build_order[1:])
return {}
......
import json
import logging
from typing import Optional
from typing import Tuple, Optional, List
import aio_pika
......@@ -24,11 +24,12 @@ async def init_channel(loop) -> None:
channel = await connection.channel()
async def import_package_task(package_id: int, import_id: int):
async def import_package_task(package_id: int, import_id: int, dependents: List[Tuple[int, int]]):
msg_body = {
"message": "import_package",
"package_id": package_id,
"import_id": import_id,
"dependents": dependents,
}
encoded = json.dumps(msg_body).encode()
......
import datetime
from tortoise.transactions import atomic
from distrobuild.models import Build, BuildStatus, Package
......@@ -19,6 +21,9 @@ async def do(package: Package, build: Build):
build.status = BuildStatus.BUILDING
await build.save()
package.last_import = datetime.datetime.now()
await package.save()
# noinspection DuplicatedCode
async def task(package_id: int, build_id):
......
import datetime
from typing import List, Tuple
from tortoise.transactions import atomic
from distrobuild.models import BuildStatus, Package, Import, ImportCommit
......@@ -27,7 +29,7 @@ async def do(package: Package, package_import: Import):
# noinspection DuplicatedCode
async def task(package_id: int, import_id: int):
async def task(package_id: int, import_id: int, dependents: List[Tuple[int, int]]):
package = await Package.filter(id=package_id).get()
package_import = await Import.filter(id=import_id).get()
try:
......@@ -40,3 +42,6 @@ async def task(package_id: int, import_id: int):
finally:
await package_import.save()
await package.save()
if len(dependents) > 0:
await task(dependents[0][0], dependents[0][1], dependents[1:])
......@@ -26,7 +26,7 @@ async def consume_messages(i: int):
body = json.loads(message.body.decode())
if body["message"] == "import_package":
await import_package.task(body["package_id"], body["import_id"])
await import_package.task(body["package_id"], body["import_id"], body["dependents"])
elif body["message"] == "build_package":
await build_package.task(body["package_id"], body["build_id"])
else:
......
......@@ -3,5 +3,5 @@ CREATE TABLE IF NOT EXISTS "aerich" (
"id" SERIAL NOT NULL PRIMARY KEY,
"version" VARCHAR(255) NOT NULL,
"app" VARCHAR(20) NOT NULL,
"content" TEXT NOT NULL
"content" JSONB NOT NULL
);
......@@ -8,7 +8,8 @@ CREATE TABLE IF NOT EXISTS "packages" (
"is_module" BOOL NOT NULL DEFAULT False,
"is_package" BOOL NOT NULL DEFAULT False,
"part_of_module" BOOL NOT NULL DEFAULT False,
"latest_dist_commit" TEXT,
"last_import" TIMESTAMPTZ,
"last_build" TIMESTAMPTZ,
"el8" BOOL NOT NULL DEFAULT False,
"repo" VARCHAR(10)
);
......@@ -17,8 +18,8 @@ CREATE TABLE IF NOT EXISTS "packagemodule" (
"id" BIGSERIAL NOT NULL PRIMARY KEY,
"created_at" TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
"updated_at" TIMESTAMPTZ,
"package_id" BIGINT NOT NULL REFERENCES "packages" ("id") ON DELETE RESTRICT,
"module_parent_package_id" BIGINT NOT NULL REFERENCES "packages" ("id") ON DELETE RESTRICT
"module_parent_package_id" BIGINT NOT NULL REFERENCES "packages" ("id") ON DELETE RESTRICT,
"package_id" BIGINT NOT NULL REFERENCES "packages" ("id") ON DELETE RESTRICT
);;
-- downgrade --
DROP TABLE IF EXISTS "packages";
......
......@@ -7,6 +7,8 @@ CREATE TABLE IF NOT EXISTS "builds" (
"mbs" BOOL NOT NULL DEFAULT False,
"koji_id" BIGINT,
"mbs_id" BIGINT,
"commit" VARCHAR(255) NOT NULL,
"branch" VARCHAR(255) NOT NULL,
"package_id" BIGINT NOT NULL REFERENCES "packages" ("id") ON DELETE CASCADE
);
COMMENT ON COLUMN "builds"."status" IS 'QUEUED: QUEUED\nBUILDING: BUILDING\nFAILED: FAILED\nSUCCEEDED: SUCCEEDED\nCANCELLED: CANCELLED';;
......@@ -16,14 +18,17 @@ CREATE TABLE IF NOT EXISTS "imports" (
"updated_at" TIMESTAMPTZ,
"status" VARCHAR(9) NOT NULL,
"version" INT NOT NULL,
"commit" VARCHAR(255),
"module" BOOL NOT NULL DEFAULT False,
"package_id" BIGINT NOT NULL REFERENCES "packages" ("id") ON DELETE CASCADE
);
COMMENT ON COLUMN "imports"."status" IS 'QUEUED: QUEUED\nBUILDING: BUILDING\nFAILED: FAILED\nSUCCEEDED: SUCCEEDED\nCANCELLED: CANCELLED';;
ALTER TABLE "packages" ADD "last_import" TIMESTAMPTZ;
ALTER TABLE "packages" DROP COLUMN "latest_dist_commit";
CREATE TABLE IF NOT EXISTS "import_commits" (
"id" BIGSERIAL NOT NULL PRIMARY KEY,
"commit" VARCHAR(255) NOT NULL,
"branch" VARCHAR(255) NOT NULL,
"import__id" BIGINT NOT NULL REFERENCES "imports" ("id") ON DELETE CASCADE
);;
-- downgrade --
ALTER TABLE "packages" ADD "latest_dist_commit" TEXT;
ALTER TABLE "packages" DROP COLUMN "last_import";
DROP TABLE IF EXISTS "builds";
DROP TABLE IF EXISTS "imports";
DROP TABLE IF EXISTS "import_commits";
-- upgrade --
ALTER TABLE "imports" ADD "module" BOOL NOT NULL DEFAULT False;
-- downgrade --
ALTER TABLE "imports" DROP COLUMN "module";
-- upgrade --
ALTER TABLE "imports" DROP COLUMN "commit";
CREATE TABLE IF NOT EXISTS "import_commits" (
"id" BIGSERIAL NOT NULL PRIMARY KEY,
"commit" VARCHAR(255) NOT NULL,
"import__id" BIGINT NOT NULL REFERENCES "imports" ("id") ON DELETE CASCADE
);;
-- downgrade --
ALTER TABLE "imports" ADD "commit" VARCHAR(255);
DROP TABLE IF EXISTS "import_commits";
-- upgrade --
ALTER TABLE "builds" ADD "commit" VARCHAR(255) NOT NULL;
ALTER TABLE "builds" ADD "branch" VARCHAR(255) NOT NULL;
ALTER TABLE "import_commits" ADD "branch" VARCHAR(255) NOT NULL;
-- downgrade --
ALTER TABLE "builds" DROP COLUMN "commit";
ALTER TABLE "builds" DROP COLUMN "branch";
ALTER TABLE "import_commits" DROP COLUMN "branch";
......@@ -11,4 +11,4 @@ pydantic[dotenv]==1.7.3
python-dotenv==0.15.0
python-gitlab==2.6.0
aio-pika==6.8.0
schedule==1.0.0
python-multipart==0.0.5
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment