add build order support and last_build column. some restructuring too

distrobuild/models/__init__.py

-from enum import Enum
+from distrobuild.models.enums import BuildStatus, Repo
+from distrobuild.models.package import Package, PackageModule
+from distrobuild.models.build import Build, Import, ImportCommit
 
-from tortoise import Model, fields
-
-
-class Repo(str, Enum):
-    BASEOS = "BASEOS"
-    APPSTREAM = "APPSTREAM"
-    POWERTOOLS = "POWERTOOLS"
-
-
-class BuildStatus(str, Enum):
-    QUEUED = "QUEUED"
-    BUILDING = "BUILDING"
-    FAILED = "FAILED"
-    SUCCEEDED = "SUCCEEDED"
-    CANCELLED = "CANCELLED"
-
-
-class Package(Model):
-    id = fields.BigIntField(pk=True)
-    created_at = fields.DatetimeField(auto_now_add=True)
-    updated_at = fields.DatetimeField(auto_add=True, null=True)
-    name = fields.CharField(max_length=255)
-    responsible_user_id = fields.CharField(max_length=255, null=True)
-    is_module = fields.BooleanField(default=False)
-    is_package = fields.BooleanField(default=False)
-    part_of_module = fields.BooleanField(default=False)
-    last_import = fields.DatetimeField(null=True)
-
-    el8 = fields.BooleanField(default=False)
-    repo = fields.CharEnumField(Repo, null=True)
-
-    class Meta:
-        table = "packages"
-
-    class PydanticMeta:
-        backward_relations = False
-
-
-class PackageModule(Model):
-    id = fields.BigIntField(pk=True)
-    created_at = fields.DatetimeField(auto_now_add=True)
-    updated_at = fields.DatetimeField(auto_add=True, null=True)
-    package = fields.ForeignKeyField("distrobuild.Package", on_delete="RESTRICT", related_name="m_subpackages")
-    module_parent_package = fields.ForeignKeyField("distrobuild.Package", on_delete="RESTRICT",
-                                                   related_name="m_module_parent_pacakges")
-
-    class PydanticMeta:
-        backward_relations = False
-
-
-class Build(Model):
-    id = fields.BigIntField(pk=True)
-    created_at = fields.DatetimeField(auto_now_add=True)
-    updated_at = fields.DatetimeField(auto_add=True, null=True)
-    package = fields.ForeignKeyField("distrobuild.Package", on_delete="CASCADE")
-    status = fields.CharEnumField(BuildStatus)
-    mbs = fields.BooleanField(default=False)
-    koji_id = fields.BigIntField(null=True)
-    mbs_id = fields.BigIntField(null=True)
-    commit = fields.CharField(max_length=255)
-    branch = fields.CharField(max_length=255)
-
-    class Meta:
-        table = "builds"
-
-
-class Import(Model):
-    id = fields.BigIntField(pk=True)
-    created_at = fields.DatetimeField(auto_now_add=True)
-    updated_at = fields.DatetimeField(auto_add=True, null=True)
-    package = fields.ForeignKeyField("distrobuild.Package", on_delete="CASCADE")
-    status = fields.CharEnumField(BuildStatus)
-    version = fields.IntField()
-    module = fields.BooleanField(default=False)
-
-    commits: fields.ReverseRelation["ImportCommit"] = fields.ReverseRelation
-
-    class Meta:
-        table = "imports"
-
-    class PydanticMeta:
-        backward_relations = False
-
-
-class ImportCommit(Model):
-    id = fields.BigIntField(pk=True)
-    commit = fields.CharField(max_length=255)
-    branch = fields.CharField(max_length=255)
-    import_ = fields.ForeignKeyField("distrobuild.Import", on_delete="CASCADE")
-
-    class Meta:
-        table = "import_commits"

distrobuild/models/build.py

+from tortoise import Model, fields
+from distrobuild.models.enums import BuildStatus
+
+
+class Build(Model):
+    id = fields.BigIntField(pk=True)
+    created_at = fields.DatetimeField(auto_now_add=True)
+    updated_at = fields.DatetimeField(auto_add=True, null=True)
+    package = fields.ForeignKeyField("distrobuild.Package", on_delete="CASCADE")
+    status = fields.CharEnumField(BuildStatus)
+    mbs = fields.BooleanField(default=False)
+    koji_id = fields.BigIntField(null=True)
+    mbs_id = fields.BigIntField(null=True)
+    commit = fields.CharField(max_length=255)
+    branch = fields.CharField(max_length=255)
+
+    class Meta:
+        table = "builds"
+
+
+class Import(Model):
+    id = fields.BigIntField(pk=True)
+    created_at = fields.DatetimeField(auto_now_add=True)
+    updated_at = fields.DatetimeField(auto_add=True, null=True)
+    package = fields.ForeignKeyField("distrobuild.Package", on_delete="CASCADE")
+    status = fields.CharEnumField(BuildStatus)
+    version = fields.IntField()
+    module = fields.BooleanField(default=False)
+
+    commits: fields.ReverseRelation["ImportCommit"] = fields.ReverseRelation
+
+    class Meta:
+        table = "imports"
+
+    class PydanticMeta:
+        backward_relations = False
+
+
+class ImportCommit(Model):
+    id = fields.BigIntField(pk=True)
+    commit = fields.CharField(max_length=255)
+    branch = fields.CharField(max_length=255)
+    import_ = fields.ForeignKeyField("distrobuild.Import", on_delete="CASCADE")
+
+    class Meta:
+        table = "import_commits"

distrobuild/models/enums.py

+from enum import Enum
+
+
+class Repo(str, Enum):
+    BASEOS = "BASEOS"
+    APPSTREAM = "APPSTREAM"
+    POWERTOOLS = "POWERTOOLS"
+
+
+class BuildStatus(str, Enum):
+    QUEUED = "QUEUED"
+    BUILDING = "BUILDING"
+    FAILED = "FAILED"
+    SUCCEEDED = "SUCCEEDED"
+    CANCELLED = "CANCELLED"

distrobuild/models/package.py

+from tortoise import Model, fields
+from distrobuild.models.enums import Repo
+
+
+class Package(Model):
+    id = fields.BigIntField(pk=True)
+    created_at = fields.DatetimeField(auto_now_add=True)
+    updated_at = fields.DatetimeField(auto_add=True, null=True)
+    name = fields.CharField(max_length=255)
+    responsible_user_id = fields.CharField(max_length=255, null=True)
+    is_module = fields.BooleanField(default=False)
+    is_package = fields.BooleanField(default=False)
+    part_of_module = fields.BooleanField(default=False)
+    last_import = fields.DatetimeField(null=True)
+    last_build = fields.DatetimeField(null=True)
+
+    el8 = fields.BooleanField(default=False)
+    repo = fields.CharEnumField(Repo, null=True)
+
+    class Meta:
+        table = "packages"
+
+    class PydanticMeta:
+        backward_relations = False
+
+
+class PackageModule(Model):
+    id = fields.BigIntField(pk=True)
+    created_at = fields.DatetimeField(auto_now_add=True)
+    updated_at = fields.DatetimeField(auto_add=True, null=True)
+    package = fields.ForeignKeyField("distrobuild.Package", on_delete="RESTRICT", related_name="m_subpackages")
+    module_parent_package = fields.ForeignKeyField("distrobuild.Package", on_delete="RESTRICT",
+                                                   related_name="m_module_parent_pacakges")
+
+    class PydanticMeta:
+        backward_relations = False

distrobuild/routes/bootstrap.py

@@ -6,11 +6,13 @@ from distrobuild.models import Repo
 
 router = APIRouter(prefix="/bootstrap")
 
+
 @router.post("/modules")
 async def bootstrap_modules():
     await process_module_dump()
     return JSONResponse(content={})
 
+
 @router.post("/{repo}")
 async def bootstrap_repo(repo: Repo):
     await process_repo_dump(repo)

distrobuild/routes/build.py

-from typing import Optional, List
+from typing import Optional, List, Tuple
 
 from fastapi import APIRouter, Depends, HTTPException
 from fastapi.responses import PlainTextResponse
@@ -92,30 +92,37 @@ async def batch_queue_build(body: BatchBuildRequest):
     return {}
 
 
-@router.post("/imports/", status_code=202)
-async def import_package_route(body: BuildRequest):
-    filters = gen_body_filters(body)
-    package = await Package.filter(**filters).get_or_none()
-    if not package:
-        raise HTTPException(404, detail="package does not exist")
+async def create_build_order(package: Package) -> List[Tuple[int, int]]:
+    pkg_list = []
 
     if package.is_package:
         package_import = await Import.create(package_id=package.id, status=BuildStatus.QUEUED, version=8)
-        await import_package_task(package.id, package_import.id)
+        pkg_list.append((package.id, package_import.id))
 
     if package.is_module:
         subpackages = await PackageModule.filter(module_parent_package_id=package.id).all()
-        all_packages_imported = True
         for subpackage in subpackages:
             imports = await Import.filter(package_id=subpackage.package_id).all()
             if not imports or len(imports) == 0:
-                all_packages_imported = False
-                await import_package_route(BuildRequest(package_id=subpackage.package_id))
+                subpackage_package = await Package.filter(id=subpackage.package_id).get()
+                pkg_list += await create_build_order(subpackage_package)
+
+        package_module_import = await Import.create(package_id=package.id, status=BuildStatus.QUEUED, version=8,
+                                                    module=True)
+        pkg_list.append((package.id, package_module_import.id))
+
+    return pkg_list
+
+
+@router.post("/imports/", status_code=202)
+async def import_package_route(body: BuildRequest):
+    filters = gen_body_filters(body)
+    package = await Package.filter(**filters).get_or_none()
+    if not package:
+        raise HTTPException(404, detail="package does not exist")
 
-        if all_packages_imported:
-            package_import = await Import.create(package_id=package.id, status=BuildStatus.QUEUED, version=8,
-                                                 module=True)
-            await import_package_task(package.id, package_import.id)
+    build_order = await create_build_order(package)
+    await import_package_task(build_order[0][0], build_order[0][1], build_order[1:])
 
     return {}
 

distrobuild_scheduler/__init__.py

 import json
 import logging
 
-from typing import Optional
+from typing import Tuple, Optional, List
 
 import aio_pika
 
@@ -24,11 +24,12 @@ async def init_channel(loop) -> None:
     channel = await connection.channel()
 
 
-async def import_package_task(package_id: int, import_id: int):
+async def import_package_task(package_id: int, import_id: int, dependents: List[Tuple[int, int]]):
     msg_body = {
         "message": "import_package",
         "package_id": package_id,
         "import_id": import_id,
+        "dependents": dependents,
     }
     encoded = json.dumps(msg_body).encode()
 

distrobuild_scheduler/build_package.py

+import datetime
+
 from tortoise.transactions import atomic
 
 from distrobuild.models import Build, BuildStatus, Package
@@ -19,6 +21,9 @@ async def do(package: Package, build: Build):
         build.status = BuildStatus.BUILDING
         await build.save()
 
+        package.last_import = datetime.datetime.now()
+        await package.save()
+
 
 # noinspection DuplicatedCode
 async def task(package_id: int, build_id):

distrobuild_scheduler/import_package.py

 import datetime
 
+from typing import List, Tuple
+
 from tortoise.transactions import atomic
 
 from distrobuild.models import BuildStatus, Package, Import, ImportCommit
@@ -27,7 +29,7 @@ async def do(package: Package, package_import: Import):
 
 
 # noinspection DuplicatedCode
-async def task(package_id: int, import_id: int):
+async def task(package_id: int, import_id: int, dependents: List[Tuple[int, int]]):
     package = await Package.filter(id=package_id).get()
     package_import = await Import.filter(id=import_id).get()
     try:
@@ -40,3 +42,6 @@ async def task(package_id: int, import_id: int):
     finally:
         await package_import.save()
         await package.save()
+
+    if len(dependents) > 0:
+        await task(dependents[0][0], dependents[0][1], dependents[1:])

distrobuild_scheduler/main.py

@@ -26,7 +26,7 @@ async def consume_messages(i: int):
                 body = json.loads(message.body.decode())
 
                 if body["message"] == "import_package":
-                    await import_package.task(body["package_id"], body["import_id"])
+                    await import_package.task(body["package_id"], body["import_id"], body["dependents"])
                 elif body["message"] == "build_package":
                     await build_package.task(body["package_id"], body["build_id"])
                 else:

migrations/distrobuild/0_20210202163648_init.sql → migrations/distrobuild/0_20210220012242_init.sql

@@ -3,5 +3,5 @@ CREATE TABLE IF NOT EXISTS "aerich" (
     "id" SERIAL NOT NULL PRIMARY KEY,
     "version" VARCHAR(255) NOT NULL,
     "app" VARCHAR(20) NOT NULL,
-    "content" TEXT NOT NULL
+    "content" JSONB NOT NULL
 );

migrations/distrobuild/1_20210214144718_packages.sql → migrations/distrobuild/1_20210220012254_packages.sql

@@ -8,7 +8,8 @@ CREATE TABLE IF NOT EXISTS "packages" (
     "is_module" BOOL NOT NULL  DEFAULT False,
     "is_package" BOOL NOT NULL  DEFAULT False,
     "part_of_module" BOOL NOT NULL  DEFAULT False,
-    "latest_dist_commit" TEXT,
+    "last_import" TIMESTAMPTZ,
+    "last_build" TIMESTAMPTZ,
     "el8" BOOL NOT NULL  DEFAULT False,
     "repo" VARCHAR(10)
 );
@@ -17,8 +18,8 @@ CREATE TABLE IF NOT EXISTS "packagemodule" (
     "id" BIGSERIAL NOT NULL PRIMARY KEY,
     "created_at" TIMESTAMPTZ NOT NULL  DEFAULT CURRENT_TIMESTAMP,
     "updated_at" TIMESTAMPTZ,
-    "package_id" BIGINT NOT NULL REFERENCES "packages" ("id") ON DELETE RESTRICT,
-    "module_parent_package_id" BIGINT NOT NULL REFERENCES "packages" ("id") ON DELETE RESTRICT
+    "module_parent_package_id" BIGINT NOT NULL REFERENCES "packages" ("id") ON DELETE RESTRICT,
+    "package_id" BIGINT NOT NULL REFERENCES "packages" ("id") ON DELETE RESTRICT
 );;
 -- downgrade --
 DROP TABLE IF EXISTS "packages";

migrations/distrobuild/2_20210215041746_builds.sql → migrations/distrobuild/2_20210220012302_builds.sql

@@ -7,6 +7,8 @@ CREATE TABLE IF NOT EXISTS "builds" (
     "mbs" BOOL NOT NULL  DEFAULT False,
     "koji_id" BIGINT,
     "mbs_id" BIGINT,
+    "commit" VARCHAR(255) NOT NULL,
+    "branch" VARCHAR(255) NOT NULL,
     "package_id" BIGINT NOT NULL REFERENCES "packages" ("id") ON DELETE CASCADE
 );
 COMMENT ON COLUMN "builds"."status" IS 'QUEUED: QUEUED\nBUILDING: BUILDING\nFAILED: FAILED\nSUCCEEDED: SUCCEEDED\nCANCELLED: CANCELLED';;
@@ -16,14 +18,17 @@ CREATE TABLE IF NOT EXISTS "imports" (
     "updated_at" TIMESTAMPTZ,
     "status" VARCHAR(9) NOT NULL,
     "version" INT NOT NULL,
-    "commit" VARCHAR(255),
+    "module" BOOL NOT NULL  DEFAULT False,
     "package_id" BIGINT NOT NULL REFERENCES "packages" ("id") ON DELETE CASCADE
 );
 COMMENT ON COLUMN "imports"."status" IS 'QUEUED: QUEUED\nBUILDING: BUILDING\nFAILED: FAILED\nSUCCEEDED: SUCCEEDED\nCANCELLED: CANCELLED';;
-ALTER TABLE "packages" ADD "last_import" TIMESTAMPTZ;
-ALTER TABLE "packages" DROP COLUMN "latest_dist_commit";
+CREATE TABLE IF NOT EXISTS "import_commits" (
+    "id" BIGSERIAL NOT NULL PRIMARY KEY,
+    "commit" VARCHAR(255) NOT NULL,
+    "branch" VARCHAR(255) NOT NULL,
+    "import__id" BIGINT NOT NULL REFERENCES "imports" ("id") ON DELETE CASCADE
+);;
 -- downgrade --
-ALTER TABLE "packages" ADD "latest_dist_commit" TEXT;
-ALTER TABLE "packages" DROP COLUMN "last_import";
 DROP TABLE IF EXISTS "builds";
 DROP TABLE IF EXISTS "imports";
+DROP TABLE IF EXISTS "import_commits";

migrations/distrobuild/3_20210215153317_add_module_to_imports.sql

--- upgrade --
-ALTER TABLE "imports" ADD "module" BOOL NOT NULL  DEFAULT False;
--- downgrade --
-ALTER TABLE "imports" DROP COLUMN "module";

migrations/distrobuild/4_20210215155319_support_multiple_commits_per_import.sql

--- upgrade --
-ALTER TABLE "imports" DROP COLUMN "commit";
-CREATE TABLE IF NOT EXISTS "import_commits" (
-    "id" BIGSERIAL NOT NULL PRIMARY KEY,
-    "commit" VARCHAR(255) NOT NULL,
-    "import__id" BIGINT NOT NULL REFERENCES "imports" ("id") ON DELETE CASCADE
-);;
--- downgrade --
-ALTER TABLE "imports" ADD "commit" VARCHAR(255);
-DROP TABLE IF EXISTS "import_commits";

migrations/distrobuild/5_20210219030232_add_commit_to_build.sql

--- upgrade --
-ALTER TABLE "builds" ADD "commit" VARCHAR(255) NOT NULL;
-ALTER TABLE "builds" ADD "branch" VARCHAR(255) NOT NULL;
-ALTER TABLE "import_commits" ADD "branch" VARCHAR(255) NOT NULL;
--- downgrade --
-ALTER TABLE "builds" DROP COLUMN "commit";
-ALTER TABLE "builds" DROP COLUMN "branch";
-ALTER TABLE "import_commits" DROP COLUMN "branch";

requirements.txt

@@ -11,4 +11,4 @@ pydantic[dotenv]==1.7.3
 python-dotenv==0.15.0
 python-gitlab==2.6.0
 aio-pika==6.8.0
-schedule==1.0.0
+python-multipart==0.0.5