Add gitlabify and move sigul disabling to each statement. Also fix module...

Add gitlabify and move sigul disabling to each statement. Also fix module builds for specific situations.

distrobuild/bootstrap/__init__.py

@@ -17,7 +17,11 @@ async def process_repo_dump(repo: Repo, responsible_username: str) -> None:
         lines = f.readlines()
         for line in lines:
             package_name = line.strip()
-            if await Package.filter(name=package_name).get_or_none():
+            existing_package = await Package.filter(name=package_name).get_or_none()
+            if existing_package:
+                existing_package.repo = repo
+                existing_package.is_package = True
+                await existing_package.save()
                 continue
             await _internal_create_package(name=package_name,
                                            is_package=True,

distrobuild/common/__init__.py

@@ -7,10 +7,10 @@ from distrobuild.settings import settings
 
 
 def gen_body_filters(body: dict) -> dict:
-    if body["package_id"]:
-        return {"id": body["package_id"]}
-    if body["package_name"]:
+    if body.get("package_name"):
         return {"name": body["package_name"]}
+    if body.get("package_id"):
+        return {"id": body["package_id"]}
 
 
 async def create_import_order(package: Package, username: str) -> List[Tuple[int, int]]:

distrobuild/mbs/__init__.py

@@ -28,6 +28,9 @@ class MBSClient:
             return r.json()
 
     async def build(self, token: str, name: str, branch: str, commit: str) -> int:
+        scmurl = f"https://{settings.gitlab_host}{settings.repo_prefix}/modules/{name}?#{commit}"
+        print(scmurl)
+
         client = httpx.AsyncClient()
         async with client:
             r = await client.post(
@@ -36,7 +39,7 @@ class MBSClient:
                     "Authorization": f"Bearer {token}"
                 },
                 json={
-                    "scmurl": f"https://{settings.gitlab_host}{settings.repo_prefix}/modules/{name}?#{commit}",
+                    "scmurl": scmurl,
                     "branch": branch,
                 }
             )
@@ -45,4 +48,5 @@ class MBSClient:
                 raise MBSConflictException()
 
             data = r.json()
+            print(data)
             return data["id"]

distrobuild/routes/builds.py

@@ -55,7 +55,7 @@ async def queue_build(request: Request, body: Dict[str, BuildRequest]):
     if package.repo == Repo.MODULAR_CANDIDATE:
         raise HTTPException(401, detail="modular subpackages cannot be built, build the main module")
 
-    if package.part_of_module:
+    if package.part_of_module and not package.is_module:
         raise HTTPException(401, detail="this package is part of a module. build the main module")
 
     extras = {
@@ -63,7 +63,7 @@ async def queue_build(request: Request, body: Dict[str, BuildRequest]):
     }
     token = None
     package_modules = await PackageModule.filter(package_id=package.id)
-    if len(package_modules) > 0:
+    if len(package_modules) > 0 or package.is_module:
         extras["mbs"] = True
         token = message_cipher.encrypt(request.session.get("token").encode()).decode()
 
@@ -78,7 +78,7 @@ async def queue_build(request: Request, body: Dict[str, BuildRequest]):
     for import_commit in import_commits:
         if "-beta" not in import_commit.branch:
             build = await Build.create(package_id=package.id, status=BuildStatus.QUEUED,
-                                       executor_username=user["preferred_username"], point_release="8_4",
+                                       executor_username=user["preferred_username"], point_release="8_3",
                                        import_commit_id=import_commit.id, **extras)
             await build_package_task(package.id, build.id, token)
 

distrobuild/settings.py

@@ -26,6 +26,7 @@ class Settings(BaseSettings):
     mbs_url: str
 
     # sigul
+    disable_sigul: bool = False
     sigul_config_file: str = "/etc/distrobuild/sigul.conf"
     sigul_passphrase: str
     sigul_key_name: str = "signing"

distrobuild/srpmproc.py

@@ -41,6 +41,7 @@ async def import_project(import_id: int, source_rpm: str, module_mode: bool = Fa
         args.append(single_tag)
 
     f = open(f"/tmp/import-{import_id}.log", "w")
+
     proc = await asyncio.create_subprocess_exec("srpmproc", *args, stdout=asyncio.subprocess.PIPE, stderr=f)
 
     last_line = ""

distrobuild_scheduler/build_package.py

@@ -10,6 +10,8 @@ from distrobuild.models import Build, BuildStatus, Package
 from distrobuild.session import koji_session, mbs_client
 from distrobuild.settings import settings
 
+from distrobuild_scheduler.utils import gitlabify
+
 
 @atomic()
 async def do(package: Package, build: Build, token: Optional[str]):
@@ -24,7 +26,7 @@ async def do(package: Package, build: Build, token: Optional[str]):
             target = tags.base() if not build.force_tag else build.force_tag
 
             host = f"git+https://{settings.gitlab_host}{settings.repo_prefix}"
-            source = f"{host}/rpms/{package.name}.git?#{build.import_commit.commit}"
+            source = f"{host}/rpms/{gitlabify(package.name)}.git?#{build.import_commit.commit}"
             task_id = koji_session.build(source, target)
 
             build.koji_id = task_id

distrobuild_scheduler/import_package.py

@@ -9,6 +9,8 @@ from distrobuild.session import koji_session, gl
 from distrobuild.settings import settings
 from distrobuild import srpmproc
 
+from distrobuild_scheduler.utils import gitlabify
+
 
 @atomic()
 async def do(package: Package, package_import: Import):
@@ -27,7 +29,7 @@ async def do(package: Package, package_import: Import):
     package.last_import = datetime.datetime.now()
 
     mode = "modules" if package_import.module else "rpms"
-    project = gl.projects.get(f"{settings.repo_prefix.removeprefix('/')}/{mode}/{package.name}")
+    project = gl.projects.get(f"{settings.repo_prefix.removeprefix('/')}/{mode}/{gitlabify(package.name)}")
     project.visibility = "public"
     project.save()
 

distrobuild_scheduler/main.py

@@ -46,7 +46,8 @@ async def main(loop):
         await Tortoise.init(config=TORTOISE_ORM)
         await init_channel(loop)
 
-        await check_sigul_key()
+        if not settings.disable_sigul:
+            await check_sigul_key()
 
         schedule_periodic_tasks()
 

distrobuild_scheduler/periodic_tasks.py

@@ -27,13 +27,14 @@ async def atomic_check_build_status():
                 await package.save()
 
                 # sign artifacts
-                build_tasks = koji_session.listBuilds(taskID=build.koji_id)
-                for build_task in build_tasks:
-                    build_rpms = koji_session.listBuildRPMs(build_task["build_id"])
-                    for rpm in build_rpms:
-                        nvr_arch = "%s.%s" % (rpm["nvr"], rpm["arch"])
-                        await sign_koji_package(nvr_arch)
-                        koji_session.writeSignedRPM(nvr_arch, settings.sigul_key_id)
+                if not settings.disable_sigul:
+                    build_tasks = koji_session.listBuilds(taskID=build.koji_id)
+                    for build_task in build_tasks:
+                        build_rpms = koji_session.listBuildRPMs(build_task["build_id"])
+                        for rpm in build_rpms:
+                            nvr_arch = "%s.%s" % (rpm["nvr"], rpm["arch"])
+                            await sign_koji_package(nvr_arch)
+                            koji_session.writeSignedRPM(nvr_arch, settings.sigul_key_id)
             elif task_info["state"] == koji.TASK_STATES["CANCELED"]:
                 build.status = BuildStatus.CANCELLED
                 await build.save()
@@ -65,6 +66,7 @@ async def atomic_check_build_status():
 async def check_build_status():
     while True:
         logger.debug("[*] Running periodic task: check_build_status")
+
         await atomic_check_build_status()
 
         # run every 5 minutes

distrobuild_scheduler/utils.py

+def gitlabify(name: str) -> str:
+    return name.replace("+", "plus")