more fixes

Author: thesayyn

MODULE.bazel

@@ -5,6 +5,7 @@ module(
     compatibility_level = 1,
 )
 
+bazel_dep(name = "rules_cc", version = "0.2.8")
 bazel_dep(name = "platforms", version = "0.0.10")
 bazel_dep(name = "bazel_features", version = "1.20.0")
 bazel_dep(name = "bazel_skylib", version = "1.5.0")

apt/extensions.bzl

@@ -27,17 +27,27 @@ def _parse_source(src):
         components = components,
     )
 
-def _distroless_extension(module_ctx):
+def _distroless_extension(mctx):
     root_direct_deps = []
     root_direct_dev_deps = []
     reproducible = False
 
-    for mod in module_ctx.modules:
-        deb_repo = deb_repository.new(module_ctx)
+    for mod in mctx.modules:
+
+        lockf = lockfile.empty(mctx)
+
+        # if mod.is_root:
+
+        if len(mod.tags.lock):
+            lock = mod.tags.lock[0]
+            lockf = lockfile.from_json(mctx, mctx.read(lock.into))
+
+        deb_repo = deb_repository.new(mctx, lockf.facts())
         resolver = dependency_resolver.new(deb_repo)
-        lockf = lockfile.empty(module_ctx)
+
 
         for sl in mod.tags.sources_list:
+            continue
             uris = [uri.removeprefix("mirror+") for uri in sl.uris]
             architectures = sl.architectures
 
@@ -58,6 +68,7 @@ def _distroless_extension(module_ctx):
         sources = lockf.sources()
         dependency_sets = lockf.dependency_sets()
         for install in mod.tags.install:
+            continue
             dependency_set = dependency_sets.setdefault(install.dependency_set, {
                 "sets": {},
             })
@@ -77,8 +88,8 @@ def _distroless_extension(module_ctx):
                     arch = architectures.pop()
                     resolved_count = 0
 
-                    module_ctx.report_progress("Resolving %s:%s" % (dep_constraint, arch))
-                    (package, dependencies, unmet_dependencies) = resolver.resolve_all(
+                    mctx.report_progress("Resolving %s:%s" % (dep_constraint, arch))
+                    (package, dependencies, unmet_dependencies, warnings) = resolver.resolve_all(
                         name = constraint["name"],
                         version = constraint["version"],
                         arch = arch,
@@ -93,9 +104,12 @@ def _distroless_extension(module_ctx):
                             "   3 - Ensure that an apt.source_list added for the specified architecture.",
                         )
 
+                    for warning in warnings:
+                        util.warning(mctx, warning)
+
+
                     if len(unmet_dependencies):
-                        # buildifier: disable=print
-                        util.warning(module_ctx, "Following dependencies could not be resolved for %s: %s" % (constraint["name"], ",".join([up[0] for up in unmet_dependencies])))
+                        util.warning(mctx, "Following dependencies could not be resolved for %s: %s" % (constraint["name"], ",".join([up[0] for up in unmet_dependencies])))
 
                     lockf.add_package(package)
 
@@ -110,15 +124,15 @@ def _distroless_extension(module_ctx):
                     arch_set[lockfile.short_package_key(package)] = package["Version"]
 
                     # For cases where architecture for the package is not specified we need
-                    # to first find out which source contains the package. and in order to do
+                    # to first find out which source contains the package. in order to do
                     # that we first need to resolve the package for amd64 architecture.
                     # Once the repository is found, then resolve the package for all the
                     # architectures the repository supports.
                     if not constraint["arch"] and arch == "amd64":
                         source = sources[package["Dist"]]
                         architectures = [a for a in source["architectures"] if a != "amd64"]
 
-                module_ctx.report_progress("Resolved %d packages for %s" % (resolved_count, arch))
+                mctx.report_progress("Resolved %d packages for %s" % (resolved_count, arch))
 
         # Generate a hub repo for every dependency set
         lock_content = lockf.as_json()
@@ -131,23 +145,36 @@ def _distroless_extension(module_ctx):
 
         # Generate a repo per package which will be aliased by hub repo.
         for (package_key, package) in lockf.packages().items():
+            # dependent_packages = None
+            # if package["name"].endswith("-dev")
+            #     packages = lockf.packages()
+            #     dependent_packages = json.encode([
+
+            #     ])
+
             deb_import(
                 name = util.sanitize(package_key),
+                target_name = util.sanitize(package_key),
                 urls = [
                     uri + "/" + package["filename"]
                     for uri in sources[package["suite"]]["uris"]
                 ],
                 sha256 = package["sha256"],
                 mergedusr = False,
-                depends_on = ["@" + util.sanitize(dep_key) for dep_key in package["depends_on"]],
+                depends_on = package["depends_on"],
+                package_name = package["name"]
             )
 
-        lock_tmp = module_ctx.path("apt.lock.json")
-        lockf.write(lock_tmp)
-        lockf_wksp = module_ctx.path(Label("@//:apt.lock.json"))
-        module_ctx.execute(
-            ["mv", lock_tmp, lockf_wksp],
-        )
+
+
+
+        for lock in mod.tags.lock:
+            lock_tmp = mctx.path("apt.lock.json")
+            lockf.write(lock_tmp)
+            lockf_wksp = mctx.path(lock.into)
+            mctx.execute(
+                ["cp", "-f", lock_tmp, lockf_wksp],
+            )
 
 _doc = """
 Module extension to create Debian repositories.
@@ -261,11 +288,21 @@ install = tag_class(
     },
 )
 
+
+lock = tag_class(
+    attrs = {
+        "into": attr.label(
+            mandatory = True,
+        ),
+    },
+)
+
 apt = module_extension(
     doc = _doc,
     implementation = _distroless_extension,
     tag_classes = {
         "install": install,
         "sources_list": sources_list,
+        "lock": lock,
     },
 )

apt/private/apt_cursed_symlink.bzl

@@ -0,0 +1,34 @@
+load("@bazel_skylib//lib:paths.bzl", "paths")
+
+def _apt_cursed_symlink(ctx):
+    own_path = ctx.attr.own_path.removeprefix(".")
+    own_dirname = own_path[:own_path.rfind("/")+1]
+    candidate_full_path = own_dirname + ctx.attr.candidate_path
+
+    found = None
+
+    for file in ctx.files.candidates:
+        if file.path.endswith(candidate_full_path):
+            found = file
+            break
+
+    if not found:
+        fail("Failed to find the candidate so library. file an issue.")
+
+    ctx.actions.symlink(
+        output = ctx.outputs.out,
+        target_file = file
+    )
+    return DefaultInfo(
+        files = depset([ctx.outputs.out])
+    )
+
+apt_cursed_symlink = rule(
+    implementation = _apt_cursed_symlink,
+    attrs = {
+        "candidates": attr.label_list(),
+        "candidate_path": attr.string(),
+        "own_path": attr.string(),
+        "out": attr.output()
+    }
+)

apt/private/apt_deb_repository.bzl

@@ -12,7 +12,7 @@ def _get_auth(ctx, urls):
         netrc = read_user_netrc(ctx)
     return use_netrc(netrc, urls, {})
 
-def _fetch_package_index(rctx, urls, dist, comp, arch, integrity):
+def _fetch_package_index(mctx, urls, dist, comp, arch, integrity):
     target_triple = "{dist}/{comp}/{arch}".format(dist = dist, comp = comp, arch = arch)
 
     # See https://linux.die.net/man/1/xz , https://linux.die.net/man/1/gzip , and https://linux.die.net/man/1/bzip2
@@ -30,7 +30,7 @@ def _fetch_package_index(rctx, urls, dist, comp, arch, integrity):
     failed_attempts = []
 
     url = None
-    base_auth = _get_auth(rctx, urls)
+    base_auth = _get_auth(mctx, urls)
     for url in urls:
         download = None
         for (ext, cmd) in supported_extensions:
@@ -39,7 +39,7 @@ def _fetch_package_index(rctx, urls, dist, comp, arch, integrity):
             auth = {}
             if url in base_auth:
                 auth = {dist_url: base_auth[url]}
-            download = rctx.download(
+            download = mctx.download(
                 url = dist_url,
                 output = output,
                 integrity = integrity,
@@ -163,6 +163,7 @@ def _package(state, name, version, arch):
 
 def _fetch_and_parse_sources(state):
     mctx = state.mctx
+    facts = state.facts
     for source in state.sources:
         (urls, dist, components, architectures) = source
 
@@ -175,19 +176,25 @@ def _fetch_and_parse_sources(state):
                 # on misconfigured HTTP servers)
                 urls = [url.rstrip("/") for url in urls]
 
+                fact_key = dist + "/" + comp + "/" + arch
+                fact_value = facts.get(fact_key, "")
+
                 # TODO: make parallel
                 mctx.report_progress("Fetching package index: {}/{} for {}".format(dist, comp, arch))
-                (output, _, _) = _fetch_package_index(mctx, urls, dist, comp, arch, "")
+                (output, url, integrity) = _fetch_package_index(mctx, urls, dist, comp, arch, fact_value)
+
+                facts[fact_key] = integrity
 
                 mctx.report_progress("Parsing package index: {}/{} for {}".format(dist, comp, arch))
                 _parse_repository(state, mctx.read(output), urls, dist)
 
-def _create(mctx):
+def _create(mctx, facts):
     state = struct(
         mctx = mctx,
         sources = list(),
         packages = dict(),
         virtual_packages = dict(),
+        facts = facts
     )
 
     return struct(

apt/private/apt_dep_resolver.bzl

@@ -17,6 +17,8 @@ def _resolve_package(state, name, version, arch):
         )
     ]
 
+    warning = None
+
     if len(candidates) == 1:
         return candidates[0]
 
@@ -36,11 +38,10 @@ def _resolve_package(state, name, version, arch):
 
         # Otherwise, we can't disambiguate the virtual package providers so
         # choose none and warn.
-        # buildifier: disable=print
-        print("\nMultiple candidates for virtual package '{}': {}".format(
+        warning = "Multiple candidates for virtual package '{}': {}".format(
             name,
-            [package["Package"] for package in candidates],
-        ))
+            ", ".join([package["Package"] for package in candidates]),
+        )
 
     # Get available versions of the package
     versions_by_arch = state.repository.package_versions(name = name, arch = arch)
@@ -68,7 +69,7 @@ def _resolve_package(state, name, version, arch):
     if not package:
         package = state.repository.package(name = name, version = selected_version, arch = "all")
 
-    return package
+    return (package, warning)
 
 _ITERATION_MAX_ = 2147483646
 
@@ -85,6 +86,8 @@ def _resolve_all(state, name, version, arch, include_transitive = True):
     dependency_group = []
     stack = [(name, version, -1)]
 
+    warnings = []
+
     for i in range(0, _ITERATION_MAX_ + 1):
         if not len(stack):
             break
@@ -98,7 +101,9 @@ def _resolve_all(state, name, version, arch, include_transitive = True):
             continue
 
         # TODO: only resolve in specified suites
-        package = _resolve_package(state, name, version, arch)
+        (package, warning) = _resolve_package(state, name, version, arch)
+        if warning:
+            warnings.append(warning)
 
         # If this package is not found and is part of a dependency group, then just skip it.
         if not package and dependency_group_idx > -1:
@@ -158,7 +163,7 @@ def _resolve_all(state, name, version, arch, include_transitive = True):
         if not met:
             unmet_dependencies.append((dep, None))
 
-    return (root_package, dependencies, unmet_dependencies)
+    return (root_package, dependencies, unmet_dependencies, warnings)
 
 def _create_resolution(repository):
     state = struct(repository = repository)

apt/private/deb_cc_export.bzl

@@ -0,0 +1,24 @@
+"normalization rules"
+
+# buildifier: disable=function-docstring-args
+def deb_cc_export(name, src, outs, **kwargs):
+    """Private. DO NOT USE."""
+    if len(outs) == 0:
+        native.filegroup(name = name, srcs = [], **kwargs)
+        return
+    toolchains = ["@bsd_tar_toolchains//:resolved_toolchain"]
+
+    cmd = """
+$(BSDTAR_BIN) -xf "$<" -C $(RULEDIR) {}  \
+""".format(
+    " ".join(outs)
+)
+    native.genrule(
+        name = name,
+        srcs = [src],
+        outs = [out.removeprefix("./") for out in outs],
+        cmd = cmd,
+        toolchains = toolchains,
+        output_to_bindir = True,
+        **kwargs
+    )