From 0b8b916cc219e2bd7fa0b4520d90563d51e6b164 Mon Sep 17 00:00:00 2001 From: Elvis Wianda Date: Sun, 19 Jan 2025 23:25:24 -0500 Subject: [PATCH 1/9] Handle symlinks --- lib/private/tar.bzl | 65 +++++++++ lib/tar.bzl | 28 ++-- lib/tests/tar/BUILD.bazel | 72 ++++++++++ lib/tests/tar/node_modules_tree.bzl | 51 +++++++ tools/mtree/BUILD.bazel | 14 ++ tools/mtree/main.go | 197 ++++++++++++++++++++++++++++ 6 files changed, 409 insertions(+), 18 deletions(-) create mode 100644 lib/tests/tar/node_modules_tree.bzl create mode 100644 tools/mtree/BUILD.bazel create mode 100644 tools/mtree/main.go diff --git a/lib/private/tar.bzl b/lib/private/tar.bzl index 21958fe2d..88583b8a6 100644 --- a/lib/private/tar.bzl +++ b/lib/private/tar.bzl @@ -446,6 +446,71 @@ def _mtree_impl(ctx): return DefaultInfo(files = depset([out]), runfiles = ctx.runfiles([out])) +def _mtree_mutate_impl(ctx): + srcs_runfiles = [ + src[DefaultInfo].default_runfiles.files + for src in ctx.attr.srcs + ] + args = ctx.actions.args() + bsdtar = ctx.toolchains[TAR_TOOLCHAIN_TYPE] + mtree_generator = ctx.executable.mtree_generator.path + + out_mtree = ctx.outputs.out + args.add("--input", ctx.file.mtree) + args.add("--output", out_mtree) + args.add("--bin_dir", ctx.bin_dir.path) + + if ctx.attr.owner: + args.add("--owner", ctx.attr.owner) + if ctx.attr.ownername: + args.add("--ownername", ctx.attr.ownername) + if ctx.attr.strip_prefix: + args.add("--strip_prefix", ctx.attr.strip_prefix) + if ctx.attr.package_dir: + args.add("--package_dir", ctx.attr.package_dir) + if ctx.attr.mtime: + args.add("--mtime", ctx.attr.mtime) + + #executable = bsdtar.tarinfo.binary, + inputs = ctx.files.srcs[:] + inputs.append(ctx.file.mtree) + ctx.actions.run( + executable = mtree_generator, + arguments = [args], + inputs = depset( + direct = inputs, + transitive = srcs_runfiles + [ + ctx.attr.mtree_generator.default_runfiles.files, + ], + ), + outputs = [out_mtree], + ) + + return [DefaultInfo(files = depset([out_mtree]))] + +mtree_mutate = rule( + implementation = _mtree_mutate_impl, + attrs = { + "mtree": attr.label(allow_single_file = True), + "awk_script": attr.label(allow_single_file = True, default = "@aspect_bazel_lib//lib/private:modify_mtree.awk"), + "srcs": attr.label_list(allow_files = True), + "strip_prefix": attr.string(), + "package_dir": attr.string(), + "mtime": attr.string(), + "owner": attr.string(), + "ownername": attr.string(), + "out": attr.output(), + "mtree_generator": attr.label( + default = Label("//tools/mtree:mtree"), + executable = True, + cfg = "exec", + ), + }, + toolchains = [ + TAR_TOOLCHAIN_TYPE, + "@aspect_bazel_lib//lib:coreutils_toolchain_type", + ], +) tar_lib = struct( attrs = _tar_attrs, implementation = _tar_impl, diff --git a/lib/tar.bzl b/lib/tar.bzl index f7db08b21..16047c24e 100644 --- a/lib/tar.bzl +++ b/lib/tar.bzl @@ -55,7 +55,7 @@ TODO: load("@bazel_skylib//lib:types.bzl", "types") load("//lib:expand_template.bzl", "expand_template") load("//lib:utils.bzl", "propagate_common_rule_attributes") -load("//lib/private:tar.bzl", _tar = "tar", _tar_lib = "tar_lib") +load("//lib/private:tar.bzl", _mutate_mtree = "mtree_mutate", _tar = "tar", _tar_lib = "tar_lib") mtree_spec = rule( doc = "Create an mtree specification to map a directory hierarchy. See https://man.freebsd.org/cgi/man.cgi?mtree(8)", @@ -157,23 +157,15 @@ def mtree_mutate( awk_script: may be overridden to change the script containing the modification logic. **kwargs: additional named parameters to genrule """ - vars = [] - if strip_prefix: - vars.append("-v strip_prefix='{}'".format(strip_prefix)) - if package_dir: - vars.append("-v package_dir='{}'".format(package_dir)) - if mtime: - vars.append("-v mtime='{}'".format(mtime)) - if owner: - vars.append("-v owner='{}'".format(owner)) - if ownername: - vars.append("-v ownername='{}'".format(ownername)) - - native.genrule( + _mutate_mtree( name = name, - srcs = [mtree], - outs = [name + ".mtree"], - cmd = "awk {} -f $(execpath {}) <$< >$@".format(" ".join(vars), awk_script), - tools = [awk_script], + mtree = mtree, + strip_prefix = strip_prefix, + package_dir = package_dir, + mtime = str(mtime) if mtime else None, + owner = owner, + ownername = ownername, + awk_script = awk_script, + out = "{}.mtree".format(name), **kwargs ) diff --git a/lib/tests/tar/BUILD.bazel b/lib/tests/tar/BUILD.bazel index b79ceb1f9..ef880d27b 100644 --- a/lib/tests/tar/BUILD.bazel +++ b/lib/tests/tar/BUILD.bazel @@ -4,6 +4,7 @@ load("@aspect_bazel_lib//lib:tar.bzl", "mtree_mutate", "mtree_spec", "tar") load("@aspect_bazel_lib//lib:testing.bzl", "assert_archive_contains") load("@bazel_skylib//rules:write_file.bzl", "write_file") load(":asserts.bzl", "assert_tar_listing", "assert_unused_listing") +load(":node_modules_tree.bzl", "node_modules_tree") # The examples below work with both source files and generated files. # Here we generate a file to use in the examples. @@ -503,3 +504,74 @@ assert_tar_listing( "-rwxr-xr-x 0 0 0 7 Jan 1 2023 tests/tar/generated.txt", ], ) + +############# +# Example 17: mtree_mutate preserves symlinks +node_modules_tree( + name = "e17_node_modules", +) + +mtree_spec( + name = "mtree17", + srcs = [ + ":e17_node_modules", + ], +) + +assert_tar_listing( + name = "test_17_before_processing", + actual = ":mtree17", + expected = [ + "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/", + "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests", + "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar", + "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/node_modules", + "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/node_modules/.pnpm", + "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/node_modules/.pnpm/a@0.0.0", + "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/node_modules/.pnpm/a@0.0.0/node_modules", + "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/node_modules/.pnpm/a@0.0.0/node_modules/a", + "-rwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/node_modules/.pnpm/a@0.0.0/node_modules/a/package.json", + "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/node_modules/.pnpm/b@0.0.0", + "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/node_modules/.pnpm/b@0.0.0/node_modules", + "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/node_modules/.pnpm/b@0.0.0/node_modules/a", + "-rwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/node_modules/.pnpm/b@0.0.0/node_modules/a/package.json", + "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/node_modules/a", + "-rwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/node_modules/a/package.json", + "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/dir", + "-rwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/dir/a", + "-rwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/dir/b", + ], +) + +mtree_mutate( + name = "resolve_symlinks", + srcs = [ + ":e17_node_modules", + ], + mtree = ":mtree17", +) + +assert_tar_listing( + name = "test_17_after_processing", + actual = ":resolve_symlinks", + expected = [ + "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/", + "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests", + "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar", + "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/node_modules", + "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/node_modules/.pnpm", + "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/node_modules/.pnpm/a@0.0.0", + "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/node_modules/.pnpm/a@0.0.0/node_modules", + "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/node_modules/.pnpm/a@0.0.0/node_modules/a", + "-rwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/node_modules/.pnpm/a@0.0.0/node_modules/a/package.json", + "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/node_modules/.pnpm/b@0.0.0", + "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/node_modules/.pnpm/b@0.0.0/node_modules", + "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/node_modules/.pnpm/b@0.0.0/node_modules/a", + "l--------- 0 0 0 0 Dec 31 1969 lib/tests/tar/node_modules/.pnpm/b@0.0.0/node_modules/a/package.json -> lib/tests/tar/node_modules/.pnpm/a@0.0.0/node_modules/a/package.json", + "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/node_modules/a", + "l--------- 0 0 0 0 Dec 31 1969 lib/tests/tar/node_modules/a/package.json -> lib/tests/tar/node_modules/.pnpm/a@0.0.0/node_modules/a/package.json", + "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/dir", + "-rwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/dir/a", + "l--------- 0 0 0 0 Dec 31 1969 lib/tests/tar/dir/b -> lib/tests/tar/dir/a", + ], +) diff --git a/lib/tests/tar/node_modules_tree.bzl b/lib/tests/tar/node_modules_tree.bzl new file mode 100644 index 000000000..6460db729 --- /dev/null +++ b/lib/tests/tar/node_modules_tree.bzl @@ -0,0 +1,51 @@ +# https://github.com/bazelbuild/rules_pkg/pull/609 +def impl(ctx): + # packages + # - a + # - b depends on a + store_a = ctx.actions.declare_directory("node_modules/.pnpm/a@0.0.0/node_modules/a") + store_b = ctx.actions.declare_directory("node_modules/.pnpm/b@0.0.0/node_modules/b") + + ctx.actions.run_shell( + outputs = [store_a, store_b], + command = "echo 'test' > %s/package.json" % store_a.path, + ) + + dep_symlink_b_to_a = ctx.actions.declare_directory("node_modules/.pnpm/b@0.0.0/node_modules/a") + + ctx.actions.symlink( + output = dep_symlink_b_to_a, + target_file = store_a, + ) + + node_modules_a = ctx.actions.declare_directory("node_modules/a") + ctx.actions.symlink( + output = node_modules_a, + target_file = store_a, + ) + + # single file + a = ctx.actions.declare_file("dir/a") + ctx.actions.run_shell( + outputs = [a], + command = "echo 'test' > %s" % a.path, + ) + + b = ctx.actions.declare_file("dir/b") + ctx.actions.symlink( + output = b, + target_file = a, + ) + + return DefaultInfo(files = depset([ + store_a, + store_b, + dep_symlink_b_to_a, + node_modules_a, + a, + b, + ])) + +node_modules_tree = rule( + implementation = impl, +) diff --git a/tools/mtree/BUILD.bazel b/tools/mtree/BUILD.bazel new file mode 100644 index 000000000..84cddefdf --- /dev/null +++ b/tools/mtree/BUILD.bazel @@ -0,0 +1,14 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") + +go_library( + name = "mtree_lib", + srcs = ["main.go"], + importpath = "github.com/bazel-contrib/bazel-lib/tools/mtree", + visibility = ["//visibility:private"], +) + +go_binary( + name = "mtree", + embed = [":mtree_lib"], + visibility = ["//visibility:public"], +) diff --git a/tools/mtree/main.go b/tools/mtree/main.go new file mode 100644 index 000000000..e92fbebee --- /dev/null +++ b/tools/mtree/main.go @@ -0,0 +1,197 @@ +package main + +import ( + "bufio" + "flag" + "fmt" + "os" + "path/filepath" + "regexp" + "strings" +) + +func main() { + // Define command-line flags + inputFile := flag.String("input", "", "Path to the input file (required)") + outputFile := flag.String("output", "", "Path to the output file (required)") + strip_prefix := flag.String("strip_prefix", "", "Prefix to strip from paths") + package_dir := flag.String("package_dir", "", "Directory to prepend to paths") + mtime := flag.String("mtime", "", "Modify time for mtree entries") + owner := flag.String("owner", "", "Owner ID for mtree entries") + ownername := flag.String("ownername", "", "Owner name for mtree entries") + bin_dir := flag.String("bin_dir", "", "Directory to check for symlink resolution") + flag.Parse() + + // Check if required flags are provided + if *inputFile == "" || *outputFile == "" { + flag.Usage() + fmt.Println("Error: Both -input and -output flags are required.") + os.Exit(1) + } + + // Open input file + file, err := os.Open(*inputFile) + if err != nil { + fmt.Fprintf(os.Stderr, "Error opening input file: %v\n", err) + return + } + defer file.Close() + + // Create output file + outFile, err := os.Create(*outputFile) + if err != nil { + fmt.Fprintf(os.Stderr, "Error creating output file: %v\n", err) + return + } + defer outFile.Close() + + scanner := bufio.NewScanner(file) + + // Create map to hold paths + paths := make(map[string]string) + + // First pass to gather paths + for scanner.Scan() { + line := scanner.Text() + fields := strings.Fields(line) + if len(fields) > 0 && strings.Contains(line, "content=") { + contentIndex := strings.Index(line, "content=") + if contentIndex != -1 { + contentValue := line[contentIndex+len("content="):] + endIndex := strings.Index(contentValue, " ") + if endIndex == -1 { + endIndex = len(contentValue) + } + paths[contentValue[:endIndex]] = fields[0] + } + } + } + + // Resolve symlinks + resolvedPaths, originalPaths := resolveAllSymlinks(paths, *bin_dir) + + // Reset scanner for second pass + file.Seek(0, 0) // Reset file pointer to the start + scanner = bufio.NewScanner(file) + + for scanner.Scan() { + line := scanner.Text() + fields := strings.Fields(line) + if len(fields) == 0 { + continue // Skip empty lines + } + + // Handle strip_prefix + if *strip_prefix != "" { + if fields[0] == *strip_prefix { + continue // Skip if this line declares the directory which is now the root + } else if strings.HasPrefix(fields[0], *strip_prefix+"/") { + fields[0] = strings.TrimPrefix(fields[0], *strip_prefix+"/") + + // Check if this is a directory at the root level + components := strings.Split(fields[0], "/") + if strings.Contains(line, "type=dir") && len(components) == 1 { + if !strings.HasPrefix(line, " ") { + fields[0] += "/" // If the line doesn't start with a space, append a slash + } else { + continue // Skip root directory entries with only orphaned keywords + } + } + } else { + continue // Skip lines that declare paths under a parent directory + } + line = strings.Join(fields, " ") + } + + // Handle mtime + if *mtime != "" { + line = regexp.MustCompile(`time=[0-9\.]+`).ReplaceAllString(line, "time="+*mtime) + } + + // Handle owner + if *owner != "" { + line = regexp.MustCompile(`uid=[0-9]+`).ReplaceAllString(line, "uid="+*owner) + } + + // Handle ownername + if *ownername != "" { + line = regexp.MustCompile(`uname=[^ ]+`).ReplaceAllString(line, "uname="+*ownername) + + } + + // Handle package_dir + if *package_dir != "" { + fields[0] = filepath.Join(*package_dir, fields[0]) + line = strings.Join(fields, " ") + } + + // Handle symlinks + if strings.Contains(line, "type=file") && strings.Contains(line, "content=") { + if resolvedPath, exists := resolvedPaths[fields[0]]; exists { + newLine := fields[0] + " type=link link=" + resolvedPath + fmt.Fprintln(outFile, newLine) + continue + } else if _, exists := originalPaths[fields[0]]; exists { + // If it's an original path, keep the line as is but update content + line = strings.Replace(line, "content=", "content="+originalPaths[fields[0]], 1) + } + } + + fmt.Fprintln(outFile, line) + } + + if err := scanner.Err(); err != nil { + fmt.Fprintf(os.Stderr, "Error reading input: %v\n", err) + } +} + +func resolveAllSymlinks(paths map[string]string, allowedPrefix string) (map[string]string, map[string]string) { + resolvedPaths := make(map[string]string) + originalPaths := make(map[string]string) + for key, value := range paths { + resolved, err := resolveSymlink(key, allowedPrefix) + if err != nil { + fmt.Printf("Error resolving symlink for %s: %v\n", key, err) + continue + } + if resolved == "" { + originalPaths[key] = value // Keep the original if not a symlink or outside prefix + } else { + resolvedPaths[value] = paths[resolved] + } + } + return resolvedPaths, originalPaths +} + +// resolveSymlink resolves a symlink and verifies its relationship to the allowed prefix. +func resolveSymlink(path string, allowedPrefix string) (string, error) { + info, err := os.Lstat(path) + if err != nil { + return "", err + } + + if info.Mode()&os.ModeSymlink == 0 { + return "", nil + } + + resolved, err := filepath.EvalSymlinks(path) + if err != nil { + return "", err + } + + index := strings.LastIndex(resolved, allowedPrefix) + if index == -1 { + return "", nil + } + + resolvedSuffix := resolved[index+len(allowedPrefix):] + if resolvedSuffix == "" { + return "", nil + } + resolvedPath := filepath.Join(allowedPrefix, resolvedSuffix) + if resolvedPath == path { + return "", nil + } + + return resolvedPath, nil +} From 0e4d8de5c8bc927cba186e95aeb69e41ef203da6 Mon Sep 17 00:00:00 2001 From: Elvis Wianda Date: Mon, 20 Jan 2025 16:46:05 -0500 Subject: [PATCH 2/9] Handle symlinks --- lib/private/modify_mtree.awk | 72 ++++++++++++- lib/private/tar.bzl | 65 ------------ lib/tar.bzl | 48 +++++++-- lib/tests/tar/BUILD.bazel | 1 + tools/mtree/BUILD.bazel | 14 --- tools/mtree/main.go | 197 ----------------------------------- 6 files changed, 110 insertions(+), 287 deletions(-) delete mode 100644 tools/mtree/BUILD.bazel delete mode 100644 tools/mtree/main.go diff --git a/lib/private/modify_mtree.awk b/lib/private/modify_mtree.awk index a333821f3..6db995787 100644 --- a/lib/private/modify_mtree.awk +++ b/lib/private/modify_mtree.awk @@ -45,5 +45,75 @@ if (package_dir != "") { sub(/^/, package_dir "/") } - print; + if (preserve_symlinks != "") { + # By default Bazel reports symlinks as regular file/dir therefore mtree_spec has no way of knowing that a file + # is a symlink. This is a problem when we want to preserve symlinks especially for symlink sensitive applications + # such as nodejs with pnpm. To work around this we need to determine if a file a symlink and if so, we need to + # determine where the symlink points to by calling readlink repeatedly until we get the final destination. + # + # We then need to decide if it's a symlink based on how many times we had to call readlink and where we ended up. + # + # Unlike Bazels own symlinks, which points out of the sandbox symlinks, symlinks created by ctx.actions.symlink + # stays within the bazel sandbox so it's possible to detect those. + # + # See https://github.com/bazelbuild/rules_pkg/pull/609 + + symlink = "" + if ($0 ~ /type=file/ && $0 ~ /content=/) { + match($0, /content=[^ ]+/) + content_field = substr($0, RSTART, RLENGTH) + split(content_field, parts, "=") + path = parts[2] + # Store paths for look up + symlink_map[path] = $1 + # Resolve the symlink if it exists + resolved_path = "" + cmd = "readlink -f " path + cmd | getline resolved_path + close(cmd) + + if (resolved_path) { + if (resolved_path ~ bin_dir) { + # Strip down the resolved path to start from bin_dir + sub("^.*" bin_dir, bin_dir, resolved_path) + if (path != resolved_path) { + # Replace the content field with the new path + symlink = resolved_path + } + } + } + } + if (symlink != "") { + line_array[NR] = $1 SUBSEP resolved_path + } + else { + line_array[NR] = $0 # Store other lines too, with an empty path + } + } + + else { + + print; # Print immediately if symlinks are not preserved + + } +} +END { + if (preserve_symlinks != "") { + # Process symlinks if needed + for (i = 1; i <= NR; i++) { + line = line_array[i] + if (index(line, SUBSEP) > 0) { # Check if this path was a symlink + split(line, fields, SUBSEP) + field0 = fields[1] + resolved_path = fields[2] + linked_to = symlink_map[resolved_path] + # Adjust the line for symlink using the map we created + new_line = field0 " type=link link=" linked_to + print new_line + } else { + # Print the original line if no symlink adjustment was needed + print line + } + } + } } diff --git a/lib/private/tar.bzl b/lib/private/tar.bzl index 88583b8a6..21958fe2d 100644 --- a/lib/private/tar.bzl +++ b/lib/private/tar.bzl @@ -446,71 +446,6 @@ def _mtree_impl(ctx): return DefaultInfo(files = depset([out]), runfiles = ctx.runfiles([out])) -def _mtree_mutate_impl(ctx): - srcs_runfiles = [ - src[DefaultInfo].default_runfiles.files - for src in ctx.attr.srcs - ] - args = ctx.actions.args() - bsdtar = ctx.toolchains[TAR_TOOLCHAIN_TYPE] - mtree_generator = ctx.executable.mtree_generator.path - - out_mtree = ctx.outputs.out - args.add("--input", ctx.file.mtree) - args.add("--output", out_mtree) - args.add("--bin_dir", ctx.bin_dir.path) - - if ctx.attr.owner: - args.add("--owner", ctx.attr.owner) - if ctx.attr.ownername: - args.add("--ownername", ctx.attr.ownername) - if ctx.attr.strip_prefix: - args.add("--strip_prefix", ctx.attr.strip_prefix) - if ctx.attr.package_dir: - args.add("--package_dir", ctx.attr.package_dir) - if ctx.attr.mtime: - args.add("--mtime", ctx.attr.mtime) - - #executable = bsdtar.tarinfo.binary, - inputs = ctx.files.srcs[:] - inputs.append(ctx.file.mtree) - ctx.actions.run( - executable = mtree_generator, - arguments = [args], - inputs = depset( - direct = inputs, - transitive = srcs_runfiles + [ - ctx.attr.mtree_generator.default_runfiles.files, - ], - ), - outputs = [out_mtree], - ) - - return [DefaultInfo(files = depset([out_mtree]))] - -mtree_mutate = rule( - implementation = _mtree_mutate_impl, - attrs = { - "mtree": attr.label(allow_single_file = True), - "awk_script": attr.label(allow_single_file = True, default = "@aspect_bazel_lib//lib/private:modify_mtree.awk"), - "srcs": attr.label_list(allow_files = True), - "strip_prefix": attr.string(), - "package_dir": attr.string(), - "mtime": attr.string(), - "owner": attr.string(), - "ownername": attr.string(), - "out": attr.output(), - "mtree_generator": attr.label( - default = Label("//tools/mtree:mtree"), - executable = True, - cfg = "exec", - ), - }, - toolchains = [ - TAR_TOOLCHAIN_TYPE, - "@aspect_bazel_lib//lib:coreutils_toolchain_type", - ], -) tar_lib = struct( attrs = _tar_attrs, implementation = _tar_impl, diff --git a/lib/tar.bzl b/lib/tar.bzl index 16047c24e..a682d2e11 100644 --- a/lib/tar.bzl +++ b/lib/tar.bzl @@ -55,7 +55,7 @@ TODO: load("@bazel_skylib//lib:types.bzl", "types") load("//lib:expand_template.bzl", "expand_template") load("//lib:utils.bzl", "propagate_common_rule_attributes") -load("//lib/private:tar.bzl", _mutate_mtree = "mtree_mutate", _tar = "tar", _tar_lib = "tar_lib") +load("//lib/private:tar.bzl", _tar = "tar", _tar_lib = "tar_lib") mtree_spec = rule( doc = "Create an mtree specification to map a directory hierarchy. See https://man.freebsd.org/cgi/man.cgi?mtree(8)", @@ -137,6 +137,8 @@ def tar(name, mtree = "auto", stamp = 0, **kwargs): def mtree_mutate( name, mtree, + srcs = None, + preserve_symlinks = False, strip_prefix = None, package_dir = None, mtime = None, @@ -148,6 +150,8 @@ def mtree_mutate( Args: name: name of the target, output will be `[name].mtree`. + srcs: source files to be used when resolving symlinks. required if `preserve_symlinks` is set to True. + preserve_symlinks: preserve symlinks mtree: input mtree file, typically created by `mtree_spec`. strip_prefix: prefix to remove from all paths in the tar. Files and directories not under this prefix are dropped. package_dir: directory prefix to add to all paths in the tar. @@ -155,17 +159,41 @@ def mtree_mutate( owner: new uid for all entries. ownername: new uname for all entries. awk_script: may be overridden to change the script containing the modification logic. + **kwargs: additional named parameters to genrule """ - _mutate_mtree( + vars = [] + if strip_prefix: + vars.append("-v strip_prefix='{}'".format(strip_prefix)) + if package_dir: + vars.append("-v package_dir='{}'".format(package_dir)) + if mtime: + vars.append("-v mtime='{}'".format(mtime)) + if owner: + vars.append("-v owner='{}'".format(owner)) + if ownername: + vars.append("-v ownername='{}'".format(ownername)) + if preserve_symlinks: + vars.append("-v preserve_symlinks=1") + if not srcs: + fail("preserve_symlinks requires srcs to be set in order to resolve symlinks") + + # Check if srcs is of type list + if srcs and not types.is_list(srcs): + srcs = [srcs] + + native.genrule( name = name, - mtree = mtree, - strip_prefix = strip_prefix, - package_dir = package_dir, - mtime = str(mtime) if mtime else None, - owner = owner, - ownername = ownername, - awk_script = awk_script, - out = "{}.mtree".format(name), + srcs = [mtree] + (srcs or []), + outs = [name + ".mtree"], + cmd = """ + awk {variables} \ + -v bin_dir=$(BINDIR) \ + -f $(execpath {awk_script}) $(execpath {mtree_spec}) >$@""".format( + variables = " ".join(vars), + awk_script = awk_script, + mtree_spec = mtree, + ), + tools = [awk_script], **kwargs ) diff --git a/lib/tests/tar/BUILD.bazel b/lib/tests/tar/BUILD.bazel index ef880d27b..0d9bccb59 100644 --- a/lib/tests/tar/BUILD.bazel +++ b/lib/tests/tar/BUILD.bazel @@ -549,6 +549,7 @@ mtree_mutate( ":e17_node_modules", ], mtree = ":mtree17", + preserve_symlinks = True, ) assert_tar_listing( diff --git a/tools/mtree/BUILD.bazel b/tools/mtree/BUILD.bazel deleted file mode 100644 index 84cddefdf..000000000 --- a/tools/mtree/BUILD.bazel +++ /dev/null @@ -1,14 +0,0 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") - -go_library( - name = "mtree_lib", - srcs = ["main.go"], - importpath = "github.com/bazel-contrib/bazel-lib/tools/mtree", - visibility = ["//visibility:private"], -) - -go_binary( - name = "mtree", - embed = [":mtree_lib"], - visibility = ["//visibility:public"], -) diff --git a/tools/mtree/main.go b/tools/mtree/main.go deleted file mode 100644 index e92fbebee..000000000 --- a/tools/mtree/main.go +++ /dev/null @@ -1,197 +0,0 @@ -package main - -import ( - "bufio" - "flag" - "fmt" - "os" - "path/filepath" - "regexp" - "strings" -) - -func main() { - // Define command-line flags - inputFile := flag.String("input", "", "Path to the input file (required)") - outputFile := flag.String("output", "", "Path to the output file (required)") - strip_prefix := flag.String("strip_prefix", "", "Prefix to strip from paths") - package_dir := flag.String("package_dir", "", "Directory to prepend to paths") - mtime := flag.String("mtime", "", "Modify time for mtree entries") - owner := flag.String("owner", "", "Owner ID for mtree entries") - ownername := flag.String("ownername", "", "Owner name for mtree entries") - bin_dir := flag.String("bin_dir", "", "Directory to check for symlink resolution") - flag.Parse() - - // Check if required flags are provided - if *inputFile == "" || *outputFile == "" { - flag.Usage() - fmt.Println("Error: Both -input and -output flags are required.") - os.Exit(1) - } - - // Open input file - file, err := os.Open(*inputFile) - if err != nil { - fmt.Fprintf(os.Stderr, "Error opening input file: %v\n", err) - return - } - defer file.Close() - - // Create output file - outFile, err := os.Create(*outputFile) - if err != nil { - fmt.Fprintf(os.Stderr, "Error creating output file: %v\n", err) - return - } - defer outFile.Close() - - scanner := bufio.NewScanner(file) - - // Create map to hold paths - paths := make(map[string]string) - - // First pass to gather paths - for scanner.Scan() { - line := scanner.Text() - fields := strings.Fields(line) - if len(fields) > 0 && strings.Contains(line, "content=") { - contentIndex := strings.Index(line, "content=") - if contentIndex != -1 { - contentValue := line[contentIndex+len("content="):] - endIndex := strings.Index(contentValue, " ") - if endIndex == -1 { - endIndex = len(contentValue) - } - paths[contentValue[:endIndex]] = fields[0] - } - } - } - - // Resolve symlinks - resolvedPaths, originalPaths := resolveAllSymlinks(paths, *bin_dir) - - // Reset scanner for second pass - file.Seek(0, 0) // Reset file pointer to the start - scanner = bufio.NewScanner(file) - - for scanner.Scan() { - line := scanner.Text() - fields := strings.Fields(line) - if len(fields) == 0 { - continue // Skip empty lines - } - - // Handle strip_prefix - if *strip_prefix != "" { - if fields[0] == *strip_prefix { - continue // Skip if this line declares the directory which is now the root - } else if strings.HasPrefix(fields[0], *strip_prefix+"/") { - fields[0] = strings.TrimPrefix(fields[0], *strip_prefix+"/") - - // Check if this is a directory at the root level - components := strings.Split(fields[0], "/") - if strings.Contains(line, "type=dir") && len(components) == 1 { - if !strings.HasPrefix(line, " ") { - fields[0] += "/" // If the line doesn't start with a space, append a slash - } else { - continue // Skip root directory entries with only orphaned keywords - } - } - } else { - continue // Skip lines that declare paths under a parent directory - } - line = strings.Join(fields, " ") - } - - // Handle mtime - if *mtime != "" { - line = regexp.MustCompile(`time=[0-9\.]+`).ReplaceAllString(line, "time="+*mtime) - } - - // Handle owner - if *owner != "" { - line = regexp.MustCompile(`uid=[0-9]+`).ReplaceAllString(line, "uid="+*owner) - } - - // Handle ownername - if *ownername != "" { - line = regexp.MustCompile(`uname=[^ ]+`).ReplaceAllString(line, "uname="+*ownername) - - } - - // Handle package_dir - if *package_dir != "" { - fields[0] = filepath.Join(*package_dir, fields[0]) - line = strings.Join(fields, " ") - } - - // Handle symlinks - if strings.Contains(line, "type=file") && strings.Contains(line, "content=") { - if resolvedPath, exists := resolvedPaths[fields[0]]; exists { - newLine := fields[0] + " type=link link=" + resolvedPath - fmt.Fprintln(outFile, newLine) - continue - } else if _, exists := originalPaths[fields[0]]; exists { - // If it's an original path, keep the line as is but update content - line = strings.Replace(line, "content=", "content="+originalPaths[fields[0]], 1) - } - } - - fmt.Fprintln(outFile, line) - } - - if err := scanner.Err(); err != nil { - fmt.Fprintf(os.Stderr, "Error reading input: %v\n", err) - } -} - -func resolveAllSymlinks(paths map[string]string, allowedPrefix string) (map[string]string, map[string]string) { - resolvedPaths := make(map[string]string) - originalPaths := make(map[string]string) - for key, value := range paths { - resolved, err := resolveSymlink(key, allowedPrefix) - if err != nil { - fmt.Printf("Error resolving symlink for %s: %v\n", key, err) - continue - } - if resolved == "" { - originalPaths[key] = value // Keep the original if not a symlink or outside prefix - } else { - resolvedPaths[value] = paths[resolved] - } - } - return resolvedPaths, originalPaths -} - -// resolveSymlink resolves a symlink and verifies its relationship to the allowed prefix. -func resolveSymlink(path string, allowedPrefix string) (string, error) { - info, err := os.Lstat(path) - if err != nil { - return "", err - } - - if info.Mode()&os.ModeSymlink == 0 { - return "", nil - } - - resolved, err := filepath.EvalSymlinks(path) - if err != nil { - return "", err - } - - index := strings.LastIndex(resolved, allowedPrefix) - if index == -1 { - return "", nil - } - - resolvedSuffix := resolved[index+len(allowedPrefix):] - if resolvedSuffix == "" { - return "", nil - } - resolvedPath := filepath.Join(allowedPrefix, resolvedSuffix) - if resolvedPath == path { - return "", nil - } - - return resolvedPath, nil -} From 6340e9f7402fdad36bc0ef83972ef60f4cfaa93a Mon Sep 17 00:00:00 2001 From: Elvis Wianda Date: Wed, 22 Jan 2025 00:24:56 -0500 Subject: [PATCH 3/9] Demonstrate failure when srcs is a binary rule --- lib/tests/tar/BUILD.bazel | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/lib/tests/tar/BUILD.bazel b/lib/tests/tar/BUILD.bazel index 0d9bccb59..b1f47b00c 100644 --- a/lib/tests/tar/BUILD.bazel +++ b/lib/tests/tar/BUILD.bazel @@ -2,6 +2,7 @@ load("@aspect_bazel_lib//lib:copy_directory.bzl", "copy_directory") load("@aspect_bazel_lib//lib:diff_test.bzl", "diff_test") load("@aspect_bazel_lib//lib:tar.bzl", "mtree_mutate", "mtree_spec", "tar") load("@aspect_bazel_lib//lib:testing.bzl", "assert_archive_contains") +load("@bazel_skylib//rules:native_binary.bzl", "native_binary") load("@bazel_skylib//rules:write_file.bzl", "write_file") load(":asserts.bzl", "assert_tar_listing", "assert_unused_listing") load(":node_modules_tree.bzl", "node_modules_tree") @@ -511,10 +512,26 @@ node_modules_tree( name = "e17_node_modules", ) +write_file( + name = "executable", + out = "executable.sh", + content = [ + "#!/usr/bin/env bash", + ], + is_executable = True, +) + +native_binary( + name = "e17_binary", + src = ":executable", + out = "native_binary_bin", + data = [":e17_node_modules"], +) + mtree_spec( name = "mtree17", srcs = [ - ":e17_node_modules", + ":e17_binary", ], ) @@ -546,7 +563,7 @@ assert_tar_listing( mtree_mutate( name = "resolve_symlinks", srcs = [ - ":e17_node_modules", + ":e17_binary", ], mtree = ":mtree17", preserve_symlinks = True, From ae0e702981fd8e7907e29653bd4fda5fbd4b1a7c Mon Sep 17 00:00:00 2001 From: Elvis Wianda Date: Thu, 23 Jan 2025 15:20:38 -0500 Subject: [PATCH 4/9] Implement mtree mutation rule --- lib/private/tar.bzl | 88 +++++++++++++++++++++++++++++ lib/tar.bzl | 48 ++++++---------- lib/tests/tar/BUILD.bazel | 78 +++++++++++++++---------- lib/tests/tar/node_modules_tree.bzl | 8 ++- 4 files changed, 158 insertions(+), 64 deletions(-) diff --git a/lib/private/tar.bzl b/lib/private/tar.bzl index 21958fe2d..71083d522 100644 --- a/lib/private/tar.bzl +++ b/lib/private/tar.bzl @@ -125,6 +125,43 @@ _mtree_attrs = { "srcs": attr.label_list(doc = "Files that are placed into the tar", allow_files = True), "out": attr.output(doc = "Resulting specification file to write"), } +_mutate_mtree_attrs = { + "mtree": attr.label( + allow_single_file = True, + doc = "Specifies the path to the mtree file, which describes the directory structure and metadata for the tar file. Must be a single file.", + ), + "awk_script": attr.label( + allow_single_file = True, + default = "@aspect_bazel_lib//lib/private:modify_mtree.awk", + doc = "Path to an AWK script used to modify the mtree file. By default, it uses the modify_mtree.awk script.", + ), + "srcs": attr.label_list( + allow_files = True, + doc = "Files, directories, or other targets whose default outputs will used to create symlinks", + ), + "preserve_symlinks": attr.bool( + default = False, + doc = "If True, symbolic links in the source files are preserved in the tar file. If False, the links are resolved to their actual targets.", + ), + "strip_prefix": attr.string( + doc = "A prefix to strip from the paths of files and directories when they are added to the tar file.", + ), + "package_dir": attr.string( + doc = "Specifies a base directory within the tar file where all files will be placed. Sets the root directory for the tar contents.", + ), + "mtime": attr.string( + doc = "Specifies the modification time (mtime) to be applied to all files in the tar file. Used for deterministic builds.", + ), + "owner": attr.string( + doc = "Specifies the numeric user ID (UID) for the owner of the files in the tar archive.", + ), + "ownername": attr.string( + doc = "Specifies the name of the owner of the files in the tar archive. Used alongside 'owner'.", + ), + "out": attr.output( + doc = "The output of the mutation, a new mtree file.", + ), +} def _add_compression_args(compress, args): if compress == "bzip2": @@ -446,6 +483,57 @@ def _mtree_impl(ctx): return DefaultInfo(files = depset([out]), runfiles = ctx.runfiles([out])) +def _mtree_mutate_impl(ctx): + srcs_runfiles = [ + src[DefaultInfo].default_runfiles.files + for src in ctx.attr.srcs + ] + args = ctx.actions.args() + + out_mtree = ctx.outputs.out + + # Use bin directory to determine if symlink is within or outside the sandbox + args.add("-v bin_dir={}".format(ctx.bin_dir.path)) + + if ctx.attr.owner: + args.add("-v owner={}".format(ctx.attr.owner)) + if ctx.attr.ownername: + args.add("-v ownername={}".format(ctx.attr.ownername)) + if ctx.attr.strip_prefix: + args.add("-v strip_prefix={}".format(ctx.attr.strip_prefix)) + if ctx.attr.package_dir: + args.add("-v package_dir={}".format(ctx.attr.package_dir)) + if ctx.attr.mtime: + args.add("-v mtime={}".format(ctx.attr.mtime)) + if ctx.attr.preserve_symlinks: + args.add("-v preserve_symlinks=1") + + inputs = ctx.files.srcs[:] + inputs.append(ctx.file.mtree) + inputs.append(ctx.file.awk_script) + ctx.actions.run_shell( + command = """ + awk $@ -f {awk_script} {mtree} > {out_mtree} + """.format( + awk_script = ctx.file.awk_script.path, + mtree = ctx.file.mtree.path, + out_mtree = out_mtree.path, + ), + arguments = [args], + inputs = depset( + direct = inputs, + transitive = srcs_runfiles, + ), + outputs = [out_mtree], + ) + + return [DefaultInfo(files = depset([out_mtree]))] + +mtree_mutate = rule( + implementation = _mtree_mutate_impl, + attrs = _mutate_mtree_attrs, +) + tar_lib = struct( attrs = _tar_attrs, implementation = _tar_impl, diff --git a/lib/tar.bzl b/lib/tar.bzl index a682d2e11..a97480d23 100644 --- a/lib/tar.bzl +++ b/lib/tar.bzl @@ -55,7 +55,7 @@ TODO: load("@bazel_skylib//lib:types.bzl", "types") load("//lib:expand_template.bzl", "expand_template") load("//lib:utils.bzl", "propagate_common_rule_attributes") -load("//lib/private:tar.bzl", _tar = "tar", _tar_lib = "tar_lib") +load("//lib/private:tar.bzl", _mutate_mtree = "mtree_mutate", _tar = "tar", _tar_lib = "tar_lib") mtree_spec = rule( doc = "Create an mtree specification to map a directory hierarchy. See https://man.freebsd.org/cgi/man.cgi?mtree(8)", @@ -150,50 +150,34 @@ def mtree_mutate( Args: name: name of the target, output will be `[name].mtree`. - srcs: source files to be used when resolving symlinks. required if `preserve_symlinks` is set to True. - preserve_symlinks: preserve symlinks mtree: input mtree file, typically created by `mtree_spec`. + srcs: list of files to resolve symlinks for. + preserve_symlinks: whether to preserve symlinks in the tar. strip_prefix: prefix to remove from all paths in the tar. Files and directories not under this prefix are dropped. package_dir: directory prefix to add to all paths in the tar. mtime: new modification time for all entries. owner: new uid for all entries. ownername: new uname for all entries. awk_script: may be overridden to change the script containing the modification logic. - **kwargs: additional named parameters to genrule """ - vars = [] - if strip_prefix: - vars.append("-v strip_prefix='{}'".format(strip_prefix)) - if package_dir: - vars.append("-v package_dir='{}'".format(package_dir)) - if mtime: - vars.append("-v mtime='{}'".format(mtime)) - if owner: - vars.append("-v owner='{}'".format(owner)) - if ownername: - vars.append("-v ownername='{}'".format(ownername)) - if preserve_symlinks: - vars.append("-v preserve_symlinks=1") - if not srcs: - fail("preserve_symlinks requires srcs to be set in order to resolve symlinks") + if preserve_symlinks and not srcs: + fail("preserve_symlinks requires srcs to be set in order to resolve symlinks") # Check if srcs is of type list if srcs and not types.is_list(srcs): srcs = [srcs] - - native.genrule( + _mutate_mtree( name = name, - srcs = [mtree] + (srcs or []), - outs = [name + ".mtree"], - cmd = """ - awk {variables} \ - -v bin_dir=$(BINDIR) \ - -f $(execpath {awk_script}) $(execpath {mtree_spec}) >$@""".format( - variables = " ".join(vars), - awk_script = awk_script, - mtree_spec = mtree, - ), - tools = [awk_script], + mtree = mtree, + srcs = srcs, + preserve_symlinks = preserve_symlinks, + strip_prefix = strip_prefix, + package_dir = package_dir, + mtime = str(mtime) if mtime else None, + owner = owner, + ownername = ownername, + awk_script = awk_script, + out = "{}.mtree".format(name), **kwargs ) diff --git a/lib/tests/tar/BUILD.bazel b/lib/tests/tar/BUILD.bazel index b1f47b00c..0238468a9 100644 --- a/lib/tests/tar/BUILD.bazel +++ b/lib/tests/tar/BUILD.bazel @@ -542,21 +542,30 @@ assert_tar_listing( "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/", "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests", "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar", - "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/node_modules", - "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/node_modules/.pnpm", - "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/node_modules/.pnpm/a@0.0.0", - "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/node_modules/.pnpm/a@0.0.0/node_modules", - "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/node_modules/.pnpm/a@0.0.0/node_modules/a", - "-rwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/node_modules/.pnpm/a@0.0.0/node_modules/a/package.json", - "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/node_modules/.pnpm/b@0.0.0", - "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/node_modules/.pnpm/b@0.0.0/node_modules", - "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/node_modules/.pnpm/b@0.0.0/node_modules/a", - "-rwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/node_modules/.pnpm/b@0.0.0/node_modules/a/package.json", - "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/node_modules/a", - "-rwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/node_modules/a/package.json", - "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/dir", - "-rwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/dir/a", - "-rwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/dir/b", + "-rwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/native_binary_bin", + "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/native_binary_bin.runfiles", + "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/native_binary_bin.runfiles/_main/", + "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/native_binary_bin.runfiles/_main/lib", + "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests", + "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar", + "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/node_modules", + "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/node_modules/.pnpm", + "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/node_modules/.pnpm/a@0.0.0", + "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/node_modules/.pnpm/a@0.0.0/node_modules", + "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/node_modules/.pnpm/a@0.0.0/node_modules/a", + "-rwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/node_modules/.pnpm/a@0.0.0/node_modules/a/package.json", + "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/node_modules/.pnpm/b@0.0.0", + "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/node_modules/.pnpm/b@0.0.0/node_modules", + "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/node_modules/.pnpm/b@0.0.0/node_modules/a", + "-rwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/node_modules/.pnpm/b@0.0.0/node_modules/a/package.json", + "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/node_modules/a", + "-rwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/node_modules/a/package.json", + "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/dir", + "-rwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/dir/a", + "-rwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/dir/b", + "-rwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/executable.sh", + "-rwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/native_binary_bin", + "-rwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/native_binary_bin.runfiles/_repo_mapping", ], ) @@ -576,20 +585,29 @@ assert_tar_listing( "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/", "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests", "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar", - "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/node_modules", - "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/node_modules/.pnpm", - "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/node_modules/.pnpm/a@0.0.0", - "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/node_modules/.pnpm/a@0.0.0/node_modules", - "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/node_modules/.pnpm/a@0.0.0/node_modules/a", - "-rwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/node_modules/.pnpm/a@0.0.0/node_modules/a/package.json", - "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/node_modules/.pnpm/b@0.0.0", - "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/node_modules/.pnpm/b@0.0.0/node_modules", - "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/node_modules/.pnpm/b@0.0.0/node_modules/a", - "l--------- 0 0 0 0 Dec 31 1969 lib/tests/tar/node_modules/.pnpm/b@0.0.0/node_modules/a/package.json -> lib/tests/tar/node_modules/.pnpm/a@0.0.0/node_modules/a/package.json", - "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/node_modules/a", - "l--------- 0 0 0 0 Dec 31 1969 lib/tests/tar/node_modules/a/package.json -> lib/tests/tar/node_modules/.pnpm/a@0.0.0/node_modules/a/package.json", - "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/dir", - "-rwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/dir/a", - "l--------- 0 0 0 0 Dec 31 1969 lib/tests/tar/dir/b -> lib/tests/tar/dir/a", + "l--------- 0 0 0 0 Dec 31 1969 lib/tests/tar/native_binary_bin -> lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/executable.sh", + "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/native_binary_bin.runfiles", + "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/native_binary_bin.runfiles/_main/", + "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/native_binary_bin.runfiles/_main/lib", + "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests", + "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar", + "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/node_modules", + "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/node_modules/.pnpm", + "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/node_modules/.pnpm/a@0.0.0", + "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/node_modules/.pnpm/a@0.0.0/node_modules", + "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/node_modules/.pnpm/a@0.0.0/node_modules/a", + "-rwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/node_modules/.pnpm/a@0.0.0/node_modules/a/package.json", + "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/node_modules/.pnpm/b@0.0.0", + "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/node_modules/.pnpm/b@0.0.0/node_modules", + "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/node_modules/.pnpm/b@0.0.0/node_modules/a", + "l--------- 0 0 0 0 Dec 31 1969 lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/node_modules/.pnpm/b@0.0.0/node_modules/a/package.json -> lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/node_modules/.pnpm/a@0.0.0/node_modules/a/package.json", + "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/node_modules/a", + "l--------- 0 0 0 0 Dec 31 1969 lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/node_modules/a/package.json -> lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/node_modules/.pnpm/a@0.0.0/node_modules/a/package.json", + "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/dir", + "-rwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/dir/a", + "l--------- 0 0 0 0 Dec 31 1969 lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/dir/b -> lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/dir/a", + "-rwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/executable.sh", + "l--------- 0 0 0 0 Dec 31 1969 lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/native_binary_bin -> lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/executable.sh", + "-rwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/native_binary_bin.runfiles/_repo_mapping", ], ) diff --git a/lib/tests/tar/node_modules_tree.bzl b/lib/tests/tar/node_modules_tree.bzl index 6460db729..00bfc1444 100644 --- a/lib/tests/tar/node_modules_tree.bzl +++ b/lib/tests/tar/node_modules_tree.bzl @@ -1,5 +1,9 @@ # https://github.com/bazelbuild/rules_pkg/pull/609 -def impl(ctx): +""" +This rule creates a tree of files and directories that represents a node_modules +""" + +def _impl(ctx): # packages # - a # - b depends on a @@ -47,5 +51,5 @@ def impl(ctx): ])) node_modules_tree = rule( - implementation = impl, + implementation = _impl, ) From 672f785b7d8441fc07c3a383fe2e3f7c1155dbc8 Mon Sep 17 00:00:00 2001 From: Elvis Wianda Date: Fri, 24 Jan 2025 16:32:11 -0500 Subject: [PATCH 5/9] Add fallback to readlink to handle relative symlinks --- docs/tar.md | 5 ++++- lib/private/modify_mtree.awk | 23 ++++++++++++++++++----- lib/tar.bzl | 2 +- 3 files changed, 23 insertions(+), 7 deletions(-) diff --git a/docs/tar.md b/docs/tar.md index ccdda5288..eee1a0413 100644 --- a/docs/tar.md +++ b/docs/tar.md @@ -103,7 +103,8 @@ Rule that executes BSD `tar`. Most users should use the [`tar`](#tar) macro, rat ## mtree_mutate 
-mtree_mutate(name, mtree, strip_prefix, package_dir, mtime, owner, ownername, awk_script, kwargs)
+mtree_mutate(name, mtree, srcs, preserve_symlinks, strip_prefix, package_dir, mtime, owner,
+             ownername, awk_script, kwargs)
Modify metadata in an mtree file. @@ -115,6 +116,8 @@ Modify metadata in an mtree file. | :------------- | :------------- | :------------- | | name | name of the target, output will be `[name].mtree`. | none | | mtree | input mtree file, typically created by `mtree_spec`. | none | +| srcs | list of files to resolve symlinks for. | `None` | +| preserve_symlinks | `EXPERIMENTAL!` We may remove or change it at any point without further notice. Flag to determine whether to preserve symlinks in the tar. | `False` | | strip_prefix | prefix to remove from all paths in the tar. Files and directories not under this prefix are dropped. | `None` | | package_dir | directory prefix to add to all paths in the tar. | `None` | | mtime | new modification time for all entries. | `None` | diff --git a/lib/private/modify_mtree.awk b/lib/private/modify_mtree.awk index 6db995787..cbdd9b3be 100644 --- a/lib/private/modify_mtree.awk +++ b/lib/private/modify_mtree.awk @@ -68,16 +68,24 @@ symlink_map[path] = $1 # Resolve the symlink if it exists resolved_path = "" - cmd = "readlink -f " path + cmd = "readlink -f \"" path "\"" cmd | getline resolved_path close(cmd) + # If readlink -f fails use readlink for relative links + if (resolved_path == "") { + cmd = "readlink \"" path "\"" + cmd | getline resolved_path + close(cmd) + } + if (resolved_path) { - if (resolved_path ~ bin_dir) { + if (resolved_path ~ bin_dir || resolved_path ~ /\.\.\//) { # Strip down the resolved path to start from bin_dir sub("^.*" bin_dir, bin_dir, resolved_path) - if (path != resolved_path) { - # Replace the content field with the new path + # If the resolved path is different from the original path, + # or if it's a relative path + if (path != resolved_path || resolved_path ~ /\.\.\//) { symlink = resolved_path } } @@ -106,7 +114,12 @@ END { split(line, fields, SUBSEP) field0 = fields[1] resolved_path = fields[2] - linked_to = symlink_map[resolved_path] + if (resolved_path in symlink_map) { + linked_to = symlink_map[resolved_path] + } + else { + linked_to = resolved_path + } # Adjust the line for symlink using the map we created new_line = field0 " type=link link=" linked_to print new_line diff --git a/lib/tar.bzl b/lib/tar.bzl index a97480d23..5e8ab5114 100644 --- a/lib/tar.bzl +++ b/lib/tar.bzl @@ -152,7 +152,7 @@ def mtree_mutate( name: name of the target, output will be `[name].mtree`. mtree: input mtree file, typically created by `mtree_spec`. srcs: list of files to resolve symlinks for. - preserve_symlinks: whether to preserve symlinks in the tar. + preserve_symlinks: `EXPERIMENTAL!` We may remove or change it at any point without further notice. Flag to determine whether to preserve symlinks in the tar. strip_prefix: prefix to remove from all paths in the tar. Files and directories not under this prefix are dropped. package_dir: directory prefix to add to all paths in the tar. mtime: new modification time for all entries. From f8247c1c6e1698b466a9f7e76533a6cc877a7d43 Mon Sep 17 00:00:00 2001 From: Elvis Wianda Date: Wed, 29 Jan 2025 01:36:18 -0500 Subject: [PATCH 6/9] Change from relative to absolute paths --- lib/private/modify_mtree.awk | 9 ++++++++- lib/tests/tar/BUILD.bazel | 10 +++++----- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/lib/private/modify_mtree.awk b/lib/private/modify_mtree.awk index cbdd9b3be..60a8d2c77 100644 --- a/lib/private/modify_mtree.awk +++ b/lib/private/modify_mtree.awk @@ -1,4 +1,9 @@ # Edits mtree files. See the modify_mtree macro in /lib/tar.bzl. +function make_relative_link(symlink, target) { + command = "realpath -s --relative-to=\"" symlink "\" \"" target "\"" + command | getline relative + return relative +} { if (strip_prefix != "") { if ($1 == strip_prefix) { @@ -115,9 +120,11 @@ END { field0 = fields[1] resolved_path = fields[2] if (resolved_path in symlink_map) { - linked_to = symlink_map[resolved_path] + mapped_link = symlink_map[resolved_path] + linked_to = make_relative_link(field0, mapped_link) } else { + # Already a relative path linked_to = resolved_path } # Adjust the line for symlink using the map we created diff --git a/lib/tests/tar/BUILD.bazel b/lib/tests/tar/BUILD.bazel index 0238468a9..7860ca935 100644 --- a/lib/tests/tar/BUILD.bazel +++ b/lib/tests/tar/BUILD.bazel @@ -585,7 +585,7 @@ assert_tar_listing( "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/", "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests", "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar", - "l--------- 0 0 0 0 Dec 31 1969 lib/tests/tar/native_binary_bin -> lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/executable.sh", + "l--------- 0 0 0 0 Dec 31 1969 lib/tests/tar/native_binary_bin -> ../native_binary_bin.runfiles/_main/lib/tests/tar/executable.sh", "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/native_binary_bin.runfiles", "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/native_binary_bin.runfiles/_main/", "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/native_binary_bin.runfiles/_main/lib", @@ -600,14 +600,14 @@ assert_tar_listing( "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/node_modules/.pnpm/b@0.0.0", "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/node_modules/.pnpm/b@0.0.0/node_modules", "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/node_modules/.pnpm/b@0.0.0/node_modules/a", - "l--------- 0 0 0 0 Dec 31 1969 lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/node_modules/.pnpm/b@0.0.0/node_modules/a/package.json -> lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/node_modules/.pnpm/a@0.0.0/node_modules/a/package.json", + "l--------- 0 0 0 0 Dec 31 1969 lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/node_modules/.pnpm/b@0.0.0/node_modules/a/package.json -> ../../../../a@0.0.0/node_modules/a/package.json", "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/node_modules/a", - "l--------- 0 0 0 0 Dec 31 1969 lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/node_modules/a/package.json -> lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/node_modules/.pnpm/a@0.0.0/node_modules/a/package.json", + "l--------- 0 0 0 0 Dec 31 1969 lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/node_modules/a/package.json -> ../../.pnpm/a@0.0.0/node_modules/a/package.json", "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/dir", "-rwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/dir/a", - "l--------- 0 0 0 0 Dec 31 1969 lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/dir/b -> lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/dir/a", + "l--------- 0 0 0 0 Dec 31 1969 lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/dir/b -> ../a", "-rwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/executable.sh", - "l--------- 0 0 0 0 Dec 31 1969 lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/native_binary_bin -> lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/executable.sh", + "l--------- 0 0 0 0 Dec 31 1969 lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/native_binary_bin -> ../executable.sh", "-rwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/native_binary_bin.runfiles/_repo_mapping", ], ) From 9c5a54465e37c7c5fd3846bb995f592da9c49e8c Mon Sep 17 00:00:00 2001 From: Elvis Wianda Date: Wed, 29 Jan 2025 09:41:40 -0500 Subject: [PATCH 7/9] resolve relative to readlink path --- lib/private/modify_mtree.awk | 2 +- lib/tests/tar/BUILD.bazel | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/private/modify_mtree.awk b/lib/private/modify_mtree.awk index 60a8d2c77..e7e1a5920 100644 --- a/lib/private/modify_mtree.awk +++ b/lib/private/modify_mtree.awk @@ -121,7 +121,7 @@ END { resolved_path = fields[2] if (resolved_path in symlink_map) { mapped_link = symlink_map[resolved_path] - linked_to = make_relative_link(field0, mapped_link) + linked_to = make_relative_link(resolved_path, mapped_link) } else { # Already a relative path diff --git a/lib/tests/tar/BUILD.bazel b/lib/tests/tar/BUILD.bazel index 7860ca935..4afb50c59 100644 --- a/lib/tests/tar/BUILD.bazel +++ b/lib/tests/tar/BUILD.bazel @@ -585,7 +585,7 @@ assert_tar_listing( "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/", "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests", "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar", - "l--------- 0 0 0 0 Dec 31 1969 lib/tests/tar/native_binary_bin -> ../native_binary_bin.runfiles/_main/lib/tests/tar/executable.sh", + "l--------- 0 0 0 0 Dec 31 1969 lib/tests/tar/native_binary_bin -> ../../../../../../../lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/executable.sh", "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/native_binary_bin.runfiles", "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/native_binary_bin.runfiles/_main/", "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/native_binary_bin.runfiles/_main/lib", @@ -600,14 +600,14 @@ assert_tar_listing( "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/node_modules/.pnpm/b@0.0.0", "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/node_modules/.pnpm/b@0.0.0/node_modules", "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/node_modules/.pnpm/b@0.0.0/node_modules/a", - "l--------- 0 0 0 0 Dec 31 1969 lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/node_modules/.pnpm/b@0.0.0/node_modules/a/package.json -> ../../../../a@0.0.0/node_modules/a/package.json", + "l--------- 0 0 0 0 Dec 31 1969 lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/node_modules/.pnpm/b@0.0.0/node_modules/a/package.json -> ../../../../../../../../../../../../lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/node_modules/.pnpm/a@0.0.0/node_modules/a/package.json", "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/node_modules/a", - "l--------- 0 0 0 0 Dec 31 1969 lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/node_modules/a/package.json -> ../../.pnpm/a@0.0.0/node_modules/a/package.json", + "l--------- 0 0 0 0 Dec 31 1969 lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/node_modules/a/package.json -> ../../../../../../../../../../../../lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/node_modules/.pnpm/a@0.0.0/node_modules/a/package.json", "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/dir", "-rwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/dir/a", - "l--------- 0 0 0 0 Dec 31 1969 lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/dir/b -> ../a", + "l--------- 0 0 0 0 Dec 31 1969 lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/dir/b -> ../../../../../../../../lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/dir/a", "-rwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/executable.sh", - "l--------- 0 0 0 0 Dec 31 1969 lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/native_binary_bin -> ../executable.sh", + "l--------- 0 0 0 0 Dec 31 1969 lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/native_binary_bin -> ../../../../../../../../lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/dir/a", "-rwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/native_binary_bin.runfiles/_repo_mapping", ], ) From 908484a3129b22aa216cf34cfb67f2792523143e Mon Sep 17 00:00:00 2001 From: Elvis Wianda Date: Thu, 30 Jan 2025 09:54:36 -0500 Subject: [PATCH 8/9] Drop use of realpath in favour of awk implementation --- lib/private/modify_mtree.awk | 50 ++++++++++++++++++++++++++++++++---- lib/tests/tar/BUILD.bazel | 10 ++++---- 2 files changed, 50 insertions(+), 10 deletions(-) diff --git a/lib/private/modify_mtree.awk b/lib/private/modify_mtree.awk index e7e1a5920..f1abbe1cb 100644 --- a/lib/private/modify_mtree.awk +++ b/lib/private/modify_mtree.awk @@ -1,9 +1,46 @@ # Edits mtree files. See the modify_mtree macro in /lib/tar.bzl. -function make_relative_link(symlink, target) { - command = "realpath -s --relative-to=\"" symlink "\" \"" target "\"" - command | getline relative - return relative +function common_sections(path1, path2, i, segments1, segments2, min_length, common_path) { + # Normalize paths (remove leading/trailing slashes) + gsub(/^\/|\/$/, "", path1) + gsub(/^\/|\/$/, "", path2) + + # Split paths into arrays + split(path1, segments1, "/") + split(path2, segments2, "/") + + # Determine the shortest path length + min_length = (length(segments1) < length(segments2)) ? length(segments1) : length(segments2) + + # Find common sections + common_path = "" + for (i = 1; i <= min_length; i++) { + if (segments1[i] != segments2[i]) { + break + } + common_path = (common_path == "" ? segments1[i] : common_path "/" segments1[i]) + } + + return common_path } +function make_relative_link(path1, path2, i, common, target, relative_path, back_steps) { + # Find the common path + common = common_sections(path1, path2) + + # Remove common prefix from both paths + target = substr(path1, length(common) + 2) # "+2" to remove trailing "/" + relative_path = substr(path2, length(common) + 2) + + # Count directories to go up from path2 + back_steps = "../" + split(relative_path, path2_segments, "/") + for (i = 1; i < length(path2_segments); i++) { + back_steps = back_steps "../" + } + + # Construct the relative symlink + return back_steps target +} + { if (strip_prefix != "") { if ($1 == strip_prefix) { @@ -64,6 +101,7 @@ function make_relative_link(symlink, target) { # See https://github.com/bazelbuild/rules_pkg/pull/609 symlink = "" + symlink_content = "" if ($0 ~ /type=file/ && $0 ~ /content=/) { match($0, /content=[^ ]+/) content_field = substr($0, RSTART, RLENGTH) @@ -92,6 +130,7 @@ function make_relative_link(symlink, target) { # or if it's a relative path if (path != resolved_path || resolved_path ~ /\.\.\//) { symlink = resolved_path + symlink_content = path } } } @@ -121,7 +160,8 @@ END { resolved_path = fields[2] if (resolved_path in symlink_map) { mapped_link = symlink_map[resolved_path] - linked_to = make_relative_link(resolved_path, mapped_link) + + linked_to = make_relative_link(mapped_link, field0) } else { # Already a relative path diff --git a/lib/tests/tar/BUILD.bazel b/lib/tests/tar/BUILD.bazel index 4afb50c59..7860ca935 100644 --- a/lib/tests/tar/BUILD.bazel +++ b/lib/tests/tar/BUILD.bazel @@ -585,7 +585,7 @@ assert_tar_listing( "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/", "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests", "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar", - "l--------- 0 0 0 0 Dec 31 1969 lib/tests/tar/native_binary_bin -> ../../../../../../../lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/executable.sh", + "l--------- 0 0 0 0 Dec 31 1969 lib/tests/tar/native_binary_bin -> ../native_binary_bin.runfiles/_main/lib/tests/tar/executable.sh", "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/native_binary_bin.runfiles", "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/native_binary_bin.runfiles/_main/", "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/native_binary_bin.runfiles/_main/lib", @@ -600,14 +600,14 @@ assert_tar_listing( "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/node_modules/.pnpm/b@0.0.0", "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/node_modules/.pnpm/b@0.0.0/node_modules", "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/node_modules/.pnpm/b@0.0.0/node_modules/a", - "l--------- 0 0 0 0 Dec 31 1969 lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/node_modules/.pnpm/b@0.0.0/node_modules/a/package.json -> ../../../../../../../../../../../../lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/node_modules/.pnpm/a@0.0.0/node_modules/a/package.json", + "l--------- 0 0 0 0 Dec 31 1969 lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/node_modules/.pnpm/b@0.0.0/node_modules/a/package.json -> ../../../../a@0.0.0/node_modules/a/package.json", "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/node_modules/a", - "l--------- 0 0 0 0 Dec 31 1969 lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/node_modules/a/package.json -> ../../../../../../../../../../../../lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/node_modules/.pnpm/a@0.0.0/node_modules/a/package.json", + "l--------- 0 0 0 0 Dec 31 1969 lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/node_modules/a/package.json -> ../../.pnpm/a@0.0.0/node_modules/a/package.json", "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/dir", "-rwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/dir/a", - "l--------- 0 0 0 0 Dec 31 1969 lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/dir/b -> ../../../../../../../../lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/dir/a", + "l--------- 0 0 0 0 Dec 31 1969 lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/dir/b -> ../a", "-rwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/executable.sh", - "l--------- 0 0 0 0 Dec 31 1969 lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/native_binary_bin -> ../../../../../../../../lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/dir/a", + "l--------- 0 0 0 0 Dec 31 1969 lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/native_binary_bin -> ../executable.sh", "-rwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/native_binary_bin.runfiles/_repo_mapping", ], ) From e80bdf0177621b49fa908f85d9878776ffa26d2d Mon Sep 17 00:00:00 2001 From: Elvis Wianda Date: Fri, 31 Jan 2025 12:48:58 -0500 Subject: [PATCH 9/9] Retain all fields and change on type=link and s/content/link/g --- lib/private/modify_mtree.awk | 17 ++++++++++++----- lib/tests/tar/BUILD.bazel | 10 +++++----- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/lib/private/modify_mtree.awk b/lib/private/modify_mtree.awk index f1abbe1cb..c442e11ad 100644 --- a/lib/private/modify_mtree.awk +++ b/lib/private/modify_mtree.awk @@ -23,6 +23,9 @@ function common_sections(path1, path2, i, segments1, segments2, min_length, comm return common_path } function make_relative_link(path1, path2, i, common, target, relative_path, back_steps) { + # A similar starlark implementation + # https://github.com/bazelbuild/bazel-skylib/blob/7209de9148e98dc20425cf83747613f23d40827b/lib/paths.bzl#L217 + # Find the common path common = common_sections(path1, path2) @@ -136,7 +139,8 @@ function make_relative_link(path1, path2, i, common, target, relative_path, back } } if (symlink != "") { - line_array[NR] = $1 SUBSEP resolved_path + # Store the original line with the resolved path + line_array[NR] = $0 SUBSEP $1 SUBSEP resolved_path } else { line_array[NR] = $0 # Store other lines too, with an empty path @@ -156,8 +160,9 @@ END { line = line_array[i] if (index(line, SUBSEP) > 0) { # Check if this path was a symlink split(line, fields, SUBSEP) - field0 = fields[1] - resolved_path = fields[2] + original_line = fields[1] + field0 = fields[2] + resolved_path = fields[3] if (resolved_path in symlink_map) { mapped_link = symlink_map[resolved_path] @@ -168,8 +173,10 @@ END { linked_to = resolved_path } # Adjust the line for symlink using the map we created - new_line = field0 " type=link link=" linked_to - print new_line + sub(/type=[^ ]+/, "type=link", original_line) + sub(/content=[^ ]+/, "link=" linked_to, original_line) + print original_line + } else { # Print the original line if no symlink adjustment was needed print line diff --git a/lib/tests/tar/BUILD.bazel b/lib/tests/tar/BUILD.bazel index 7860ca935..92eacfea5 100644 --- a/lib/tests/tar/BUILD.bazel +++ b/lib/tests/tar/BUILD.bazel @@ -585,7 +585,7 @@ assert_tar_listing( "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/", "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests", "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar", - "l--------- 0 0 0 0 Dec 31 1969 lib/tests/tar/native_binary_bin -> ../native_binary_bin.runfiles/_main/lib/tests/tar/executable.sh", + "lrwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/native_binary_bin -> ../native_binary_bin.runfiles/_main/lib/tests/tar/executable.sh", "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/native_binary_bin.runfiles", "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/native_binary_bin.runfiles/_main/", "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/native_binary_bin.runfiles/_main/lib", @@ -600,14 +600,14 @@ assert_tar_listing( "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/node_modules/.pnpm/b@0.0.0", "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/node_modules/.pnpm/b@0.0.0/node_modules", "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/node_modules/.pnpm/b@0.0.0/node_modules/a", - "l--------- 0 0 0 0 Dec 31 1969 lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/node_modules/.pnpm/b@0.0.0/node_modules/a/package.json -> ../../../../a@0.0.0/node_modules/a/package.json", + "lrwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/node_modules/.pnpm/b@0.0.0/node_modules/a/package.json -> ../../../../a@0.0.0/node_modules/a/package.json", "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/node_modules/a", - "l--------- 0 0 0 0 Dec 31 1969 lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/node_modules/a/package.json -> ../../.pnpm/a@0.0.0/node_modules/a/package.json", + "lrwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/node_modules/a/package.json -> ../../.pnpm/a@0.0.0/node_modules/a/package.json", "drwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/dir", "-rwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/dir/a", - "l--------- 0 0 0 0 Dec 31 1969 lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/dir/b -> ../a", + "lrwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/dir/b -> ../a", "-rwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/executable.sh", - "l--------- 0 0 0 0 Dec 31 1969 lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/native_binary_bin -> ../executable.sh", + "lrwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/native_binary_bin.runfiles/_main/lib/tests/tar/native_binary_bin -> ../executable.sh", "-rwxr-xr-x 0 0 0 0 Jan 1 2023 lib/tests/tar/native_binary_bin.runfiles/_repo_mapping", ], )