compiler: resolve library paths in the frontend

search_strategy is no longer passed to Compilation at all; instead it is
used in the CLI code only.

When using Zig CLI mode, `-l` no longer has the ability to link
statically; use positional arguments for this.

The CLI has a small abstraction around library resolution handling which
is used to remove some code duplication regarding static libraries, as
well as handle the difference between zig cc CLI mode and zig CLI mode.

Thanks to this, system libraries are now included in the cache hash, and
thus changes to them will correctly cause cache misses.

In the future, lib_dirs should no longer be passed to Compilation at
all, because it is a frontend-only concept.

Previously, -search_paths_first and -search_dylibs_first were
Darwin-only arguments; they now work the same for all targets. Same
thing with --sysroot.

Improved the error reporting for failure to find a system library. An
example error now looks like this:

```
$ zig build-exe test.zig -lfoo -L. -L/a -target x86_64-macos --sysroot /home/andy/local
error: unable to find Dynamic system library 'foo' using strategy 'no_fallback'. search paths:
  ./libfoo.tbd
  ./libfoo.dylib
  ./libfoo.so
  /home/andy/local/a/libfoo.tbd
  /home/andy/local/a/libfoo.dylib
  /home/andy/local/a/libfoo.so
  /a/libfoo.tbd
  /a/libfoo.dylib
  /a/libfoo.so
```

closes #14963

Author: andrewrk

src/Compilation.zig

@@ -448,6 +448,7 @@ pub const ClangPreprocessorMode = enum {
     stdout,
 };
 
+pub const Framework = link.Framework;
 pub const SystemLib = link.SystemLib;
 pub const CacheMode = link.CacheMode;
 
@@ -505,7 +506,7 @@ pub const InitOptions = struct {
     c_source_files: []const CSourceFile = &[0]CSourceFile{},
     link_objects: []LinkObject = &[0]LinkObject{},
     framework_dirs: []const []const u8 = &[0][]const u8{},
-    frameworks: std.StringArrayHashMapUnmanaged(SystemLib) = .{},
+    frameworks: std.StringArrayHashMapUnmanaged(Framework) = .{},
     system_lib_names: []const []const u8 = &.{},
     system_lib_infos: []const SystemLib = &.{},
     /// These correspond to the WASI libc emulated subcomponents including:
@@ -644,8 +645,6 @@ pub const InitOptions = struct {
     entitlements: ?[]const u8 = null,
     /// (Darwin) size of the __PAGEZERO segment
     pagezero_size: ?u64 = null,
-    /// (Darwin) search strategy for system libraries
-    search_strategy: ?link.File.MachO.SearchStrategy = null,
     /// (Darwin) set minimum space for future expansion of the load commands
     headerpad_size: ?u32 = null,
     /// (Darwin) set enough space as if all paths were MATPATHLEN
@@ -1567,7 +1566,6 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation {
             .install_name = options.install_name,
             .entitlements = options.entitlements,
             .pagezero_size = options.pagezero_size,
-            .search_strategy = options.search_strategy,
             .headerpad_size = options.headerpad_size,
             .headerpad_max_install_names = options.headerpad_max_install_names,
             .dead_strip_dylibs = options.dead_strip_dylibs,
@@ -1727,15 +1725,18 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation {
 
             // When linking mingw-w64 there are some import libs we always need.
             for (mingw.always_link_libs) |name| {
-                try comp.bin_file.options.system_libs.put(comp.gpa, name, .{});
+                try comp.bin_file.options.system_libs.put(comp.gpa, name, .{
+                    .needed = false,
+                    .weak = false,
+                    .path = name,
+                });
             }
         }
         // Generate Windows import libs.
         if (target.os.tag == .windows) {
             const count = comp.bin_file.options.system_libs.count();
             try comp.work_queue.ensureUnusedCapacity(count);
-            var i: usize = 0;
-            while (i < count) : (i += 1) {
+            for (0..count) |i| {
                 comp.work_queue.writeItemAssumeCapacity(.{ .windows_import_lib = i });
             }
         }
@@ -2377,7 +2378,7 @@ fn addNonIncrementalStuffToCacheManifest(comp: *Compilation, man: *Cache.Manifes
     }
     man.hash.addOptionalBytes(comp.bin_file.options.soname);
     man.hash.addOptional(comp.bin_file.options.version);
-    link.hashAddSystemLibs(&man.hash, comp.bin_file.options.system_libs);
+    try link.hashAddSystemLibs(man, comp.bin_file.options.system_libs);
     man.hash.addListOfBytes(comp.bin_file.options.force_undefined_symbols.keys());
     man.hash.addOptional(comp.bin_file.options.allow_shlib_undefined);
     man.hash.add(comp.bin_file.options.bind_global_refs_locally);
@@ -2395,10 +2396,9 @@ fn addNonIncrementalStuffToCacheManifest(comp: *Compilation, man: *Cache.Manifes
 
     // Mach-O specific stuff
     man.hash.addListOfBytes(comp.bin_file.options.framework_dirs);
-    link.hashAddSystemLibs(&man.hash, comp.bin_file.options.frameworks);
+    link.hashAddFrameworks(&man.hash, comp.bin_file.options.frameworks);
     try man.addOptionalFile(comp.bin_file.options.entitlements);
     man.hash.addOptional(comp.bin_file.options.pagezero_size);
-    man.hash.addOptional(comp.bin_file.options.search_strategy);
     man.hash.addOptional(comp.bin_file.options.headerpad_size);
     man.hash.add(comp.bin_file.options.headerpad_max_install_names);
     man.hash.add(comp.bin_file.options.dead_strip_dylibs);

src/link.zig

@@ -21,7 +21,16 @@ const Type = @import("type.zig").Type;
 const TypedValue = @import("TypedValue.zig");
 
 /// When adding a new field, remember to update `hashAddSystemLibs`.
+/// These are *always* dynamically linked. Static libraries will be
+/// provided as positional arguments.
 pub const SystemLib = struct {
+    needed: bool,
+    weak: bool,
+    path: []const u8,
+};
+
+/// When adding a new field, remember to update `hashAddFrameworks`.
+pub const Framework = struct {
     needed: bool = false,
     weak: bool = false,
 };
@@ -31,11 +40,23 @@ pub const SortSection = enum { name, alignment };
 pub const CacheMode = enum { incremental, whole };
 
 pub fn hashAddSystemLibs(
-    hh: *Cache.HashHelper,
+    man: *Cache.Manifest,
     hm: std.StringArrayHashMapUnmanaged(SystemLib),
+) !void {
+    const keys = hm.keys();
+    man.hash.addListOfBytes(keys);
+    for (hm.values()) |value| {
+        man.hash.add(value.needed);
+        man.hash.add(value.weak);
+        _ = try man.addFile(value.path, null);
+    }
+}
+
+pub fn hashAddFrameworks(
+    hh: *Cache.HashHelper,
+    hm: std.StringArrayHashMapUnmanaged(Framework),
 ) void {
     const keys = hm.keys();
-    hh.add(keys.len);
     hh.addListOfBytes(keys);
     for (hm.values()) |value| {
         hh.add(value.needed);
@@ -183,9 +204,12 @@ pub const Options = struct {
 
     objects: []Compilation.LinkObject,
     framework_dirs: []const []const u8,
-    frameworks: std.StringArrayHashMapUnmanaged(SystemLib),
+    frameworks: std.StringArrayHashMapUnmanaged(Framework),
+    /// These are *always* dynamically linked. Static libraries will be
+    /// provided as positional arguments.
     system_libs: std.StringArrayHashMapUnmanaged(SystemLib),
     wasi_emulated_libs: []const wasi_libc.CRTFile,
+    // TODO: remove this. libraries are resolved by the frontend.
     lib_dirs: []const []const u8,
     rpath_list: []const []const u8,
 
@@ -225,9 +249,6 @@ pub const Options = struct {
     /// (Darwin) size of the __PAGEZERO segment
     pagezero_size: ?u64 = null,
 
-    /// (Darwin) search strategy for system libraries
-    search_strategy: ?File.MachO.SearchStrategy = null,
-
     /// (Darwin) set minimum space for future expansion of the load commands
     headerpad_size: ?u32 = null,
 

src/link/Coff/lld.zig

@@ -88,7 +88,7 @@ pub fn linkWithLLD(self: *Coff, comp: *Compilation, prog_node: *std.Progress.Nod
                 }
             }
         }
-        link.hashAddSystemLibs(&man.hash, self.base.options.system_libs);
+        try link.hashAddSystemLibs(&man, self.base.options.system_libs);
         man.hash.addListOfBytes(self.base.options.force_undefined_symbols.keys());
         man.hash.addOptional(self.base.options.subsystem);
         man.hash.add(self.base.options.is_test);

src/link/Elf.zig

@@ -1428,7 +1428,7 @@ fn linkWithLLD(self: *Elf, comp: *Compilation, prog_node: *std.Progress.Node) !v
         }
         man.hash.addOptionalBytes(self.base.options.soname);
         man.hash.addOptional(self.base.options.version);
-        link.hashAddSystemLibs(&man.hash, self.base.options.system_libs);
+        try link.hashAddSystemLibs(&man, self.base.options.system_libs);
         man.hash.addListOfBytes(self.base.options.force_undefined_symbols.keys());
         man.hash.add(allow_shlib_undefined);
         man.hash.add(self.base.options.bind_global_refs_locally);
@@ -1824,8 +1824,8 @@ fn linkWithLLD(self: *Elf, comp: *Compilation, prog_node: *std.Progress.Node) !v
             argv.appendAssumeCapacity("--as-needed");
             var as_needed = true;
 
-            for (system_libs, 0..) |link_lib, i| {
-                const lib_as_needed = !system_libs_values[i].needed;
+            for (system_libs_values) |lib_info| {
+                const lib_as_needed = !lib_info.needed;
                 switch ((@as(u2, @intFromBool(lib_as_needed)) << 1) | @intFromBool(as_needed)) {
                     0b00, 0b11 => {},
                     0b01 => {
@@ -1842,9 +1842,7 @@ fn linkWithLLD(self: *Elf, comp: *Compilation, prog_node: *std.Progress.Node) !v
                 // libraries and not static libraries (the check for that needs to be earlier),
                 // but they could be full paths to .so files, in which case we
                 // want to avoid prepending "-l".
-                const ext = Compilation.classifyFileExt(link_lib);
-                const arg = if (ext == .shared_library) link_lib else try std.fmt.allocPrint(arena, "-l{s}", .{link_lib});
-                argv.appendAssumeCapacity(arg);
+                argv.appendAssumeCapacity(lib_info.path);
             }
 
             if (!as_needed) {

src/link/MachO.zig

@@ -58,11 +58,6 @@ const Rebase = @import("MachO/dyld_info/Rebase.zig");
 
 pub const base_tag: File.Tag = File.Tag.macho;
 
-pub const SearchStrategy = enum {
-    paths_first,
-    dylibs_first,
-};
-
 /// Mode of operation of the linker.
 pub const Mode = enum {
     /// Incremental mode will preallocate segments/sections and is compatible with
@@ -840,7 +835,11 @@ pub fn resolveLibSystem(
         // re-exports every single symbol definition.
         for (search_dirs) |dir| {
             if (try resolveLib(arena, dir, "System", ".tbd")) |full_path| {
-                try out_libs.put(full_path, .{ .needed = true });
+                try out_libs.put(full_path, .{
+                    .needed = true,
+                    .weak = false,
+                    .path = full_path,
+                });
                 libsystem_available = true;
                 break :blk;
             }
@@ -850,8 +849,16 @@ pub fn resolveLibSystem(
         for (search_dirs) |dir| {
             if (try resolveLib(arena, dir, "System", ".dylib")) |libsystem_path| {
                 if (try resolveLib(arena, dir, "c", ".dylib")) |libc_path| {
-                    try out_libs.put(libsystem_path, .{ .needed = true });
-                    try out_libs.put(libc_path, .{ .needed = true });
+                    try out_libs.put(libsystem_path, .{
+                        .needed = true,
+                        .weak = false,
+                        .path = libsystem_path,
+                    });
+                    try out_libs.put(libc_path, .{
+                        .needed = true,
+                        .weak = false,
+                        .path = libc_path,
+                    });
                     libsystem_available = true;
                     break :blk;
                 }
@@ -865,7 +872,11 @@ pub fn resolveLibSystem(
         const full_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{
             "libc", "darwin", libsystem_name,
         });
-        try out_libs.put(full_path, .{ .needed = true });
+        try out_libs.put(full_path, .{
+            .needed = true,
+            .weak = false,
+            .path = full_path,
+        });
     }
 }
 

src/link/MachO/zld.zig

@@ -3410,21 +3410,20 @@ pub fn linkWithZld(macho_file: *MachO, comp: *Compilation, prog_node: *std.Progr
         // installation sources because they are always a product of the compiler version + target information.
         man.hash.add(stack_size);
         man.hash.addOptional(options.pagezero_size);
-        man.hash.addOptional(options.search_strategy);
         man.hash.addOptional(options.headerpad_size);
         man.hash.add(options.headerpad_max_install_names);
         man.hash.add(gc_sections);
         man.hash.add(options.dead_strip_dylibs);
         man.hash.add(options.strip);
         man.hash.addListOfBytes(options.lib_dirs);
         man.hash.addListOfBytes(options.framework_dirs);
-        link.hashAddSystemLibs(&man.hash, options.frameworks);
+        link.hashAddFrameworks(&man.hash, options.frameworks);
         man.hash.addListOfBytes(options.rpath_list);
         if (is_dyn_lib) {
             man.hash.addOptionalBytes(options.install_name);
             man.hash.addOptional(options.version);
         }
-        link.hashAddSystemLibs(&man.hash, options.system_libs);
+        try link.hashAddSystemLibs(&man, options.system_libs);
         man.hash.addOptionalBytes(options.sysroot);
         man.hash.addListOfBytes(options.force_undefined_symbols.keys());
         try man.addOptionalFile(options.entitlements);
@@ -3550,84 +3549,20 @@ pub fn linkWithZld(macho_file: *MachO, comp: *Compilation, prog_node: *std.Progr
             try positionals.append(comp.libcxx_static_lib.?.full_object_path);
         }
 
-        // Shared and static libraries passed via `-l` flag.
-        var candidate_libs = std.StringArrayHashMap(link.SystemLib).init(arena);
-
-        const system_lib_names = options.system_libs.keys();
-        for (system_lib_names) |system_lib_name| {
-            // By this time, we depend on these libs being dynamically linked libraries and not static libraries
-            // (the check for that needs to be earlier), but they could be full paths to .dylib files, in which
-            // case we want to avoid prepending "-l".
-            if (Compilation.classifyFileExt(system_lib_name) == .shared_library) {
-                try positionals.append(system_lib_name);
-                continue;
-            }
-
-            const system_lib_info = options.system_libs.get(system_lib_name).?;
-            try candidate_libs.put(system_lib_name, .{
-                .needed = system_lib_info.needed,
-                .weak = system_lib_info.weak,
-            });
-        }
-
-        var lib_dirs = std.ArrayList([]const u8).init(arena);
-        for (options.lib_dirs) |dir| {
-            if (try MachO.resolveSearchDir(arena, dir, options.sysroot)) |search_dir| {
-                try lib_dirs.append(search_dir);
-            } else {
-                log.warn("directory not found for '-L{s}'", .{dir});
-            }
+        {
+            // Add all system library paths to positionals.
+            const vals = options.system_libs.values();
+            try positionals.ensureUnusedCapacity(vals.len);
+            for (vals) |info| positionals.appendAssumeCapacity(info.path);
         }
 
         var libs = std.StringArrayHashMap(link.SystemLib).init(arena);
 
-        // Assume ld64 default -search_paths_first if no strategy specified.
-        const search_strategy = options.search_strategy orelse .paths_first;
-        outer: for (candidate_libs.keys()) |lib_name| {
-            switch (search_strategy) {
-                .paths_first => {
-                    // Look in each directory for a dylib (stub first), and then for archive
-                    for (lib_dirs.items) |dir| {
-                        for (&[_][]const u8{ ".tbd", ".dylib", ".a" }) |ext| {
-                            if (try MachO.resolveLib(arena, dir, lib_name, ext)) |full_path| {
-                                try libs.put(full_path, candidate_libs.get(lib_name).?);
-                                continue :outer;
-                            }
-                        }
-                    } else {
-                        log.warn("library not found for '-l{s}'", .{lib_name});
-                        lib_not_found = true;
-                    }
-                },
-                .dylibs_first => {
-                    // First, look for a dylib in each search dir
-                    for (lib_dirs.items) |dir| {
-                        for (&[_][]const u8{ ".tbd", ".dylib" }) |ext| {
-                            if (try MachO.resolveLib(arena, dir, lib_name, ext)) |full_path| {
-                                try libs.put(full_path, candidate_libs.get(lib_name).?);
-                                continue :outer;
-                            }
-                        }
-                    } else for (lib_dirs.items) |dir| {
-                        if (try MachO.resolveLib(arena, dir, lib_name, ".a")) |full_path| {
-                            try libs.put(full_path, candidate_libs.get(lib_name).?);
-                        } else {
-                            log.warn("library not found for '-l{s}'", .{lib_name});
-                            lib_not_found = true;
-                        }
-                    }
-                },
-            }
-        }
-
-        if (lib_not_found) {
-            log.warn("Library search paths:", .{});
-            for (lib_dirs.items) |dir| {
-                log.warn("  {s}", .{dir});
-            }
+        for (options.system_libs.values()) |v| {
+            try libs.put(v.path, v);
         }
 
-        try MachO.resolveLibSystem(arena, comp, options.sysroot, target, lib_dirs.items, &libs);
+        try MachO.resolveLibSystem(arena, comp, options.sysroot, target, options.lib_dirs, &libs);
 
         // frameworks
         var framework_dirs = std.ArrayList([]const u8).init(arena);
@@ -3647,6 +3582,7 @@ pub fn linkWithZld(macho_file: *MachO, comp: *Compilation, prog_node: *std.Progr
                         try libs.put(full_path, .{
                             .needed = info.needed,
                             .weak = info.weak,
+                            .path = full_path,
                         });
                         continue :outer;
                     }
@@ -3698,11 +3634,6 @@ pub fn linkWithZld(macho_file: *MachO, comp: *Compilation, prog_node: *std.Progr
                 try argv.append(try std.fmt.allocPrint(arena, "0x{x}", .{pagezero_size}));
             }
 
-            if (options.search_strategy) |strat| switch (strat) {
-                .paths_first => try argv.append("-search_paths_first"),
-                .dylibs_first => try argv.append("-search_dylibs_first"),
-            };
-
             if (options.headerpad_size) |headerpad_size| {
                 try argv.append("-headerpad_size");
                 try argv.append(try std.fmt.allocPrint(arena, "0x{x}", .{headerpad_size}));