Put libraries into $store/lib

cabal-install/Distribution/Client/PackageHash.hs

@@ -33,7 +33,7 @@ import Distribution.Package
          ( PackageId, PackageIdentifier(..), mkComponentId
          , PkgconfigName )
 import Distribution.System
-         ( Platform, OS(Windows), buildOS )
+         ( Platform, OS(Windows, OSX), buildOS )
 import Distribution.PackageDescription
          ( FlagAssignment, showFlagValue )
 import Distribution.Simple.Compiler
@@ -82,6 +82,7 @@ import System.IO         (withBinaryFile, IOMode(..))
 hashedInstalledPackageId :: PackageHashInputs -> InstalledPackageId
 hashedInstalledPackageId
   | buildOS == Windows = hashedInstalledPackageIdShort
+  | buildOS == OSX     = hashedInstalledPackageIdVeryShort
   | otherwise          = hashedInstalledPackageIdLong
 
 -- | Calculate a 'InstalledPackageId' for a package using our nix-style
@@ -135,6 +136,41 @@ hashedInstalledPackageIdShort pkghashinputs@PackageHashInputs{pkgHashPkgId} =
     truncateStr n s | length s <= n = s
                     | otherwise     = take (n-1) s ++ "_"
 
+-- | On macOS we shorten the name very aggressively.  The mach-o linker on
+-- macOS has a limited load command size, to which the name of the lirbary
+-- as well as it's relative path (\@rpath) entry count.  To circumvent this,
+-- on macOS the libraries are not stored as
+--  @store/<libraryname>/libHS<libraryname>.dylib@
+-- where libraryname contains the librarys name, version and abi hash, but in
+--  @store/lib/libHS<very short libraryname>.dylib@
+-- where the very short library name drops all vowels from the package name,
+-- and truncates the hash to 4 bytes.
+--
+-- We therefore only need one \@rpath entry to @store/lib@ instead of one
+-- \@rpath entry for each library. And the reduced library name saves some
+-- additional space.
+--
+-- This however has two major drawbacks:
+-- 1) Packages can easier collide due to the shortened hash.
+-- 2) The lirbaries are *not* prefix relocateable anymore as they all end up
+--    in the same @store/lib@ folder.
+--
+-- The ultimate solution would have to include generating proxy dynamic
+-- libraries on macOS, such that the proxy libraries and the linked libraries
+-- stay under the load command limit, and the recursive linker is still able
+-- to link all of them.
+hashedInstalledPackageIdVeryShort :: PackageHashInputs -> InstalledPackageId
+hashedInstalledPackageIdVeryShort pkghashinputs@PackageHashInputs{pkgHashPkgId} =
+  mkComponentId $
+    intercalate "-"
+      [ filter (not . flip elem "aeiou") (display name)
+      , display version
+      , showHashValue (truncateHash (hashPackageHashInputs pkghashinputs))
+      ]
+  where
+    PackageIdentifier name version = pkgHashPkgId
+    truncateHash (HashValue h) = HashValue (BS.take 4 h)
+
 -- | All the information that contribues to a package's hash, and thus its
 -- 'InstalledPackageId'.
 --

cabal-install/Distribution/Client/ProjectBuilding.hs

@@ -35,6 +35,10 @@ module Distribution.Client.ProjectBuilding (
     BuildFailureReason(..),
   ) where
 
+#if !MIN_VERSION_base(4,8,0)
+import Control.Applicative ((<$>))
+#endif
+
 import           Distribution.Client.PackageHash (renderPackageHashInputs)
 import           Distribution.Client.RebuildMonad
 import           Distribution.Client.ProjectConfig
@@ -88,6 +92,7 @@ import qualified Data.Map as Map
 import           Data.Set (Set)
 import qualified Data.Set as Set
 import qualified Data.ByteString.Lazy as LBS
+import           Data.List (isPrefixOf)
 
 import           Control.Monad
 import           Control.Exception
@@ -97,6 +102,12 @@ import           System.FilePath
 import           System.IO
 import           System.Directory
 
+#if !MIN_VERSION_directory(1,2,5)
+listDirectory :: FilePath -> IO [FilePath]
+listDirectory path =
+  (filter f) <$> (getDirectoryContents path)
+  where f filename = filename /= "." && filename /= ".."
+#endif
 
 ------------------------------------------------------------------------------
 -- * Overall building strategy.
@@ -938,9 +949,25 @@ buildAndInstallUnpackedPackage verbosity
             LBS.writeFile
               (entryDir </> "cabal-hash.txt")
               (renderPackageHashInputs (packageHashInputs pkgshared pkg))
+
+            -- Ensure that there are no files in `tmpDir`, that are not in `entryDir`
+            -- While this breaks the prefix-relocatable property of the lirbaries
+            -- it is necessary on macOS to stay under the load command limit of the
+            -- macOS mach-o linker. See also @PackageHash.hashedInstalledPackageIdVeryShort@.
+            otherFiles <- filter (not . isPrefixOf entryDir) <$> listFilesRecursive tmpDir 
             -- here's where we could keep track of the installed files ourselves
             -- if we wanted to by making a manifest of the files in the tmp dir
-            return entryDir
+            return (entryDir, otherFiles)
+            where
+              listFilesRecursive :: FilePath -> IO [FilePath]
+              listFilesRecursive path = do
+                files <- fmap (path </>) <$> (listDirectory path)
+                allFiles <- forM files $ \file -> do
+                  isDir <- doesDirectoryExist file
+                  if isDir
+                    then listFilesRecursive file
+                    else return [file]
+                return (concat allFiles)
 
           registerPkg
             | not (elabRequiresRegistration pkg) =

cabal-install/Distribution/Client/ProjectPlanning.hs

@@ -2934,15 +2934,22 @@ storePackageInstallDirs :: StoreDirLayout
                         -> CompilerId
                         -> InstalledPackageId
                         -> InstallDirs.InstallDirs FilePath
-storePackageInstallDirs StoreDirLayout{storePackageDirectory}
+storePackageInstallDirs StoreDirLayout{ storePackageDirectory
+                                      , storeDirectory }
                         compid ipkgid =
     InstallDirs.InstallDirs {..}
   where
+    store        = storeDirectory compid
     prefix       = storePackageDirectory compid (newSimpleUnitId ipkgid)
     bindir       = prefix </> "bin"
     libdir       = prefix </> "lib"
     libsubdir    = ""
-    dynlibdir    = libdir
+    -- Note: on macOS, we place libraries into
+    --       @store/lib@ to work around the load
+    --       command size limit of macOSs mach-o linker.
+    --       See also @PackageHash.hashedInstalledPackageIdVeryShort@
+    dynlibdir    | buildOS == OSX = store </> "lib"
+                 | otherwise      = libdir
     flibdir      = libdir
     libexecdir   = prefix </> "libexec"
     libexecsubdir= ""

cabal-install/Distribution/Client/Store.hs

@@ -39,6 +39,7 @@ import           Distribution.Text
 import           Data.Set (Set)
 import qualified Data.Set as Set
 import           Control.Exception
+import           Control.Monad (forM_)
 import           System.FilePath
 import           System.Directory
 import           System.IO
@@ -171,7 +172,7 @@ newStoreEntry :: Verbosity
               -> StoreDirLayout
               -> CompilerId
               -> UnitId
-              -> (FilePath -> IO FilePath) -- ^ Action to place files.
+              -> (FilePath -> IO (FilePath, [FilePath])) -- ^ Action to place files.
               -> IO ()                     -- ^ Register action, if necessary.
               -> IO NewStoreEntryOutcome
 newStoreEntry verbosity storeDirLayout@StoreDirLayout{..}
@@ -182,7 +183,7 @@ newStoreEntry verbosity storeDirLayout@StoreDirLayout{..}
     withTempIncomingDir storeDirLayout compid $ \incomingTmpDir -> do
 
       -- Write all store entry files within the temp dir and return the prefix.
-      incomingEntryDir <- copyFiles incomingTmpDir
+      (incomingEntryDir, otherFiles) <- copyFiles incomingTmpDir
 
       -- Take a lock named after the 'UnitId' in question.
       withIncomingUnitIdLock verbosity storeDirLayout compid unitid $ do
@@ -207,6 +208,10 @@ newStoreEntry verbosity storeDirLayout@StoreDirLayout{..}
 
             -- Atomically rename the temp dir to the final store entry location.
             renameDirectory incomingEntryDir finalEntryDir
+            forM_ otherFiles $ \file -> do
+              let finalStoreFile = storeDirectory compid </> makeRelative (incomingTmpDir </> (dropDrive (storeDirectory compid))) file
+              createDirectoryIfMissing True (takeDirectory finalStoreFile)
+              renameFile file finalStoreFile
 
             debug verbosity $
               "Installed store entry " ++ display compid </> display unitid

cabal-install/changelog

@@ -17,6 +17,9 @@
 	dependencies (#4575, #4669).
 	* `--store-dir` option can be used to configure the location of
 	the build global build store.
+	* On macOS, `new-build` will place dynamic libraries into
+	`store/lib` and aggressively shorten their names in an effort to
+	stay within the load command size limits of macOSs mach-o linker.
 
 2.0.TBD
 	* Turn `allow-{newer,older}` in `cabal.project` files into an

cabal-install/tests/UnitTests/Distribution/Client/Store.hs

@@ -51,7 +51,7 @@ testInstallSerial =
           let destprefix = dir </> "prefix"
           createDirectory destprefix
           writeFile (destprefix </> file) content
-          return destprefix
+          return (destprefix,[])
 
     assertNewStoreEntry tmp storeDirLayout compid unitid1
                         (copyFiles "file1" "content-foo") (return ())
@@ -137,7 +137,7 @@ testInstallParallel =
 
 assertNewStoreEntry :: FilePath -> StoreDirLayout
                     -> CompilerId -> UnitId
-                    -> (FilePath -> IO FilePath) -> IO ()
+                    -> (FilePath -> IO (FilePath,[FilePath])) -> IO ()
                     -> NewStoreEntryOutcome
                     -> Assertion
 assertNewStoreEntry tmp storeDirLayout compid unitid