diff --git a/.gitignore b/.gitignore index bca7f6fdbfe..3d695480cd0 100644 --- a/.gitignore +++ b/.gitignore @@ -28,3 +28,6 @@ venv /docsgen/arduino-cli.exe /docs/rpc/*.md /docs/commands/*.md + +# Delve debugger binary file +__debug_bin diff --git a/arduino/sketch/sketch.go b/arduino/sketch/sketch.go index 5de5ffda9be..9bc725a022f 100644 --- a/arduino/sketch/sketch.go +++ b/arduino/sketch/sketch.go @@ -294,7 +294,7 @@ func CheckForPdeFiles(sketch *paths.Path) []*paths.Path { // DefaultBuildPath generates the default build directory for a given sketch. // The build path is in a temporary directory and is unique for each sketch. func (s *Sketch) DefaultBuildPath() *paths.Path { - return paths.TempDir().Join("arduino", "sketch-"+s.Hash()) + return paths.TempDir().Join("arduino", "sketches", s.Hash()) } // Hash generate a unique hash for the given sketch. diff --git a/arduino/sketch/sketch_test.go b/arduino/sketch/sketch_test.go index dabace257a9..f6de4a11261 100644 --- a/arduino/sketch/sketch_test.go +++ b/arduino/sketch/sketch_test.go @@ -286,7 +286,7 @@ func TestNewSketchFolderSymlink(t *testing.T) { } func TestGenBuildPath(t *testing.T) { - want := paths.TempDir().Join("arduino", "sketch-ACBD18DB4CC2F85CEDEF654FCCC4A4D8") + want := paths.TempDir().Join("arduino", "sketches", "ACBD18DB4CC2F85CEDEF654FCCC4A4D8") assert.True(t, (&Sketch{FullPath: paths.New("foo")}).DefaultBuildPath().EquivalentTo(want)) assert.Equal(t, "ACBD18DB4CC2F85CEDEF654FCCC4A4D8", (&Sketch{FullPath: paths.New("foo")}).Hash()) } diff --git a/buildcache/build_cache.go b/buildcache/build_cache.go new file mode 100644 index 00000000000..dc690e2cbc2 --- /dev/null +++ b/buildcache/build_cache.go @@ -0,0 +1,113 @@ +// This file is part of arduino-cli. +// +// Copyright 2020 ARDUINO SA (http://www.arduino.cc/) +// +// This software is released under the GNU General Public License version 3, +// which covers the main part of arduino-cli. +// The terms of this license can be found at: +// https://www.gnu.org/licenses/gpl-3.0.en.html +// +// You can be released from the requirements of the above licenses by purchasing +// a commercial license. Buying such a license is mandatory if you want to +// modify or otherwise use the software for commercial activities involving the +// Arduino software without disclosing the source code of your own applications. +// To purchase a commercial license, send an email to license@arduino.cc. + +package buildcache + +import ( + "time" + + "github.com/arduino/go-paths-helper" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" +) + +const ( + createDirErrCode = 1 + fileWriteErrCode = 2 +) + +type cacheError struct { + Code int + wrappedErr error +} + +func (e cacheError) Error() string { + return e.wrappedErr.Error() +} + +func (e cacheError) Unwrap() error { + return e.wrappedErr +} + +func (e cacheError) Is(target error) bool { + te, ok := target.(cacheError) + return ok && te.Code == e.Code +} + +var ( + // CreateDirErr error occurred when creating the cache directory + CreateDirErr = cacheError{Code: createDirErrCode} + // FileWriteErr error occurred when writing the placeholder file + FileWriteErr = cacheError{Code: fileWriteErrCode} +) + +const lastUsedFileName = ".last-used" + +// BuildCache represents a cache of built files (sketches and cores), it's designed +// to work on directories. Given a directory as "base" it handles direct subdirectories as +// keys +type BuildCache struct { + baseDir *paths.Path +} + +// GetOrCreate retrieves or creates the cache directory at the given path +// If the cache already exists the lifetime of the cache is extended. +func (bc *BuildCache) GetOrCreate(key string) (*paths.Path, error) { + keyDir := bc.baseDir.Join(key) + if err := keyDir.MkdirAll(); err != nil { + return nil, cacheError{createDirErrCode, err} + } + + if err := keyDir.Join(lastUsedFileName).WriteFile([]byte{}); err != nil { + return nil, cacheError{fileWriteErrCode, err} + } + return keyDir, nil +} + +// Purge removes all cache directories within baseDir that have expired +// To know how long ago a directory has been last used +// it checks into the .last-used file. +func (bc *BuildCache) Purge(ttl time.Duration) { + files, err := bc.baseDir.ReadDir() + if err != nil { + return + } + for _, file := range files { + if file.IsDir() { + removeIfExpired(file, ttl) + } + } +} + +// New instantiates a build cache +func New(baseDir *paths.Path) *BuildCache { + return &BuildCache{baseDir} +} + +func removeIfExpired(dir *paths.Path, ttl time.Duration) { + fileInfo, err := dir.Join(lastUsedFileName).Stat() + if err != nil { + return + } + lifeExpectancy := ttl - time.Since(fileInfo.ModTime()) + if lifeExpectancy > 0 { + return + } + logrus.Tracef(`Purging cache directory "%s". Expired by %s`, dir, lifeExpectancy.Abs()) + err = dir.RemoveAll() + if err != nil { + logrus.Tracef(`Error while pruning cache directory "%s": %s`, dir, errors.WithStack(err)) + } +} diff --git a/buildcache/build_cache_test.go b/buildcache/build_cache_test.go new file mode 100644 index 00000000000..9e79f927a22 --- /dev/null +++ b/buildcache/build_cache_test.go @@ -0,0 +1,78 @@ +// This file is part of arduino-cli. +// +// Copyright 2020 ARDUINO SA (http://www.arduino.cc/) +// +// This software is released under the GNU General Public License version 3, +// which covers the main part of arduino-cli. +// The terms of this license can be found at: +// https://www.gnu.org/licenses/gpl-3.0.en.html +// +// You can be released from the requirements of the above licenses by purchasing +// a commercial license. Buying such a license is mandatory if you want to +// modify or otherwise use the software for commercial activities involving the +// Arduino software without disclosing the source code of your own applications. +// To purchase a commercial license, send an email to license@arduino.cc. + +package buildcache + +import ( + "testing" + "time" + + "github.com/arduino/go-paths-helper" + "github.com/stretchr/testify/require" +) + +func Test_UpdateLastUsedFileNotExisting(t *testing.T) { + testBuildDir := paths.New(t.TempDir(), "sketches", "xxx") + require.NoError(t, testBuildDir.MkdirAll()) + timeBeforeUpdating := time.Unix(0, 0) + requireCorrectUpdate(t, testBuildDir, timeBeforeUpdating) +} + +func Test_UpdateLastUsedFileExisting(t *testing.T) { + testBuildDir := paths.New(t.TempDir(), "sketches", "xxx") + require.NoError(t, testBuildDir.MkdirAll()) + + // create the file + preExistingFile := testBuildDir.Join(lastUsedFileName) + require.NoError(t, preExistingFile.WriteFile([]byte{})) + timeBeforeUpdating := time.Now().Add(-time.Second) + preExistingFile.Chtimes(time.Now(), timeBeforeUpdating) + requireCorrectUpdate(t, testBuildDir, timeBeforeUpdating) +} + +func requireCorrectUpdate(t *testing.T, dir *paths.Path, prevModTime time.Time) { + _, err := New(dir.Parent()).GetOrCreate(dir.Base()) + require.NoError(t, err) + expectedFile := dir.Join(lastUsedFileName) + fileInfo, err := expectedFile.Stat() + require.Nil(t, err) + require.Greater(t, fileInfo.ModTime(), prevModTime) +} + +func TestPurge(t *testing.T) { + ttl := time.Minute + + dirToPurge := paths.New(t.TempDir(), "root") + + lastUsedTimesByDirPath := map[*paths.Path]time.Time{ + (dirToPurge.Join("old")): time.Now().Add(-ttl - time.Hour), + (dirToPurge.Join("fresh")): time.Now().Add(-ttl + time.Minute), + } + + // create the metadata files + for dirPath, lastUsedTime := range lastUsedTimesByDirPath { + require.NoError(t, dirPath.MkdirAll()) + infoFilePath := dirPath.Join(lastUsedFileName).Canonical() + require.NoError(t, infoFilePath.WriteFile([]byte{})) + // make sure access time does not matter + accesstime := time.Now() + require.NoError(t, infoFilePath.Chtimes(accesstime, lastUsedTime)) + } + + New(dirToPurge).Purge(ttl) + + require.False(t, dirToPurge.Join("old").Exist()) + require.True(t, dirToPurge.Join("fresh").Exist()) +} diff --git a/commands/compile/compile.go b/commands/compile/compile.go index 092b69c4238..664368e067c 100644 --- a/commands/compile/compile.go +++ b/commands/compile/compile.go @@ -27,9 +27,11 @@ import ( "github.com/arduino/arduino-cli/arduino/cores" "github.com/arduino/arduino-cli/arduino/cores/packagemanager" "github.com/arduino/arduino-cli/arduino/sketch" + "github.com/arduino/arduino-cli/buildcache" "github.com/arduino/arduino-cli/commands" "github.com/arduino/arduino-cli/configuration" "github.com/arduino/arduino-cli/i18n" + "github.com/arduino/arduino-cli/inventory" "github.com/arduino/arduino-cli/legacy/builder" "github.com/arduino/arduino-cli/legacy/builder/types" rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1" @@ -135,6 +137,11 @@ func Compile(ctx context.Context, req *rpc.CompileRequest, outStream, errStream if err = builderCtx.BuildPath.MkdirAll(); err != nil { return nil, &arduino.PermissionDeniedError{Message: tr("Cannot create build directory"), Cause: err} } + + buildcache.New(builderCtx.BuildPath.Parent()).GetOrCreate(builderCtx.BuildPath.Base()) + // cache is purged after compilation to not remove entries that might be required + defer maybePurgeBuildCache() + builderCtx.CompilationDatabase = bldr.NewCompilationDatabase( builderCtx.BuildPath.Join("compile_commands.json"), ) @@ -153,7 +160,7 @@ func Compile(ctx context.Context, req *rpc.CompileRequest, outStream, errStream builderCtx.CustomBuildProperties = append(req.GetBuildProperties(), securityKeysOverride...) if req.GetBuildCachePath() == "" { - builderCtx.CoreBuildCachePath = paths.TempDir().Join("arduino", "core-cache") + builderCtx.CoreBuildCachePath = paths.TempDir().Join("arduino", "cores") } else { buildCachePath, err := paths.New(req.GetBuildCachePath()).Abs() if err != nil { @@ -287,3 +294,24 @@ func Compile(ctx context.Context, req *rpc.CompileRequest, outStream, errStream return r, nil } + +// maybePurgeBuildCache runs the build files cache purge if the policy conditions are met. +func maybePurgeBuildCache() { + + compilationsBeforePurge := configuration.Settings.GetUint("build_cache.compilations_before_purge") + // 0 means never purge + if compilationsBeforePurge == 0 { + return + } + compilationSinceLastPurge := inventory.Store.GetUint("build_cache.compilation_count_since_last_purge") + compilationSinceLastPurge++ + inventory.Store.Set("build_cache.compilation_count_since_last_purge", compilationSinceLastPurge) + defer inventory.WriteStore() + if compilationsBeforePurge == 0 || compilationSinceLastPurge < compilationsBeforePurge { + return + } + inventory.Store.Set("build_cache.compilation_count_since_last_purge", 0) + cacheTTL := configuration.Settings.GetDuration("build_cache.ttl").Abs() + buildcache.New(paths.TempDir().Join("arduino", "cores")).Purge(cacheTTL) + buildcache.New(paths.TempDir().Join("arduino", "sketches")).Purge(cacheTTL) +} diff --git a/configuration/defaults.go b/configuration/defaults.go index 323eb65203b..be1a0088a62 100644 --- a/configuration/defaults.go +++ b/configuration/defaults.go @@ -18,6 +18,7 @@ package configuration import ( "path/filepath" "strings" + "time" "github.com/spf13/viper" ) @@ -41,6 +42,8 @@ func SetDefaults(settings *viper.Viper) { // Sketch compilation settings.SetDefault("sketch.always_export_binaries", false) + settings.SetDefault("build_cache.ttl", time.Hour*24*30) + settings.SetDefault("build_cache.compilations_before_purge", 10) // daemon settings settings.SetDefault("daemon.port", "50051") diff --git a/docs/configuration.md b/docs/configuration.md index 16cacfcc5a6..f60402bf212 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -33,6 +33,12 @@ to the sketch folder. This is the equivalent of using the [`--export-binaries`][arduino-cli compile options] flag. - `updater` - configuration options related to Arduino CLI updates - `enable_notification` - set to `false` to disable notifications of new Arduino CLI releases, defaults to `true` +- `build_cache` configuration options related to the compilation cache + - `compilations_before_purge` - interval, in number of compilations, at which the cache is purged, defaults to `10`. + When `0` the cache is never purged. + - `ttl` - cache expiration time of build folders. If the cache is hit by a compilation the corresponding build files + lifetime is renewed. The value format must be a valid input for + [time.ParseDuration()](https://pkg.go.dev/time#ParseDuration), defaults to `720h` (30 days). ## Configuration methods diff --git a/internal/integrationtest/arduino-cli.go b/internal/integrationtest/arduino-cli.go index 5750189bead..92bce9e85bc 100644 --- a/internal/integrationtest/arduino-cli.go +++ b/internal/integrationtest/arduino-cli.go @@ -112,6 +112,7 @@ func NewArduinoCliWithinEnvironment(env *Environment, config *ArduinoCLIConfig) "ARDUINO_DATA_DIR": cli.dataDir.String(), "ARDUINO_DOWNLOADS_DIR": cli.stagingDir.String(), "ARDUINO_SKETCHBOOK_DIR": cli.sketchbookDir.String(), + "ARDUINO_BUILD_CACHE_COMPILATIONS_BEFORE_PURGE": "0", } env.RegisterCleanUpCallback(cli.CleanUp) return cli diff --git a/internal/integrationtest/compile_1/compile_test.go b/internal/integrationtest/compile_1/compile_test.go index 13cb87d8ecf..998fe7c0483 100644 --- a/internal/integrationtest/compile_1/compile_test.go +++ b/internal/integrationtest/compile_1/compile_test.go @@ -20,8 +20,10 @@ import ( "encoding/hex" "encoding/json" "os" + "sort" "strings" "testing" + "time" "github.com/arduino/arduino-cli/internal/integrationtest" "github.com/arduino/go-paths-helper" @@ -47,6 +49,7 @@ func TestCompile(t *testing.T) { {"WithoutFqbn", compileWithoutFqbn}, {"ErrorMessage", compileErrorMessage}, {"WithSimpleSketch", compileWithSimpleSketch}, + {"WithCachePurgeNeeded", compileWithCachePurgeNeeded}, {"OutputFlagDefaultPath", compileOutputFlagDefaultPath}, {"WithSketchWithSymlinkSelfloop", compileWithSketchWithSymlinkSelfloop}, {"BlacklistedSketchname", compileBlacklistedSketchname}, @@ -112,6 +115,35 @@ func compileErrorMessage(t *testing.T, env *integrationtest.Environment, cli *in } func compileWithSimpleSketch(t *testing.T, env *integrationtest.Environment, cli *integrationtest.ArduinoCLI) { + compileWithSimpleSketchCustomEnv(t, env, cli, cli.GetDefaultEnv()) +} + +func compileWithCachePurgeNeeded(t *testing.T, env *integrationtest.Environment, cli *integrationtest.ArduinoCLI) { + // create directories that must be purged + baseDir := paths.TempDir().Join("arduino", "sketches") + + // purge case: last used file too old + oldDir1 := baseDir.Join("test_old_sketch_1") + require.NoError(t, oldDir1.MkdirAll()) + require.NoError(t, oldDir1.Join(".last-used").WriteFile([]byte{})) + require.NoError(t, oldDir1.Join(".last-used").Chtimes(time.Now(), time.Unix(0, 0))) + // no purge case: last used file not existing + missingFileDir := baseDir.Join("test_sketch_2") + require.NoError(t, missingFileDir.MkdirAll()) + + defer oldDir1.RemoveAll() + defer missingFileDir.RemoveAll() + + customEnv := cli.GetDefaultEnv() + customEnv["ARDUINO_BUILD_CACHE_COMPILATIONS_BEFORE_PURGE"] = "1" + compileWithSimpleSketchCustomEnv(t, env, cli, customEnv) + + // check that purge has been run + require.NoFileExists(t, oldDir1.String()) + require.DirExists(t, missingFileDir.String()) +} + +func compileWithSimpleSketchCustomEnv(t *testing.T, env *integrationtest.Environment, cli *integrationtest.ArduinoCLI, customEnv map[string]string) { sketchName := "CompileIntegrationTest" sketchPath := cli.SketchbookDir().Join(sketchName) defer sketchPath.RemoveAll() @@ -127,7 +159,7 @@ func compileWithSimpleSketch(t *testing.T, env *integrationtest.Environment, cli require.NoError(t, err) // Build sketch for arduino:avr:uno with json output - stdout, _, err = cli.Run("compile", "-b", fqbn, sketchPath.String(), "--format", "json") + stdout, _, err = cli.RunWithCustomEnv(customEnv, "compile", "-b", fqbn, sketchPath.String(), "--format", "json") require.NoError(t, err) // check is a valid json and contains requested data var compileOutput map[string]interface{} @@ -140,7 +172,7 @@ func compileWithSimpleSketch(t *testing.T, env *integrationtest.Environment, cli md5 := md5.Sum(([]byte(sketchPath.String()))) sketchPathMd5 := strings.ToUpper(hex.EncodeToString(md5[:])) require.NotEmpty(t, sketchPathMd5) - buildDir := paths.TempDir().Join("arduino", "sketch-"+sketchPathMd5) + buildDir := paths.TempDir().Join("arduino", "sketches", sketchPathMd5) require.FileExists(t, buildDir.Join(sketchName+".ino.eep").String()) require.FileExists(t, buildDir.Join(sketchName+".ino.elf").String()) require.FileExists(t, buildDir.Join(sketchName+".ino.hex").String()) @@ -374,7 +406,7 @@ func compileWithOutputDirFlag(t *testing.T, env *integrationtest.Environment, cl md5 := md5.Sum(([]byte(sketchPath.String()))) sketchPathMd5 := strings.ToUpper(hex.EncodeToString(md5[:])) require.NotEmpty(t, sketchPathMd5) - buildDir := paths.TempDir().Join("arduino", "sketch-"+sketchPathMd5) + buildDir := paths.TempDir().Join("arduino", "sketches", sketchPathMd5) require.FileExists(t, buildDir.Join(sketchName+".ino.eep").String()) require.FileExists(t, buildDir.Join(sketchName+".ino.elf").String()) require.FileExists(t, buildDir.Join(sketchName+".ino.hex").String()) @@ -441,7 +473,7 @@ func compileWithCustomBuildPath(t *testing.T, env *integrationtest.Environment, md5 := md5.Sum(([]byte(sketchPath.String()))) sketchPathMd5 := strings.ToUpper(hex.EncodeToString(md5[:])) require.NotEmpty(t, sketchPathMd5) - buildDir := paths.TempDir().Join("arduino", "sketch-"+sketchPathMd5) + buildDir := paths.TempDir().Join("arduino", "sketches", sketchPathMd5) require.NoFileExists(t, buildDir.Join(sketchName+".ino.eep").String()) require.NoFileExists(t, buildDir.Join(sketchName+".ino.elf").String()) require.NoFileExists(t, buildDir.Join(sketchName+".ino.hex").String()) @@ -975,7 +1007,7 @@ func compileWithInvalidBuildOptionJson(t *testing.T, env *integrationtest.Enviro md5 := md5.Sum(([]byte(sketchPath.String()))) sketchPathMd5 := strings.ToUpper(hex.EncodeToString(md5[:])) require.NotEmpty(t, sketchPathMd5) - buildDir := paths.TempDir().Join("arduino", "sketch-"+sketchPathMd5) + buildDir := paths.TempDir().Join("arduino", "sketches", sketchPathMd5) _, _, err = cli.Run("compile", "-b", fqbn, sketchPath.String(), "--verbose") require.NoError(t, err) @@ -1008,18 +1040,41 @@ func compileWithRelativeBuildPath(t *testing.T, env *integrationtest.Environment absoluteBuildPath := cli.SketchbookDir().Join("build_path") builtFiles, err := absoluteBuildPath.ReadDir() require.NoError(t, err) - require.Contains(t, builtFiles[8].String(), sketchName+".ino.eep") - require.Contains(t, builtFiles[9].String(), sketchName+".ino.elf") - require.Contains(t, builtFiles[10].String(), sketchName+".ino.hex") - require.Contains(t, builtFiles[11].String(), sketchName+".ino.with_bootloader.bin") - require.Contains(t, builtFiles[12].String(), sketchName+".ino.with_bootloader.hex") - require.Contains(t, builtFiles[0].String(), "build.options.json") - require.Contains(t, builtFiles[1].String(), "compile_commands.json") - require.Contains(t, builtFiles[2].String(), "core") - require.Contains(t, builtFiles[3].String(), "includes.cache") - require.Contains(t, builtFiles[4].String(), "libraries") - require.Contains(t, builtFiles[6].String(), "preproc") - require.Contains(t, builtFiles[7].String(), "sketch") + + expectedFiles := []string{ + sketchName + ".ino.eep", + sketchName + ".ino.elf", + sketchName + ".ino.hex", + sketchName + ".ino.with_bootloader.bin", + sketchName + ".ino.with_bootloader.hex", + "build.options.json", + "compile_commands.json", + "core", + "includes.cache", + "libraries", + "preproc", + "sketch", + } + + foundFiles := []string{} + for _, builtFile := range builtFiles { + if sliceIncludes(expectedFiles, builtFile.Base()) { + foundFiles = append(foundFiles, builtFile.Base()) + } + } + sort.Strings(expectedFiles) + sort.Strings(foundFiles) + require.Equal(t, expectedFiles, foundFiles) +} + +// TODO: remove this when a generic library is introduced +func sliceIncludes[T comparable](slice []T, target T) bool { + for _, e := range slice { + if e == target { + return true + } + } + return false } func compileWithFakeSecureBootCore(t *testing.T, env *integrationtest.Environment, cli *integrationtest.ArduinoCLI) { diff --git a/internal/integrationtest/compile_2/compile_test.go b/internal/integrationtest/compile_2/compile_test.go index dc67aeaf2b0..f37f23eab19 100644 --- a/internal/integrationtest/compile_2/compile_test.go +++ b/internal/integrationtest/compile_2/compile_test.go @@ -145,7 +145,7 @@ func recompileWithDifferentLibrary(t *testing.T, env *integrationtest.Environmen md5 := md5.Sum(([]byte(sketchPath.String()))) sketchPathMd5 := strings.ToUpper(hex.EncodeToString(md5[:])) require.NotEmpty(t, sketchPathMd5) - buildDir := paths.TempDir().Join("arduino", "sketch-"+sketchPathMd5) + buildDir := paths.TempDir().Join("arduino", "sketches", sketchPathMd5) // Compile sketch using library not managed by CLI stdout, _, err := cli.Run("compile", "-b", fqbn, "--library", manuallyInstalledLibPath.String(), sketchPath.String(), "-v") diff --git a/internal/integrationtest/core/core_test.go b/internal/integrationtest/core/core_test.go index f8279580b18..94e47b374f9 100644 --- a/internal/integrationtest/core/core_test.go +++ b/internal/integrationtest/core/core_test.go @@ -253,7 +253,7 @@ func TestCoreInstallEsp32(t *testing.T) { md5 := md5.Sum(([]byte(sketchPath.String()))) sketchPathMd5 := strings.ToUpper(hex.EncodeToString(md5[:])) require.NotEmpty(t, sketchPathMd5) - buildDir := paths.TempDir().Join("arduino", "sketch-"+sketchPathMd5) + buildDir := paths.TempDir().Join("arduino", "sketches", sketchPathMd5) require.FileExists(t, buildDir.Join(sketchName+".ino.partitions.bin").String()) } diff --git a/internal/integrationtest/upload_mock/upload_mock_test.go b/internal/integrationtest/upload_mock/upload_mock_test.go index 208e48fd772..ec0875d8706 100644 --- a/internal/integrationtest/upload_mock/upload_mock_test.go +++ b/internal/integrationtest/upload_mock/upload_mock_test.go @@ -697,7 +697,7 @@ func TestUploadSketch(t *testing.T) { func generateBuildDir(sketchPath *paths.Path, t *testing.T) *paths.Path { md5 := md5.Sum(([]byte(sketchPath.String()))) sketchPathMd5 := strings.ToUpper(hex.EncodeToString(md5[:])) - buildDir := paths.TempDir().Join("arduino", "sketch-"+sketchPathMd5) + buildDir := paths.TempDir().Join("arduino", "sketches", sketchPathMd5) require.NoError(t, buildDir.MkdirAll()) require.NoError(t, buildDir.ToAbs()) return buildDir diff --git a/legacy/builder/add_additional_entries_to_context.go b/legacy/builder/add_additional_entries_to_context.go index 984b686a04c..64ee1ee8a67 100644 --- a/legacy/builder/add_additional_entries_to_context.go +++ b/legacy/builder/add_additional_entries_to_context.go @@ -35,7 +35,7 @@ func (*AddAdditionalEntriesToContext) Run(ctx *types.Context) error { if err != nil { return errors.WithStack(err) } - librariesBuildPath, err := buildPath.Join("libraries").Abs() + librariesBuildPath, err := buildPath.Join(constants.FOLDER_LIBRARIES).Abs() if err != nil { return errors.WithStack(err) } diff --git a/legacy/builder/constants/constants.go b/legacy/builder/constants/constants.go index e1266930fe3..5d7d3bb45a5 100644 --- a/legacy/builder/constants/constants.go +++ b/legacy/builder/constants/constants.go @@ -47,6 +47,7 @@ const FOLDER_CORE = "core" const FOLDER_PREPROC = "preproc" const FOLDER_SKETCH = "sketch" const FOLDER_TOOLS = "tools" +const FOLDER_LIBRARIES = "libraries" const LIBRARY_ALL_ARCHS = "*" const LIBRARY_EMAIL = "email" const LIBRARY_FOLDER_ARCH = "arch" diff --git a/legacy/builder/phases/core_builder.go b/legacy/builder/phases/core_builder.go index 76cb731062c..5b928915321 100644 --- a/legacy/builder/phases/core_builder.go +++ b/legacy/builder/phases/core_builder.go @@ -16,9 +16,11 @@ package phases import ( + "fmt" "os" "strings" + "github.com/arduino/arduino-cli/buildcache" "github.com/arduino/arduino-cli/i18n" "github.com/arduino/arduino-cli/legacy/builder/builder_utils" "github.com/arduino/arduino-cli/legacy/builder/constants" @@ -91,10 +93,17 @@ func compileCore(ctx *types.Context, buildPath *paths.Path, buildCachePath *path realCoreFolder := coreFolder.Parent().Parent() var targetArchivedCore *paths.Path + var buildCacheErr error if buildCachePath != nil { - archivedCoreName := GetCachedCoreArchiveFileName(buildProperties.Get(constants.BUILD_PROPERTIES_FQBN), + archivedCoreName := GetCachedCoreArchiveDirName(buildProperties.Get(constants.BUILD_PROPERTIES_FQBN), buildProperties.Get("compiler.optimization_flags"), realCoreFolder) - targetArchivedCore = buildCachePath.Join(archivedCoreName) + targetArchivedCore = buildCachePath.Join(archivedCoreName, "core.a") + _, buildCacheErr = buildcache.New(buildCachePath).GetOrCreate(archivedCoreName) + + if errors.Is(buildCacheErr, buildcache.CreateDirErr) { + return nil, nil, fmt.Errorf(tr("creating core cache folder: %s", err)) + } + canUseArchivedCore := !ctx.OnlyUpdateCompilationDatabase && !ctx.Clean && !builder_utils.CoreOrReferencedCoreHasChanged(realCoreFolder, targetCoreFolder, targetArchivedCore) @@ -137,19 +146,19 @@ func compileCore(ctx *types.Context, buildPath *paths.Path, buildCachePath *path return archiveFile, variantObjectFiles, nil } -// GetCachedCoreArchiveFileName returns the filename to be used to store +// GetCachedCoreArchiveDirName returns the directory name to be used to store // the global cached core.a. -func GetCachedCoreArchiveFileName(fqbn string, optimizationFlags string, coreFolder *paths.Path) string { +func GetCachedCoreArchiveDirName(fqbn string, optimizationFlags string, coreFolder *paths.Path) string { fqbnToUnderscore := strings.Replace(fqbn, ":", "_", -1) fqbnToUnderscore = strings.Replace(fqbnToUnderscore, "=", "_", -1) if absCoreFolder, err := coreFolder.Abs(); err == nil { coreFolder = absCoreFolder } // silently continue if absolute path can't be detected hash := utils.MD5Sum([]byte(coreFolder.String() + optimizationFlags)) - realName := "core_" + fqbnToUnderscore + "_" + hash + ".a" + realName := fqbnToUnderscore + "_" + hash if len(realName) > 100 { - // avoid really long names, simply hash the final part - realName = "core_" + utils.MD5Sum([]byte(fqbnToUnderscore+"_"+hash)) + ".a" + // avoid really long names, simply hash the name + realName = utils.MD5Sum([]byte(fqbnToUnderscore + "_" + hash)) } return realName } diff --git a/legacy/builder/test/builder_test.go b/legacy/builder/test/builder_test.go index c7b91a18473..16887bfc679 100644 --- a/legacy/builder/test/builder_test.go +++ b/legacy/builder/test/builder_test.go @@ -382,10 +382,13 @@ func TestBuilderCacheCoreAFile(t *testing.T) { // Pick timestamp of cached core coreFolder := paths.New("downloaded_hardware", "arduino", "avr") - coreFileName := phases.GetCachedCoreArchiveFileName(ctx.FQBN.String(), ctx.OptimizationFlags, coreFolder) - cachedCoreFile := ctx.CoreBuildCachePath.Join(coreFileName) + coreFileName := phases.GetCachedCoreArchiveDirName(ctx.FQBN.String(), ctx.OptimizationFlags, coreFolder) + cachedCoreFile := ctx.CoreBuildCachePath.Join(coreFileName, "core.a") coreStatBefore, err := cachedCoreFile.Stat() require.NoError(t, err) + lastUsedFile := ctx.CoreBuildCachePath.Join(coreFileName, ".last-used") + _, err = lastUsedFile.Stat() + require.NoError(t, err) // Run build again, to verify that the builder skips rebuilding core.a err = bldr.Run(ctx)