diff --git a/.gitignore b/.gitignore index fdd1cde..d3aa067 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,3 @@ cslt-tool .vscode -build/ \ No newline at end of file +lib* \ No newline at end of file diff --git a/README.md b/README.md index 5464512..b3788bc 100644 --- a/README.md +++ b/README.md @@ -1,33 +1,59 @@ # cslt-tool -cslt-tool is a convenient wrapper of [arduino-cli](https://github.com/arduino/arduino-cli), it compiles Arduino sketches outputting object files and a json file in a `build/` directory -The json contains information regarding libraries and core to use in order to build the sketch. The result is achieved by parsing the verbose output of `arduino-cli`. +`cslt-tool` is a convenient wrapper of [arduino-cli](https://github.com/arduino/arduino-cli), it compiles Arduino sketches outputting a precompiled library in the current working directory. +It generates a json file in the `extras/` folder that contains information regarding libraries and core to use in order to build the sketch. The result is achieved by parsing the verbose output of `arduino-cli` and by using [GNU ar](https://sourceware.org/binutils/docs/binutils/ar.html) to generate an archive of the object files. -## Requisites -In order to run this tool you have to install first the [arduino-cli](https://github.com/arduino/arduino-cli) and have `arduino-cli` binary in your path, otherwise `cslt-tool` won't work. -Please use a version of the cli that has [this](https://github.com/arduino/arduino-cli/pull/1608) change, version > 0.20.2 +## Prequisites +In order to run this tool you have to install first the [Arduino CLI](https://github.com/arduino/arduino-cli) and have `arduino-cli` binary in your `$PATH`, otherwise `cslt-tool` won't work. +Please use a version of the Arduino CLI that has [this](https://github.com/arduino/arduino-cli/pull/1608) change (version > 0.20.2). + +Another requirement is [`gcc-ar`](https://sourceware.org/binutils/docs/binutils/ar.html) (installable with `apt-get install gcc`) in your `$PATH`. ## Build it -In order to build it just use `go build` +In order to build `cslt-tool` just use `go build` ## Usage `./cslt-tool compile -b ` -This is an example execution: -``` bash -$ ./cslt-tool compile -b arduino:samd:mkrwan1310 /home/umberto/getdeveui -INFO[0001] arduino-cli version: git-snapshot -INFO[0001] running: arduino-cli compile -b arduino:samd:mkrwan1310 /home/umberto/getdeveui -v --format json -INFO[0002] copied file to /home/umberto/Nextcloud/8tb/Lavoro/cslt-tool/build/getdeveui.ino.cpp.o -INFO[0002] created new file in: /home/umberto/Nextcloud/8tb/Lavoro/cslt-tool/build/result.json +[![asciicast](https://asciinema.org/a/463342.svg)](https://asciinema.org/a/463342) + +For example, running `./cslt-tool compile -b arduino:samd:mkrwifi1010 sketch/sketch.ino` should produce a library with the following structure, in the current working directory: ``` -The structure of the `build` forder is the following: +libsketch/ +├── examples +│ └── sketch +│ └── sketch.ino <-- the actual sketch we are going to compile with the arduino-cli later +├── extras +│ └── result.json +├── library.properties +└── src + ├── cortex-m0plus + │ └── libsketch.a + └── libsketch.h ``` -build/ -├── getdeveui.ino.cpp.o -└── result.json + +This is an example execution: ``` -And the content of `build/result.json` is: +$ ./cslt-tool compile -b arduino:samd:mkrwifi1010 sketch/sketch.ino +INFO[0000] arduino-cli version: git-snapshot +INFO[0000] GNU ar (GNU Binutils) 2.37 +INFO[0000] the ino file path is sketch/sketch.ino +INFO[0000] created sketch/main.cpp +INFO[0000] replaced setup() and loop() functions in sketch/sketch.ino +INFO[0000] running: arduino-cli compile -b arduino:samd:mkrwifi1010 sketch/sketch.ino -v --format json +INFO[0000] running: arduino-cli compile -b arduino:samd:mkrwifi1010 sketch/sketch.ino --show-properties +INFO[0001] removed sketch/main.cpp +INFO[0001] created sketch/sketch.ino +INFO[0001] restored sketch/sketch.ino +INFO[0001] created libsketch/library.properties +INFO[0001] created libsketch/src/libsketch.h +INFO[0001] created libsketch/examples/sketch/sketch.ino +INFO[0001] running: gcc-ar rcs libsketch/src/cortex-m0plus/libsketch.a /tmp/arduino-sketch-E4D76B1781E9EB73A7B3491CAC68F374/sketch/sketch.ino.cpp.o +INFO[0001] created libsketch/src/cortex-m0plus/libsketch.a +INFO[0001] created libsketch/extras/result.json +``` + +And the content of `libsketch/extras/result.json` is: ```json { "coreInfo": { @@ -36,9 +62,42 @@ And the content of `build/result.json` is: }, "libsInfo": [ { - "name": "MKRWAN", - "version": "1.1.0" + "name": "WiFiNINA", + "version": "1.8.13", + "provides_includes": [ + "WiFiNINA.h" + ] + }, + { + "name": "SPI", + "version": "1.0", + "provides_includes": [ + "SPI.h" + ] } ] } +``` + +## How to compile the precompiled sketch +In order to compile the sketch you have first to install manually the libraries and the core listed in the `/extras/result.json` file. + +You can install a library with [`arduino-cli lib install LIBRARY[@VERSION_NUMBER]`](https://arduino.github.io/arduino-cli/latest/commands/arduino-cli_lib_install/). + +You can install a core with [`arduino-cli core install PACKAGER:ARCH[@VERSION]`](https://arduino.github.io/arduino-cli/latest/commands/arduino-cli_core_install/). + +After completing that operation you can compile it with: + +`arduino-cli compile -b /examples/sketch/sketch.ino --library `. + +It's important to use the `--library` flag to include the precompiled library generated with cslt-tool otherwise the Arduino CLI won't find it. + +For example a legit execution looks like this: +``` +$ arduino-cli compile -b arduino:samd:mkrwifi1010 libsketch/examples/sketch/sketch.ino --library libsketch/ + +Library libsketch has been declared precompiled: +Using precompiled library in libsketch/src/cortex-m0plus +Sketch uses 14636 bytes (5%) of program storage space. Maximum is 262144 bytes. +Global variables use 3224 bytes (9%) of dynamic memory, leaving 29544 bytes for local variables. Maximum is 32768 bytes. ``` \ No newline at end of file diff --git a/cmd/compile.go b/cmd/compile.go index f98b9c3..195082b 100644 --- a/cmd/compile.go +++ b/cmd/compile.go @@ -5,9 +5,11 @@ Copyright © 2021 NAME HERE package cmd import ( + "bytes" "encoding/json" "os" "os/exec" + "strings" "github.com/arduino/go-paths-helper" "github.com/sirupsen/logrus" @@ -31,8 +33,9 @@ type BuilderResult struct { // UsedLibrary contains information regarding the library used during the compile process type UsedLibrary struct { - Name string `json:"name"` - Version string `json:"version"` + Name string `json:"name"` + Version string `json:"version"` + ProvidesIncludes []string `json:"provides_includes"` } // BuildPlatform contains information regarding the platform used during the compile process @@ -77,51 +80,63 @@ func compileSketch(cmd *cobra.Command, args []string) { json.Unmarshal(cmdOutput, &unmarshalledOutput) logrus.Infof("arduino-cli version: %s", unmarshalledOutput["VersionString"]) - // let's call arduino-cli compile and parse the verbose output - logrus.Infof("running: arduino-cli compile -b %s %s -v --format json", fqbn, args[0]) - cmdOutput, err = exec.Command("arduino-cli", "compile", "-b", fqbn, args[0], "-v", "--format", "json").Output() + // let's check if gcc-ar version + cmdOutput, err = exec.Command("gcc-ar", "--version").CombinedOutput() if err != nil { + logrus.Warn("Before running this tool be sure to have \"gcc-ar\" installed in your $PATH") logrus.Fatal(err) } - objFilesPaths, returnJson := parseOutput(cmdOutput) + // print the version of ar + logrus.Infof(strings.Split(string(cmdOutput), "\n")[0]) - workingDir, err := paths.Getwd() + // check if the path of the sketch passed as args[0] is valid and get the path of the main sketch.ino (in case the sketch dir is specified) + inoPath := getInoSketchPath(args[0]) + + // create a main.cpp file in the same dir of the sketch.ino + createMainCpp(inoPath) + + // replace setup() with _setup() and loop() with _loop() in the user's sketch.ino file + oldSketchContent := patchSketch(inoPath) + + // let's call arduino-cli compile and parse the verbose output + cmdArgs := []string{"compile", "-b", fqbn, inoPath.String(), "-v", "--format", "json"} + logrus.Infof("running: arduino-cli %s", strings.Join(cmdArgs, " ")) + cmdOutput, err = exec.Command("arduino-cli", cmdArgs...).Output() if err != nil { logrus.Fatal(err) } - buildDir := workingDir.Join("build") - if !buildDir.Exist() { - if err = buildDir.Mkdir(); err != nil { - logrus.Fatal(err) - } - } - // Copy the object files from the `/arduino-sketch_stuff/sketch` folder - for _, objFilePath := range objFilesPaths { - destObjFilePath := buildDir.Join(objFilePath.Base()) - if err = objFilePath.CopyTo(destObjFilePath); err != nil { - logrus.Errorf("error copying object file: %s", err) - } else { - logrus.Infof("copied file to %s", destObjFilePath) - } - } + objFilePaths, returnJson := parseCliCompileOutput(cmdOutput) - // save the result.json in the build dir - jsonFilePath := buildDir.Join("result.json") - if jsonContents, err := json.MarshalIndent(returnJson, "", " "); err != nil { - logrus.Errorf("error serializing json: %s", err) - } else if err := jsonFilePath.WriteFile(jsonContents); err != nil { - logrus.Errorf("error writing result.json: %s", err) - } else { - logrus.Infof("created new file in: %s", jsonFilePath) + // this is done to get the {build.mcu} used later to create the lib dir structure + // the --show-properties will only print on stdout and not compile + // the json output is currently broken with this flag, see https://github.com/arduino/arduino-cli/issues/1628 + cmdArgs = []string{"compile", "-b", fqbn, inoPath.String(), "--show-properties"} + logrus.Infof("running: arduino-cli %s", strings.Join(cmdArgs, " ")) + cmdOutput, err = exec.Command("arduino-cli", cmdArgs...).Output() + if err != nil { + logrus.Fatal(err) } + + buildMcu := parseCliCompileOutputShowProp(cmdOutput) + + // remove main.cpp file, we don't need it anymore + removeMainCpp(inoPath) + + // restore the sketch content, this allows us to rerun cslt-tool if we want without breaking the compile process + createFile(inoPath, string(oldSketchContent)) + logrus.Infof("restored %s", inoPath.String()) + + sketchName := strings.TrimSuffix(inoPath.Base(), inoPath.Ext()) + // let's create the library corresponding to the precompiled sketch + createLib(sketchName, buildMcu, returnJson, objFilePaths) } -// parseOutput function takes cmdOutToParse as argument, +// parseCliCompileOutput function takes cmdOutToParse as argument, // cmdOutToParse is the json output captured from the command run // the function extracts and returns the paths of the .o files // (generated during the compile phase) and a ReturnJson object -func parseOutput(cmdOutToParse []byte) ([]*paths.Path, *ResultJson) { +func parseCliCompileOutput(cmdOutToParse []byte) (*paths.PathList, *ResultJson) { var compileOutput CompileOutput err := json.Unmarshal(cmdOutToParse, &compileOutput) if err != nil { @@ -136,7 +151,7 @@ func parseOutput(cmdOutToParse []byte) ([]*paths.Path, *ResultJson) { if err != nil { logrus.Fatal(err) } else if len(sketchFilesPaths) == 0 { - logrus.Fatalf("empty directory: %s", sketchDir) + logrus.Fatalf("empty directory: %s", sketchDir.String()) } sketchFilesPaths.FilterSuffix(".o") @@ -145,5 +160,235 @@ func parseOutput(cmdOutToParse []byte) ([]*paths.Path, *ResultJson) { LibsInfo: compileOutput.BuilderResult.UsedLibraries, } - return sketchFilesPaths, &returnJson + return &sketchFilesPaths, &returnJson +} + +// parseCliCompileOutputShowProp function takes cmdOutToParse as argument, +// cmdOutToParse is the output of the command run +// the function extract the value corresponding to `build.mcu` key +// that string is returned if it's found. Otherwise the program exits +func parseCliCompileOutputShowProp(cmdOutToParse []byte) string { + cmdOut := string(cmdOutToParse) + lines := strings.Split(cmdOut, "\n") + for _, line := range lines { + if strings.Contains(line, "build.mcu") { // the line should be something like: 'build.mcu=cortex-m0plus' + if mcuLine := strings.Split(line, "="); len(mcuLine) == 2 { + return mcuLine[1] + } + } + } + logrus.Fatal("cannot find \"build.mcu\" in arduino-cli output") + return "" +} + +// getInoSketchPath function will take argSketchPath as argument. +// and will return the path to the ino sketch +// it will run some checks along the way, +// we need the main ino file because we need to replace setup() and loop() functions in it +func getInoSketchPath(argSketchPath string) (inoPath *paths.Path) { + sketchPath := paths.New(argSketchPath) + if !sketchPath.Exist() { + logrus.Fatalf("the path %s do not exist!", sketchPath.String()) + } + if sketchPath.Ext() == ".ino" { + inoPath = sketchPath + } else { // if there are multiple .ino files in the sketchPath we need to know which is the one containing setup() and loop() functions + files, _ := sketchPath.ReadDir() + files.FilterSuffix(".ino") + if len(files) == 0 { + logrus.Fatal("the sketch path specified does not contain an .ino file") + } else if len(files) > 1 { + logrus.Fatalf("the sketch path specified contains multiple .ino files:\n%s\nIn order to make the magic please use the path of the .ino file containing the setup() and loop() functions", strings.Join(files.AsStrings(), "\n")) + } + inoPath = files[0] + } + logrus.Infof("the ino file path is %s", inoPath.String()) + return inoPath +} + +// createMainCpp function, as the name suggests. will create a main.cpp file inside inoPath +// we do this because setup() and loop() functions will be replaced inside the ino file, in order to allow the linking afterwards +// creating this file is mandatory, we include also Arduino.h because it's a step done by the builder during the building phase, but only for ino files +func createMainCpp(inoPath *paths.Path) { + // the main.cpp contains the following: + mainCpp := `#include "Arduino.h" +void _setup(); +void _loop(); + +void setup() { +_setup(); +} + +void loop() { +_loop(); +}` + mainCppPath := inoPath.Parent().Join("main.cpp") + createFile(mainCppPath, mainCpp) +} + +// removeMainCpp function, as the name suggests. will remove a main.cpp file inside inoPath +// we do this after the compile has been completed, this way we can rerun cslt-tool again. +// If we do not remove this file and run the compile again it will fail because a main.cpp file with the same definitions is already present +func removeMainCpp(inoPath *paths.Path) { + mainCppPath := inoPath.Parent().Join("main.cpp") + if err := os.Remove(mainCppPath.String()); err != nil { + logrus.Warn(err) + } else { + logrus.Infof("removed %s", mainCppPath.String()) + } +} + +// patchSketch function will modify the content of the inoPath sketch passed as argument, +// the old unmodified sketch content is returned as oldSketchContent, +// we do this to allow the compile process to succeed +func patchSketch(inoPath *paths.Path) (oldSketchContent []byte) { + oldSketchContent, err := os.ReadFile(inoPath.String()) + if err != nil { + logrus.Fatal(err) + } + if bytes.Contains(oldSketchContent, []byte("_setup()")) || bytes.Contains(oldSketchContent, []byte("_loop()")) { + logrus.Warnf("already patched %s, skipping", inoPath.String()) + } else { + newSketchContent := bytes.Replace(oldSketchContent, []byte("void setup()"), []byte("void _setup()"), -1) + newSketchContent = bytes.Replace(newSketchContent, []byte("void loop()"), []byte("void _loop()"), -1) + if err = os.WriteFile(inoPath.String(), newSketchContent, 0644); err != nil { + logrus.Fatal(err) + } + logrus.Infof("replaced setup() and loop() functions in %s", inoPath.String()) + } + return oldSketchContent +} + +// createLib function will take care of creating the library directory structure and files required, for the precompiled library to be recognized as such. +// sketchName is the name of the sketch without the .ino extension. We use this for the name of the lib. +// buildMcu is the name of the MCU of the board we have compiled for. The library specifications (https://arduino.github.io/arduino-cli/0.20/library-specification/#precompiled-binaries) requires that the precompiled archive is stored inside a folder with the name of the MCU used during the compile. +// returnJson is the ResultJson object containing informations regarding core and libraries used during the compile process. +// objFilePaths is a paths.PathList containing the paths.Paths to all the sketch related object files produced during the compile phase. +func createLib(sketchName string, buildMcu string, returnJson *ResultJson, objFilePaths *paths.PathList) { + // we are going to leverage the precompiled library infrastructure to make the linking work. + // this type of lib, as the type suggest, is already compiled so it only gets linked during the linking phase of a sketch + // but we have to create a library folder structure in the current directory: + + // libsketch/ + // ├── examples + // │ └── sketch + // │ └── sketch.ino <-- the actual sketch we are going to compile with the arduino-cli later + // ├── extras + // │ └── result.json + // ├── library.properties + // └── src + // ├── cortex-m0plus + // │ └── libsketch.a + // └── libsketch.h + + // let's create the dir structure + workingDir, err := paths.Getwd() + if err != nil { + logrus.Fatal(err) + } + libDir := workingDir.Join("lib" + sketchName) + if libDir.Exist() { // if the dir already exixst we clean it before + os.RemoveAll(libDir.String()) + logrus.Warnf("removed %s", libDir.String()) + } + if err = libDir.Mkdir(); err != nil { + logrus.Fatal(err) + } + srcDir := libDir.Join("src").Join(buildMcu) + if err = srcDir.MkdirAll(); err != nil { + logrus.Fatal(err) + } + exampleDir := libDir.Join("examples").Join(sketchName) + if err = exampleDir.MkdirAll(); err != nil { + logrus.Fatal(err) + } + extraDir := libDir.Join("extras") + if err = extraDir.Mkdir(); err != nil { + logrus.Fatal(err) + } + + // let's create the files + + // create a library.properties file in the root dir of the lib + // the library.properties contains the following: + libraryProperties := `name=` + sketchName + ` +author=TODO +maintainer=TODO +sentence=This technically is not a library but a precompiled sketch. The result is produced using ` + os.Args[0] + ` +paragraph= +url=https://github.com/arduino/cslt-tool +version=1.0.0 +precompiled=true` + + libraryPropertyPath := libDir.Join("library.properties") + createFile(libraryPropertyPath, libraryProperties) + + // we calculate the #include part to append at the beginning of the header file here with all the libraries used by the original sketch + var librariesIncludes []string + for _, lib := range returnJson.LibsInfo { + for _, include := range lib.ProvidesIncludes { + librariesIncludes = append(librariesIncludes, "#include \""+include+"\"") + } + } + + // create the header file in the src/ dir + // This file has predeclarations of _setup() and _loop() functions declared originally in the main.cpp file (which is not included in the .a archive), + // It is the counterpart of libsketch.a + // the libsketch.h contains the following: + libsketchHeader := strings.Join(librariesIncludes, "\n") + ` +void _setup(); +void _loop();` + + libsketchFilePath := srcDir.Parent().Join("lib" + sketchName + ".h") + createFile(libsketchFilePath, libsketchHeader) + + // create the sketch file in the example dir of the lib + // This one will include the libsketch.h and basically is the replacement of main.cpp + // the sketch.ino contains the following: + sketchFile := `#include <` + "lib" + sketchName + `.h> +void setup() { + _setup(); +} +void loop() { + _loop(); +}` + sketchFilePath := exampleDir.Join(sketchName + ".ino") + createFile(sketchFilePath, sketchFile) + + // run gcc-ar to create an archive containing all the object files except the main.cpp.o (we don't need it because we have created a substitute of it before ⬆️) + // we exclude the main.cpp.o because we are going to link the archive libsketch.a against sketchName.ino + objFilePaths.FilterOutPrefix("main.cpp") + archivePath := srcDir.Join("lib" + sketchName + ".a") + cmdArgs := append([]string{"rcs", archivePath.String()}, objFilePaths.AsStrings()...) + logrus.Infof("running: gcc-ar %s", strings.Join(cmdArgs, " ")) + cmdOutput, err := exec.Command("gcc-ar", cmdArgs...).CombinedOutput() + if err != nil { + logrus.Fatal(err) + } + if len(cmdOutput) != 0 { + logrus.Info(string(cmdOutput)) + } else { + logrus.Infof("created %s", archivePath.String()) + } + // save the result.json in the library extra dir + jsonFilePath := extraDir.Join("result.json") + if jsonContents, err := json.MarshalIndent(returnJson, "", " "); err != nil { + logrus.Errorf("error serializing json: %s", err) + } else if err := jsonFilePath.WriteFile(jsonContents); err != nil { + logrus.Errorf("error writing %s: %s", jsonFilePath.Base(), err) + } else { + logrus.Infof("created %s", jsonFilePath.String()) + } +} + +// createFile is an helper function useful to create a file, +// it takes filePath and fileContent as arguments, +// filePath points to the location where to save the file +// fileContent,as the name suggests, include the content of the file +func createFile(filePath *paths.Path, fileContent string) { + err := os.WriteFile(filePath.String(), []byte(fileContent), 0644) + if err != nil { + logrus.Fatal(err) + } + logrus.Infof("created %s", filePath.String()) }