mage: build with OpenSSL

3 build options have been added:

- static with statically linked OpenSSL;
- shared with dynamically linked OpenSSL;
- nossl  without OpenSSL;

Closes #308

Author: oleg-jukovec

.github/actions/prepare-ce-test-env/action.yml

@@ -42,5 +42,5 @@ runs:
         tarantool-version: '${{ inputs.tarantool-version }}'
 
     - name: Build tt
-      run: mage build
+      run: mage build static
       shell: bash

.github/actions/prepare-ee-test-env/action.yml

@@ -64,5 +64,5 @@ runs:
         shell: bash
 
       - name: Build tt
-        run: mage build
+        run: mage build static
         shell: bash

.github/workflows/full-ci.yml

@@ -134,7 +134,7 @@ jobs:
         uses: docker-practice/actions-setup-docker@master
 
       - name: Build tt
-        run: mage build
+        run: mage build static
 
       - name: Install tarantool
         run: brew install tarantool

.github/workflows/publish.yml

@@ -28,6 +28,14 @@ jobs:
           cd mage
           go run bootstrap.go
 
+      - name: Build OpenSSL 3.0
+        run: |
+          wget https://github.com/openssl/openssl/releases/download/openssl-3.0.8/openssl-3.0.8.tar.gz
+          tar -xvf openssl-3.0.8.tar.gz
+          cd openssl-3.0.8/
+          ./Configure --prefix=${GITHUB_WORKSPACE}/openssl no-shared
+          make && make install
+
       - name: Setup GoReleaser
         run: |
           curl -O -L https://github.com/goreleaser/goreleaser/releases/download/v1.12.3/goreleaser_1.12.3_amd64.deb
@@ -45,6 +53,8 @@ jobs:
 
       - name: Build packages
         env:
+          CGO_LDFLAGS: "-L${{ env.GITHUB_WORKSPACE }}/openssl/lib64"
+          CGO_CFLAGS: "-I${{ env.GITHUB_WORKSPACE }}/openssl/include"
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
           goreleaser release ${{ steps.set-goreleaser-flags.outputs.GORELEASER_FLAGS }}

.github/workflows/tests.yml

@@ -124,7 +124,7 @@ jobs:
           pip3 install pytest tarantool requests psutil pyyaml netifaces
 
       - name: Build tt
-        run: mage build
+        run: mage build static
 
       - name: Install tarantool
         run: brew install tarantool

.goreleaser.yml

@@ -12,12 +12,15 @@ builds:
     dir: cli
 
     env:
-      - CGO_ENABLED=0
+      - CGO_ENABLED=1
 
-    flags:
-      - -tags=go_tarantool_ssl_disable
+    tags:
+      - netgo
+      - osusergo
+      - openssl_static
 
     ldflags:
+      - -linkmode=external -extldflags -static
       - -s -w
       - -X github.com/tarantool/tt/cli/version.gitTag={{ .Tag }}
       - -X github.com/tarantool/tt/cli/version.gitCommit={{ .ShortCommit }}

README.rst

@@ -37,7 +37,27 @@ Build
 
    git clone https://github.com/tarantool/tt --recursive
    cd tt
-   mage build
+
+You can build a binary with statically linked OpenSSL. This build type is used
+for releases:
+
+.. code-block:: bash
+
+   mage build static
+
+You can build a binary with dynamically linked OpenSSL for development
+purposes:
+
+.. code-block:: bash
+
+   mage build shared
+
+Finnally, you can build the binary without OpenSSL and TLS support for
+development purposes:
+
+.. code-block:: bash
+
+   mage build nossl
 
 ~~~~~~~~~~~~
 Dependencies

go.mod

@@ -21,7 +21,7 @@ require (
 	github.com/spf13/cobra v1.3.0
 	github.com/stretchr/testify v1.7.1
 	github.com/tarantool/cartridge-cli v0.0.0-20220605082730-53e6a5be9a61
-	github.com/tarantool/go-tarantool v1.9.0
+	github.com/tarantool/go-tarantool v1.10.1-0.20230309143354-e257ff30dd4d
 	github.com/vmihailenco/msgpack/v5 v5.3.5
 	github.com/yuin/gopher-lua v0.0.0-20220504180219-658193537a64
 	golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd
@@ -69,7 +69,7 @@ require (
 	github.com/sirupsen/logrus v1.8.1 // indirect
 	github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/tarantool/go-openssl v0.0.8-0.20220711094538-d93c1eff4f49 // indirect
+	github.com/tarantool/go-openssl v0.0.8-0.20230307065445-720eeb389195 // indirect
 	github.com/tklauser/go-sysconf v0.3.4 // indirect
 	github.com/tklauser/numcpus v0.2.1 // indirect
 	github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect

go.sum

@@ -686,6 +686,7 @@ github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN
 github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs=
 github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
+github.com/markphelps/optional v0.10.0/go.mod h1:Fvjs1vxcm7/wDqJPFGEiEM1RuxFl9GCyxQlj9M9YMAQ=
 github.com/marstr/guid v1.1.0/go.mod h1:74gB1z2wpxxInTG6yaqA7KrtM0NZ+RbrcqDvYHefzho=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
 github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
@@ -987,12 +988,12 @@ github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69
 github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
 github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
 github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
-github.com/tarantool/go-openssl v0.0.8-0.20220711094538-d93c1eff4f49 h1:rZYYi1cI3QXZ3yRFZd2ItYM1XA2BaJqP0buDroMbjNo=
-github.com/tarantool/go-openssl v0.0.8-0.20220711094538-d93c1eff4f49/go.mod h1:M7H4xYSbzqpW/ZRBMyH0eyqQBsnhAMfsYk5mv0yid7A=
+github.com/tarantool/go-openssl v0.0.8-0.20230307065445-720eeb389195 h1:/AN3eUPsTlvF6W+Ng/8ZjnSU6o7L0H4Wb9GMks6RkzU=
+github.com/tarantool/go-openssl v0.0.8-0.20230307065445-720eeb389195/go.mod h1:M7H4xYSbzqpW/ZRBMyH0eyqQBsnhAMfsYk5mv0yid7A=
 github.com/tarantool/go-prompt v0.2.6-tarantool h1:/dYMRBuM5nE3mleka/mqJWPf8SrJ151U+OqDlTzvES0=
 github.com/tarantool/go-prompt v0.2.6-tarantool/go.mod h1:8enZKIgoGFEQu2XPBK79TguJG2XF3SR4QU2iYI28NSo=
-github.com/tarantool/go-tarantool v1.9.0 h1:qUotwiMNZhi9AvogF3uyswEN2NAFcKJOhcvXiE/P+Rs=
-github.com/tarantool/go-tarantool v1.9.0/go.mod h1:oPjvZNKaN4iKbf8YPo3pGpxzk6cRZF1neAxgq8WcBh8=
+github.com/tarantool/go-tarantool v1.10.1-0.20230309143354-e257ff30dd4d h1:qkKjWaxX8GPdHSP2WETv096vm/MH2iBPOZa6hnJ780s=
+github.com/tarantool/go-tarantool v1.10.1-0.20230309143354-e257ff30dd4d/go.mod h1:oMCTx0UAjEkHjdVdgnh3DjwUxs0xgyvIcaOBV8i4CF4=
 github.com/tchap/go-patricia v2.2.6+incompatible/go.mod h1:bmLyhP68RS6kStMGxByiQ23RP/odRBOTVjwp2cDyi6I=
 github.com/tj/assert v0.0.0-20171129193455-018094318fb0/go.mod h1:mZ9/Rh9oLWpLLDRpvE+3b7gP/C2YyLFYxNmcLnPTMe0=
 github.com/tj/assert v0.0.3 h1:Df/BlaZ20mq6kuai7f5z2TvPFiwC3xaWJSDQNiIS3Rk=

magefile.go

@@ -38,14 +38,24 @@ var (
 		"-X ${PACKAGE}/version.versionLabel=${VERSION_LABEL}",
 		"-X ${PACKAGE}/configure.defaultConfigPath=${CONFIG_PATH}",
 	}
-
+	staticLdflags = []string{
+		"-linkmode=external", "-extldflags", "-static",
+	}
 	goExecutableName     = "go"
 	pythonExecutableName = "python3"
 	ttExecutableName     = "tt"
 
 	generateModePath = filepath.Join(packagePath, "codegen", "generate_code.go")
 )
 
+type BuildType string
+
+const (
+	BuildTypeShared BuildType = "shared"
+	BuildTypeStatic BuildType = "static"
+	BuildTypeNossl  BuildType = "nossl"
+)
+
 func init() {
 	var err error
 
@@ -60,7 +70,6 @@ func init() {
 			panic(err)
 		}
 	}
-
 	// We want to use Go 1.11 modules even if the source lives inside GOPATH.
 	// The default is "auto".
 	os.Setenv("GO111MODULE", "on")
@@ -144,17 +153,35 @@ func PatchCC() error {
 }
 
 // Building tt executable.
-func Build() error {
+func Build(env string) error {
 	fmt.Println("Building tt...")
 
 	mg.Deps(PatchCC)
 	mg.Deps(GenerateGoCode)
 
+	buildLdflags := make([]string, len(ldflags))
+	copy(buildLdflags, ldflags)
+	tags := "-tags=netgo,osusergo"
+	switch BuildType(env) {
+	case BuildTypeStatic:
+		if runtime.GOOS != "darwin" {
+			buildLdflags = append(buildLdflags, staticLdflags...)
+		}
+		tags = tags + ",openssl_static"
+	case BuildTypeShared:
+	case BuildTypeNossl:
+		tags = tags + ",go_tarantool_ssl_disable"
+	default:
+		return fmt.Errorf("Unsupported build type: %s, supported: "+
+			"%s, %s, %s\n",
+			env, BuildTypeStatic, BuildTypeShared, BuildTypeNossl)
+	}
+
 	err := sh.RunWith(
 		getBuildEnvironment(), goExecutableName, "build",
 		"-o", ttExecutableName,
-		"-tags=go_tarantool_ssl_disable",
-		"-ldflags", strings.Join(ldflags, " "),
+		tags,
+		"-ldflags", strings.Join(buildLdflags, " "),
 		"-asmflags", asmflags,
 		"-gcflags", gcflags,
 		packagePath,
@@ -210,11 +237,11 @@ func Unit() error {
 	mg.Deps(GenerateGoCode)
 
 	if mg.Verbose() {
-		return sh.RunV(goExecutableName, "test", "-v", "-tags", "go_tarantool_ssl_disable",
+		return sh.RunV(goExecutableName, "test", "-v",
 			fmt.Sprintf("%s/...", packagePath))
 	}
 
-	return sh.RunV(goExecutableName, "test", "-tags", "go_tarantool_ssl_disable", fmt.Sprintf("%s/...", packagePath))
+	return sh.RunV(goExecutableName, "test", fmt.Sprintf("%s/...", packagePath))
 }
 
 // Run unit tests with a Tarantool instance integration.
@@ -225,11 +252,11 @@ func UnitFull() error {
 
 	if mg.Verbose() {
 		return sh.RunV(goExecutableName, "test", "-v", fmt.Sprintf("%s/...", packagePath),
-			"-tags", "integration,go_tarantool_ssl_disable")
+			"-tags", "integration")
 	}
 
 	return sh.RunV(goExecutableName, "test", fmt.Sprintf("%s/...", packagePath),
-		"-tags", "integration,go_tarantool_ssl_disable")
+		"-tags", "integration")
 }
 
 // Run integration tests, excluding slow tests.
@@ -320,6 +347,6 @@ func getBuildEnvironment() map[string]string {
 		"VERSION_LABEL": os.Getenv("VERSION_LABEL"),
 		"PWD":           currentDir,
 		"CONFIG_PATH":   getDefaultConfigPath(),
-		"CGO_ENABLED":   "0",
+		"CGO_ENABLED":   "1",
 	}
 }

test/conftest.py

@@ -41,7 +41,7 @@ def tt_cmd(session_tmpdir):
     build_env = os.environ.copy()
     build_env["TTEXE"] = tt_path
 
-    process = subprocess.run(["mage", "-v", "build"], cwd=tt_base_path, env=build_env)
+    process = subprocess.run(["mage", "-v", "build", "static"], cwd=tt_base_path, env=build_env)
     assert process.returncode == 0, "Failed to build Tarantool CLI executable"
 
     return tt_path