Skip to content

Commit 2ad7fb3

Browse files
hdinizhdiniz
authored
add cli httpclient with support for proxy configuration (#672)
* add configuration for cli httpclient * add httpclient roundtripper with proxy and user-agent * add cli httpclient * set downloader user-agent tp equal cli httpclient * use cli httpclient on board list command * add unit tests for httpclient * add copyright notice * upgrade downloader to v2 with httpclient injection * change go.mod to point to upstream downloader project v2
1 parent 06bc37e commit 2ad7fb3

File tree

15 files changed

+245
-46
lines changed

15 files changed

+245
-46
lines changed

‎arduino/cores/packagemanager/download.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ import (
1919
"fmt"
2020

2121
"github.com/arduino/arduino-cli/arduino/cores"
22-
"go.bug.st/downloader"
22+
"go.bug.st/downloader/v2"
2323
semver "go.bug.st/relaxed-semver"
2424
)
2525

‎arduino/libraries/librariesmanager/download.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ package librariesmanager
1818
import (
1919
"net/url"
2020

21-
"go.bug.st/downloader"
21+
"go.bug.st/downloader/v2"
2222
)
2323

2424
// LibraryIndexURL is the URL where to get library index.

‎arduino/resources/helpers.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ import (
2020
"os"
2121

2222
"github.com/arduino/go-paths-helper"
23-
"go.bug.st/downloader"
23+
"go.bug.st/downloader/v2"
2424
)
2525

2626
// ArchivePath returns the path of the Archive of the specified DownloadResource relative

‎arduino/resources/helpers_test.go

+5-2
Original file line numberDiff line numberDiff line change
@@ -23,9 +23,10 @@ import (
2323
"strings"
2424
"testing"
2525

26+
"github.com/arduino/arduino-cli/httpclient"
2627
"github.com/arduino/go-paths-helper"
2728
"github.com/stretchr/testify/require"
28-
"go.bug.st/downloader"
29+
"go.bug.st/downloader/v2"
2930
)
3031

3132
type EchoHandler struct{}
@@ -53,7 +54,9 @@ func TestDownloadApplyUserAgentHeaderUsingConfig(t *testing.T) {
5354
URL: srv.URL,
5455
}
5556

56-
d, err := r.Download(tmp, &downloader.Config{RequestHeaders: http.Header{"User-Agent": []string{goldUserAgentValue}}})
57+
httpClient := httpclient.NewWithConfig(&httpclient.Config{UserAgent: goldUserAgentValue})
58+
59+
d, err := r.Download(tmp, &downloader.Config{HttpClient: *httpClient})
5760
require.NoError(t, err)
5861
err = d.Run()
5962
require.NoError(t, err)

‎arduino/resources/resources_test.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ import (
2222

2323
"github.com/arduino/go-paths-helper"
2424
"github.com/stretchr/testify/require"
25-
"go.bug.st/downloader"
25+
"go.bug.st/downloader/v2"
2626
)
2727

2828
func TestDownloadAndChecksums(t *testing.T) {

‎cli/globals/globals.go

-18
Original file line numberDiff line numberDiff line change
@@ -16,11 +16,8 @@
1616
package globals
1717

1818
import (
19-
"fmt"
20-
"net/http"
2119
"os"
2220
"path/filepath"
23-
"runtime"
2421

2522
"github.com/arduino/arduino-cli/version"
2623
)
@@ -31,18 +28,3 @@ var (
3128
// DefaultIndexURL is the default index url
3229
DefaultIndexURL = "https://downloads.arduino.cc/packages/package_index.json"
3330
)
34-
35-
// NewHTTPClientHeader returns the http.Header object that must be used by the clients inside the downloaders
36-
// and adds the subComponent if specified
37-
func NewHTTPClientHeader(subComponent string) http.Header {
38-
if subComponent != "" {
39-
subComponent = " " + subComponent
40-
}
41-
userAgentValue := fmt.Sprintf("%s/%s%s (%s; %s; %s) Commit:%s",
42-
VersionInfo.Application,
43-
VersionInfo.VersionString,
44-
subComponent,
45-
runtime.GOARCH, runtime.GOOS, runtime.Version(),
46-
VersionInfo.Commit)
47-
return http.Header{"User-Agent": []string{userAgentValue}}
48-
}

‎commands/board/list.go

+8-3
Original file line numberDiff line numberDiff line change
@@ -23,8 +23,8 @@ import (
2323
"regexp"
2424
"sync"
2525

26-
"github.com/arduino/arduino-cli/cli/globals"
2726
"github.com/arduino/arduino-cli/commands"
27+
"github.com/arduino/arduino-cli/httpclient"
2828
rpc "github.com/arduino/arduino-cli/rpc/commands"
2929
"github.com/pkg/errors"
3030
"github.com/segmentio/stats/v4"
@@ -51,12 +51,17 @@ func apiByVidPid(vid, pid string) ([]*rpc.BoardListItem, error) {
5151
url := fmt.Sprintf("%s/%s/%s", vidPidURL, vid, pid)
5252
retVal := []*rpc.BoardListItem{}
5353
req, _ := http.NewRequest("GET", url, nil)
54-
req.Header = globals.NewHTTPClientHeader("")
5554
req.Header.Set("Content-Type", "application/json")
5655

5756
// TODO: use proxy if set
5857

59-
if res, err := http.DefaultClient.Do(req); err == nil {
58+
httpClient, err := httpclient.New()
59+
60+
if err != nil {
61+
return nil, errors.Wrap(err, "failed to initialize http client")
62+
}
63+
64+
if res, err := httpClient.Do(req); err == nil {
6065
if res.StatusCode >= 400 {
6166
if res.StatusCode == 404 {
6267
return nil, ErrNotFound

‎commands/download.go

+9-15
Original file line numberDiff line numberDiff line change
@@ -16,30 +16,24 @@
1616
package commands
1717

1818
import (
19-
"net/url"
2019
"time"
2120

22-
"github.com/arduino/arduino-cli/cli/globals"
21+
"github.com/arduino/arduino-cli/httpclient"
2322
rpc "github.com/arduino/arduino-cli/rpc/commands"
24-
"github.com/pkg/errors"
25-
"github.com/sirupsen/logrus"
26-
"github.com/spf13/viper"
27-
"go.bug.st/downloader"
23+
"go.bug.st/downloader/v2"
2824
)
2925

3026
// GetDownloaderConfig returns the downloader configuration based on
3127
// current settings.
3228
func GetDownloaderConfig() (*downloader.Config, error) {
33-
res := &downloader.Config{
34-
RequestHeaders: globals.NewHTTPClientHeader(viper.GetString("network.user_agent_ext")),
29+
30+
httpClient, err := httpclient.New()
31+
if err != nil {
32+
return nil, err
3533
}
36-
if viper.IsSet("network.proxy") {
37-
proxy := viper.GetString("network.proxy")
38-
if _, err := url.Parse(proxy); err != nil {
39-
return nil, errors.New("Invalid network.proxy '" + proxy + "': " + err.Error())
40-
}
41-
res.ProxyURL = proxy
42-
logrus.Infof("Using proxy %s", proxy)
34+
35+
res := &downloader.Config{
36+
HttpClient: *httpClient,
4337
}
4438
return res, nil
4539
}

‎commands/instances.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ import (
3333
paths "github.com/arduino/go-paths-helper"
3434
"github.com/sirupsen/logrus"
3535
"github.com/spf13/viper"
36-
"go.bug.st/downloader"
36+
"go.bug.st/downloader/v2"
3737
)
3838

3939
// this map contains all the running Arduino Core Services instances

‎go.mod

+1-1
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ require (
3838
github.com/spf13/viper v1.6.2
3939
github.com/stretchr/testify v1.4.0
4040
go.bug.st/cleanup v1.0.0
41-
go.bug.st/downloader v1.2.0
41+
go.bug.st/downloader/v2 v2.0.1
4242
go.bug.st/relaxed-semver v0.0.0-20190922224835-391e10178d18
4343
go.bug.st/serial v1.0.0
4444
go.bug.st/serial.v1 v0.0.0-20180827123349-5f7892a7bb45 // indirect

‎go.sum

+2-2
Original file line numberDiff line numberDiff line change
@@ -205,8 +205,8 @@ github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q
205205
github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
206206
go.bug.st/cleanup v1.0.0 h1:XVj1HZxkBXeq3gMT7ijWUpHyIC1j8XAoNSyQ06CskgA=
207207
go.bug.st/cleanup v1.0.0/go.mod h1:EqVmTg2IBk4znLbPD28xne3abjsJftMdqqJEjhn70bk=
208-
go.bug.st/downloader v1.2.0 h1:YmXFTcTnm0v8WzAWHn2DyV46c/Izlc/gReXubc2oBho=
209-
go.bug.st/downloader v1.2.0/go.mod h1:l+RPbNbrTB+MoAIp8nrZsP22nRPDy26XJZQqmm4gNT4=
208+
go.bug.st/downloader/v2 v2.0.1 h1:F/ZgVDrUHo2tHIeOVbcl4rgGaMmQgAEjn5c8yi5t+Iw=
209+
go.bug.st/downloader/v2 v2.0.1/go.mod h1:VZW2V1iGKV8rJL2ZEGIDzzBeKowYv34AedJz13RzVII=
210210
go.bug.st/relaxed-semver v0.0.0-20190922224835-391e10178d18 h1:F1qxtaFuewctYc/SsHRn+Q7Dtwi+yJGPgVq8YLtQz98=
211211
go.bug.st/relaxed-semver v0.0.0-20190922224835-391e10178d18/go.mod h1:Cx1VqMtEhE9pIkEyUj3LVVVPkv89dgW8aCKrRPDR/uE=
212212
go.bug.st/serial v1.0.0 h1:ogEPzrllCsnG00EqKRjeYvPRsO7NJW6DqykzkdD6E/k=

‎httpclient/httpclient.go

+40
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
// This file is part of arduino-cli.
2+
//
3+
// Copyright 2020 ARDUINO SA (http://www.arduino.cc/)
4+
//
5+
// This software is released under the GNU General Public License version 3,
6+
// which covers the main part of arduino-cli.
7+
// The terms of this license can be found at:
8+
// https://www.gnu.org/licenses/gpl-3.0.en.html
9+
//
10+
// You can be released from the requirements of the above licenses by purchasing
11+
// a commercial license. Buying such a license is mandatory if you want to
12+
// modify or otherwise use the software for commercial activities involving the
13+
// Arduino software without disclosing the source code of your own applications.
14+
// To purchase a commercial license, send an email to [email protected].
15+
16+
package httpclient
17+
18+
import (
19+
"net/http"
20+
)
21+
22+
// New returns a default http client for use in the cli API calls
23+
func New() (*http.Client, error) {
24+
config, err := DefaultConfig()
25+
26+
if err != nil {
27+
return nil, err
28+
}
29+
30+
return NewWithConfig(config), nil
31+
}
32+
33+
// NewWithConfig creates a http client for use in the cli API calls with a given configuration
34+
func NewWithConfig(config *Config) *http.Client {
35+
transport := newHTTPClientTransport(config)
36+
37+
return &http.Client{
38+
Transport: transport,
39+
}
40+
}

‎httpclient/httpclient_config.go

+64
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
// This file is part of arduino-cli.
2+
//
3+
// Copyright 2020 ARDUINO SA (http://www.arduino.cc/)
4+
//
5+
// This software is released under the GNU General Public License version 3,
6+
// which covers the main part of arduino-cli.
7+
// The terms of this license can be found at:
8+
// https://www.gnu.org/licenses/gpl-3.0.en.html
9+
//
10+
// You can be released from the requirements of the above licenses by purchasing
11+
// a commercial license. Buying such a license is mandatory if you want to
12+
// modify or otherwise use the software for commercial activities involving the
13+
// Arduino software without disclosing the source code of your own applications.
14+
// To purchase a commercial license, send an email to [email protected].
15+
16+
package httpclient
17+
18+
import (
19+
"errors"
20+
"fmt"
21+
"net/url"
22+
"runtime"
23+
24+
"github.com/arduino/arduino-cli/cli/globals"
25+
"github.com/spf13/viper"
26+
)
27+
28+
// Config is the configuration of the http client
29+
type Config struct {
30+
UserAgent string
31+
Proxy *url.URL
32+
}
33+
34+
// DefaultConfig returns the default http client config
35+
func DefaultConfig() (*Config, error) {
36+
var proxy *url.URL
37+
var err error
38+
if viper.IsSet("network.proxy") {
39+
proxyConfig := viper.GetString("network.proxy")
40+
if proxy, err = url.Parse(proxyConfig); err != nil {
41+
return nil, errors.New("Invalid network.proxy '" + proxyConfig + "': " + err.Error())
42+
}
43+
}
44+
45+
return &Config{
46+
UserAgent: UserAgent(),
47+
Proxy: proxy,
48+
}, nil
49+
}
50+
51+
// UserAgent returns the user agent for the cli http client
52+
func UserAgent() string {
53+
subComponent := viper.GetString("network.user_agent_ext")
54+
if subComponent != "" {
55+
subComponent = " " + subComponent
56+
}
57+
58+
return fmt.Sprintf("%s/%s%s (%s; %s; %s) Commit:%s",
59+
globals.VersionInfo.Application,
60+
globals.VersionInfo.VersionString,
61+
subComponent,
62+
runtime.GOARCH, runtime.GOOS, runtime.Version(),
63+
globals.VersionInfo.Commit)
64+
}

‎httpclient/httpclient_test.go

+70
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
// This file is part of arduino-cli.
2+
//
3+
// Copyright 2020 ARDUINO SA (http://www.arduino.cc/)
4+
//
5+
// This software is released under the GNU General Public License version 3,
6+
// which covers the main part of arduino-cli.
7+
// The terms of this license can be found at:
8+
// https://www.gnu.org/licenses/gpl-3.0.en.html
9+
//
10+
// You can be released from the requirements of the above licenses by purchasing
11+
// a commercial license. Buying such a license is mandatory if you want to
12+
// modify or otherwise use the software for commercial activities involving the
13+
// Arduino software without disclosing the source code of your own applications.
14+
// To purchase a commercial license, send an email to [email protected].
15+
16+
package httpclient
17+
18+
import (
19+
"fmt"
20+
"io/ioutil"
21+
"net/http"
22+
"net/http/httptest"
23+
"net/url"
24+
"testing"
25+
26+
"github.com/stretchr/testify/require"
27+
)
28+
29+
func TestUserAgentHeader(t *testing.T) {
30+
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
31+
fmt.Fprint(w, r.Header.Get("User-Agent"))
32+
}))
33+
defer ts.Close()
34+
35+
client := NewWithConfig(&Config{
36+
UserAgent: "test-user-agent",
37+
})
38+
39+
request, err := http.NewRequest("GET", ts.URL, nil)
40+
require.NoError(t, err)
41+
42+
response, err := client.Do(request)
43+
require.NoError(t, err)
44+
45+
b, err := ioutil.ReadAll(response.Body)
46+
require.NoError(t, err)
47+
48+
require.Equal(t, "test-user-agent", string(b))
49+
}
50+
51+
func TestProxy(t *testing.T) {
52+
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
53+
w.WriteHeader(http.StatusNoContent)
54+
}))
55+
defer ts.Close()
56+
57+
proxyURL, err := url.Parse(ts.URL)
58+
require.NoError(t, err)
59+
60+
client := NewWithConfig(&Config{
61+
Proxy: proxyURL,
62+
})
63+
64+
request, err := http.NewRequest("GET", "http://arduino.cc", nil)
65+
require.NoError(t, err)
66+
67+
response, err := client.Do(request)
68+
require.NoError(t, err)
69+
require.Equal(t, http.StatusNoContent, response.StatusCode)
70+
}

0 commit comments

Comments
 (0)