diff --git a/.gitignore b/.gitignore index 31083e9..58ba97e 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,7 @@ libchdb.tar.gz # Test binary, built with `go test -c` *.test +*.db # Output of the go coverage tool, specifically when used with LiteIDE *.out diff --git a/Makefile b/Makefile index be53ad7..5276e2b 100644 --- a/Makefile +++ b/Makefile @@ -7,10 +7,10 @@ install: curl -sL https://lib.chdb.io | bash test: - CGO_ENABLED=1 go test -v -coverprofile=coverage.out ./... + go test -v -coverprofile=coverage.out ./... run: - CGO_ENABLED=1 go run main.go + go run main.go build: - CGO_ENABLED=1 go build -ldflags '-extldflags "-Wl,-rpath,/usr/local/lib"' -o chdb-go main.go + go build -o chdb-go main.go diff --git a/chdb-purego/binding.go b/chdb-purego/binding.go index 9c684a6..e2330cf 100644 --- a/chdb-purego/binding.go +++ b/chdb-purego/binding.go @@ -39,7 +39,7 @@ var ( freeResult func(result *local_result) queryStableV2 func(argc int, argv []string) *local_result_v2 freeResultV2 func(result *local_result_v2) - connectChdb func(argc int, argv []string) **chdb_conn + connectChdb func(argc int, argv []*byte) **chdb_conn closeConn func(conn **chdb_conn) queryConn func(conn *chdb_conn, query string, format string) *local_result_v2 ) diff --git a/chdb-purego/chdb.go b/chdb-purego/chdb.go index c4171ba..6a0cdef 100644 --- a/chdb-purego/chdb.go +++ b/chdb-purego/chdb.go @@ -3,7 +3,12 @@ package chdbpurego import ( "errors" "fmt" + "os" + "path/filepath" + "strings" "unsafe" + + "golang.org/x/sys/unix" ) type result struct { @@ -141,12 +146,76 @@ func (c *connection) Ready() bool { return false } +// NewConnection is the low level function to create a new connection to the chdb server. +// using NewConnectionFromConnString is recommended. +// +// Deprecated: Use NewConnectionFromConnString instead. This function will be removed in a future version. +// // Session will keep the state of query. // If path is None, it will create a temporary directory and use it as the database path // and the temporary directory will be removed when the session is closed. // You can also pass in a path to create a database at that path where will keep your data. +// This is a thin wrapper around the connect_chdb C API. +// the argc and argv should be like: +// - argc = 1, argv = []string{"--path=/tmp/chdb"} +// - argc = 2, argv = []string{"--path=/tmp/chdb", "--readonly=1"} // -// You can also use a connection string to pass in the path and other parameters. +// Important: +// - There can be only one session at a time. If you want to create a new session, you need to close the existing one. +// - Creating a new session will close the existing one. +// - You need to ensure that the path exists before creating a new session. Or you can use NewConnectionFromConnString. +func NewConnection(argc int, argv []string) (ChdbConn, error) { + var new_argv []string + if (argc > 0 && argv[0] != "clickhouse") || argc == 0 { + new_argv = make([]string, argc+1) + new_argv[0] = "clickhouse" + copy(new_argv[1:], argv) + } else { + new_argv = argv + } + + // Remove ":memory:" if it is the only argument + if len(new_argv) == 2 && (new_argv[1] == ":memory:" || new_argv[1] == "file::memory:") { + new_argv = new_argv[:1] + } + + // Convert string slice to C-style char pointers in one step + c_argv := make([]*byte, len(new_argv)) + for i, str := range new_argv { + // Convert string to []byte and append null terminator + bytes := append([]byte(str), 0) + // Use &bytes[0] to get pointer to first byte + c_argv[i] = &bytes[0] + } + + // debug print new_argv + // for _, arg := range new_argv { + // fmt.Println("arg: ", arg) + // } + + var conn **chdb_conn + var err error + func() { + defer func() { + if r := recover(); r != nil { + err = fmt.Errorf("C++ exception: %v", r) + } + }() + conn = connectChdb(len(new_argv), c_argv) + }() + + if err != nil { + return nil, err + } + + if conn == nil { + return nil, fmt.Errorf("could not create a chdb connection") + } + return newChdbConn(conn), nil +} + +// NewConnectionFromConnString creates a new connection to the chdb server using a connection string. +// You can use a connection string to pass in the path and other parameters. // Examples: // - ":memory:" (for in-memory database) // - "test.db" (for relative path) @@ -169,10 +238,99 @@ func (c *connection) Ready() bool { // Important: // - There can be only one session at a time. If you want to create a new session, you need to close the existing one. // - Creating a new session will close the existing one. -func NewConnection(argc int, argv []string) (ChdbConn, error) { - conn := connectChdb(argc, argv) - if conn == nil { - return nil, fmt.Errorf("could not create a chdb connection") +func NewConnectionFromConnString(conn_string string) (ChdbConn, error) { + if conn_string == "" || conn_string == ":memory:" { + return NewConnection(0, []string{}) } - return newChdbConn(conn), nil + + // Handle file: prefix + workingStr := conn_string + if strings.HasPrefix(workingStr, "file:") { + workingStr = workingStr[5:] + // Handle triple slash for absolute paths + if strings.HasPrefix(workingStr, "///") { + workingStr = workingStr[2:] // Remove two slashes, keep one + } + } + + // Split path and parameters + var path string + var params []string + if queryPos := strings.Index(workingStr, "?"); queryPos != -1 { + path = workingStr[:queryPos] + paramStr := workingStr[queryPos+1:] + + // Parse parameters + for _, param := range strings.Split(paramStr, "&") { + if param == "" { + continue + } + if eqPos := strings.Index(param, "="); eqPos != -1 { + key := param[:eqPos] + value := param[eqPos+1:] + if key == "mode" && value == "ro" { + params = append(params, "--readonly=1") + } else if key == "udf_path" && value != "" { + params = append(params, "--") + params = append(params, "--user_scripts_path="+value) + params = append(params, "--user_defined_executable_functions_config="+value+"/*.xml") + } else { + params = append(params, "--"+key+"="+value) + } + } else { + params = append(params, "--"+param) + } + } + } else { + path = workingStr + } + + // Convert relative paths to absolute if needed + if path != "" && !strings.HasPrefix(path, "/") && path != ":memory:" { + absPath, err := filepath.Abs(path) + if err != nil { + return nil, fmt.Errorf("failed to resolve path: %s", path) + } + path = absPath + } + + // Check if path exists and handle directory creation/permissions + if path != "" && path != ":memory:" { + // Check if path exists + _, err := os.Stat(path) + if os.IsNotExist(err) { + // Create directory if it doesn't exist + if err := os.MkdirAll(path, 0755); err != nil { + return nil, fmt.Errorf("failed to create directory: %s", path) + } + } else if err != nil { + return nil, fmt.Errorf("failed to check directory: %s", path) + } + + // Check write permissions if not in readonly mode + isReadOnly := false + for _, param := range params { + if param == "--readonly=1" { + isReadOnly = true + break + } + } + + if !isReadOnly { + // Check write permissions by attempting to create a file + if err := unix.Access(path, unix.W_OK); err != nil { + return nil, fmt.Errorf("no write permission for directory: %s", path) + } + } + } + + // Build arguments array + argv := make([]string, 0, len(params)+2) + argv = append(argv, "clickhouse") + if path != "" && path != ":memory:" { + argv = append(argv, "--path="+path) + } + argv = append(argv, params...) + + return NewConnection(len(argv), argv) } diff --git a/chdb-purego/chdb_test.go b/chdb-purego/chdb_test.go new file mode 100644 index 0000000..d5192a4 --- /dev/null +++ b/chdb-purego/chdb_test.go @@ -0,0 +1,167 @@ +package chdbpurego + +import ( + "os" + "path/filepath" + "testing" +) + +func TestNewConnection(t *testing.T) { + tests := []struct { + name string + argc int + argv []string + wantErr bool + }{ + { + name: "empty args", + argc: 0, + argv: []string{}, + wantErr: false, + }, + { + name: "memory database", + argc: 1, + argv: []string{":memory:"}, + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + conn, err := NewConnection(tt.argc, tt.argv) + if (err != nil) != tt.wantErr { + t.Errorf("NewConnection() error = %v, wantErr %v", err, tt.wantErr) + return + } + if conn == nil && !tt.wantErr { + t.Error("NewConnection() returned nil connection without error") + return + } + if conn != nil { + defer conn.Close() + if !conn.Ready() { + t.Error("NewConnection() returned connection that is not ready") + } + } + }) + } +} + +func TestNewConnectionFromConnString(t *testing.T) { + // Create a temporary directory for testing + tmpDir, err := os.MkdirTemp("", "chdb_test_*") + if err != nil { + t.Fatalf("Failed to create temp dir: %v", err) + } + defer os.RemoveAll(tmpDir) + + tests := []struct { + name string + connStr string + wantErr bool + checkPath bool + }{ + { + name: "empty string", + connStr: "", + wantErr: false, + }, + { + name: "memory database", + connStr: ":memory:", + wantErr: false, + }, + { + name: "memory database with params", + connStr: ":memory:?verbose&log-level=test", + wantErr: false, + }, + { + name: "relative path", + connStr: "test.db", + wantErr: false, + checkPath: true, + }, + { + name: "file prefix", + connStr: "file:test.db", + wantErr: false, + checkPath: true, + }, + { + name: "absolute path", + connStr: filepath.Join(tmpDir, "test.db"), + wantErr: false, + checkPath: true, + }, + { + name: "file prefix with absolute path", + connStr: "file:" + filepath.Join(tmpDir, "test.db"), + wantErr: false, + checkPath: true, + }, + // { + // name: "readonly mode with existing dir", + // connStr: filepath.Join(tmpDir, "readonly.db") + "?mode=ro", + // wantErr: false, + // checkPath: true, + // }, + // { + // name: "readonly mode with non-existing dir", + // connStr: filepath.Join(tmpDir, "new_readonly.db") + "?mode=ro", + // wantErr: true, + // checkPath: true, + // }, + { + name: "write mode with existing dir", + connStr: filepath.Join(tmpDir, "write.db"), + wantErr: false, + checkPath: true, + }, + { + name: "write mode with non-existing dir", + connStr: filepath.Join(tmpDir, "new_write.db"), + wantErr: false, + checkPath: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + conn, err := NewConnectionFromConnString(tt.connStr) + if (err != nil) != tt.wantErr { + t.Errorf("NewConnectionFromConnString() error = %v, wantErr %v", err, tt.wantErr) + return + } + if conn == nil && !tt.wantErr { + t.Error("NewConnectionFromConnString() returned nil connection without error") + return + } + if conn != nil { + defer conn.Close() + if !conn.Ready() { + t.Error("NewConnectionFromConnString() returned connection that is not ready") + } + + // Test a simple query to verify the connection works + result, err := conn.Query("SELECT 1", "CSV") + if err != nil { + t.Errorf("Query failed: %v", err) + return + } + if result == nil { + t.Error("Query returned nil result") + return + } + if result.Error() != nil { + t.Errorf("Query result has error: %v", result.Error()) + return + } + if result.String() != "1\n" { + t.Errorf("Query result = %v, want %v", result.String(), "1\n") + } + } + }) + } +} diff --git a/chdb/session.go b/chdb/session.go index 3ed1128..374d367 100644 --- a/chdb/session.go +++ b/chdb/session.go @@ -1,7 +1,6 @@ package chdb import ( - "fmt" "os" "path/filepath" @@ -40,9 +39,8 @@ func NewSession(paths ...string) (*Session, error) { } path = tempDir isTemp = true - } - connStr := fmt.Sprintf("file:%s/chdb.db", path) + connStr := path conn, err := initConnection(connStr) if err != nil { @@ -77,6 +75,8 @@ func (s *Session) Close() { func (s *Session) Cleanup() { // Remove the session directory, no matter if it is temporary or not _ = os.RemoveAll(s.path) + s.conn.Close() + globalSession = nil } // Path returns the path of the session. diff --git a/chdb/wrapper.go b/chdb/wrapper.go index 9c19fed..53619a0 100644 --- a/chdb/wrapper.go +++ b/chdb/wrapper.go @@ -10,6 +10,7 @@ func Query(queryStr string, outputFormats ...string) (result chdbpurego.ChdbResu if len(outputFormats) > 0 { outputFormat = outputFormats[0] } + // tempSession, err := initConnection(":memory:?verbose&log-level=test") tempSession, err := initConnection(":memory:") if err != nil { return nil, err @@ -19,7 +20,5 @@ func Query(queryStr string, outputFormats ...string) (result chdbpurego.ChdbResu } func initConnection(connStr string) (result chdbpurego.ChdbConn, err error) { - argv := []string{connStr} - // Call NewConnection with the constructed arguments - return chdbpurego.NewConnection(len(argv), argv) + return chdbpurego.NewConnectionFromConnString(connStr) } diff --git a/chdb/wrapper_test.go b/chdb/wrapper_test.go index c97680a..7556e0d 100644 --- a/chdb/wrapper_test.go +++ b/chdb/wrapper_test.go @@ -1,6 +1,7 @@ package chdb import ( + "fmt" "testing" ) @@ -43,6 +44,7 @@ func TestQueryToBuffer(t *testing.T) { // Call queryToBuffer result, err := Query(tc.queryStr, tc.outputFormat) + fmt.Println("result: ", result) // Verify if tc.expectedErrMsg != "" {