From a54ca890d49e299578a90703edcddc9ffbef7325 Mon Sep 17 00:00:00 2001 From: Andrei Goncear Date: Thu, 16 Jan 2025 08:59:58 +0000 Subject: [PATCH 01/22] bump chdb v3, implemented first part of connection based query --- chdb/connection.go | 80 +++++++++++++++++++++++++ chdb/connection_test.go | 118 +++++++++++++++++++++++++++++++++++++ chdb/driver/driver.go | 41 +++++++++---- chdb/driver/driver_test.go | 57 ++++++++++++++++++ chdb/wrapper.go | 13 ++++ chdbstable/chdb.go | 55 +++++++++++++++++ chdbstable/chdb.h | 83 +++++++++++++++++++++++++- 7 files changed, 435 insertions(+), 12 deletions(-) create mode 100644 chdb/connection.go create mode 100644 chdb/connection_test.go diff --git a/chdb/connection.go b/chdb/connection.go new file mode 100644 index 0000000..bba1732 --- /dev/null +++ b/chdb/connection.go @@ -0,0 +1,80 @@ +package chdb + +import ( + "fmt" + "os" + + "github.com/chdb-io/chdb-go/chdbstable" +) + +type Connection struct { + conn *chdbstable.ChdbConn + connStr string + path string + isTemp bool +} + +// NewSession creates a new session with the given path. +// If path is empty, a temporary directory is created. +// Note: The temporary directory is removed when Close is called. +func NewConnection(paths ...string) (*Connection, error) { + path := "" + if len(paths) > 0 { + path = paths[0] + } + isTemp := false + if path == "" { + // Create a temporary directory + tempDir, err := os.MkdirTemp("", "chdb_") + if err != nil { + return nil, err + } + path = tempDir + isTemp = true + + } + connStr := fmt.Sprintf("file:%s/chdb.db", path) + + conn, err := initConnection(connStr) + if err != nil { + return nil, err + } + return &Connection{connStr: connStr, path: path, isTemp: isTemp, conn: conn}, nil +} + +// Query calls queryToBuffer with a default output format of "CSV" if not provided. +func (s *Connection) Query(queryStr string, outputFormats ...string) (result *chdbstable.LocalResult, err error) { + outputFormat := "CSV" // Default value + if len(outputFormats) > 0 { + outputFormat = outputFormats[0] + } + + return connQueryToBuffer(s.conn, queryStr, outputFormat) +} + +// Close closes the session and removes the temporary directory +// +// temporary directory is created when NewSession was called with an empty path. +func (s *Connection) Close() { + // Remove the temporary directory if it starts with "chdb_" + s.conn.Close() + if s.isTemp { + s.Cleanup() + } +} + +// Cleanup closes the session and removes the directory. +func (s *Connection) Cleanup() { + // Remove the session directory, no matter if it is temporary or not + _ = os.RemoveAll(s.path) +} + +// Path returns the path of the session. +func (s *Connection) Path() string { + return s.path +} + +// IsTemp returns whether the session is temporary. +func (s *Connection) IsTemp() bool { + return s.isTemp +} diff --git a/chdb/connection_test.go b/chdb/connection_test.go new file mode 100644 index 0000000..e6b2d7a --- /dev/null +++ b/chdb/connection_test.go @@ -0,0 +1,118 @@ +package chdb + +import ( + "os" + "path/filepath" + "testing" +) + +// TestNewconnection tests the creation of a new connection. +func TestNewConnection(t *testing.T) { + connection, err := NewConnection() + if err != nil { + t.Fatalf("Failed to create new connection: %s", err) + } + defer connection.Cleanup() + + // Check if the connection directory exists + if _, err := os.Stat(connection.Path()); os.IsNotExist(err) { + t.Errorf("connection directory does not exist: %s", connection.Path()) + } + + // Check if the connection is temporary + if !connection.IsTemp() { + t.Errorf("Expected connection to be temporary") + } +} + +// TestconnectionClose tests the Close method of the connection. +func TestConnectionClose(t *testing.T) { + connection, _ := NewConnection() + defer connection.Cleanup() // Cleanup in case Close fails + + // Close the connection + connection.Close() + + // Check if the connection directory has been removed + if _, err := os.Stat(connection.Path()); !os.IsNotExist(err) { + t.Errorf("connection directory should be removed after Close: %s", connection.Path()) + } +} + +// TestconnectionCleanup tests the Cleanup method of the connection. +func TestConnectionCleanup(t *testing.T) { + connection, _ := NewConnection() + + // Cleanup the connection + connection.Cleanup() + + // Check if the connection directory has been removed + if _, err := os.Stat(connection.Path()); !os.IsNotExist(err) { + t.Errorf("connection directory should be removed after Cleanup: %s", connection.Path()) + } +} + +// TestQuery tests the Query method of the connection. +func TestQueryOnConnection(t *testing.T) { + path := filepath.Join(os.TempDir(), "chdb_test") + defer os.RemoveAll(path) + connection, _ := NewConnection(path) + defer connection.Cleanup() + + connection.Query("CREATE DATABASE IF NOT EXISTS testdb; " + + "CREATE TABLE IF NOT EXISTS testdb.testtable (id UInt32) ENGINE = MergeTree() ORDER BY id;") + + connection.Query(" INSERT INTO testdb.testtable VALUES (1), (2), (3);") + + ret, err := connection.Query("SELECT * FROM testtable;") + if err != nil { + t.Errorf("Query failed: %s", err) + } + t.Errorf("result is: %s", string(ret.Buf())) + if string(ret.Buf()) != "1\n2\n3\n" { + t.Errorf("Query result should be 1\n2\n3\n, got %s", string(ret.Buf())) + } +} + +func TestQueryOnConnection2(t *testing.T) { + path := filepath.Join(os.TempDir(), "chdb_test") + defer os.RemoveAll(path) + connection, _ := NewConnection(path) + defer connection.Cleanup() + + ret, err := connection.Query("SELECT number+1 from system.numbers limit 3") + if err != nil { + t.Errorf("Query failed: %s", err) + } + if string(ret.Buf()) != "1\n2\n3\n" { + t.Errorf("Query result should be 1\n2\n3\n, got %s", string(ret.Buf())) + } +} + +func TestConnectionPathAndIsTemp(t *testing.T) { + // Create a new connection and check its Path and IsTemp + connection, _ := NewConnection() + defer connection.Cleanup() + + if connection.Path() == "" { + t.Errorf("connection path should not be empty") + } + + if !connection.IsTemp() { + t.Errorf("connection should be temporary") + } + + // Create a new connection with a specific path and check its Path and IsTemp + path := filepath.Join(os.TempDir(), "chdb_test2") + defer os.RemoveAll(path) + connection, _ = NewConnection(path) + defer connection.Cleanup() + + if connection.Path() != path { + t.Errorf("connection path should be %s, got %s", path, connection.Path()) + } + + if connection.IsTemp() { + t.Errorf("connection should not be temporary") + } +} diff --git a/chdb/driver/driver.go b/chdb/driver/driver.go index 22f7a57..0181cf5 100644 --- a/chdb/driver/driver.go +++ b/chdb/driver/driver.go @@ -27,6 +27,7 @@ const ( const ( sessionOptionKey = "session" + connectionOptionKey = "connection" udfPathOptionKey = "udfPath" driverTypeKey = "driverType" useUnsafeStringReaderKey = "useUnsafeStringReader" @@ -136,11 +137,13 @@ func (e *execResult) RowsAffected() (int64, error) { type queryHandle func(string, ...string) (*chdbstable.LocalResult, error) type connector struct { - udfPath string - driverType DriverType - bufferSize int - useUnsafe bool - session *chdb.Session + udfPath string + driverType DriverType + bufferSize int + useUnsafe bool + session *chdb.Session + connection *chdb.Connection + useConnection bool } // Connect returns a connection to a database. @@ -186,6 +189,17 @@ func NewConnect(opts map[string]string) (ret *connector, err error) { return nil, err } } + connectionStr, ok := opts[connectionOptionKey] + if ok { + if ret.session != nil { + return nil, fmt.Errorf("could not use both session & connection. please use one of the two") + } + ret.connection, err = chdb.NewConnection(connectionStr) + if err != nil { + return nil, err + } + ret.useConnection = true + } driverType, ok := opts[driverTypeKey] if ok { ret.driverType = parseDriverType(driverType) @@ -238,12 +252,14 @@ func (d Driver) OpenConnector(name string) (driver.Connector, error) { } type conn struct { - udfPath string - driverType DriverType - bufferSize int - useUnsafe bool - session *chdb.Session - QueryFun queryHandle + udfPath string + driverType DriverType + bufferSize int + useUnsafe bool + useConnection bool + session *chdb.Session + connection *chdb.Connection + QueryFun queryHandle } func prepareValues(values []driver.Value) []driver.NamedValue { @@ -267,6 +283,9 @@ func (c *conn) SetupQueryFun() { if c.session != nil { c.QueryFun = c.session.Query } + if c.connection != nil { + c.QueryFun = c.connection.Query + } } func (c *conn) Query(query string, values []driver.Value) (driver.Rows, error) { diff --git a/chdb/driver/driver_test.go b/chdb/driver/driver_test.go index ebbfcb1..799da09 100644 --- a/chdb/driver/driver_test.go +++ b/chdb/driver/driver_test.go @@ -168,6 +168,63 @@ func TestDbWithSession(t *testing.T) { } } +func TestDbWithConnection(t *testing.T) { + connectionDir, err := os.MkdirTemp("", "unittest-connectiondata") + if err != nil { + t.Fatalf("create temp directory fail, err: %s", err) + } + defer os.RemoveAll(connectionDir) + connection, err := chdb.NewConnection(connectionDir) + if err != nil { + t.Fatalf("new connection fail, err: %s", err) + } + defer connection.Cleanup() + + connection.Query("CREATE DATABASE IF NOT EXISTS testdb; " + + "CREATE TABLE IF NOT EXISTS testdb.testtable (id UInt32) ENGINE = MergeTree() ORDER BY id;") + + connection.Query("USE testdb; INSERT INTO testtable VALUES (1), (2), (3);") + + ret, err := connection.Query("SELECT * FROM testtable;") + if err != nil { + t.Fatalf("Query fail, err: %s", err) + } + if string(ret.Buf()) != "1\n2\n3\n" { + t.Errorf("Query result should be 1\n2\n3\n, got %s", string(ret.Buf())) + } + db, err := sql.Open("chdb", fmt.Sprintf("connection=%s", connectionDir)) + if err != nil { + t.Fatalf("open db fail, err: %s", err) + } + if db.Ping() != nil { + t.Fatalf("ping db fail, err: %s", err) + } + rows, err := db.Query("select * from testtable;") + if err != nil { + t.Fatalf("exec create function fail, err: %s", err) + } + defer rows.Close() + cols, err := rows.Columns() + if err != nil { + t.Fatalf("get result columns fail, err: %s", err) + } + if len(cols) != 1 { + t.Fatalf("result columns length shoule be 3, actual: %d", len(cols)) + } + var bar = 0 + var count = 1 + for rows.Next() { + err = rows.Scan(&bar) + if err != nil { + t.Fatalf("scan fail, err: %s", err) + } + if bar != count { + t.Fatalf("result is not match, want: %d actual: %d", count, bar) + } + count++ + } +} + func TestQueryRow(t *testing.T) { sessionDir, err := os.MkdirTemp("", "unittest-sessiondata") if err != nil { diff --git a/chdb/wrapper.go b/chdb/wrapper.go index be5ab63..ad0eb40 100644 --- a/chdb/wrapper.go +++ b/chdb/wrapper.go @@ -40,3 +40,16 @@ func queryToBuffer(queryStr, outputFormat, path, udfPath string) (result *chdbst // Call QueryStable with the constructed arguments return chdbstable.QueryStable(len(argv), argv) } + +func initConnection(connStr string) (result *chdbstable.ChdbConn, err error) { + argv := []string{connStr} + // Call NewConnection with the constructed arguments + return chdbstable.NewConnection(len(argv), argv) +} + +func connQueryToBuffer(conn *chdbstable.ChdbConn, queryStr, outputFormat string) (result *chdbstable.LocalResult, err error) { + if outputFormat == "" { + outputFormat = "CSV" + } + return conn.QueryConn(queryStr, outputFormat) +} diff --git a/chdbstable/chdb.go b/chdbstable/chdb.go index c02aba2..48fb506 100644 --- a/chdbstable/chdb.go +++ b/chdbstable/chdb.go @@ -8,6 +8,7 @@ package chdbstable import "C" import ( "errors" + "fmt" "runtime" "unsafe" ) @@ -29,6 +30,10 @@ type LocalResult struct { cResult *C.struct_local_result_v2 } +type ChdbConn struct { + conn *C.struct_chdb_conn +} + // newLocalResult creates a new LocalResult and sets a finalizer to free C memory. func newLocalResult(cResult *C.struct_local_result_v2) *LocalResult { result := &LocalResult{cResult: cResult} @@ -36,6 +41,30 @@ func newLocalResult(cResult *C.struct_local_result_v2) *LocalResult { return result } +// newChdbConn creates a new ChdbConn and sets a finalizer to close the connection (and thus free the memory) +func newChdbConn(conn *C.struct_chdb_conn) *ChdbConn { + result := &ChdbConn{conn: conn} + runtime.SetFinalizer(result, closeChdbConn) + return result +} + +func NewConnection(argc int, argv []string) (*ChdbConn, error) { + cArgv := make([]*C.char, len(argv)) + for i, s := range argv { + cArgv[i] = C.CString(s) + defer C.free(unsafe.Pointer(cArgv[i])) + } + conn := C.connect_chdb(C.int(argc), &cArgv[0]) + if conn == nil { + return nil, fmt.Errorf("could not create a chdb connection") + } + return newChdbConn(*conn), nil +} + +func closeChdbConn(conn *ChdbConn) { + C.close_conn(&conn.conn) +} + // freeLocalResult is called by the garbage collector. func freeLocalResult(result *LocalResult) { C.free_result_v2(result.cResult) @@ -62,6 +91,32 @@ func QueryStable(argc int, argv []string) (result *LocalResult, err error) { return newLocalResult(cResult), nil } +// QueryStable calls the C function query_conn. +func (c *ChdbConn) QueryConn(queryStr string, formatStr string) (result *LocalResult, err error) { + + query := C.CString(queryStr) + format := C.CString(formatStr) + // free the strings in the C heap + defer C.free(unsafe.Pointer(query)) + defer C.free(unsafe.Pointer(format)) + + cResult := C.query_conn(c.conn, query, format) + if cResult == nil { + // According to the C ABI of chDB v1.2.0, the C function query_stable_v2 + // returns nil if the query returns no data. This is not an error. We + // will change this behavior in the future. + return newLocalResult(cResult), nil + } + if cResult.error_message != nil { + return nil, &ChdbError{msg: C.GoString(cResult.error_message)} + } + return newLocalResult(cResult), nil +} + +func (c *ChdbConn) Close() { + C.close_conn(&c.conn) +} + // Accessor methods to access fields of the local_result_v2 struct. func (r *LocalResult) Buf() []byte { if r.cResult == nil { diff --git a/chdbstable/chdb.h b/chdbstable/chdb.h index 821e3e8..ebc2009 100644 --- a/chdbstable/chdb.h +++ b/chdbstable/chdb.h @@ -1,10 +1,15 @@ #pragma once #ifdef __cplusplus +# include # include # include +# include +# include +# include extern "C" { #else +# include # include # include #endif @@ -20,6 +25,18 @@ struct local_result uint64_t bytes_read; }; +#ifdef __cplusplus +struct local_result_v2 +{ + char * buf = nullptr; + size_t len = 0; + void * _vec = nullptr; // std::vector *, for freeing + double elapsed = 0.0; + uint64_t rows_read = 0; + uint64_t bytes_read = 0; + char * error_message = nullptr; +}; +#else struct local_result_v2 { char * buf; @@ -30,6 +47,7 @@ struct local_result_v2 uint64_t bytes_read; char * error_message; }; +#endif CHDB_EXPORT struct local_result * query_stable(int argc, char ** argv); CHDB_EXPORT void free_result(struct local_result * result); @@ -38,5 +56,68 @@ CHDB_EXPORT struct local_result_v2 * query_stable_v2(int argc, char ** argv); CHDB_EXPORT void free_result_v2(struct local_result_v2 * result); #ifdef __cplusplus -} +struct query_request +{ + std::string query; + std::string format; +}; + +struct query_queue +{ + std::mutex mutex; + std::condition_variable query_cv; // For query submission + std::condition_variable result_cv; // For query result retrieval + query_request current_query; + local_result_v2 * current_result = nullptr; + bool has_query = false; + bool shutdown = false; + bool cleanup_done = false; +}; #endif + +/** + * Connection structure for chDB + * Contains server instance, connection state, and query processing queue + */ +struct chdb_conn +{ + void * server; /* ClickHouse LocalServer instance */ + bool connected; /* Connection state flag */ + void * queue; /* Query processing queue */ +}; + +/** + * Creates a new chDB connection. + * Only one active connection is allowed per process. + * Creating a new connection with different path requires closing existing connection. + * + * @param argc Number of command-line arguments + * @param argv Command-line arguments array (--path= to specify database location) + * @return Pointer to connection pointer, or NULL on failure + * @note Default path is ":memory:" if not specified + */ +CHDB_EXPORT struct chdb_conn ** connect_chdb(int argc, char ** argv); + +/** + * Closes an existing chDB connection and cleans up resources. + * Thread-safe function that handles connection shutdown and cleanup. + * + * @param conn Pointer to connection pointer to close + */ +CHDB_EXPORT void close_conn(struct chdb_conn ** conn); + +/** + * Executes a query on the given connection. + * Thread-safe function that handles query execution in a separate thread. + * + * @param conn Connection to execute query on + * @param query SQL query string to execute + * @param format Output format string (e.g., "CSV", default format) + * @return Query result structure containing output or error message + * @note Returns error result if connection is invalid or closed + */ +CHDB_EXPORT struct local_result_v2 * query_conn(struct chdb_conn * conn, const char * query, const char * format); + +#ifdef __cplusplus +} +#endif \ No newline at end of file From 0b87dd849384b4cc601c9d540c693c7c98186a1f Mon Sep 17 00:00:00 2001 From: Andrei Goncear Date: Thu, 16 Jan 2025 09:31:55 +0000 Subject: [PATCH 02/22] add tests for `connect` method --- chdb/driver/driver.go | 6 +++-- chdb/driver/driver_test.go | 54 ++++++++++++++++++++++++++++++++++++-- 2 files changed, 56 insertions(+), 4 deletions(-) diff --git a/chdb/driver/driver.go b/chdb/driver/driver.go index 0181cf5..6a1fe44 100644 --- a/chdb/driver/driver.go +++ b/chdb/driver/driver.go @@ -153,7 +153,9 @@ func (c *connector) Connect(ctx context.Context) (driver.Conn, error) { } cc := &conn{ udfPath: c.udfPath, session: c.session, - driverType: c.driverType, bufferSize: c.bufferSize, + connection: c.connection, + useConnection: c.useConnection, + driverType: c.driverType, bufferSize: c.bufferSize, useUnsafe: c.useUnsafe, } cc.SetupQueryFun() @@ -353,7 +355,7 @@ func (c *conn) QueryContext(ctx context.Context, query string, args []driver.Nam } buf := result.Buf() - if buf == nil { + if len(buf) == 0 { return nil, fmt.Errorf("result is nil") } return c.driverType.PrepareRows(result, buf, c.bufferSize, c.useUnsafe) diff --git a/chdb/driver/driver_test.go b/chdb/driver/driver_test.go index 799da09..223eb26 100644 --- a/chdb/driver/driver_test.go +++ b/chdb/driver/driver_test.go @@ -185,14 +185,14 @@ func TestDbWithConnection(t *testing.T) { connection.Query("USE testdb; INSERT INTO testtable VALUES (1), (2), (3);") - ret, err := connection.Query("SELECT * FROM testtable;") + ret, err := connection.Query("SELECT * FROM testdb.testtable;") if err != nil { t.Fatalf("Query fail, err: %s", err) } if string(ret.Buf()) != "1\n2\n3\n" { t.Errorf("Query result should be 1\n2\n3\n, got %s", string(ret.Buf())) } - db, err := sql.Open("chdb", fmt.Sprintf("connection=%s", connectionDir)) + db, err := sql.Open("chdb", fmt.Sprintf("connection=file:%s/chdb.db", connectionDir)) if err != nil { t.Fatalf("open db fail, err: %s", err) } @@ -225,6 +225,56 @@ func TestDbWithConnection(t *testing.T) { } } +func TestDbWithConnectionSqlDriverOnly(t *testing.T) { + connectionDir, err := os.MkdirTemp("", "unittest-connectiondata") + if err != nil { + t.Fatalf("create temp directory fail, err: %s", err) + } + defer os.RemoveAll(connectionDir) + db, err := sql.Open("chdb", fmt.Sprintf("connection=file:%s/chdb.db", connectionDir)) + if err != nil { + t.Fatalf("open db fail, err: %s", err) + } + if db.Ping() != nil { + t.Fatalf("ping db fail, err: %s", err) + } + + _, err = db.Exec("CREATE DATABASE IF NOT EXISTS testdb; " + + "CREATE TABLE IF NOT EXISTS testdb.testtable (id UInt32) ENGINE = MergeTree() ORDER BY id;") + if err != nil { + t.Fatalf("could not create database & table: %s", err) + } + _, err = db.Exec("INSERT INTO testdb.testtable VALUES (1), (2), (3);") + if err != nil { + t.Fatalf("could not insert rows in the table: %s", err) + } + + rows, err := db.Query("select * from testdb.testtable;") + if err != nil { + t.Fatalf("exec create function fail, err: %s", err) + } + defer rows.Close() + cols, err := rows.Columns() + if err != nil { + t.Fatalf("get result columns fail, err: %s", err) + } + if len(cols) != 1 { + t.Fatalf("result columns length shoule be 3, actual: %d", len(cols)) + } + var bar = 0 + var count = 1 + for rows.Next() { + err = rows.Scan(&bar) + if err != nil { + t.Fatalf("scan fail, err: %s", err) + } + if bar != count { + t.Fatalf("result is not match, want: %d actual: %d", count, bar) + } + count++ + } +} + func TestQueryRow(t *testing.T) { sessionDir, err := os.MkdirTemp("", "unittest-sessiondata") if err != nil { From 5fa7af5e68c8a5515427b2f263fe1c4a194714ab Mon Sep 17 00:00:00 2001 From: Andrei Goncear Date: Thu, 16 Jan 2025 09:37:37 +0000 Subject: [PATCH 03/22] fix comments on connection.go --- chdb/connection.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/chdb/connection.go b/chdb/connection.go index bba1732..a18c687 100644 --- a/chdb/connection.go +++ b/chdb/connection.go @@ -14,7 +14,7 @@ type Connection struct { isTemp bool } -// NewSession creates a new session with the given path. +// NewConnection creates a new connection with the given path. // If path is empty, a temporary directory is created. // Note: The temporary directory is removed when Close is called. func NewConnection(paths ...string) (*Connection, error) { @@ -52,9 +52,9 @@ func (s *Connection) Query(queryStr string, outputFormats ...string) (result *ch return connQueryToBuffer(s.conn, queryStr, outputFormat) } -// Close closes the session and removes the temporary directory +// Close closes the connection and removes the temporary directory // -// temporary directory is created when NewSession was called with an empty path. +// temporary directory is created when Newconnection was called with an empty path. func (s *Connection) Close() { // Remove the temporary directory if it starts with "chdb_" s.conn.Close() @@ -63,18 +63,18 @@ func (s *Connection) Close() { } } -// Cleanup closes the session and removes the directory. +// Cleanup closes the connection and removes the directory. func (s *Connection) Cleanup() { - // Remove the session directory, no matter if it is temporary or not + // Remove the connection directory, no matter if it is temporary or not _ = os.RemoveAll(s.path) } -// Path returns the path of the session. +// Path returns the path of the connection. func (s *Connection) Path() string { return s.path } -// IsTemp returns whether the session is temporary. +// IsTemp returns whether the connection is temporary. func (s *Connection) IsTemp() bool { return s.isTemp } From de314c6b5edd6ff20f228ab8e835a1729051ff7c Mon Sep 17 00:00:00 2001 From: Andrei Goncear Date: Thu, 16 Jan 2025 09:43:31 +0000 Subject: [PATCH 04/22] fix flaky test --- chdb/connection_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/chdb/connection_test.go b/chdb/connection_test.go index e6b2d7a..f77831e 100644 --- a/chdb/connection_test.go +++ b/chdb/connection_test.go @@ -64,11 +64,10 @@ func TestQueryOnConnection(t *testing.T) { connection.Query(" INSERT INTO testdb.testtable VALUES (1), (2), (3);") - ret, err := connection.Query("SELECT * FROM testtable;") + ret, err := connection.Query("SELECT * FROM testdb.testtable;") if err != nil { t.Errorf("Query failed: %s", err) } - t.Errorf("result is: %s", string(ret.Buf())) if string(ret.Buf()) != "1\n2\n3\n" { t.Errorf("Query result should be 1\n2\n3\n, got %s", string(ret.Buf())) } From 8980a7af863116ec95fcd53492702a230fbab348 Mon Sep 17 00:00:00 2001 From: Andrei Goncear Date: Thu, 16 Jan 2025 09:46:31 +0000 Subject: [PATCH 05/22] add parquet with connection test --- chdb/driver/parquet_test.go | 57 +++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/chdb/driver/parquet_test.go b/chdb/driver/parquet_test.go index 44fdcc1..e88d459 100644 --- a/chdb/driver/parquet_test.go +++ b/chdb/driver/parquet_test.go @@ -104,3 +104,60 @@ func TestDBWithParquetSession(t *testing.T) { count++ } } + +func TestDBWithParquetConnection(t *testing.T) { + connectionDir, err := os.MkdirTemp("", "unittest-connectiondata") + if err != nil { + t.Fatalf("create temp directory fail, err: %s", err) + } + defer os.RemoveAll(connectionDir) + connection, err := chdb.NewConnection(connectionDir) + if err != nil { + t.Fatalf("new connection fail, err: %s", err) + } + defer connection.Cleanup() + + connection.Query("CREATE DATABASE IF NOT EXISTS testdb; " + + "CREATE TABLE IF NOT EXISTS testdb.testtable (id UInt32) ENGINE = MergeTree() ORDER BY id;") + + connection.Query("INSERT INTO testdb.testtable VALUES (1), (2), (3);") + + ret, err := connection.Query("SELECT * FROM testdb.testtable;") + if err != nil { + t.Fatalf("Query fail, err: %s", err) + } + if string(ret.Buf()) != "1\n2\n3\n" { + t.Errorf("Query result should be 1\n2\n3\n, got %s", string(ret.Buf())) + } + db, err := sql.Open("chdb", fmt.Sprintf("connection=file:%s/chdb.db;driverType=%s", connectionDir, "PARQUET")) + if err != nil { + t.Fatalf("open db fail, err: %s", err) + } + if db.Ping() != nil { + t.Fatalf("ping db fail, err: %s", err) + } + rows, err := db.Query("select * from testdb.testtable;") + if err != nil { + t.Fatalf("exec create function fail, err: %s", err) + } + defer rows.Close() + cols, err := rows.Columns() + if err != nil { + t.Fatalf("get result columns fail, err: %s", err) + } + if len(cols) != 1 { + t.Fatalf("result columns length shoule be 3, actual: %d", len(cols)) + } + var bar = 0 + var count = 1 + for rows.Next() { + err = rows.Scan(&bar) + if err != nil { + t.Fatalf("scan fail, err: %s", err) + } + if bar != count { + t.Fatalf("result is not match, want: %d actual: %d", count, bar) + } + count++ + } +} From b2ba138627550c8779b3b0c5d425fe233edc5376 Mon Sep 17 00:00:00 2001 From: Andrei Goncear Date: Mon, 10 Feb 2025 09:36:55 +0000 Subject: [PATCH 06/22] change session mode to sqlite like api (v3.0.0) --- chdb/connection.go | 80 ------------------- chdb/connection_test.go | 117 ---------------------------- chdb/driver/driver.go | 53 +++++-------- chdb/driver/driver_test.go | 150 ++++++++++++++++++------------------ chdb/driver/parquet_test.go | 47 +++-------- chdb/session.go | 40 ++++++++-- chdb/session_test.go | 50 ------------ 7 files changed, 141 insertions(+), 396 deletions(-) delete mode 100644 chdb/connection.go delete mode 100644 chdb/connection_test.go diff --git a/chdb/connection.go b/chdb/connection.go deleted file mode 100644 index a18c687..0000000 --- a/chdb/connection.go +++ /dev/null @@ -1,80 +0,0 @@ -package chdb - -import ( - "fmt" - "os" - - "github.com/chdb-io/chdb-go/chdbstable" -) - -type Connection struct { - conn *chdbstable.ChdbConn - connStr string - path string - isTemp bool -} - -// NewConnection creates a new connection with the given path. -// If path is empty, a temporary directory is created. -// Note: The temporary directory is removed when Close is called. -func NewConnection(paths ...string) (*Connection, error) { - path := "" - if len(paths) > 0 { - path = paths[0] - } - isTemp := false - if path == "" { - // Create a temporary directory - tempDir, err := os.MkdirTemp("", "chdb_") - if err != nil { - return nil, err - } - path = tempDir - isTemp = true - - } - connStr := fmt.Sprintf("file:%s/chdb.db", path) - - conn, err := initConnection(connStr) - if err != nil { - return nil, err - } - return &Connection{connStr: connStr, path: path, isTemp: isTemp, conn: conn}, nil -} - -// Query calls queryToBuffer with a default output format of "CSV" if not provided. -func (s *Connection) Query(queryStr string, outputFormats ...string) (result *chdbstable.LocalResult, err error) { - outputFormat := "CSV" // Default value - if len(outputFormats) > 0 { - outputFormat = outputFormats[0] - } - - return connQueryToBuffer(s.conn, queryStr, outputFormat) -} - -// Close closes the connection and removes the temporary directory -// -// temporary directory is created when Newconnection was called with an empty path. -func (s *Connection) Close() { - // Remove the temporary directory if it starts with "chdb_" - s.conn.Close() - if s.isTemp { - s.Cleanup() - } -} - -// Cleanup closes the connection and removes the directory. -func (s *Connection) Cleanup() { - // Remove the connection directory, no matter if it is temporary or not - _ = os.RemoveAll(s.path) -} - -// Path returns the path of the connection. -func (s *Connection) Path() string { - return s.path -} - -// IsTemp returns whether the connection is temporary. -func (s *Connection) IsTemp() bool { - return s.isTemp -} diff --git a/chdb/connection_test.go b/chdb/connection_test.go deleted file mode 100644 index f77831e..0000000 --- a/chdb/connection_test.go +++ /dev/null @@ -1,117 +0,0 @@ -package chdb - -import ( - "os" - "path/filepath" - "testing" -) - -// TestNewconnection tests the creation of a new connection. -func TestNewConnection(t *testing.T) { - connection, err := NewConnection() - if err != nil { - t.Fatalf("Failed to create new connection: %s", err) - } - defer connection.Cleanup() - - // Check if the connection directory exists - if _, err := os.Stat(connection.Path()); os.IsNotExist(err) { - t.Errorf("connection directory does not exist: %s", connection.Path()) - } - - // Check if the connection is temporary - if !connection.IsTemp() { - t.Errorf("Expected connection to be temporary") - } -} - -// TestconnectionClose tests the Close method of the connection. -func TestConnectionClose(t *testing.T) { - connection, _ := NewConnection() - defer connection.Cleanup() // Cleanup in case Close fails - - // Close the connection - connection.Close() - - // Check if the connection directory has been removed - if _, err := os.Stat(connection.Path()); !os.IsNotExist(err) { - t.Errorf("connection directory should be removed after Close: %s", connection.Path()) - } -} - -// TestconnectionCleanup tests the Cleanup method of the connection. -func TestConnectionCleanup(t *testing.T) { - connection, _ := NewConnection() - - // Cleanup the connection - connection.Cleanup() - - // Check if the connection directory has been removed - if _, err := os.Stat(connection.Path()); !os.IsNotExist(err) { - t.Errorf("connection directory should be removed after Cleanup: %s", connection.Path()) - } -} - -// TestQuery tests the Query method of the connection. -func TestQueryOnConnection(t *testing.T) { - path := filepath.Join(os.TempDir(), "chdb_test") - defer os.RemoveAll(path) - connection, _ := NewConnection(path) - defer connection.Cleanup() - - connection.Query("CREATE DATABASE IF NOT EXISTS testdb; " + - "CREATE TABLE IF NOT EXISTS testdb.testtable (id UInt32) ENGINE = MergeTree() ORDER BY id;") - - connection.Query(" INSERT INTO testdb.testtable VALUES (1), (2), (3);") - - ret, err := connection.Query("SELECT * FROM testdb.testtable;") - if err != nil { - t.Errorf("Query failed: %s", err) - } - if string(ret.Buf()) != "1\n2\n3\n" { - t.Errorf("Query result should be 1\n2\n3\n, got %s", string(ret.Buf())) - } -} - -func TestQueryOnConnection2(t *testing.T) { - path := filepath.Join(os.TempDir(), "chdb_test") - defer os.RemoveAll(path) - connection, _ := NewConnection(path) - defer connection.Cleanup() - - ret, err := connection.Query("SELECT number+1 from system.numbers limit 3") - if err != nil { - t.Errorf("Query failed: %s", err) - } - if string(ret.Buf()) != "1\n2\n3\n" { - t.Errorf("Query result should be 1\n2\n3\n, got %s", string(ret.Buf())) - } -} - -func TestConnectionPathAndIsTemp(t *testing.T) { - // Create a new connection and check its Path and IsTemp - connection, _ := NewConnection() - defer connection.Cleanup() - - if connection.Path() == "" { - t.Errorf("connection path should not be empty") - } - - if !connection.IsTemp() { - t.Errorf("connection should be temporary") - } - - // Create a new connection with a specific path and check its Path and IsTemp - path := filepath.Join(os.TempDir(), "chdb_test2") - defer os.RemoveAll(path) - connection, _ = NewConnection(path) - defer connection.Cleanup() - - if connection.Path() != path { - t.Errorf("connection path should be %s, got %s", path, connection.Path()) - } - - if connection.IsTemp() { - t.Errorf("connection should not be temporary") - } -} diff --git a/chdb/driver/driver.go b/chdb/driver/driver.go index 6a1fe44..bd0c101 100644 --- a/chdb/driver/driver.go +++ b/chdb/driver/driver.go @@ -27,7 +27,6 @@ const ( const ( sessionOptionKey = "session" - connectionOptionKey = "connection" udfPathOptionKey = "udfPath" driverTypeKey = "driverType" useUnsafeStringReaderKey = "useUnsafeStringReader" @@ -137,13 +136,11 @@ func (e *execResult) RowsAffected() (int64, error) { type queryHandle func(string, ...string) (*chdbstable.LocalResult, error) type connector struct { - udfPath string - driverType DriverType - bufferSize int - useUnsafe bool - session *chdb.Session - connection *chdb.Connection - useConnection bool + udfPath string + driverType DriverType + bufferSize int + useUnsafe bool + session *chdb.Session } // Connect returns a connection to a database. @@ -153,9 +150,7 @@ func (c *connector) Connect(ctx context.Context) (driver.Conn, error) { } cc := &conn{ udfPath: c.udfPath, session: c.session, - connection: c.connection, - useConnection: c.useConnection, - driverType: c.driverType, bufferSize: c.bufferSize, + driverType: c.driverType, bufferSize: c.bufferSize, useUnsafe: c.useUnsafe, } cc.SetupQueryFun() @@ -191,17 +186,6 @@ func NewConnect(opts map[string]string) (ret *connector, err error) { return nil, err } } - connectionStr, ok := opts[connectionOptionKey] - if ok { - if ret.session != nil { - return nil, fmt.Errorf("could not use both session & connection. please use one of the two") - } - ret.connection, err = chdb.NewConnection(connectionStr) - if err != nil { - return nil, err - } - ret.useConnection = true - } driverType, ok := opts[driverTypeKey] if ok { ret.driverType = parseDriverType(driverType) @@ -230,6 +214,12 @@ func NewConnect(opts map[string]string) (ret *connector, err error) { if ok { ret.udfPath = udfPath } + if ret.session == nil { + ret.session, err = chdb.NewSession() + if err != nil { + return nil, err + } + } return } @@ -254,14 +244,13 @@ func (d Driver) OpenConnector(name string) (driver.Connector, error) { } type conn struct { - udfPath string - driverType DriverType - bufferSize int - useUnsafe bool - useConnection bool - session *chdb.Session - connection *chdb.Connection - QueryFun queryHandle + udfPath string + driverType DriverType + bufferSize int + useUnsafe bool + session *chdb.Session + + QueryFun queryHandle } func prepareValues(values []driver.Value) []driver.NamedValue { @@ -285,9 +274,7 @@ func (c *conn) SetupQueryFun() { if c.session != nil { c.QueryFun = c.session.Query } - if c.connection != nil { - c.QueryFun = c.connection.Query - } + } func (c *conn) Query(query string, values []driver.Value) (driver.Rows, error) { diff --git a/chdb/driver/driver_test.go b/chdb/driver/driver_test.go index 223eb26..d765e09 100644 --- a/chdb/driver/driver_test.go +++ b/chdb/driver/driver_test.go @@ -9,6 +9,39 @@ import ( "github.com/chdb-io/chdb-go/chdb" ) +var ( + session *chdb.Session +) + +func globalSetup() error { + sess, err := chdb.NewSession() + if err != nil { + return err + } + session = sess + return nil +} + +func globalTeardown() { + session.Cleanup() + session.Close() +} + +func TestMain(m *testing.M) { + if err := globalSetup(); err != nil { + fmt.Println("Global setup failed:", err) + os.Exit(1) + } + // Run all tests. + exitCode := m.Run() + + // Global teardown: clean up any resources here. + globalTeardown() + + // Exit with the code returned by m.Run(). + os.Exit(exitCode) +} + func TestDb(t *testing.T) { db, err := sql.Open("chdb", "") if err != nil { @@ -112,37 +145,27 @@ func TestDbWithOpt(t *testing.T) { } func TestDbWithSession(t *testing.T) { - sessionDir, err := os.MkdirTemp("", "unittest-sessiondata") - if err != nil { - t.Fatalf("create temp directory fail, err: %s", err) - } - defer os.RemoveAll(sessionDir) - session, err := chdb.NewSession(sessionDir) - if err != nil { - t.Fatalf("new session fail, err: %s", err) - } - defer session.Cleanup() - session.Query("CREATE DATABASE IF NOT EXISTS testdb; " + - "CREATE TABLE IF NOT EXISTS testdb.testtable (id UInt32) ENGINE = MergeTree() ORDER BY id;") + session.Query( + "CREATE TABLE IF NOT EXISTS TestDbWithSession (id UInt32) ENGINE = MergeTree() ORDER BY id;") - session.Query("USE testdb; INSERT INTO testtable VALUES (1), (2), (3);") + session.Query("INSERT INTO TestDbWithSession VALUES (1), (2), (3);") - ret, err := session.Query("SELECT * FROM testtable;") + ret, err := session.Query("SELECT * FROM TestDbWithSession;") if err != nil { t.Fatalf("Query fail, err: %s", err) } if string(ret.Buf()) != "1\n2\n3\n" { t.Errorf("Query result should be 1\n2\n3\n, got %s", string(ret.Buf())) } - db, err := sql.Open("chdb", fmt.Sprintf("session=%s", sessionDir)) + db, err := sql.Open("chdb", fmt.Sprintf("session=%s", session.ConnStr())) if err != nil { t.Fatalf("open db fail, err: %s", err) } if db.Ping() != nil { t.Fatalf("ping db fail, err: %s", err) } - rows, err := db.Query("select * from testtable;") + rows, err := db.Query("select * from TestDbWithSession;") if err != nil { t.Fatalf("exec create function fail, err: %s", err) } @@ -169,37 +192,27 @@ func TestDbWithSession(t *testing.T) { } func TestDbWithConnection(t *testing.T) { - connectionDir, err := os.MkdirTemp("", "unittest-connectiondata") - if err != nil { - t.Fatalf("create temp directory fail, err: %s", err) - } - defer os.RemoveAll(connectionDir) - connection, err := chdb.NewConnection(connectionDir) - if err != nil { - t.Fatalf("new connection fail, err: %s", err) - } - defer connection.Cleanup() - connection.Query("CREATE DATABASE IF NOT EXISTS testdb; " + - "CREATE TABLE IF NOT EXISTS testdb.testtable (id UInt32) ENGINE = MergeTree() ORDER BY id;") + session.Query( + "CREATE TABLE IF NOT EXISTS TestDbWithConnection (id UInt32) ENGINE = MergeTree() ORDER BY id;") - connection.Query("USE testdb; INSERT INTO testtable VALUES (1), (2), (3);") + session.Query("INSERT INTO TestDbWithConnection VALUES (1), (2), (3);") - ret, err := connection.Query("SELECT * FROM testdb.testtable;") + ret, err := session.Query("SELECT * FROM TestDbWithConnection;") if err != nil { t.Fatalf("Query fail, err: %s", err) } if string(ret.Buf()) != "1\n2\n3\n" { t.Errorf("Query result should be 1\n2\n3\n, got %s", string(ret.Buf())) } - db, err := sql.Open("chdb", fmt.Sprintf("connection=file:%s/chdb.db", connectionDir)) + db, err := sql.Open("chdb", fmt.Sprintf("session=%s", session.ConnStr())) if err != nil { t.Fatalf("open db fail, err: %s", err) } if db.Ping() != nil { t.Fatalf("ping db fail, err: %s", err) } - rows, err := db.Query("select * from testtable;") + rows, err := db.Query("select * from TestDbWithConnection;") if err != nil { t.Fatalf("exec create function fail, err: %s", err) } @@ -226,12 +239,7 @@ func TestDbWithConnection(t *testing.T) { } func TestDbWithConnectionSqlDriverOnly(t *testing.T) { - connectionDir, err := os.MkdirTemp("", "unittest-connectiondata") - if err != nil { - t.Fatalf("create temp directory fail, err: %s", err) - } - defer os.RemoveAll(connectionDir) - db, err := sql.Open("chdb", fmt.Sprintf("connection=file:%s/chdb.db", connectionDir)) + db, err := sql.Open("chdb", fmt.Sprintf("session=%s", session.ConnStr())) if err != nil { t.Fatalf("open db fail, err: %s", err) } @@ -239,17 +247,17 @@ func TestDbWithConnectionSqlDriverOnly(t *testing.T) { t.Fatalf("ping db fail, err: %s", err) } - _, err = db.Exec("CREATE DATABASE IF NOT EXISTS testdb; " + - "CREATE TABLE IF NOT EXISTS testdb.testtable (id UInt32) ENGINE = MergeTree() ORDER BY id;") + _, err = db.Exec( + "CREATE TABLE IF NOT EXISTS TestDbWithConnectionSqlDriverOnly (id UInt32) ENGINE = MergeTree() ORDER BY id;") if err != nil { t.Fatalf("could not create database & table: %s", err) } - _, err = db.Exec("INSERT INTO testdb.testtable VALUES (1), (2), (3);") + _, err = db.Exec("INSERT INTO TestDbWithConnectionSqlDriverOnly VALUES (1), (2), (3);") if err != nil { t.Fatalf("could not insert rows in the table: %s", err) } - rows, err := db.Query("select * from testdb.testtable;") + rows, err := db.Query("select * from TestDbWithConnectionSqlDriverOnly;") if err != nil { t.Fatalf("exec create function fail, err: %s", err) } @@ -276,36 +284,27 @@ func TestDbWithConnectionSqlDriverOnly(t *testing.T) { } func TestQueryRow(t *testing.T) { - sessionDir, err := os.MkdirTemp("", "unittest-sessiondata") - if err != nil { - t.Fatalf("create temp directory fail, err: %s", err) - } - defer os.RemoveAll(sessionDir) - session, err := chdb.NewSession(sessionDir) - if err != nil { - t.Fatalf("new session fail, err: %s", err) - } - defer session.Cleanup() - session.Query("CREATE DATABASE IF NOT EXISTS testdb; " + - "CREATE TABLE IF NOT EXISTS testdb.testtable (id UInt32) ENGINE = MergeTree() ORDER BY id;") - session.Query("USE testdb; INSERT INTO testtable VALUES (1), (2), (3);") + session.Query( + "CREATE TABLE IF NOT EXISTS TestQueryRow (id UInt32) ENGINE = MergeTree() ORDER BY id;") - ret, err := session.Query("SELECT * FROM testtable;") + session.Query(" INSERT INTO TestQueryRow VALUES (1), (2), (3);") + + ret, err := session.Query("SELECT * FROM TestQueryRow;") if err != nil { t.Fatalf("Query fail, err: %s", err) } if string(ret.Buf()) != "1\n2\n3\n" { t.Errorf("Query result should be 1\n2\n3\n, got %s", string(ret.Buf())) } - db, err := sql.Open("chdb", fmt.Sprintf("session=%s", sessionDir)) + db, err := sql.Open("chdb", fmt.Sprintf("session=%s", session.ConnStr())) if err != nil { t.Fatalf("open db fail, err: %s", err) } if db.Ping() != nil { t.Fatalf("ping db fail, err: %s", err) } - rows := db.QueryRow("select * from testtable;") + rows := db.QueryRow("select * from TestQueryRow;") var bar = 0 var count = 1 @@ -324,20 +323,11 @@ func TestQueryRow(t *testing.T) { } func TestExec(t *testing.T) { - sessionDir, err := os.MkdirTemp("", "unittest-sessiondata") - if err != nil { - t.Fatalf("create temp directory fail, err: %s", err) - } - defer os.RemoveAll(sessionDir) - session, err := chdb.NewSession(sessionDir) - if err != nil { - t.Fatalf("new session fail, err: %s", err) - } - defer session.Cleanup() - session.Query("CREATE DATABASE IF NOT EXISTS testdb; " + - "CREATE TABLE IF NOT EXISTS testdb.testtable (id UInt32) ENGINE = MergeTree() ORDER BY id;") - db, err := sql.Open("chdb", fmt.Sprintf("session=%s", sessionDir)) + session.Query( + "CREATE TABLE IF NOT EXISTS TestExec (id UInt32) ENGINE = MergeTree() ORDER BY id;") + + db, err := sql.Open("chdb", fmt.Sprintf("session=%s", session.ConnStr())) if err != nil { t.Fatalf("open db fail, err: %s", err) } @@ -345,11 +335,25 @@ func TestExec(t *testing.T) { t.Fatalf("ping db fail, err: %s", err) } - _, err = db.Exec("INSERT INTO testdb.testtable VALUES (1), (2), (3);") + tables, err := db.Query("SHOW TABLES;") + if err != nil { + t.Fatalf(err.Error()) + } + defer tables.Close() + for tables.Next() { + var tblName string + if err := tables.Scan(&tblName); err != nil { + t.Fatal(err) + } + t.Log(tblName) + fmt.Printf("tblName: %v\n", tblName) + } + + _, err = db.Exec("INSERT INTO TestExec VALUES (1), (2), (3);") if err != nil { t.Fatalf("exec failed, err: %s", err) } - rows := db.QueryRow("select * from testdb.testtable;") + rows := db.QueryRow("select * from TestExec;") var bar = 0 var count = 1 diff --git a/chdb/driver/parquet_test.go b/chdb/driver/parquet_test.go index e88d459..d21293c 100644 --- a/chdb/driver/parquet_test.go +++ b/chdb/driver/parquet_test.go @@ -3,10 +3,7 @@ package chdbdriver import ( "database/sql" "fmt" - "os" "testing" - - "github.com/chdb-io/chdb-go/chdb" ) func TestDbWithParquet(t *testing.T) { @@ -49,37 +46,27 @@ func TestDbWithParquet(t *testing.T) { } func TestDBWithParquetSession(t *testing.T) { - sessionDir, err := os.MkdirTemp("", "unittest-sessiondata") - if err != nil { - t.Fatalf("create temp directory fail, err: %s", err) - } - defer os.RemoveAll(sessionDir) - session, err := chdb.NewSession(sessionDir) - if err != nil { - t.Fatalf("new session fail, err: %s", err) - } - defer session.Cleanup() - session.Query("CREATE DATABASE IF NOT EXISTS testdb; " + - "CREATE TABLE IF NOT EXISTS testdb.testtable (id UInt32) ENGINE = MergeTree() ORDER BY id;") + session.Query( + "CREATE TABLE IF NOT EXISTS TestDBWithParquetSession (id UInt32) ENGINE = MergeTree() ORDER BY id;") - session.Query("INSERT INTO testdb.testtable VALUES (1), (2), (3);") + session.Query("INSERT INTO TestDBWithParquetSession VALUES (1), (2), (3);") - ret, err := session.Query("SELECT * FROM testdb.testtable;") + ret, err := session.Query("SELECT * FROM TestDBWithParquetSession;") if err != nil { t.Fatalf("Query fail, err: %s", err) } if string(ret.Buf()) != "1\n2\n3\n" { t.Errorf("Query result should be 1\n2\n3\n, got %s", string(ret.Buf())) } - db, err := sql.Open("chdb", fmt.Sprintf("session=%s;driverType=%s", sessionDir, "PARQUET")) + db, err := sql.Open("chdb", fmt.Sprintf("session=%s;driverType=%s", session.ConnStr(), "PARQUET")) if err != nil { t.Fatalf("open db fail, err: %s", err) } if db.Ping() != nil { t.Fatalf("ping db fail, err: %s", err) } - rows, err := db.Query("select * from testdb.testtable;") + rows, err := db.Query("select * from TestDBWithParquetSession;") if err != nil { t.Fatalf("exec create function fail, err: %s", err) } @@ -106,37 +93,27 @@ func TestDBWithParquetSession(t *testing.T) { } func TestDBWithParquetConnection(t *testing.T) { - connectionDir, err := os.MkdirTemp("", "unittest-connectiondata") - if err != nil { - t.Fatalf("create temp directory fail, err: %s", err) - } - defer os.RemoveAll(connectionDir) - connection, err := chdb.NewConnection(connectionDir) - if err != nil { - t.Fatalf("new connection fail, err: %s", err) - } - defer connection.Cleanup() - connection.Query("CREATE DATABASE IF NOT EXISTS testdb; " + - "CREATE TABLE IF NOT EXISTS testdb.testtable (id UInt32) ENGINE = MergeTree() ORDER BY id;") + session.Query( + "CREATE TABLE IF NOT EXISTS TestDBWithParquetConnection (id UInt32) ENGINE = MergeTree() ORDER BY id;") - connection.Query("INSERT INTO testdb.testtable VALUES (1), (2), (3);") + session.Query("INSERT INTO TestDBWithParquetConnection VALUES (1), (2), (3);") - ret, err := connection.Query("SELECT * FROM testdb.testtable;") + ret, err := session.Query("SELECT * FROM TestDBWithParquetConnection;") if err != nil { t.Fatalf("Query fail, err: %s", err) } if string(ret.Buf()) != "1\n2\n3\n" { t.Errorf("Query result should be 1\n2\n3\n, got %s", string(ret.Buf())) } - db, err := sql.Open("chdb", fmt.Sprintf("connection=file:%s/chdb.db;driverType=%s", connectionDir, "PARQUET")) + db, err := sql.Open("chdb", fmt.Sprintf("session=%s;driverType=%s", session.ConnStr(), "PARQUET")) if err != nil { t.Fatalf("open db fail, err: %s", err) } if db.Ping() != nil { t.Fatalf("ping db fail, err: %s", err) } - rows, err := db.Query("select * from testdb.testtable;") + rows, err := db.Query("select * from TestDBWithParquetConnection;") if err != nil { t.Fatalf("exec create function fail, err: %s", err) } diff --git a/chdb/session.go b/chdb/session.go index 93d5fcb..a93f9d7 100644 --- a/chdb/session.go +++ b/chdb/session.go @@ -1,38 +1,55 @@ package chdb import ( - "io/ioutil" + "fmt" "os" "path/filepath" "github.com/chdb-io/chdb-go/chdbstable" ) +var ( + globalSession *Session +) + type Session struct { - path string - isTemp bool + conn *chdbstable.ChdbConn + connStr string + path string + isTemp bool } // NewSession creates a new session with the given path. // If path is empty, a temporary directory is created. // Note: The temporary directory is removed when Close is called. func NewSession(paths ...string) (*Session, error) { + if globalSession != nil { + return globalSession, nil + } + path := "" if len(paths) > 0 { path = paths[0] } - + isTemp := false if path == "" { // Create a temporary directory - tempDir, err := ioutil.TempDir("", "chdb_") + tempDir, err := os.MkdirTemp("", "chdb_") if err != nil { return nil, err } path = tempDir - return &Session{path: path, isTemp: true}, nil + isTemp = true + } + connStr := fmt.Sprintf("file:%s/chdb.db", path) - return &Session{path: path, isTemp: false}, nil + conn, err := initConnection(connStr) + if err != nil { + return nil, err + } + globalSession = &Session{connStr: connStr, path: path, isTemp: isTemp, conn: conn} + return globalSession, nil } // Query calls queryToBuffer with a default output format of "CSV" if not provided. @@ -41,7 +58,8 @@ func (s *Session) Query(queryStr string, outputFormats ...string) (result *chdbs if len(outputFormats) > 0 { outputFormat = outputFormats[0] } - return queryToBuffer(queryStr, outputFormat, s.path, "") + + return connQueryToBuffer(s.conn, queryStr, outputFormat) } // Close closes the session and removes the temporary directory @@ -49,9 +67,11 @@ func (s *Session) Query(queryStr string, outputFormats ...string) (result *chdbs // temporary directory is created when NewSession was called with an empty path. func (s *Session) Close() { // Remove the temporary directory if it starts with "chdb_" + s.conn.Close() if s.isTemp && filepath.Base(s.path)[:5] == "chdb_" { s.Cleanup() } + globalSession = nil } // Cleanup closes the session and removes the directory. @@ -65,6 +85,10 @@ func (s *Session) Path() string { return s.path } +func (s *Session) ConnStr() string { + return s.connStr +} + // IsTemp returns whether the session is temporary. func (s *Session) IsTemp() bool { return s.isTemp diff --git a/chdb/session_test.go b/chdb/session_test.go index f99ea62..c414a12 100644 --- a/chdb/session_test.go +++ b/chdb/session_test.go @@ -2,7 +2,6 @@ package chdb import ( "os" - "path/filepath" "testing" ) @@ -51,52 +50,3 @@ func TestSessionCleanup(t *testing.T) { t.Errorf("Session directory should be removed after Cleanup: %s", session.Path()) } } - -// TestQuery tests the Query method of the session. -func TestQuery(t *testing.T) { - path := filepath.Join(os.TempDir(), "chdb_test") - defer os.RemoveAll(path) - session, _ := NewSession(path) - defer session.Cleanup() - - session.Query("CREATE DATABASE IF NOT EXISTS testdb; " + - "CREATE TABLE IF NOT EXISTS testdb.testtable (id UInt32) ENGINE = MergeTree() ORDER BY id;") - - session.Query("USE testdb; INSERT INTO testtable VALUES (1), (2), (3);") - - ret, err := session.Query("SELECT * FROM testtable;") - if err != nil { - t.Errorf("Query failed: %s", err) - } - if string(ret.Buf()) != "1\n2\n3\n" { - t.Errorf("Query result should be 1\n2\n3\n, got %s", string(ret.Buf())) - } -} - -func TestSessionPathAndIsTemp(t *testing.T) { - // Create a new session and check its Path and IsTemp - session, _ := NewSession() - defer session.Cleanup() - - if session.Path() == "" { - t.Errorf("Session path should not be empty") - } - - if !session.IsTemp() { - t.Errorf("Session should be temporary") - } - - // Create a new session with a specific path and check its Path and IsTemp - path := filepath.Join(os.TempDir(), "chdb_test2") - defer os.RemoveAll(path) - session, _ = NewSession(path) - defer session.Cleanup() - - if session.Path() != path { - t.Errorf("Session path should be %s, got %s", path, session.Path()) - } - - if session.IsTemp() { - t.Errorf("Session should not be temporary") - } -} From 9c2376b1b797d1242e57102206c95a6e7add8156 Mon Sep 17 00:00:00 2001 From: Andrei Goncear Date: Wed, 12 Feb 2025 13:45:59 +0000 Subject: [PATCH 07/22] add purego bindings --- chdb-purego/binding.go | 62 ++++++++++++ chdb-purego/chdb.go | 191 +++++++++++++++++++++++++++++++++++++ chdb-purego/helpers.go | 40 ++++++++ chdb-purego/types.go | 49 ++++++++++ chdb.h | 123 ++++++++++++++++++++++++ chdb/driver/arrow.go | 4 +- chdb/driver/driver.go | 19 ++-- chdb/driver/driver_test.go | 10 +- chdb/driver/parquet.go | 4 +- chdb/session.go | 6 +- chdb/wrapper.go | 16 ++-- go.mod | 1 + go.sum | 2 + main.go | 2 + 14 files changed, 500 insertions(+), 29 deletions(-) create mode 100644 chdb-purego/binding.go create mode 100644 chdb-purego/chdb.go create mode 100644 chdb-purego/helpers.go create mode 100644 chdb-purego/types.go create mode 100644 chdb.h diff --git a/chdb-purego/binding.go b/chdb-purego/binding.go new file mode 100644 index 0000000..601aef5 --- /dev/null +++ b/chdb-purego/binding.go @@ -0,0 +1,62 @@ +package chdbpurego + +import ( + "os" + "os/exec" + + "github.com/ebitengine/purego" +) + +func findLibrary() string { + // Env var + if envPath := os.Getenv("CHDB_LIB_PATH"); envPath != "" { + return envPath + } + + // ldconfig with Linux + if path, err := exec.LookPath("libchdb.so"); err == nil { + return path + } + + // default path + commonPaths := []string{ + "/usr/local/lib/libchdb.so", + "/opt/homebrew/lib/libchdb.dylib", + } + + for _, p := range commonPaths { + if _, err := os.Stat(p); err == nil { + return p + } + } + + //should be an error ? + return "libchdb.so" +} + +var ( + queryStable func(argc int, argv **byte) *local_result + freeResult func(result *local_result) + queryStableV2 func(argc int, argv **byte) *local_result_v2 + freeResultV2 func(result *local_result_v2) + connectChdb func(argc int, argv **byte) **chdb_conn + closeConn func(conn **chdb_conn) + queryConn func(conn *chdb_conn, query *byte, format *byte) *local_result_v2 + queryConnV2 func(conn *chdb_conn, query string, format string) *local_result_v2 +) + +func init() { + path := findLibrary() + libchdb, err := purego.Dlopen(path, purego.RTLD_NOW|purego.RTLD_GLOBAL) + if err != nil { + panic(err) + } + purego.RegisterLibFunc(&queryStable, libchdb, "query_stable") + purego.RegisterLibFunc(&freeResult, libchdb, "free_result") + purego.RegisterLibFunc(&queryStableV2, libchdb, "query_stable_v2") + purego.RegisterLibFunc(&freeResultV2, libchdb, "free_result_v2") + purego.RegisterLibFunc(&connectChdb, libchdb, "connect_chdb") + purego.RegisterLibFunc(&closeConn, libchdb, "close_conn") + purego.RegisterLibFunc(&queryConn, libchdb, "query_conn") + purego.RegisterLibFunc(&queryConnV2, libchdb, "query_conn") +} diff --git a/chdb-purego/chdb.go b/chdb-purego/chdb.go new file mode 100644 index 0000000..b9865c3 --- /dev/null +++ b/chdb-purego/chdb.go @@ -0,0 +1,191 @@ +package chdbpurego + +import ( + "errors" + "fmt" + "runtime" + "unsafe" +) + +type result struct { + localResv2 *local_result_v2 +} + +func newChdbResult(cRes *local_result_v2) ChdbResult { + res := &result{ + localResv2: cRes, + } + return res + +} + +// Buf implements ChdbResult. +func (c *result) Buf() []byte { + if c.localResv2 != nil { + if c.localResv2.buf != nil && c.localResv2.len > 0 { + return unsafe.Slice(c.localResv2.buf, c.localResv2.len) + } + } + return nil +} + +// BytesRead implements ChdbResult. +func (c *result) BytesRead() uint64 { + if c.localResv2 != nil { + return c.localResv2.bytes_read + } + return 0 +} + +// Elapsed implements ChdbResult. +func (c *result) Elapsed() float64 { + if c.localResv2 != nil { + return c.localResv2.elapsed + } + return 0 +} + +// Error implements ChdbResult. +func (c *result) Error() error { + if c.localResv2 != nil { + if c.localResv2.error_message != nil { + return errors.New(ptrToGoString(c.localResv2.error_message)) + } + } + return nil +} + +// Free implements ChdbResult. +func (c *result) Free() error { + if c.localResv2 != nil { + freeResultV2(c.localResv2) + c.localResv2 = nil + } + return nil +} + +// Len implements ChdbResult. +func (c *result) Len() int { + if c.localResv2 != nil { + return int(c.localResv2.len) + } + return 0 +} + +// RowsRead implements ChdbResult. +func (c *result) RowsRead() uint64 { + if c.localResv2 != nil { + return c.localResv2.rows_read + } + return 0 +} + +// String implements ChdbResult. +func (c *result) String() string { + ret := c.Buf() + if ret == nil { + return "" + } + return string(ret) +} + +type connection struct { + conn **chdb_conn + pinner runtime.Pinner +} + +func NewChdbConn(conn **chdb_conn) ChdbConn { + return &connection{ + conn: conn, + pinner: runtime.Pinner{}, + } +} + +// Close implements ChdbConn. +func (c *connection) Close() { + if c.conn != nil { + closeConn(c.conn) + } +} + +// Query implements ChdbConn. +func (c *connection) Query(queryStr string, formatStr string) (result ChdbResult, err error) { + + if c.conn == nil { + return nil, fmt.Errorf("invalid connection") + } + defer c.pinner.Unpin() + // qPtr := stringToPtr(queryStr, &c.pinner) + // fPtr := stringToPtr(formatStr, &c.pinner) + deref := *c.conn + // fmt.Printf("queryPtr: %p, formatPtr: %p, conn: %p\n", qPtr, fPtr, deref) + // fmt.Printf("query string: %s\n", queryStr) + // fmt.Printf("format string: %s\n", formatStr) + res := queryConnV2(deref, queryStr, formatStr) + if res == nil { + // According to the C ABI of chDB v1.2.0, the C function query_stable_v2 + // returns nil if the query returns no data. This is not an error. We + // will change this behavior in the future. + return newChdbResult(res), nil + } + if res.error_message != nil { + return nil, errors.New(ptrToGoString(res.error_message)) + } + + return newChdbResult(res), nil +} + +// Ready implements ChdbConn. +func (c *connection) Ready() bool { + if c.conn != nil { + deref := *c.conn + if deref != nil { + return deref.connected + } + } + return false +} + +func RawQuery(argc int, argv []string) (result ChdbResult, err error) { + pinner := runtime.Pinner{} + defer pinner.Unpin() + + cArgv := make([]*byte, len(argv)) + for idx, arg := range argv { + cArgv[idx] = (*byte)(unsafe.Pointer(&([]byte(arg + "\x00")[0]))) + + } + cArgvPtr := (**byte)(unsafe.Pointer(&cArgv[0])) + pinner.Pin(cArgvPtr) + + res := queryStableV2(argc, cArgvPtr) + if res == nil { + // According to the C ABI of chDB v1.2.0, the C function query_stable_v2 + // returns nil if the query returns no data. This is not an error. We + // will change this behavior in the future. + return newChdbResult(res), nil + } + if res.error_message != nil { + return nil, errors.New(ptrToGoString(res.error_message)) + } + + return newChdbResult(res), nil +} + +func NewConnection(argc int, argv []string) (ChdbConn, error) { + pinner := runtime.Pinner{} + defer pinner.Unpin() + + cArgv := make([]*byte, len(argv)) + for idx, arg := range argv { + cArgv[idx] = (*byte)(unsafe.Pointer(&([]byte(arg + "\x00")[0]))) + + } + cArgvPtr := (**byte)(unsafe.Pointer(&cArgv[0])) + pinner.Pin(cArgvPtr) + conn := connectChdb(argc, cArgvPtr) + if conn == nil { + return nil, fmt.Errorf("could not create a chdb connection") + } + return NewChdbConn(conn), nil +} diff --git a/chdb-purego/helpers.go b/chdb-purego/helpers.go new file mode 100644 index 0000000..28507d6 --- /dev/null +++ b/chdb-purego/helpers.go @@ -0,0 +1,40 @@ +package chdbpurego + +import ( + "runtime" + "unsafe" +) + +func ptrToGoString(ptr *byte) string { + if ptr == nil { + return "" + } + + var length int + for { + if *(*byte)(unsafe.Pointer(uintptr(unsafe.Pointer(ptr)) + uintptr(length))) == 0 { + break + } + length++ + } + + return string(unsafe.Slice(ptr, length)) +} + +func stringToPtr(s string, pinner *runtime.Pinner) *byte { + // Pinne for convert string to bytes + // maybe there is simpler solution but it was late when I write this code. + data := make([]byte, len(s)+1) + copy(data, s) + data[len(s)] = 0 // Null-terminator + + ptr := &data[0] + pinner.Pin(ptr) + + return (*byte)(unsafe.Pointer(ptr)) +} + +func strToBytePtr(s string) *byte { + b := append([]byte(s), 0) // Convert to []byte and append null terminator + return &b[0] // Get pointer to first byte +} diff --git a/chdb-purego/types.go b/chdb-purego/types.go new file mode 100644 index 0000000..f84a5a9 --- /dev/null +++ b/chdb-purego/types.go @@ -0,0 +1,49 @@ +package chdbpurego + +import "unsafe" + +// old local result struct. for reference: +// https://github.com/chdb-io/chdb/blob/main/programs/local/chdb.h#L29 +type local_result struct { + buf *byte + len uintptr + _vec unsafe.Pointer + elapsed float64 + rows_read uint64 + bytes_read uint64 +} + +// new local result struct. for reference: https://github.com/chdb-io/chdb/blob/main/programs/local/chdb.h#L40 +type local_result_v2 struct { + buf *byte + len uintptr + _vec unsafe.Pointer + elapsed float64 + rows_read uint64 + bytes_read uint64 + error_message *byte +} + +// clickhouse background server connection.for reference: https://github.com/chdb-io/chdb/blob/main/programs/local/chdb.h#L82 +type chdb_conn struct { + server unsafe.Pointer + connected bool + queue unsafe.Pointer +} + +type ChdbResult interface { + Buf() []byte + String() string + Len() int + Elapsed() float64 + RowsRead() uint64 + BytesRead() uint64 + Error() error + Free() error +} + +type ChdbConn interface { + Query(queryStr string, formatStr string) (result ChdbResult, err error) + Ready() bool + Close() +} diff --git a/chdb.h b/chdb.h new file mode 100644 index 0000000..1188128 --- /dev/null +++ b/chdb.h @@ -0,0 +1,123 @@ +#pragma once + +#ifdef __cplusplus +# include +# include +# include +# include +# include +# include +extern "C" { +#else +# include +# include +# include +#endif + +#define CHDB_EXPORT __attribute__((visibility("default"))) +struct local_result +{ + char * buf; + size_t len; + void * _vec; // std::vector *, for freeing + double elapsed; + uint64_t rows_read; + uint64_t bytes_read; +}; + +#ifdef __cplusplus +struct local_result_v2 +{ + char * buf = nullptr; + size_t len = 0; + void * _vec = nullptr; // std::vector *, for freeing + double elapsed = 0.0; + uint64_t rows_read = 0; + uint64_t bytes_read = 0; + char * error_message = nullptr; +}; +#else +struct local_result_v2 +{ + char * buf; + size_t len; + void * _vec; // std::vector *, for freeing + double elapsed; + uint64_t rows_read; + uint64_t bytes_read; + char * error_message; +}; +#endif + +CHDB_EXPORT struct local_result * query_stable(int argc, char ** argv); +CHDB_EXPORT void free_result(struct local_result * result); + +CHDB_EXPORT struct local_result_v2 * query_stable_v2(int argc, char ** argv); +CHDB_EXPORT void free_result_v2(struct local_result_v2 * result); + +#ifdef __cplusplus +struct query_request +{ + std::string query; + std::string format; +}; + +struct query_queue +{ + std::mutex mutex; + std::condition_variable query_cv; // For query submission + std::condition_variable result_cv; // For query result retrieval + query_request current_query; + local_result_v2 * current_result = nullptr; + bool has_query = false; + bool shutdown = false; + bool cleanup_done = false; +}; +#endif + +/** + * Connection structure for chDB + * Contains server instance, connection state, and query processing queue + */ +struct chdb_conn +{ + void * server; /* ClickHouse LocalServer instance */ + bool connected; /* Connection state flag */ + void * queue; /* Query processing queue */ +}; + +/** + * Creates a new chDB connection. + * Only one active connection is allowed per process. + * Creating a new connection with different path requires closing existing connection. + * + * @param argc Number of command-line arguments + * @param argv Command-line arguments array (--path= to specify database location) + * @return Pointer to connection pointer, or NULL on failure + * @note Default path is ":memory:" if not specified + */ +CHDB_EXPORT struct chdb_conn ** connect_chdb(int argc, char ** argv); + +/** + * Closes an existing chDB connection and cleans up resources. + * Thread-safe function that handles connection shutdown and cleanup. + * + * @param conn Pointer to connection pointer to close + */ +CHDB_EXPORT void close_conn(struct chdb_conn ** conn); + +/** + * Executes a query on the given connection. + * Thread-safe function that handles query execution in a separate thread. + * + * @param conn Connection to execute query on + * @param query SQL query string to execute + * @param format Output format string (e.g., "CSV", default format) + * @return Query result structure containing output or error message + * @note Returns error result if connection is invalid or closed + */ +CHDB_EXPORT struct local_result_v2 * query_conn(struct chdb_conn * conn, const char * query, const char * format); + +#ifdef __cplusplus +} +#endif diff --git a/chdb/driver/arrow.go b/chdb/driver/arrow.go index 3066ca0..96c6aae 100644 --- a/chdb/driver/arrow.go +++ b/chdb/driver/arrow.go @@ -11,11 +11,11 @@ import ( "github.com/apache/arrow/go/v15/arrow/decimal128" "github.com/apache/arrow/go/v15/arrow/decimal256" "github.com/apache/arrow/go/v15/arrow/ipc" - "github.com/chdb-io/chdb-go/chdbstable" + chdbpurego "github.com/chdb-io/chdb-go/chdb-purego" ) type arrowRows struct { - localResult *chdbstable.LocalResult + localResult chdbpurego.ChdbResult reader *ipc.FileReader curRecord arrow.Record curRow int64 diff --git a/chdb/driver/driver.go b/chdb/driver/driver.go index bd0c101..f2b8c91 100644 --- a/chdb/driver/driver.go +++ b/chdb/driver/driver.go @@ -10,7 +10,7 @@ import ( "strings" "github.com/chdb-io/chdb-go/chdb" - "github.com/chdb-io/chdb-go/chdbstable" + chdbpurego "github.com/chdb-io/chdb-go/chdb-purego" "github.com/huandu/go-sqlbuilder" "github.com/parquet-go/parquet-go" @@ -46,7 +46,7 @@ func (d DriverType) String() string { return "" } -func (d DriverType) PrepareRows(result *chdbstable.LocalResult, buf []byte, bufSize int, useUnsafe bool) (driver.Rows, error) { +func (d DriverType) PrepareRows(result chdbpurego.ChdbResult, buf []byte, bufSize int, useUnsafe bool) (driver.Rows, error) { switch d { case ARROW: reader, err := ipc.NewFileReader(bytes.NewReader(buf)) @@ -133,7 +133,7 @@ func (e *execResult) RowsAffected() (int64, error) { return -1, fmt.Errorf("does not support RowsAffected") } -type queryHandle func(string, ...string) (*chdbstable.LocalResult, error) +type queryHandle func(string, ...string) (chdbpurego.ChdbResult, error) type connector struct { udfPath string @@ -214,12 +214,13 @@ func NewConnect(opts map[string]string) (ret *connector, err error) { if ok { ret.udfPath = udfPath } - if ret.session == nil { - ret.session, err = chdb.NewSession() - if err != nil { - return nil, err - } - } + // if ret.session == nil { + + // ret.session, err = chdb.NewSession() + // if err != nil { + // return nil, err + // } + // } return } diff --git a/chdb/driver/driver_test.go b/chdb/driver/driver_test.go index d765e09..8321e20 100644 --- a/chdb/driver/driver_test.go +++ b/chdb/driver/driver_test.go @@ -45,21 +45,21 @@ func TestMain(m *testing.M) { func TestDb(t *testing.T) { db, err := sql.Open("chdb", "") if err != nil { - t.Errorf("open db fail, err:%s", err) + t.Fatalf("open db fail, err:%s", err) } if db.Ping() != nil { - t.Errorf("ping db fail") + t.Fatalf("ping db fail") } rows, err := db.Query(`SELECT 1,'abc'`) if err != nil { - t.Errorf("run Query fail, err:%s", err) + t.Fatalf("run Query fail, err:%s", err) } cols, err := rows.Columns() if err != nil { - t.Errorf("get result columns fail, err: %s", err) + t.Fatalf("get result columns fail, err: %s", err) } if len(cols) != 2 { - t.Errorf("select result columns length should be 2") + t.Fatalf("select result columns length should be 2") } var ( bar int diff --git a/chdb/driver/parquet.go b/chdb/driver/parquet.go index 5b1cce8..64b66fa 100644 --- a/chdb/driver/parquet.go +++ b/chdb/driver/parquet.go @@ -9,7 +9,7 @@ import ( "reflect" - "github.com/chdb-io/chdb-go/chdbstable" + chdbpurego "github.com/chdb-io/chdb-go/chdb-purego" "github.com/parquet-go/parquet-go" ) @@ -24,7 +24,7 @@ func getStringFromBytes(v parquet.Value) string { } type parquetRows struct { - localResult *chdbstable.LocalResult // result from clickhouse + localResult chdbpurego.ChdbResult // result from clickhouse reader *parquet.GenericReader[any] // parquet reader curRecord parquet.Row // TODO: delete this? buffer []parquet.Row // record buffer diff --git a/chdb/session.go b/chdb/session.go index a93f9d7..a8cca97 100644 --- a/chdb/session.go +++ b/chdb/session.go @@ -5,7 +5,7 @@ import ( "os" "path/filepath" - "github.com/chdb-io/chdb-go/chdbstable" + chdbpurego "github.com/chdb-io/chdb-go/chdb-purego" ) var ( @@ -13,7 +13,7 @@ var ( ) type Session struct { - conn *chdbstable.ChdbConn + conn chdbpurego.ChdbConn connStr string path string isTemp bool @@ -53,7 +53,7 @@ func NewSession(paths ...string) (*Session, error) { } // Query calls queryToBuffer with a default output format of "CSV" if not provided. -func (s *Session) Query(queryStr string, outputFormats ...string) (result *chdbstable.LocalResult, err error) { +func (s *Session) Query(queryStr string, outputFormats ...string) (result chdbpurego.ChdbResult, err error) { outputFormat := "CSV" // Default value if len(outputFormats) > 0 { outputFormat = outputFormats[0] diff --git a/chdb/wrapper.go b/chdb/wrapper.go index ad0eb40..e38b295 100644 --- a/chdb/wrapper.go +++ b/chdb/wrapper.go @@ -1,11 +1,11 @@ package chdb import ( - "github.com/chdb-io/chdb-go/chdbstable" + chdbpurego "github.com/chdb-io/chdb-go/chdb-purego" ) // Query calls queryToBuffer with a default output format of "CSV" if not provided. -func Query(queryStr string, outputFormats ...string) (result *chdbstable.LocalResult, err error) { +func Query(queryStr string, outputFormats ...string) (result chdbpurego.ChdbResult, err error) { outputFormat := "CSV" // Default value if len(outputFormats) > 0 { outputFormat = outputFormats[0] @@ -14,7 +14,7 @@ func Query(queryStr string, outputFormats ...string) (result *chdbstable.LocalRe } // queryToBuffer constructs the arguments for QueryStable and calls it. -func queryToBuffer(queryStr, outputFormat, path, udfPath string) (result *chdbstable.LocalResult, err error) { +func queryToBuffer(queryStr, outputFormat, path, udfPath string) (result chdbpurego.ChdbResult, err error) { argv := []string{"clickhouse", "--multiquery"} // Handle output format @@ -38,18 +38,18 @@ func queryToBuffer(queryStr, outputFormat, path, udfPath string) (result *chdbst } // Call QueryStable with the constructed arguments - return chdbstable.QueryStable(len(argv), argv) + return chdbpurego.RawQuery(len(argv), argv) } -func initConnection(connStr string) (result *chdbstable.ChdbConn, err error) { +func initConnection(connStr string) (result chdbpurego.ChdbConn, err error) { argv := []string{connStr} // Call NewConnection with the constructed arguments - return chdbstable.NewConnection(len(argv), argv) + return chdbpurego.NewConnection(len(argv), argv) } -func connQueryToBuffer(conn *chdbstable.ChdbConn, queryStr, outputFormat string) (result *chdbstable.LocalResult, err error) { +func connQueryToBuffer(conn chdbpurego.ChdbConn, queryStr, outputFormat string) (result chdbpurego.ChdbResult, err error) { if outputFormat == "" { outputFormat = "CSV" } - return conn.QueryConn(queryStr, outputFormat) + return conn.Query(queryStr, outputFormat) } diff --git a/go.mod b/go.mod index c97fee5..042d968 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( require ( github.com/andybalholm/brotli v1.1.0 // indirect + github.com/ebitengine/purego v0.8.2 // indirect github.com/goccy/go-json v0.10.3 // indirect github.com/google/flatbuffers v24.3.25+incompatible // indirect github.com/google/uuid v1.6.0 // indirect diff --git a/go.sum b/go.sum index 444020a..8b8999a 100644 --- a/go.sum +++ b/go.sum @@ -6,6 +6,8 @@ github.com/c-bata/go-prompt v0.2.6 h1:POP+nrHE+DfLYx370bedwNhsqmpCUynWPxuHi0C5vZ github.com/c-bata/go-prompt v0.2.6/go.mod h1:/LMAke8wD2FsNu9EXNdHxNLbd9MedkPnCdfpU9wwHfY= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/ebitengine/purego v0.8.2 h1:jPPGWs2sZ1UgOSgD2bClL0MJIqu58nOmIcBuXr62z1I= +github.com/ebitengine/purego v0.8.2/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA= github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/google/flatbuffers v24.3.25+incompatible h1:CX395cjN9Kke9mmalRoL3d81AtFUxJM+yDthflgJGkI= diff --git a/main.go b/main.go index 0865e8b..97626dc 100644 --- a/main.go +++ b/main.go @@ -39,6 +39,8 @@ data will lost after exit. If you want to keep the data, specify a path to a dir // If path is specified or no additional arguments, enter interactive mode if len(flag.Args()) == 0 { + t := "/tmp" + pathFlag = &t var err error var session *chdb.Session if *pathFlag != "" { From ef98543f004947bd5a8a53f97e747eb3e082d6f1 Mon Sep 17 00:00:00 2001 From: Andrei Goncear Date: Thu, 13 Feb 2025 13:17:35 +0000 Subject: [PATCH 08/22] fix function signatures --- chdb-purego/binding.go | 12 +++++------ chdb-purego/chdb.go | 49 +++++++++++------------------------------- chdb/driver/arrow.go | 2 ++ chdb/driver/driver.go | 12 +++++------ chdb/driver/parquet.go | 2 ++ 5 files changed, 28 insertions(+), 49 deletions(-) diff --git a/chdb-purego/binding.go b/chdb-purego/binding.go index 601aef5..9c684a6 100644 --- a/chdb-purego/binding.go +++ b/chdb-purego/binding.go @@ -35,14 +35,13 @@ func findLibrary() string { } var ( - queryStable func(argc int, argv **byte) *local_result + queryStable func(argc int, argv []string) *local_result freeResult func(result *local_result) - queryStableV2 func(argc int, argv **byte) *local_result_v2 + queryStableV2 func(argc int, argv []string) *local_result_v2 freeResultV2 func(result *local_result_v2) - connectChdb func(argc int, argv **byte) **chdb_conn + connectChdb func(argc int, argv []string) **chdb_conn closeConn func(conn **chdb_conn) - queryConn func(conn *chdb_conn, query *byte, format *byte) *local_result_v2 - queryConnV2 func(conn *chdb_conn, query string, format string) *local_result_v2 + queryConn func(conn *chdb_conn, query string, format string) *local_result_v2 ) func init() { @@ -54,9 +53,10 @@ func init() { purego.RegisterLibFunc(&queryStable, libchdb, "query_stable") purego.RegisterLibFunc(&freeResult, libchdb, "free_result") purego.RegisterLibFunc(&queryStableV2, libchdb, "query_stable_v2") + purego.RegisterLibFunc(&freeResultV2, libchdb, "free_result_v2") purego.RegisterLibFunc(&connectChdb, libchdb, "connect_chdb") purego.RegisterLibFunc(&closeConn, libchdb, "close_conn") purego.RegisterLibFunc(&queryConn, libchdb, "query_conn") - purego.RegisterLibFunc(&queryConnV2, libchdb, "query_conn") + } diff --git a/chdb-purego/chdb.go b/chdb-purego/chdb.go index b9865c3..ff558f1 100644 --- a/chdb-purego/chdb.go +++ b/chdb-purego/chdb.go @@ -3,7 +3,6 @@ package chdbpurego import ( "errors" "fmt" - "runtime" "unsafe" ) @@ -15,6 +14,7 @@ func newChdbResult(cRes *local_result_v2) ChdbResult { res := &result{ localResv2: cRes, } + // runtime.SetFinalizer(res, res.Free) return res } @@ -90,15 +90,15 @@ func (c *result) String() string { } type connection struct { - conn **chdb_conn - pinner runtime.Pinner + conn **chdb_conn } func NewChdbConn(conn **chdb_conn) ChdbConn { - return &connection{ - conn: conn, - pinner: runtime.Pinner{}, + c := &connection{ + conn: conn, } + // runtime.SetFinalizer(c, c.Close) + return c } // Close implements ChdbConn. @@ -114,14 +114,10 @@ func (c *connection) Query(queryStr string, formatStr string) (result ChdbResult if c.conn == nil { return nil, fmt.Errorf("invalid connection") } - defer c.pinner.Unpin() - // qPtr := stringToPtr(queryStr, &c.pinner) - // fPtr := stringToPtr(formatStr, &c.pinner) - deref := *c.conn - // fmt.Printf("queryPtr: %p, formatPtr: %p, conn: %p\n", qPtr, fPtr, deref) - // fmt.Printf("query string: %s\n", queryStr) - // fmt.Printf("format string: %s\n", formatStr) - res := queryConnV2(deref, queryStr, formatStr) + + rawConn := *c.conn + + res := queryConn(rawConn, queryStr, formatStr) if res == nil { // According to the C ABI of chDB v1.2.0, the C function query_stable_v2 // returns nil if the query returns no data. This is not an error. We @@ -147,18 +143,7 @@ func (c *connection) Ready() bool { } func RawQuery(argc int, argv []string) (result ChdbResult, err error) { - pinner := runtime.Pinner{} - defer pinner.Unpin() - - cArgv := make([]*byte, len(argv)) - for idx, arg := range argv { - cArgv[idx] = (*byte)(unsafe.Pointer(&([]byte(arg + "\x00")[0]))) - - } - cArgvPtr := (**byte)(unsafe.Pointer(&cArgv[0])) - pinner.Pin(cArgvPtr) - - res := queryStableV2(argc, cArgvPtr) + res := queryStableV2(argc, argv) if res == nil { // According to the C ABI of chDB v1.2.0, the C function query_stable_v2 // returns nil if the query returns no data. This is not an error. We @@ -173,17 +158,7 @@ func RawQuery(argc int, argv []string) (result ChdbResult, err error) { } func NewConnection(argc int, argv []string) (ChdbConn, error) { - pinner := runtime.Pinner{} - defer pinner.Unpin() - - cArgv := make([]*byte, len(argv)) - for idx, arg := range argv { - cArgv[idx] = (*byte)(unsafe.Pointer(&([]byte(arg + "\x00")[0]))) - - } - cArgvPtr := (**byte)(unsafe.Pointer(&cArgv[0])) - pinner.Pin(cArgvPtr) - conn := connectChdb(argc, cArgvPtr) + conn := connectChdb(argc, argv) if conn == nil { return nil, fmt.Errorf("could not create a chdb connection") } diff --git a/chdb/driver/arrow.go b/chdb/driver/arrow.go index 96c6aae..87ca193 100644 --- a/chdb/driver/arrow.go +++ b/chdb/driver/arrow.go @@ -36,7 +36,9 @@ func (r *arrowRows) Close() error { // ignore reader close _ = r.reader.Close() r.reader = nil + r.localResult.Free() r.localResult = nil + return nil } diff --git a/chdb/driver/driver.go b/chdb/driver/driver.go index f2b8c91..c866382 100644 --- a/chdb/driver/driver.go +++ b/chdb/driver/driver.go @@ -214,13 +214,13 @@ func NewConnect(opts map[string]string) (ret *connector, err error) { if ok { ret.udfPath = udfPath } - // if ret.session == nil { + if ret.session == nil { - // ret.session, err = chdb.NewSession() - // if err != nil { - // return nil, err - // } - // } + ret.session, err = chdb.NewSession() + if err != nil { + return nil, err + } + } return } diff --git a/chdb/driver/parquet.go b/chdb/driver/parquet.go index 64b66fa..700468e 100644 --- a/chdb/driver/parquet.go +++ b/chdb/driver/parquet.go @@ -51,7 +51,9 @@ func (r *parquetRows) Close() error { // ignore reader close _ = r.reader.Close() r.reader = nil + r.localResult.Free() r.localResult = nil + r.buffer = nil return nil } From d8a963feae040e66ef8995ddb8942d88d990d989 Mon Sep 17 00:00:00 2001 From: Andrei Goncear Date: Fri, 14 Feb 2025 11:38:56 +0000 Subject: [PATCH 09/22] remove arrow for the moment since it has a problem in the library itself --- chdb/driver/arrow_test.go | 106 -------------------------------------- chdb/driver/driver.go | 18 +++---- chdb/session.go | 3 +- 3 files changed, 11 insertions(+), 116 deletions(-) delete mode 100644 chdb/driver/arrow_test.go diff --git a/chdb/driver/arrow_test.go b/chdb/driver/arrow_test.go deleted file mode 100644 index 769db00..0000000 --- a/chdb/driver/arrow_test.go +++ /dev/null @@ -1,106 +0,0 @@ -package chdbdriver - -import ( - "database/sql" - "fmt" - "os" - "testing" - - "github.com/chdb-io/chdb-go/chdb" -) - -func TestDbWithArrow(t *testing.T) { - - db, err := sql.Open("chdb", fmt.Sprintf("driverType=%s", "ARROW")) - if err != nil { - t.Errorf("open db fail, err:%s", err) - } - if db.Ping() != nil { - t.Errorf("ping db fail") - } - rows, err := db.Query(`SELECT 1,'abc'`) - if err != nil { - t.Errorf("run Query fail, err:%s", err) - } - cols, err := rows.Columns() - if err != nil { - t.Errorf("get result columns fail, err: %s", err) - } - if len(cols) != 2 { - t.Errorf("select result columns length should be 2") - } - var ( - bar int - foo string - ) - defer rows.Close() - for rows.Next() { - err := rows.Scan(&bar, &foo) - if err != nil { - t.Errorf("scan fail, err: %s", err) - } - if bar != 1 { - t.Errorf("expected error") - } - if foo != "abc" { - t.Errorf("expected error") - } - } -} - -func TestDBWithArrowSession(t *testing.T) { - sessionDir, err := os.MkdirTemp("", "unittest-sessiondata") - if err != nil { - t.Fatalf("create temp directory fail, err: %s", err) - } - defer os.RemoveAll(sessionDir) - session, err := chdb.NewSession(sessionDir) - if err != nil { - t.Fatalf("new session fail, err: %s", err) - } - defer session.Cleanup() - - session.Query("CREATE DATABASE IF NOT EXISTS testdb; " + - "CREATE TABLE IF NOT EXISTS testdb.testtable (id UInt32) ENGINE = MergeTree() ORDER BY id;") - - session.Query("INSERT INTO testdb.testtable VALUES (1), (2), (3);") - - ret, err := session.Query("SELECT * FROM testdb.testtable;") - if err != nil { - t.Fatalf("Query fail, err: %s", err) - } - if string(ret.Buf()) != "1\n2\n3\n" { - t.Errorf("Query result should be 1\n2\n3\n, got %s", string(ret.Buf())) - } - db, err := sql.Open("chdb", fmt.Sprintf("session=%s;driverType=%s", sessionDir, "ARROW")) - if err != nil { - t.Fatalf("open db fail, err: %s", err) - } - if db.Ping() != nil { - t.Fatalf("ping db fail, err: %s", err) - } - rows, err := db.Query("select * from testdb.testtable;") - if err != nil { - t.Fatalf("exec create function fail, err: %s", err) - } - defer rows.Close() - cols, err := rows.Columns() - if err != nil { - t.Fatalf("get result columns fail, err: %s", err) - } - if len(cols) != 1 { - t.Fatalf("result columns length shoule be 3, actual: %d", len(cols)) - } - var bar = 0 - var count = 1 - for rows.Next() { - err = rows.Scan(&bar) - if err != nil { - t.Fatalf("scan fail, err: %s", err) - } - if bar != count { - t.Fatalf("result is not match, want: %d actual: %d", count, bar) - } - count++ - } -} diff --git a/chdb/driver/driver.go b/chdb/driver/driver.go index c866382..b3bbc2b 100644 --- a/chdb/driver/driver.go +++ b/chdb/driver/driver.go @@ -67,8 +67,8 @@ func (d DriverType) PrepareRows(result chdbpurego.ChdbResult, buf []byte, bufSiz func parseDriverType(s string) DriverType { switch strings.ToUpper(s) { - case "ARROW": - return ARROW + // case "ARROW": + // return ARROW case "PARQUET": return PARQUET } @@ -190,7 +190,7 @@ func NewConnect(opts map[string]string) (ret *connector, err error) { if ok { ret.driverType = parseDriverType(driverType) } else { - ret.driverType = ARROW //default to arrow + ret.driverType = PARQUET //default to parquet } bufferSize, ok := opts[driverBufferSizeKey] if ok { @@ -214,13 +214,13 @@ func NewConnect(opts map[string]string) (ret *connector, err error) { if ok { ret.udfPath = udfPath } - if ret.session == nil { + // if ret.session == nil { - ret.session, err = chdb.NewSession() - if err != nil { - return nil, err - } - } + // ret.session, err = chdb.NewSession() + // if err != nil { + // return nil, err + // } + // } return } diff --git a/chdb/session.go b/chdb/session.go index a8cca97..a8e63f7 100644 --- a/chdb/session.go +++ b/chdb/session.go @@ -58,8 +58,9 @@ func (s *Session) Query(queryStr string, outputFormats ...string) (result chdbpu if len(outputFormats) > 0 { outputFormat = outputFormats[0] } + return s.conn.Query(queryStr, outputFormat) - return connQueryToBuffer(s.conn, queryStr, outputFormat) + // return connQueryToBuffer(s.conn, queryStr, outputFormat) } // Close closes the session and removes the temporary directory From 01896db3ce6da5e8fec117511717dade7d851732 Mon Sep 17 00:00:00 2001 From: Andrei Goncear Date: Fri, 14 Feb 2025 13:15:36 +0000 Subject: [PATCH 10/22] removed arrow dependency, fix tests --- {chdbstable => chdb-purego}/chdb.h | 0 chdb/driver/arrow.go | 177 ----------------------------- chdb/driver/driver.go | 20 +--- chdb/session.go | 1 - chdb/wrapper_test.go | 88 +++++++------- chdbstable/chdb.go | 166 --------------------------- chdbstable/chdb_test.go | 75 ------------ go.mod | 14 +-- go.sum | 26 +---- 9 files changed, 54 insertions(+), 513 deletions(-) rename {chdbstable => chdb-purego}/chdb.h (100%) delete mode 100644 chdb/driver/arrow.go delete mode 100644 chdbstable/chdb.go delete mode 100644 chdbstable/chdb_test.go diff --git a/chdbstable/chdb.h b/chdb-purego/chdb.h similarity index 100% rename from chdbstable/chdb.h rename to chdb-purego/chdb.h diff --git a/chdb/driver/arrow.go b/chdb/driver/arrow.go deleted file mode 100644 index 87ca193..0000000 --- a/chdb/driver/arrow.go +++ /dev/null @@ -1,177 +0,0 @@ -package chdbdriver - -import ( - "database/sql/driver" - "fmt" - "reflect" - "time" - - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/decimal128" - "github.com/apache/arrow/go/v15/arrow/decimal256" - "github.com/apache/arrow/go/v15/arrow/ipc" - chdbpurego "github.com/chdb-io/chdb-go/chdb-purego" -) - -type arrowRows struct { - localResult chdbpurego.ChdbResult - reader *ipc.FileReader - curRecord arrow.Record - curRow int64 -} - -func (r *arrowRows) Columns() (out []string) { - sch := r.reader.Schema() - for i := 0; i < sch.NumFields(); i++ { - out = append(out, sch.Field(i).Name) - } - return -} - -func (r *arrowRows) Close() error { - if r.curRecord != nil { - r.curRecord = nil - } - // ignore reader close - _ = r.reader.Close() - r.reader = nil - r.localResult.Free() - r.localResult = nil - - return nil -} - -func (r *arrowRows) Next(dest []driver.Value) error { - if r.curRecord != nil && r.curRow == r.curRecord.NumRows() { - r.curRecord = nil - } - for r.curRecord == nil { - record, err := r.reader.Read() - if err != nil { - return err - } - if record.NumRows() == 0 { - continue - } - r.curRecord = record - r.curRow = 0 - } - - for i, col := range r.curRecord.Columns() { - if col.IsNull(int(r.curRow)) { - dest[i] = nil - continue - } - switch col := col.(type) { - case *array.Boolean: - dest[i] = col.Value(int(r.curRow)) - case *array.Int8: - dest[i] = col.Value(int(r.curRow)) - case *array.Uint8: - dest[i] = col.Value(int(r.curRow)) - case *array.Int16: - dest[i] = col.Value(int(r.curRow)) - case *array.Uint16: - dest[i] = col.Value(int(r.curRow)) - case *array.Int32: - dest[i] = col.Value(int(r.curRow)) - case *array.Uint32: - dest[i] = col.Value(int(r.curRow)) - case *array.Int64: - dest[i] = col.Value(int(r.curRow)) - case *array.Uint64: - dest[i] = col.Value(int(r.curRow)) - case *array.Float32: - dest[i] = col.Value(int(r.curRow)) - case *array.Float64: - dest[i] = col.Value(int(r.curRow)) - case *array.String: - dest[i] = col.Value(int(r.curRow)) - case *array.LargeString: - dest[i] = col.Value(int(r.curRow)) - case *array.Binary: - dest[i] = col.Value(int(r.curRow)) - case *array.LargeBinary: - dest[i] = col.Value(int(r.curRow)) - case *array.Date32: - dest[i] = col.Value(int(r.curRow)).ToTime() - case *array.Date64: - dest[i] = col.Value(int(r.curRow)).ToTime() - case *array.Time32: - dest[i] = col.Value(int(r.curRow)).ToTime(col.DataType().(*arrow.Time32Type).Unit) - case *array.Time64: - dest[i] = col.Value(int(r.curRow)).ToTime(col.DataType().(*arrow.Time64Type).Unit) - case *array.Timestamp: - dest[i] = col.Value(int(r.curRow)).ToTime(col.DataType().(*arrow.TimestampType).Unit) - case *array.Decimal128: - dest[i] = col.Value(int(r.curRow)) - case *array.Decimal256: - dest[i] = col.Value(int(r.curRow)) - default: - return fmt.Errorf( - "not yet implemented populating from columns of type " + col.DataType().String(), - ) - } - } - - r.curRow++ - return nil -} - -func (r *arrowRows) ColumnTypeDatabaseTypeName(index int) string { - return r.reader.Schema().Field(index).Type.String() -} - -func (r *arrowRows) ColumnTypeNullable(index int) (nullable, ok bool) { - return r.reader.Schema().Field(index).Nullable, true -} - -func (r *arrowRows) ColumnTypePrecisionScale(index int) (precision, scale int64, ok bool) { - typ := r.reader.Schema().Field(index).Type - switch dt := typ.(type) { - case *arrow.Decimal128Type: - return int64(dt.Precision), int64(dt.Scale), true - case *arrow.Decimal256Type: - return int64(dt.Precision), int64(dt.Scale), true - } - return 0, 0, false -} - -func (r *arrowRows) ColumnTypeScanType(index int) reflect.Type { - switch r.reader.Schema().Field(index).Type.ID() { - case arrow.BOOL: - return reflect.TypeOf(false) - case arrow.INT8: - return reflect.TypeOf(int8(0)) - case arrow.UINT8: - return reflect.TypeOf(uint8(0)) - case arrow.INT16: - return reflect.TypeOf(int16(0)) - case arrow.UINT16: - return reflect.TypeOf(uint16(0)) - case arrow.INT32: - return reflect.TypeOf(int32(0)) - case arrow.UINT32: - return reflect.TypeOf(uint32(0)) - case arrow.INT64: - return reflect.TypeOf(int64(0)) - case arrow.UINT64: - return reflect.TypeOf(uint64(0)) - case arrow.FLOAT32: - return reflect.TypeOf(float32(0)) - case arrow.FLOAT64: - return reflect.TypeOf(float64(0)) - case arrow.DECIMAL128: - return reflect.TypeOf(decimal128.Num{}) - case arrow.DECIMAL256: - return reflect.TypeOf(decimal256.Num{}) - case arrow.BINARY: - return reflect.TypeOf([]byte{}) - case arrow.STRING: - return reflect.TypeOf(string("")) - case arrow.TIME32, arrow.TIME64, arrow.DATE32, arrow.DATE64, arrow.TIMESTAMP: - return reflect.TypeOf(time.Time{}) - } - return nil -} diff --git a/chdb/driver/driver.go b/chdb/driver/driver.go index b3bbc2b..29e8347 100644 --- a/chdb/driver/driver.go +++ b/chdb/driver/driver.go @@ -13,8 +13,6 @@ import ( chdbpurego "github.com/chdb-io/chdb-go/chdb-purego" "github.com/huandu/go-sqlbuilder" "github.com/parquet-go/parquet-go" - - "github.com/apache/arrow/go/v15/arrow/ipc" ) type DriverType int @@ -48,12 +46,6 @@ func (d DriverType) String() string { func (d DriverType) PrepareRows(result chdbpurego.ChdbResult, buf []byte, bufSize int, useUnsafe bool) (driver.Rows, error) { switch d { - case ARROW: - reader, err := ipc.NewFileReader(bytes.NewReader(buf)) - if err != nil { - return nil, err - } - return &arrowRows{localResult: result, reader: reader}, nil case PARQUET: reader := parquet.NewGenericReader[any](bytes.NewReader(buf)) return &parquetRows{ @@ -214,13 +206,13 @@ func NewConnect(opts map[string]string) (ret *connector, err error) { if ok { ret.udfPath = udfPath } - // if ret.session == nil { + if ret.session == nil { - // ret.session, err = chdb.NewSession() - // if err != nil { - // return nil, err - // } - // } + ret.session, err = chdb.NewSession() + if err != nil { + return nil, err + } + } return } diff --git a/chdb/session.go b/chdb/session.go index a8e63f7..fac7152 100644 --- a/chdb/session.go +++ b/chdb/session.go @@ -60,7 +60,6 @@ func (s *Session) Query(queryStr string, outputFormats ...string) (result chdbpu } return s.conn.Query(queryStr, outputFormat) - // return connQueryToBuffer(s.conn, queryStr, outputFormat) } // Close closes the session and removes the temporary directory diff --git a/chdb/wrapper_test.go b/chdb/wrapper_test.go index 10ae5bc..2c3fee4 100644 --- a/chdb/wrapper_test.go +++ b/chdb/wrapper_test.go @@ -1,31 +1,32 @@ package chdb import ( - "os" - "path/filepath" "testing" ) func TestQueryToBuffer(t *testing.T) { // Create a temporary directory - tempDir := filepath.Join(os.TempDir(), "chdb_test") - defer os.RemoveAll(tempDir) + sess, err := NewSession() + if err != nil { + t.Fatalf("could not create session: %s", err) + } + defer sess.Close() // Define test cases testCases := []struct { - name string - queryStr string - outputFormat string - path string + name string + queryStr string + outputFormat string + udfPath string expectedErrMsg string expectedResult string }{ { - name: "Basic Query", - queryStr: "SELECT 123", - outputFormat: "CSV", - path: "", + name: "Basic Query", + queryStr: "SELECT 123", + outputFormat: "CSV", + udfPath: "", expectedErrMsg: "", expectedResult: "123\n", @@ -35,35 +36,35 @@ func TestQueryToBuffer(t *testing.T) { name: "Session Query 1", queryStr: "CREATE DATABASE IF NOT EXISTS testdb; " + "CREATE TABLE IF NOT EXISTS testdb.testtable (id UInt32) ENGINE = MergeTree() ORDER BY id;", - outputFormat: "CSV", - path: tempDir, + outputFormat: "CSV", + udfPath: "", expectedErrMsg: "", expectedResult: "", }, - // { - // name: "Session Query 2", - // queryStr: "USE testdb; INSERT INTO testtable VALUES (1), (2), (3);", - // outputFormat: "CSV", - // path: tempDir, - // udfPath: "", - // expectedErrMsg: "", - // expectedResult: "", - // }, - // { - // name: "Session Query 3", - // queryStr: "SELECT * FROM testtable;", - // outputFormat: "CSV", - // path: tempDir, - // udfPath: "", - // expectedErrMsg: "", - // expectedResult: "1\n2\n3\n", - // }, { - name: "Error Query", - queryStr: "SELECT * FROM nonexist; ", - outputFormat: "CSV", - path: tempDir, + name: "Session Query 2", + queryStr: "USE testdb; INSERT INTO testtable VALUES (1), (2), (3);", + outputFormat: "CSV", + + udfPath: "", + expectedErrMsg: "", + expectedResult: "", + }, + { + name: "Session Query 3", + queryStr: "SELECT * FROM testtable;", + outputFormat: "CSV", + + udfPath: "", + expectedErrMsg: "", + expectedResult: "1\n2\n3\n", + }, + { + name: "Error Query", + queryStr: "SELECT * FROM nonexist; ", + outputFormat: "CSV", + udfPath: "", expectedErrMsg: "Code: 60. DB::Exception: Unknown table expression identifier 'nonexist' in scope SELECT * FROM nonexist. (UNKNOWN_TABLE)", expectedResult: "", @@ -73,23 +74,24 @@ func TestQueryToBuffer(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { // Call queryToBuffer - result, err := queryToBuffer(tc.queryStr, tc.outputFormat, tc.path, tc.udfPath) + + result, err := sess.Query(tc.queryStr, tc.outputFormat) // Verify if tc.expectedErrMsg != "" { if err == nil { - t.Errorf("%v queryToBuffer() with queryStr %v, outputFormat %v, path %v, udfPath %v, expect error message: %v, got no error", - tc.name, tc.queryStr, tc.outputFormat, tc.path, tc.udfPath, tc.expectedErrMsg) + t.Errorf("%v queryToBuffer() with queryStr %v, outputFormat %v, udfPath %v, expect error message: %v, got no error", + tc.name, tc.queryStr, tc.outputFormat, tc.udfPath, tc.expectedErrMsg) } else { if err.Error() != tc.expectedErrMsg { - t.Errorf("%v queryToBuffer() with queryStr %v, outputFormat %v, path %v, udfPath %v, expect error message: %v, got error message: %v", - tc.name, tc.queryStr, tc.outputFormat, tc.path, tc.udfPath, tc.expectedErrMsg, err.Error()) + t.Errorf("%v queryToBuffer() with queryStr %v, outputFormat %v, udfPath %v, expect error message: %v, got error message: %v", + tc.name, tc.queryStr, tc.outputFormat, tc.udfPath, tc.expectedErrMsg, err.Error()) } } } else { if string(result.Buf()) != tc.expectedResult { - t.Errorf("%v queryToBuffer() with queryStr %v, outputFormat %v, path %v, udfPath %v, expect result: %v, got result: %v", - tc.name, tc.queryStr, tc.outputFormat, tc.path, tc.udfPath, tc.expectedResult, string(result.Buf())) + t.Errorf("%v queryToBuffer() with queryStr %v, outputFormat %v, udfPath %v, expect result: %v, got result: %v", + tc.name, tc.queryStr, tc.outputFormat, tc.udfPath, tc.expectedResult, string(result.Buf())) } } }) diff --git a/chdbstable/chdb.go b/chdbstable/chdb.go deleted file mode 100644 index 48fb506..0000000 --- a/chdbstable/chdb.go +++ /dev/null @@ -1,166 +0,0 @@ -package chdbstable - -/* -#cgo LDFLAGS: -L/usr/local/lib -lchdb -#include // Include the C standard library for C.free -#include "chdb.h" -*/ -import "C" -import ( - "errors" - "fmt" - "runtime" - "unsafe" -) - -// ChdbError is returned when the C function returns an error. -type ChdbError struct { - msg string -} - -func (e *ChdbError) Error() string { - return e.msg -} - -// ErrNilResult is returned when the C function returns a nil pointer. -var ErrNilResult = errors.New("chDB C function returned nil pointer") - -// LocalResult mirrors the C struct local_result_v2 in Go. -type LocalResult struct { - cResult *C.struct_local_result_v2 -} - -type ChdbConn struct { - conn *C.struct_chdb_conn -} - -// newLocalResult creates a new LocalResult and sets a finalizer to free C memory. -func newLocalResult(cResult *C.struct_local_result_v2) *LocalResult { - result := &LocalResult{cResult: cResult} - runtime.SetFinalizer(result, freeLocalResult) - return result -} - -// newChdbConn creates a new ChdbConn and sets a finalizer to close the connection (and thus free the memory) -func newChdbConn(conn *C.struct_chdb_conn) *ChdbConn { - result := &ChdbConn{conn: conn} - runtime.SetFinalizer(result, closeChdbConn) - return result -} - -func NewConnection(argc int, argv []string) (*ChdbConn, error) { - cArgv := make([]*C.char, len(argv)) - for i, s := range argv { - cArgv[i] = C.CString(s) - defer C.free(unsafe.Pointer(cArgv[i])) - } - conn := C.connect_chdb(C.int(argc), &cArgv[0]) - if conn == nil { - return nil, fmt.Errorf("could not create a chdb connection") - } - return newChdbConn(*conn), nil -} - -func closeChdbConn(conn *ChdbConn) { - C.close_conn(&conn.conn) -} - -// freeLocalResult is called by the garbage collector. -func freeLocalResult(result *LocalResult) { - C.free_result_v2(result.cResult) -} - -// QueryStable calls the C function query_stable_v2. -func QueryStable(argc int, argv []string) (result *LocalResult, err error) { - cArgv := make([]*C.char, len(argv)) - for i, s := range argv { - cArgv[i] = C.CString(s) - defer C.free(unsafe.Pointer(cArgv[i])) - } - - cResult := C.query_stable_v2(C.int(argc), &cArgv[0]) - if cResult == nil { - // According to the C ABI of chDB v1.2.0, the C function query_stable_v2 - // returns nil if the query returns no data. This is not an error. We - // will change this behavior in the future. - return newLocalResult(cResult), nil - } - if cResult.error_message != nil { - return nil, &ChdbError{msg: C.GoString(cResult.error_message)} - } - return newLocalResult(cResult), nil -} - -// QueryStable calls the C function query_conn. -func (c *ChdbConn) QueryConn(queryStr string, formatStr string) (result *LocalResult, err error) { - - query := C.CString(queryStr) - format := C.CString(formatStr) - // free the strings in the C heap - defer C.free(unsafe.Pointer(query)) - defer C.free(unsafe.Pointer(format)) - - cResult := C.query_conn(c.conn, query, format) - if cResult == nil { - // According to the C ABI of chDB v1.2.0, the C function query_stable_v2 - // returns nil if the query returns no data. This is not an error. We - // will change this behavior in the future. - return newLocalResult(cResult), nil - } - if cResult.error_message != nil { - return nil, &ChdbError{msg: C.GoString(cResult.error_message)} - } - return newLocalResult(cResult), nil -} - -func (c *ChdbConn) Close() { - C.close_conn(&c.conn) -} - -// Accessor methods to access fields of the local_result_v2 struct. -func (r *LocalResult) Buf() []byte { - if r.cResult == nil { - return nil - } - if r.cResult.buf == nil { - return nil - } - return C.GoBytes(unsafe.Pointer(r.cResult.buf), C.int(r.cResult.len)) -} - -// Stringer interface for LocalResult -func (r LocalResult) String() string { - ret := r.Buf() - if ret == nil { - return "" - } - return string(ret) -} - -func (r *LocalResult) Len() int { - if r.cResult == nil { - return 0 - } - return int(r.cResult.len) -} - -func (r *LocalResult) Elapsed() float64 { - if r.cResult == nil { - return 0 - } - return float64(r.cResult.elapsed) -} - -func (r *LocalResult) RowsRead() uint64 { - if r.cResult == nil { - return 0 - } - return uint64(r.cResult.rows_read) -} - -func (r *LocalResult) BytesRead() uint64 { - if r.cResult == nil { - return 0 - } - return uint64(r.cResult.bytes_read) -} diff --git a/chdbstable/chdb_test.go b/chdbstable/chdb_test.go deleted file mode 100644 index 1cacb84..0000000 --- a/chdbstable/chdb_test.go +++ /dev/null @@ -1,75 +0,0 @@ -package chdbstable - -import ( - "testing" -) - -// TestCase defines the structure of a test case -type TestCase struct { - name string // Name of the test case - argv []string // Arguments to pass to QueryStable - expectError bool // Whether an error is expected - expectOutput string // Expected output -} - -func TestQueryStableMultipleCases(t *testing.T) { - // Define a series of test cases - testCases := []TestCase{ - { - name: "Single Query", - argv: []string{"clickhouse", "--multiquery", "--output-format=CSV", "--query=SELECT 123;"}, - expectError: false, - expectOutput: "123\n", - }, - { - name: "Single Queries", - argv: []string{"clickhouse", "--multiquery", "--output-format=CSV", "--query=SELECT 'abc';"}, - expectError: false, - expectOutput: "\"abc\"\n", - }, - { - name: "Error Query", - argv: []string{"clickhouse", "--multiquery", "--output-format=CSV", "--query=XXX;"}, - expectError: true, - expectOutput: "", - }, - } - - // Iterate over the test cases - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - result, err := QueryStable(len(tc.argv), tc.argv) - - // Assert based on the expected outcome of the test case - if tc.expectError { - if err == nil { - t.Errorf("Expected error, but got nil") - } - } else { - if err != nil { - t.Errorf("Expected no error, but got %v", err) - } else { - if result == nil { - t.Errorf("Expected non-nil result, but got nil") - } else { - if result.cResult == nil { - t.Errorf("Expected non-nil cResult, but got nil") - } else { - if result.cResult.error_message != nil { - t.Errorf("Expected nil error_message, but got %v", result.cResult.error_message) - } else { - if result.cResult.buf == nil { - t.Errorf("Expected non-nil output, but got nil") - } else { - if tc.expectOutput != string(result.String()) { - t.Errorf("Expected output %v, but got %v", tc.expectOutput, string(result.String())) - } - } - } - } - } - } - } - }) - } -} diff --git a/go.mod b/go.mod index 042d968..e716833 100644 --- a/go.mod +++ b/go.mod @@ -3,21 +3,17 @@ module github.com/chdb-io/chdb-go go 1.21 require ( - github.com/apache/arrow/go/v15 v15.0.2 github.com/c-bata/go-prompt v0.2.6 + github.com/ebitengine/purego v0.8.2 + github.com/huandu/go-sqlbuilder v1.27.3 github.com/parquet-go/parquet-go v0.23.0 ) require ( github.com/andybalholm/brotli v1.1.0 // indirect - github.com/ebitengine/purego v0.8.2 // indirect - github.com/goccy/go-json v0.10.3 // indirect - github.com/google/flatbuffers v24.3.25+incompatible // indirect github.com/google/uuid v1.6.0 // indirect - github.com/huandu/go-sqlbuilder v1.27.3 // indirect github.com/huandu/xstrings v1.4.0 // indirect github.com/klauspost/compress v1.17.9 // indirect - github.com/klauspost/cpuid/v2 v2.2.8 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-runewidth v0.0.15 // indirect @@ -27,11 +23,5 @@ require ( github.com/pkg/term v1.2.0-beta.2 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/segmentio/encoding v0.4.0 // indirect - github.com/zeebo/xxh3 v1.0.2 // indirect - golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect - golang.org/x/mod v0.19.0 // indirect - golang.org/x/sync v0.7.0 // indirect golang.org/x/sys v0.22.0 // indirect - golang.org/x/tools v0.23.0 // indirect - golang.org/x/xerrors v0.0.0-20240716161551-93cc26a95ae9 // indirect ) diff --git a/go.sum b/go.sum index 8b8999a..fdca1d0 100644 --- a/go.sum +++ b/go.sum @@ -1,21 +1,16 @@ github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= -github.com/apache/arrow/go/v15 v15.0.2 h1:60IliRbiyTWCWjERBCkO1W4Qun9svcYoZrSLcyOsMLE= -github.com/apache/arrow/go/v15 v15.0.2/go.mod h1:DGXsR3ajT524njufqf95822i+KTh+yea1jass9YXgjA= github.com/c-bata/go-prompt v0.2.6 h1:POP+nrHE+DfLYx370bedwNhsqmpCUynWPxuHi0C5vZI= github.com/c-bata/go-prompt v0.2.6/go.mod h1:/LMAke8wD2FsNu9EXNdHxNLbd9MedkPnCdfpU9wwHfY= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/ebitengine/purego v0.8.2 h1:jPPGWs2sZ1UgOSgD2bClL0MJIqu58nOmIcBuXr62z1I= github.com/ebitengine/purego v0.8.2/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= -github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA= -github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= -github.com/google/flatbuffers v24.3.25+incompatible h1:CX395cjN9Kke9mmalRoL3d81AtFUxJM+yDthflgJGkI= -github.com/google/flatbuffers v24.3.25+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= +github.com/huandu/go-assert v1.1.6 h1:oaAfYxq9KNDi9qswn/6aE0EydfxSa+tWZC1KabNitYs= github.com/huandu/go-assert v1.1.6/go.mod h1:JuIfbmYG9ykwvuxoJ3V8TB5QP+3+ajIA54Y44TmkMxs= github.com/huandu/go-sqlbuilder v1.27.3 h1:cNVF9vQP4i7rTk6XXJIEeMbGkZbxfjcITeJzobJK44k= github.com/huandu/go-sqlbuilder v1.27.3/go.mod h1:mS0GAtrtW+XL6nM2/gXHRJax2RwSW1TraavWDFAc1JA= @@ -23,8 +18,6 @@ github.com/huandu/xstrings v1.4.0 h1:D17IlohoQq4UcpqD7fDk80P7l+lwAmlFaBHgOipl2FU github.com/huandu/xstrings v1.4.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= -github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM= -github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= @@ -60,17 +53,7 @@ github.com/segmentio/encoding v0.4.0 h1:MEBYvRqiUB2nfR2criEXWqwdY6HJOUrCn5hboVOV github.com/segmentio/encoding v0.4.0/go.mod h1:/d03Cd8PoaDeceuhUUUQWjU0KhWjrmYrWPgtJHYZSnI= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= -github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= -github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= -github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= -golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= -golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= -golang.org/x/mod v0.19.0 h1:fEdghXQSo20giMthA7cd28ZC+jts4amQ3YMXiP5oMQ8= -golang.org/x/mod v0.19.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= -golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -79,16 +62,9 @@ golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200918174421-af09f7315aff/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/tools v0.23.0 h1:SGsXPZ+2l4JsgaCKkx+FQ9YZ5XEtA1GZYuoDjenLjvg= -golang.org/x/tools v0.23.0/go.mod h1:pnu6ufv6vQkll6szChhK3C3L/ruaIv5eBeztNG8wtsI= -golang.org/x/xerrors v0.0.0-20240716161551-93cc26a95ae9 h1:LLhsEBxRTBLuKlQxFBYUOU8xyFgXv6cOTp2HASDlsDk= -golang.org/x/xerrors v0.0.0-20240716161551-93cc26a95ae9/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= -gonum.org/v1/gonum v0.12.0 h1:xKuo6hzt+gMav00meVPUlXwSdoEJP46BR+wdxQEFK2o= -gonum.org/v1/gonum v0.12.0/go.mod h1:73TDxJfAAHeA8Mk9mf8NlIppyhQNo5GLTcYeqgo2lvY= google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= From f54b767bcbbbe4a6bbc331b15418da608ece9295 Mon Sep 17 00:00:00 2001 From: Andrei Goncear Date: Fri, 14 Feb 2025 13:16:06 +0000 Subject: [PATCH 11/22] fix macos runner (github actions) --- .github/workflows/chdb.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/chdb.yml b/.github/workflows/chdb.yml index 1c07297..2f09caf 100644 --- a/.github/workflows/chdb.yml +++ b/.github/workflows/chdb.yml @@ -32,7 +32,7 @@ jobs: run: ./chdb-go "SELECT 12345" build_mac: - runs-on: macos-12 + runs-on: macos-15 steps: - uses: actions/checkout@v3 - name: Fetch library From 7a5fd4ef76b8cca966d7608ff558c3408b6b8d51 Mon Sep 17 00:00:00 2001 From: Andrei Goncear Date: Fri, 14 Feb 2025 13:23:30 +0000 Subject: [PATCH 12/22] fix wrapper_test --- chdb/wrapper_test.go | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/chdb/wrapper_test.go b/chdb/wrapper_test.go index 2c3fee4..c27f731 100644 --- a/chdb/wrapper_test.go +++ b/chdb/wrapper_test.go @@ -33,9 +33,17 @@ func TestQueryToBuffer(t *testing.T) { }, // Session { - name: "Session Query 1", - queryStr: "CREATE DATABASE IF NOT EXISTS testdb; " + - "CREATE TABLE IF NOT EXISTS testdb.testtable (id UInt32) ENGINE = MergeTree() ORDER BY id;", + name: "Session Query 1", + queryStr: "CREATE DATABASE IF NOT EXISTS testdb; ", + outputFormat: "CSV", + + udfPath: "", + expectedErrMsg: "", + expectedResult: "", + }, + { + name: "Session Query 1 bis", + queryStr: "CREATE TABLE IF NOT EXISTS testdb.testtable (id UInt32) ENGINE = MergeTree() ORDER BY id;", outputFormat: "CSV", udfPath: "", From 342e4b5a2e94a07b1f13a13e87700f78a436414d Mon Sep 17 00:00:00 2001 From: Andrei Goncear Date: Fri, 14 Feb 2025 13:41:28 +0000 Subject: [PATCH 13/22] fix tests --- chdb/wrapper.go | 42 ++++++------------------------------------ chdb/wrapper_test.go | 43 +------------------------------------------ 2 files changed, 7 insertions(+), 78 deletions(-) diff --git a/chdb/wrapper.go b/chdb/wrapper.go index e38b295..9c19fed 100644 --- a/chdb/wrapper.go +++ b/chdb/wrapper.go @@ -4,41 +4,18 @@ import ( chdbpurego "github.com/chdb-io/chdb-go/chdb-purego" ) -// Query calls queryToBuffer with a default output format of "CSV" if not provided. +// Query calls query_conn with a default in-memory session and default output format of "CSV" if not provided. func Query(queryStr string, outputFormats ...string) (result chdbpurego.ChdbResult, err error) { outputFormat := "CSV" // Default value if len(outputFormats) > 0 { outputFormat = outputFormats[0] } - return queryToBuffer(queryStr, outputFormat, "", "") -} - -// queryToBuffer constructs the arguments for QueryStable and calls it. -func queryToBuffer(queryStr, outputFormat, path, udfPath string) (result chdbpurego.ChdbResult, err error) { - argv := []string{"clickhouse", "--multiquery"} - - // Handle output format - if outputFormat == "Debug" || outputFormat == "debug" { - argv = append(argv, "--verbose", "--log-level=trace", "--output-format=CSV") - } else { - argv = append(argv, "--output-format="+outputFormat) - } - - // Handle path - if path != "" { - argv = append(argv, "--path="+path) - } - - // Add query string - argv = append(argv, "--query="+queryStr) - - // Handle user-defined functions path - if udfPath != "" { - argv = append(argv, "--", "--user_scripts_path="+udfPath, "--user_defined_executable_functions_config="+udfPath+"/*.xml") + tempSession, err := initConnection(":memory:") + if err != nil { + return nil, err } - - // Call QueryStable with the constructed arguments - return chdbpurego.RawQuery(len(argv), argv) + defer tempSession.Close() + return tempSession.Query(queryStr, outputFormat) } func initConnection(connStr string) (result chdbpurego.ChdbConn, err error) { @@ -46,10 +23,3 @@ func initConnection(connStr string) (result chdbpurego.ChdbConn, err error) { // Call NewConnection with the constructed arguments return chdbpurego.NewConnection(len(argv), argv) } - -func connQueryToBuffer(conn chdbpurego.ChdbConn, queryStr, outputFormat string) (result chdbpurego.ChdbResult, err error) { - if outputFormat == "" { - outputFormat = "CSV" - } - return conn.Query(queryStr, outputFormat) -} diff --git a/chdb/wrapper_test.go b/chdb/wrapper_test.go index c27f731..c97680a 100644 --- a/chdb/wrapper_test.go +++ b/chdb/wrapper_test.go @@ -6,11 +6,6 @@ import ( func TestQueryToBuffer(t *testing.T) { // Create a temporary directory - sess, err := NewSession() - if err != nil { - t.Fatalf("could not create session: %s", err) - } - defer sess.Close() // Define test cases testCases := []struct { @@ -31,43 +26,7 @@ func TestQueryToBuffer(t *testing.T) { expectedErrMsg: "", expectedResult: "123\n", }, - // Session - { - name: "Session Query 1", - queryStr: "CREATE DATABASE IF NOT EXISTS testdb; ", - outputFormat: "CSV", - - udfPath: "", - expectedErrMsg: "", - expectedResult: "", - }, - { - name: "Session Query 1 bis", - queryStr: "CREATE TABLE IF NOT EXISTS testdb.testtable (id UInt32) ENGINE = MergeTree() ORDER BY id;", - outputFormat: "CSV", - udfPath: "", - expectedErrMsg: "", - expectedResult: "", - }, - { - name: "Session Query 2", - queryStr: "USE testdb; INSERT INTO testtable VALUES (1), (2), (3);", - outputFormat: "CSV", - - udfPath: "", - expectedErrMsg: "", - expectedResult: "", - }, - { - name: "Session Query 3", - queryStr: "SELECT * FROM testtable;", - outputFormat: "CSV", - - udfPath: "", - expectedErrMsg: "", - expectedResult: "1\n2\n3\n", - }, { name: "Error Query", queryStr: "SELECT * FROM nonexist; ", @@ -83,7 +42,7 @@ func TestQueryToBuffer(t *testing.T) { t.Run(tc.name, func(t *testing.T) { // Call queryToBuffer - result, err := sess.Query(tc.queryStr, tc.outputFormat) + result, err := Query(tc.queryStr, tc.outputFormat) // Verify if tc.expectedErrMsg != "" { From 8a983de88c47d4f9292ecd79137a013e78ac2d8d Mon Sep 17 00:00:00 2001 From: Andrei Goncear Date: Fri, 14 Feb 2025 14:14:22 +0000 Subject: [PATCH 14/22] update documentation --- README.md | 2 +- chdb-purego/chdb.go | 34 +++++++++++- chdb-purego/chdb.h | 123 ----------------------------------------- chdb-purego/helpers.go | 19 ------- chdb-purego/types.go | 11 ++++ chdb.md | 38 ++++++++----- chdb/doc.md | 109 ++++++++++++++++++++++++++++++++++++ chdb/session.go | 4 +- lowApi.md | 115 ++++++++++++++++++++------------------ 9 files changed, 240 insertions(+), 215 deletions(-) delete mode 100644 chdb-purego/chdb.h create mode 100644 chdb/doc.md diff --git a/README.md b/README.md index 3570958..f703248 100644 --- a/README.md +++ b/README.md @@ -67,9 +67,9 @@ func main() { fmt.Println(result) tmp_path := filepath.Join(os.TempDir(), "chdb_test") - defer os.RemoveAll(tmp_path) // Stateful Query (persistent) session, _ := chdb.NewSession(tmp_path) + // session cleanup will also delete the folder defer session.Cleanup() _, err = session.Query("CREATE DATABASE IF NOT EXISTS testdb; " + diff --git a/chdb-purego/chdb.go b/chdb-purego/chdb.go index ff558f1..72bb16b 100644 --- a/chdb-purego/chdb.go +++ b/chdb-purego/chdb.go @@ -93,7 +93,7 @@ type connection struct { conn **chdb_conn } -func NewChdbConn(conn **chdb_conn) ChdbConn { +func newChdbConn(conn **chdb_conn) ChdbConn { c := &connection{ conn: conn, } @@ -131,7 +131,6 @@ func (c *connection) Query(queryStr string, formatStr string) (result ChdbResult return newChdbResult(res), nil } -// Ready implements ChdbConn. func (c *connection) Ready() bool { if c.conn != nil { deref := *c.conn @@ -142,6 +141,7 @@ func (c *connection) Ready() bool { return false } +// RawQuery will execute the given clickouse query without using any session. func RawQuery(argc int, argv []string) (result ChdbResult, err error) { res := queryStableV2(argc, argv) if res == nil { @@ -157,10 +157,38 @@ func RawQuery(argc int, argv []string) (result ChdbResult, err error) { return newChdbResult(res), nil } +// Session will keep the state of query. +// If path is None, it will create a temporary directory and use it as the database path +// and the temporary directory will be removed when the session is closed. +// You can also pass in a path to create a database at that path where will keep your data. +// +// You can also use a connection string to pass in the path and other parameters. +// Examples: +// - ":memory:" (for in-memory database) +// - "test.db" (for relative path) +// - "file:test.db" (same as above) +// - "/path/to/test.db" (for absolute path) +// - "file:/path/to/test.db" (same as above) +// - "file:test.db?param1=value1¶m2=value2" (for relative path with query params) +// - "file::memory:?verbose&log-level=test" (for in-memory database with query params) +// - "///path/to/test.db?param1=value1¶m2=value2" (for absolute path) +// +// Connection string args handling: +// +// Connection string can contain query params like "file:test.db?param1=value1¶m2=value2" +// "param1=value1" will be passed to ClickHouse engine as start up args. +// +// For more details, see `clickhouse local --help --verbose` +// Some special args handling: +// - "mode=ro" would be "--readonly=1" for clickhouse (read-only mode) +// +// Important: +// - There can be only one session at a time. If you want to create a new session, you need to close the existing one. +// - Creating a new session will close the existing one. func NewConnection(argc int, argv []string) (ChdbConn, error) { conn := connectChdb(argc, argv) if conn == nil { return nil, fmt.Errorf("could not create a chdb connection") } - return NewChdbConn(conn), nil + return newChdbConn(conn), nil } diff --git a/chdb-purego/chdb.h b/chdb-purego/chdb.h deleted file mode 100644 index ebc2009..0000000 --- a/chdb-purego/chdb.h +++ /dev/null @@ -1,123 +0,0 @@ -#pragma once - -#ifdef __cplusplus -# include -# include -# include -# include -# include -# include -extern "C" { -#else -# include -# include -# include -#endif - -#define CHDB_EXPORT __attribute__((visibility("default"))) -struct local_result -{ - char * buf; - size_t len; - void * _vec; // std::vector *, for freeing - double elapsed; - uint64_t rows_read; - uint64_t bytes_read; -}; - -#ifdef __cplusplus -struct local_result_v2 -{ - char * buf = nullptr; - size_t len = 0; - void * _vec = nullptr; // std::vector *, for freeing - double elapsed = 0.0; - uint64_t rows_read = 0; - uint64_t bytes_read = 0; - char * error_message = nullptr; -}; -#else -struct local_result_v2 -{ - char * buf; - size_t len; - void * _vec; // std::vector *, for freeing - double elapsed; - uint64_t rows_read; - uint64_t bytes_read; - char * error_message; -}; -#endif - -CHDB_EXPORT struct local_result * query_stable(int argc, char ** argv); -CHDB_EXPORT void free_result(struct local_result * result); - -CHDB_EXPORT struct local_result_v2 * query_stable_v2(int argc, char ** argv); -CHDB_EXPORT void free_result_v2(struct local_result_v2 * result); - -#ifdef __cplusplus -struct query_request -{ - std::string query; - std::string format; -}; - -struct query_queue -{ - std::mutex mutex; - std::condition_variable query_cv; // For query submission - std::condition_variable result_cv; // For query result retrieval - query_request current_query; - local_result_v2 * current_result = nullptr; - bool has_query = false; - bool shutdown = false; - bool cleanup_done = false; -}; -#endif - -/** - * Connection structure for chDB - * Contains server instance, connection state, and query processing queue - */ -struct chdb_conn -{ - void * server; /* ClickHouse LocalServer instance */ - bool connected; /* Connection state flag */ - void * queue; /* Query processing queue */ -}; - -/** - * Creates a new chDB connection. - * Only one active connection is allowed per process. - * Creating a new connection with different path requires closing existing connection. - * - * @param argc Number of command-line arguments - * @param argv Command-line arguments array (--path= to specify database location) - * @return Pointer to connection pointer, or NULL on failure - * @note Default path is ":memory:" if not specified - */ -CHDB_EXPORT struct chdb_conn ** connect_chdb(int argc, char ** argv); - -/** - * Closes an existing chDB connection and cleans up resources. - * Thread-safe function that handles connection shutdown and cleanup. - * - * @param conn Pointer to connection pointer to close - */ -CHDB_EXPORT void close_conn(struct chdb_conn ** conn); - -/** - * Executes a query on the given connection. - * Thread-safe function that handles query execution in a separate thread. - * - * @param conn Connection to execute query on - * @param query SQL query string to execute - * @param format Output format string (e.g., "CSV", default format) - * @return Query result structure containing output or error message - * @note Returns error result if connection is invalid or closed - */ -CHDB_EXPORT struct local_result_v2 * query_conn(struct chdb_conn * conn, const char * query, const char * format); - -#ifdef __cplusplus -} -#endif \ No newline at end of file diff --git a/chdb-purego/helpers.go b/chdb-purego/helpers.go index 28507d6..7c78274 100644 --- a/chdb-purego/helpers.go +++ b/chdb-purego/helpers.go @@ -1,7 +1,6 @@ package chdbpurego import ( - "runtime" "unsafe" ) @@ -20,21 +19,3 @@ func ptrToGoString(ptr *byte) string { return string(unsafe.Slice(ptr, length)) } - -func stringToPtr(s string, pinner *runtime.Pinner) *byte { - // Pinne for convert string to bytes - // maybe there is simpler solution but it was late when I write this code. - data := make([]byte, len(s)+1) - copy(data, s) - data[len(s)] = 0 // Null-terminator - - ptr := &data[0] - pinner.Pin(ptr) - - return (*byte)(unsafe.Pointer(ptr)) -} - -func strToBytePtr(s string) *byte { - b := append([]byte(s), 0) // Convert to []byte and append null terminator - return &b[0] // Get pointer to first byte -} diff --git a/chdb-purego/types.go b/chdb-purego/types.go index f84a5a9..d11ca5c 100644 --- a/chdb-purego/types.go +++ b/chdb-purego/types.go @@ -32,18 +32,29 @@ type chdb_conn struct { } type ChdbResult interface { + // Raw bytes result buffer, used for reading the result of clickhouse query Buf() []byte + // String rapresentation of the the buffer String() string + // Lenght in bytes of the buffer Len() int + // Number of seconds elapsed for the query execution Elapsed() float64 + // Amount of rows returned by the query RowsRead() uint64 + // Amount of bytes returned by the query BytesRead() uint64 + // If the query had any error during execution, here you can retrieve the cause. Error() error + // Free the query result and all the allocated memory Free() error } type ChdbConn interface { + //Query executes the given queryStr in the underlying clickhouse connection, and output the result in the given formatStr Query(queryStr string, formatStr string) (result ChdbResult, err error) + //Ready returns a boolean indicating if the connections is successfully established. Ready() bool + //Close the connection and free the underlying allocated memory Close() } diff --git a/chdb.md b/chdb.md index 32b2abf..b54d426 100644 --- a/chdb.md +++ b/chdb.md @@ -8,27 +8,28 @@ import "github.com/chdb-io/chdb-go/chdb" ## Index -- [func Query\(queryStr string, outputFormats ...string\) \*chdbstable.LocalResult](<#Query>) +- [func Query\(queryStr string, outputFormats ...string\) \(result chdbpurego.ChdbResult, err error\)](<#Query>) - [type Session](<#Session>) - [func NewSession\(paths ...string\) \(\*Session, error\)](<#NewSession>) - [func \(s \*Session\) Cleanup\(\)](<#Session.Cleanup>) - [func \(s \*Session\) Close\(\)](<#Session.Close>) + - [func \(s \*Session\) ConnStr\(\) string](<#Session.ConnStr>) - [func \(s \*Session\) IsTemp\(\) bool](<#Session.IsTemp>) - [func \(s \*Session\) Path\(\) string](<#Session.Path>) - - [func \(s \*Session\) Query\(queryStr string, outputFormats ...string\) \*chdbstable.LocalResult](<#Session.Query>) + - [func \(s \*Session\) Query\(queryStr string, outputFormats ...string\) \(result chdbpurego.ChdbResult, err error\)](<#Session.Query>) -## func [Query]() +## func [Query]() ```go -func Query(queryStr string, outputFormats ...string) *chdbstable.LocalResult +func Query(queryStr string, outputFormats ...string) (result chdbpurego.ChdbResult, err error) ``` -Query calls queryToBuffer with a default output format of "CSV" if not provided. +Query calls query\_conn with a default in\-memory session and default output format of "CSV" if not provided. -## type [Session]() +## type [Session]() @@ -39,7 +40,7 @@ type Session struct { ``` -### func [NewSession]() +### func [NewSession]() ```go func NewSession(paths ...string) (*Session, error) @@ -48,7 +49,7 @@ func NewSession(paths ...string) (*Session, error) NewSession creates a new session with the given path. If path is empty, a temporary directory is created. Note: The temporary directory is removed when Close is called. -### func \(\*Session\) [Cleanup]() +### func \(\*Session\) [Cleanup]() ```go func (s *Session) Cleanup() @@ -57,7 +58,7 @@ func (s *Session) Cleanup() Cleanup closes the session and removes the directory. -### func \(\*Session\) [Close]() +### func \(\*Session\) [Close]() ```go func (s *Session) Close() @@ -69,8 +70,17 @@ Close closes the session and removes the temporary directory temporary directory is created when NewSession was called with an empty path. ``` + +### func \(\*Session\) [ConnStr]() + +```go +func (s *Session) ConnStr() string +``` + +ConnStr returns the current connection string used for the underlying connection + -### func \(\*Session\) [IsTemp]() +### func \(\*Session\) [IsTemp]() ```go func (s *Session) IsTemp() bool @@ -79,7 +89,7 @@ func (s *Session) IsTemp() bool IsTemp returns whether the session is temporary. -### func \(\*Session\) [Path]() +### func \(\*Session\) [Path]() ```go func (s *Session) Path() string @@ -88,12 +98,12 @@ func (s *Session) Path() string Path returns the path of the session. -### func \(\*Session\) [Query]() +### func \(\*Session\) [Query]() ```go -func (s *Session) Query(queryStr string, outputFormats ...string) *chdbstable.LocalResult +func (s *Session) Query(queryStr string, outputFormats ...string) (result chdbpurego.ChdbResult, err error) ``` -Query calls queryToBuffer with a default output format of "CSV" if not provided. +Query calls \`query\_conn\` function with the current connection and a default output format of "CSV" if not provided. Generated by [gomarkdoc]() diff --git a/chdb/doc.md b/chdb/doc.md new file mode 100644 index 0000000..1cf7e97 --- /dev/null +++ b/chdb/doc.md @@ -0,0 +1,109 @@ + + +# chdb + +```go +import "github.com/chdb-io/chdb-go/chdb" +``` + +## Index + +- [func Query\(queryStr string, outputFormats ...string\) \(result chdbpurego.ChdbResult, err error\)](<#Query>) +- [type Session](<#Session>) + - [func NewSession\(paths ...string\) \(\*Session, error\)](<#NewSession>) + - [func \(s \*Session\) Cleanup\(\)](<#Session.Cleanup>) + - [func \(s \*Session\) Close\(\)](<#Session.Close>) + - [func \(s \*Session\) ConnStr\(\) string](<#Session.ConnStr>) + - [func \(s \*Session\) IsTemp\(\) bool](<#Session.IsTemp>) + - [func \(s \*Session\) Path\(\) string](<#Session.Path>) + - [func \(s \*Session\) Query\(queryStr string, outputFormats ...string\) \(result chdbpurego.ChdbResult, err error\)](<#Session.Query>) + + + +## func [Query]() + +```go +func Query(queryStr string, outputFormats ...string) (result chdbpurego.ChdbResult, err error) +``` + +Query calls query\_conn with a default in\-memory session and default output format of "CSV" if not provided. + + +## type [Session]() + + + +```go +type Session struct { + // contains filtered or unexported fields +} +``` + + +### func [NewSession]() + +```go +func NewSession(paths ...string) (*Session, error) +``` + +NewSession creates a new session with the given path. If path is empty, a temporary directory is created. Note: The temporary directory is removed when Close is called. + + +### func \(\*Session\) [Cleanup]() + +```go +func (s *Session) Cleanup() +``` + +Cleanup closes the session and removes the directory. + + +### func \(\*Session\) [Close]() + +```go +func (s *Session) Close() +``` + +Close closes the session and removes the temporary directory + +``` +temporary directory is created when NewSession was called with an empty path. +``` + + +### func \(\*Session\) [ConnStr]() + +```go +func (s *Session) ConnStr() string +``` + + + + +### func \(\*Session\) [IsTemp]() + +```go +func (s *Session) IsTemp() bool +``` + +IsTemp returns whether the session is temporary. + + +### func \(\*Session\) [Path]() + +```go +func (s *Session) Path() string +``` + +Path returns the path of the session. + + +### func \(\*Session\) [Query]() + +```go +func (s *Session) Query(queryStr string, outputFormats ...string) (result chdbpurego.ChdbResult, err error) +``` + +Query calls queryToBuffer with a default output format of "CSV" if not provided. + +Generated by [gomarkdoc]() diff --git a/chdb/session.go b/chdb/session.go index fac7152..3ed1128 100644 --- a/chdb/session.go +++ b/chdb/session.go @@ -52,14 +52,13 @@ func NewSession(paths ...string) (*Session, error) { return globalSession, nil } -// Query calls queryToBuffer with a default output format of "CSV" if not provided. +// Query calls `query_conn` function with the current connection and a default output format of "CSV" if not provided. func (s *Session) Query(queryStr string, outputFormats ...string) (result chdbpurego.ChdbResult, err error) { outputFormat := "CSV" // Default value if len(outputFormats) > 0 { outputFormat = outputFormats[0] } return s.conn.Query(queryStr, outputFormat) - } // Close closes the session and removes the temporary directory @@ -85,6 +84,7 @@ func (s *Session) Path() string { return s.path } +// ConnStr returns the current connection string used for the underlying connection func (s *Session) ConnStr() string { return s.connStr } diff --git a/lowApi.md b/lowApi.md index 08c019a..3648242 100644 --- a/lowApi.md +++ b/lowApi.md @@ -1,95 +1,104 @@ -# chdbstable +# chdbpurego ```go -import "github.com/chdb-io/chdb-go/chdbstable" +import "github.com/chdb-io/chdb-go/chdb-purego" ``` ## Index -- [type LocalResult](<#LocalResult>) - - [func QueryStable\(argc int, argv \[\]string\) \*LocalResult](<#QueryStable>) - - [func \(r \*LocalResult\) Buf\(\) \[\]byte](<#LocalResult.Buf>) - - [func \(r \*LocalResult\) BytesRead\(\) uint64](<#LocalResult.BytesRead>) - - [func \(r \*LocalResult\) Elapsed\(\) float64](<#LocalResult.Elapsed>) - - [func \(r \*LocalResult\) Len\(\) int](<#LocalResult.Len>) - - [func \(r \*LocalResult\) RowsRead\(\) uint64](<#LocalResult.RowsRead>) - - [func \(r LocalResult\) String\(\) string](<#LocalResult.String>) +- [type ChdbConn](<#ChdbConn>) + - [func NewConnection\(argc int, argv \[\]string\) \(ChdbConn, error\)](<#NewConnection>) +- [type ChdbResult](<#ChdbResult>) + - [func RawQuery\(argc int, argv \[\]string\) \(result ChdbResult, err error\)](<#RawQuery>) - -## type [LocalResult]() + +## type [ChdbConn]() + -LocalResult mirrors the C struct local\_result in Go. ```go -type LocalResult struct { - // contains filtered or unexported fields +type ChdbConn interface { + //Query executes the given queryStr in the underlying clickhouse connection, and output the result in the given formatStr + Query(queryStr string, formatStr string) (result ChdbResult, err error) + //Ready returns a boolean indicating if the connections is successfully established. + Ready() bool + //Close the connection and free the underlying allocated memory + Close() } ``` - -### func [QueryStable]() + +### func [NewConnection]() ```go -func QueryStable(argc int, argv []string) *LocalResult +func NewConnection(argc int, argv []string) (ChdbConn, error) ``` -QueryStable calls the C function query\_stable. - - -### func \(\*LocalResult\) [Buf]() +Session will keep the state of query. If path is None, it will create a temporary directory and use it as the database path and the temporary directory will be removed when the session is closed. You can also pass in a path to create a database at that path where will keep your data. -```go -func (r *LocalResult) Buf() []byte -``` +You can also use a connection string to pass in the path and other parameters. Examples: -Accessor methods to access fields of the local\_result struct. +- ":memory:" \(for in\-memory database\) +- "test.db" \(for relative path\) +- "file:test.db" \(same as above\) +- "/path/to/test.db" \(for absolute path\) +- "file:/path/to/test.db" \(same as above\) +- "file:test.db?param1=value1¶m2=value2" \(for relative path with query params\) +- "file::memory:?verbose&log-level=test" \(for in\-memory database with query params\) +- "///path/to/test.db?param1=value1¶m2=value2" \(for absolute path\) - -### func \(\*LocalResult\) [BytesRead]() +Connection string args handling: -```go -func (r *LocalResult) BytesRead() uint64 ``` +Connection string can contain query params like "file:test.db?param1=value1¶m2=value2" +"param1=value1" will be passed to ClickHouse engine as start up args. - - - -### func \(\*LocalResult\) [Elapsed]() - -```go -func (r *LocalResult) Elapsed() float64 +For more details, see `clickhouse local --help --verbose` +Some special args handling: +- "mode=ro" would be "--readonly=1" for clickhouse (read-only mode) ``` +Important: +- There can be only one session at a time. If you want to create a new session, you need to close the existing one. +- Creating a new session will close the existing one. - -### func \(\*LocalResult\) [Len]() - -```go -func (r *LocalResult) Len() int -``` - + +## type [ChdbResult]() - -### func \(\*LocalResult\) [RowsRead]() ```go -func (r *LocalResult) RowsRead() uint64 +type ChdbResult interface { + // Raw bytes result buffer, used for reading the result of clickhouse query + Buf() []byte + // String rapresentation of the the buffer + String() string + // Lenght in bytes of the buffer + Len() int + // Number of seconds elapsed for the query execution + Elapsed() float64 + // Amount of rows returned by the query + RowsRead() uint64 + // Amount of bytes returned by the query + BytesRead() uint64 + // If the query had any error during execution, here you can retrieve the cause. + Error() error + // Free the query result and all the allocated memory + Free() error +} ``` - - - -### func \(LocalResult\) [String]() + +### func [RawQuery]() ```go -func (r LocalResult) String() string +func RawQuery(argc int, argv []string) (result ChdbResult, err error) ``` -Stringer interface for LocalResult +RawQuery will execute the given clickouse query without using any session. Generated by [gomarkdoc]() From 43c572ec6baa4308272334e2ab80ce8a7fce364c Mon Sep 17 00:00:00 2001 From: Andrei Goncear Date: Fri, 14 Feb 2025 14:25:50 +0000 Subject: [PATCH 15/22] remove useless error in `Free()` method --- chdb-purego/chdb.go | 4 ++-- chdb-purego/types.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/chdb-purego/chdb.go b/chdb-purego/chdb.go index 72bb16b..4eae47d 100644 --- a/chdb-purego/chdb.go +++ b/chdb-purego/chdb.go @@ -56,12 +56,12 @@ func (c *result) Error() error { } // Free implements ChdbResult. -func (c *result) Free() error { +func (c *result) Free() { if c.localResv2 != nil { freeResultV2(c.localResv2) c.localResv2 = nil } - return nil + } // Len implements ChdbResult. diff --git a/chdb-purego/types.go b/chdb-purego/types.go index d11ca5c..ea953cb 100644 --- a/chdb-purego/types.go +++ b/chdb-purego/types.go @@ -47,7 +47,7 @@ type ChdbResult interface { // If the query had any error during execution, here you can retrieve the cause. Error() error // Free the query result and all the allocated memory - Free() error + Free() } type ChdbConn interface { From 67d0538a1a9c978975a053e2b8fbc089f80245cb Mon Sep 17 00:00:00 2001 From: Andrei Goncear Date: Fri, 14 Feb 2025 15:54:25 +0000 Subject: [PATCH 16/22] remove test hardcoded path on main.go --- main.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/main.go b/main.go index 97626dc..b4fd461 100644 --- a/main.go +++ b/main.go @@ -39,8 +39,7 @@ data will lost after exit. If you want to keep the data, specify a path to a dir // If path is specified or no additional arguments, enter interactive mode if len(flag.Args()) == 0 { - t := "/tmp" - pathFlag = &t + var err error var session *chdb.Session if *pathFlag != "" { From f1aaefc3915dcb847a3ad148d36cb85d7658d996 Mon Sep 17 00:00:00 2001 From: Andrei Goncear Date: Sun, 16 Feb 2025 09:37:35 +0000 Subject: [PATCH 17/22] fix session tests, remove RawQuery, fix docs --- chdb-purego/chdb.go | 16 --------------- chdb.md | 18 ++++++++--------- chdb/session_test.go | 47 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 56 insertions(+), 25 deletions(-) diff --git a/chdb-purego/chdb.go b/chdb-purego/chdb.go index 4eae47d..c4171ba 100644 --- a/chdb-purego/chdb.go +++ b/chdb-purego/chdb.go @@ -141,22 +141,6 @@ func (c *connection) Ready() bool { return false } -// RawQuery will execute the given clickouse query without using any session. -func RawQuery(argc int, argv []string) (result ChdbResult, err error) { - res := queryStableV2(argc, argv) - if res == nil { - // According to the C ABI of chDB v1.2.0, the C function query_stable_v2 - // returns nil if the query returns no data. This is not an error. We - // will change this behavior in the future. - return newChdbResult(res), nil - } - if res.error_message != nil { - return nil, errors.New(ptrToGoString(res.error_message)) - } - - return newChdbResult(res), nil -} - // Session will keep the state of query. // If path is None, it will create a temporary directory and use it as the database path // and the temporary directory will be removed when the session is closed. diff --git a/chdb.md b/chdb.md index b54d426..426a847 100644 --- a/chdb.md +++ b/chdb.md @@ -20,7 +20,7 @@ import "github.com/chdb-io/chdb-go/chdb" -## func [Query]() +## func [Query]() ```go func Query(queryStr string, outputFormats ...string) (result chdbpurego.ChdbResult, err error) @@ -29,7 +29,7 @@ func Query(queryStr string, outputFormats ...string) (result chdbpurego.ChdbResu Query calls query\_conn with a default in\-memory session and default output format of "CSV" if not provided. -## type [Session]() +## type [Session]() @@ -40,7 +40,7 @@ type Session struct { ``` -### func [NewSession]() +### func [NewSession]() ```go func NewSession(paths ...string) (*Session, error) @@ -49,7 +49,7 @@ func NewSession(paths ...string) (*Session, error) NewSession creates a new session with the given path. If path is empty, a temporary directory is created. Note: The temporary directory is removed when Close is called. -### func \(\*Session\) [Cleanup]() +### func \(\*Session\) [Cleanup]() ```go func (s *Session) Cleanup() @@ -58,7 +58,7 @@ func (s *Session) Cleanup() Cleanup closes the session and removes the directory. -### func \(\*Session\) [Close]() +### func \(\*Session\) [Close]() ```go func (s *Session) Close() @@ -71,7 +71,7 @@ temporary directory is created when NewSession was called with an empty path. ``` -### func \(\*Session\) [ConnStr]() +### func \(\*Session\) [ConnStr]() ```go func (s *Session) ConnStr() string @@ -80,7 +80,7 @@ func (s *Session) ConnStr() string ConnStr returns the current connection string used for the underlying connection -### func \(\*Session\) [IsTemp]() +### func \(\*Session\) [IsTemp]() ```go func (s *Session) IsTemp() bool @@ -89,7 +89,7 @@ func (s *Session) IsTemp() bool IsTemp returns whether the session is temporary. -### func \(\*Session\) [Path]() +### func \(\*Session\) [Path]() ```go func (s *Session) Path() string @@ -98,7 +98,7 @@ func (s *Session) Path() string Path returns the path of the session. -### func \(\*Session\) [Query]() +### func \(\*Session\) [Query]() ```go func (s *Session) Query(queryStr string, outputFormats ...string) (result chdbpurego.ChdbResult, err error) diff --git a/chdb/session_test.go b/chdb/session_test.go index c414a12..768e5f6 100644 --- a/chdb/session_test.go +++ b/chdb/session_test.go @@ -2,6 +2,7 @@ package chdb import ( "os" + "path/filepath" "testing" ) @@ -50,3 +51,49 @@ func TestSessionCleanup(t *testing.T) { t.Errorf("Session directory should be removed after Cleanup: %s", session.Path()) } } + +func TestQuery(t *testing.T) { + session, _ := NewSession() + defer session.Cleanup() + + session.Query("CREATE DATABASE IF NOT EXISTS testdb; " + + "CREATE TABLE IF NOT EXISTS testdb.testtable (id UInt32) ENGINE = MergeTree() ORDER BY id;") + + session.Query("USE testdb; INSERT INTO testtable VALUES (1), (2), (3);") + + ret, err := session.Query("SELECT * FROM testtable;") + if err != nil { + t.Errorf("Query failed: %s", err) + } + if string(ret.Buf()) != "1\n2\n3\n" { + t.Errorf("Query result should be 1\n2\n3\n, got %s", string(ret.Buf())) + } +} + +func TestSessionPathAndIsTemp(t *testing.T) { + // Create a new session and check its Path and IsTemp + session, _ := NewSession() + + if session.Path() == "" { + t.Errorf("Session path should not be empty") + } + + if !session.IsTemp() { + t.Errorf("Session should be temporary") + } + session.Close() + + // Create a new session with a specific path and check its Path and IsTemp + path := filepath.Join(os.TempDir(), "chdb_test2") + defer os.RemoveAll(path) + session, _ = NewSession(path) + defer session.Cleanup() + + if session.Path() != path { + t.Errorf("Session path should be %s, got %s", path, session.Path()) + } + + if session.IsTemp() { + t.Errorf("Session should not be temporary") + } +} From 51ede244570992beeaa9fa67bd1e4096d3343d5d Mon Sep 17 00:00:00 2001 From: Andrei Goncear Date: Sun, 16 Feb 2025 09:44:59 +0000 Subject: [PATCH 18/22] fix tests --- chdb/session_test.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/chdb/session_test.go b/chdb/session_test.go index 768e5f6..591d0ff 100644 --- a/chdb/session_test.go +++ b/chdb/session_test.go @@ -56,10 +56,9 @@ func TestQuery(t *testing.T) { session, _ := NewSession() defer session.Cleanup() - session.Query("CREATE DATABASE IF NOT EXISTS testdb; " + - "CREATE TABLE IF NOT EXISTS testdb.testtable (id UInt32) ENGINE = MergeTree() ORDER BY id;") + session.Query("CREATE TABLE IF NOT EXISTS testtable (id UInt32) ENGINE = MergeTree() ORDER BY id;") - session.Query("USE testdb; INSERT INTO testtable VALUES (1), (2), (3);") + session.Query("INSERT INTO testtable VALUES (1), (2), (3);") ret, err := session.Query("SELECT * FROM testtable;") if err != nil { From 54e2bdd43a674aa71432bb5b9e3a15b6ac26cfae Mon Sep 17 00:00:00 2001 From: Auxten Wang Date: Mon, 17 Feb 2025 13:07:55 +0800 Subject: [PATCH 19/22] Update session_test.go --- chdb/session_test.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/chdb/session_test.go b/chdb/session_test.go index 591d0ff..83c9ea1 100644 --- a/chdb/session_test.go +++ b/chdb/session_test.go @@ -53,7 +53,9 @@ func TestSessionCleanup(t *testing.T) { } func TestQuery(t *testing.T) { - session, _ := NewSession() + path := filepath.Join(os.TempDir(), "chdb_test") + defer os.RemoveAll(path) + session, _ := NewSession(path) defer session.Cleanup() session.Query("CREATE TABLE IF NOT EXISTS testtable (id UInt32) ENGINE = MergeTree() ORDER BY id;") From 7e1c4abf245d4c2022922cd827e84ffab8f14777 Mon Sep 17 00:00:00 2001 From: Andrei Goncear Date: Mon, 17 Feb 2025 11:02:00 +0000 Subject: [PATCH 20/22] remove flaky test --- chdb/session_test.go | 43 +++++++++++++++++++++++++------------------ 1 file changed, 25 insertions(+), 18 deletions(-) diff --git a/chdb/session_test.go b/chdb/session_test.go index 83c9ea1..325923c 100644 --- a/chdb/session_test.go +++ b/chdb/session_test.go @@ -52,24 +52,31 @@ func TestSessionCleanup(t *testing.T) { } } -func TestQuery(t *testing.T) { - path := filepath.Join(os.TempDir(), "chdb_test") - defer os.RemoveAll(path) - session, _ := NewSession(path) - defer session.Cleanup() - - session.Query("CREATE TABLE IF NOT EXISTS testtable (id UInt32) ENGINE = MergeTree() ORDER BY id;") - - session.Query("INSERT INTO testtable VALUES (1), (2), (3);") - - ret, err := session.Query("SELECT * FROM testtable;") - if err != nil { - t.Errorf("Query failed: %s", err) - } - if string(ret.Buf()) != "1\n2\n3\n" { - t.Errorf("Query result should be 1\n2\n3\n, got %s", string(ret.Buf())) - } -} +// This test is currently flaky because of this: https://github.com/chdb-io/chdb/pull/299/commits/91b0aedd8c17e74a4bb213e885d89cc9a77c99ad +// func TestQuery(t *testing.T) { + +// session, _ := NewSession() +// defer session.Cleanup() +// // time.Sleep(time.Second * 5) + +// _, err := session.Query("CREATE TABLE IF NOT EXISTS TestQuery (id UInt32) ENGINE = MergeTree() ORDER BY id;") +// if err != nil { +// t.Fatal(err) +// } + +// _, err = session.Query("INSERT INTO TestQuery VALUES (1), (2), (3);") +// if err != nil { +// t.Fatal(err) +// } +// ret, err := session.Query("SELECT * FROM TestQuery;") +// if err != nil { +// t.Fatalf("Query failed: %s", err) +// } + +// if string(ret.Buf()) != "1\n2\n3\n" { +// t.Fatalf("Query result should be 1\n2\n3\n, got %s", string(ret.Buf())) +// } +// } func TestSessionPathAndIsTemp(t *testing.T) { // Create a new session and check its Path and IsTemp From 81e31dbe16cae9b3b562e1ccde1e779febad22ac Mon Sep 17 00:00:00 2001 From: Andrei Goncear Date: Mon, 17 Feb 2025 13:22:09 +0000 Subject: [PATCH 21/22] fix flaky test with global session initializer --- chdb/session_test.go | 127 ++++++++++++++++++++++++++----------------- 1 file changed, 77 insertions(+), 50 deletions(-) diff --git a/chdb/session_test.go b/chdb/session_test.go index 325923c..dee42ce 100644 --- a/chdb/session_test.go +++ b/chdb/session_test.go @@ -1,18 +1,47 @@ package chdb import ( + "fmt" "os" "path/filepath" "testing" ) -// TestNewSession tests the creation of a new session. -func TestNewSession(t *testing.T) { - session, err := NewSession() +var ( + session *Session +) + +func globalSetup() error { + sess, err := NewSession() if err != nil { - t.Fatalf("Failed to create new session: %s", err) + return err } - defer session.Cleanup() + session = sess + return nil +} + +func globalTeardown() { + session.Cleanup() + session.Close() +} + +func TestMain(m *testing.M) { + if err := globalSetup(); err != nil { + fmt.Println("Global setup failed:", err) + os.Exit(1) + } + // Run all tests. + exitCode := m.Run() + + // Global teardown: clean up any resources here. + globalTeardown() + + // Exit with the code returned by m.Run(). + os.Exit(exitCode) +} + +// TestNewSession tests the creation of a new session. +func TestNewSession(t *testing.T) { // Check if the session directory exists if _, err := os.Stat(session.Path()); os.IsNotExist(err) { @@ -25,59 +54,30 @@ func TestNewSession(t *testing.T) { } } -// TestSessionClose tests the Close method of the session. -func TestSessionClose(t *testing.T) { - session, _ := NewSession() - defer session.Cleanup() // Cleanup in case Close fails +// This test is currently flaky because of this: https://github.com/chdb-io/chdb/pull/299/commits/91b0aedd8c17e74a4bb213e885d89cc9a77c99ad +func TestQuery(t *testing.T) { - // Close the session - session.Close() + // time.Sleep(time.Second * 5) - // Check if the session directory has been removed - if _, err := os.Stat(session.Path()); !os.IsNotExist(err) { - t.Errorf("Session directory should be removed after Close: %s", session.Path()) + _, err := session.Query("CREATE TABLE IF NOT EXISTS TestQuery (id UInt32) ENGINE = MergeTree() ORDER BY id;") + if err != nil { + t.Fatal(err) } -} -// TestSessionCleanup tests the Cleanup method of the session. -func TestSessionCleanup(t *testing.T) { - session, _ := NewSession() - - // Cleanup the session - session.Cleanup() + _, err = session.Query("INSERT INTO TestQuery VALUES (1), (2), (3);") + if err != nil { + t.Fatal(err) + } + ret, err := session.Query("SELECT * FROM TestQuery;") + if err != nil { + t.Fatalf("Query failed: %s", err) + } - // Check if the session directory has been removed - if _, err := os.Stat(session.Path()); !os.IsNotExist(err) { - t.Errorf("Session directory should be removed after Cleanup: %s", session.Path()) + if string(ret.Buf()) != "1\n2\n3\n" { + t.Fatalf("Query result should be 1\n2\n3\n, got %s", string(ret.Buf())) } } -// This test is currently flaky because of this: https://github.com/chdb-io/chdb/pull/299/commits/91b0aedd8c17e74a4bb213e885d89cc9a77c99ad -// func TestQuery(t *testing.T) { - -// session, _ := NewSession() -// defer session.Cleanup() -// // time.Sleep(time.Second * 5) - -// _, err := session.Query("CREATE TABLE IF NOT EXISTS TestQuery (id UInt32) ENGINE = MergeTree() ORDER BY id;") -// if err != nil { -// t.Fatal(err) -// } - -// _, err = session.Query("INSERT INTO TestQuery VALUES (1), (2), (3);") -// if err != nil { -// t.Fatal(err) -// } -// ret, err := session.Query("SELECT * FROM TestQuery;") -// if err != nil { -// t.Fatalf("Query failed: %s", err) -// } - -// if string(ret.Buf()) != "1\n2\n3\n" { -// t.Fatalf("Query result should be 1\n2\n3\n, got %s", string(ret.Buf())) -// } -// } - func TestSessionPathAndIsTemp(t *testing.T) { // Create a new session and check its Path and IsTemp session, _ := NewSession() @@ -105,3 +105,30 @@ func TestSessionPathAndIsTemp(t *testing.T) { t.Errorf("Session should not be temporary") } } + +// TestSessionClose tests the Close method of the session. +func TestSessionClose(t *testing.T) { + session, _ := NewSession() + defer session.Cleanup() // Cleanup in case Close fails + + // Close the session + session.Close() + + // Check if the session directory has been removed + if _, err := os.Stat(session.Path()); !os.IsNotExist(err) { + t.Errorf("Session directory should be removed after Close: %s", session.Path()) + } +} + +// TestSessionCleanup tests the Cleanup method of the session. +func TestSessionCleanup(t *testing.T) { + session, _ := NewSession() + + // Cleanup the session + session.Cleanup() + + // Check if the session directory has been removed + if _, err := os.Stat(session.Path()); !os.IsNotExist(err) { + t.Errorf("Session directory should be removed after Cleanup: %s", session.Path()) + } +} From 6a4de22ae21756deca456e22a6bcdfa33cb8d336 Mon Sep 17 00:00:00 2001 From: Andrei Goncear Date: Mon, 17 Feb 2025 13:39:54 +0000 Subject: [PATCH 22/22] fix wrong docs --- chdb/doc.md | 18 +++++++++--------- lowApi.md | 8 ++++---- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/chdb/doc.md b/chdb/doc.md index 1cf7e97..968986a 100644 --- a/chdb/doc.md +++ b/chdb/doc.md @@ -20,7 +20,7 @@ import "github.com/chdb-io/chdb-go/chdb" -## func [Query]() +## func [Query]() ```go func Query(queryStr string, outputFormats ...string) (result chdbpurego.ChdbResult, err error) @@ -29,7 +29,7 @@ func Query(queryStr string, outputFormats ...string) (result chdbpurego.ChdbResu Query calls query\_conn with a default in\-memory session and default output format of "CSV" if not provided. -## type [Session]() +## type [Session]() @@ -40,7 +40,7 @@ type Session struct { ``` -### func [NewSession]() +### func [NewSession]() ```go func NewSession(paths ...string) (*Session, error) @@ -49,7 +49,7 @@ func NewSession(paths ...string) (*Session, error) NewSession creates a new session with the given path. If path is empty, a temporary directory is created. Note: The temporary directory is removed when Close is called. -### func \(\*Session\) [Cleanup]() +### func \(\*Session\) [Cleanup]() ```go func (s *Session) Cleanup() @@ -58,7 +58,7 @@ func (s *Session) Cleanup() Cleanup closes the session and removes the directory. -### func \(\*Session\) [Close]() +### func \(\*Session\) [Close]() ```go func (s *Session) Close() @@ -71,7 +71,7 @@ temporary directory is created when NewSession was called with an empty path. ``` -### func \(\*Session\) [ConnStr]() +### func \(\*Session\) [ConnStr]() ```go func (s *Session) ConnStr() string @@ -80,7 +80,7 @@ func (s *Session) ConnStr() string -### func \(\*Session\) [IsTemp]() +### func \(\*Session\) [IsTemp]() ```go func (s *Session) IsTemp() bool @@ -89,7 +89,7 @@ func (s *Session) IsTemp() bool IsTemp returns whether the session is temporary. -### func \(\*Session\) [Path]() +### func \(\*Session\) [Path]() ```go func (s *Session) Path() string @@ -98,7 +98,7 @@ func (s *Session) Path() string Path returns the path of the session. -### func \(\*Session\) [Query]() +### func \(\*Session\) [Query]() ```go func (s *Session) Query(queryStr string, outputFormats ...string) (result chdbpurego.ChdbResult, err error) diff --git a/lowApi.md b/lowApi.md index 3648242..4829877 100644 --- a/lowApi.md +++ b/lowApi.md @@ -15,7 +15,7 @@ import "github.com/chdb-io/chdb-go/chdb-purego" -## type [ChdbConn]() +## type [ChdbConn]() @@ -31,7 +31,7 @@ type ChdbConn interface { ``` -### func [NewConnection]() +### func [NewConnection]() ```go func NewConnection(argc int, argv []string) (ChdbConn, error) @@ -67,7 +67,7 @@ Important: - Creating a new session will close the existing one. -## type [ChdbResult]() +## type [ChdbResult]() @@ -93,7 +93,7 @@ type ChdbResult interface { ``` -### func [RawQuery]() +### func [RawQuery]() ```go func RawQuery(argc int, argv []string) (result ChdbResult, err error)