optimize: caching + minor opts

lang/collect/export.go

@@ -33,7 +33,7 @@ type dependency struct {
 
 func (c *Collector) fileLine(loc Location) uniast.FileLine {
 	var rel string
-	if c.internal(loc) {
+	if c.isLocationInRepo(loc) {
 		rel, _ = filepath.Rel(c.repo, loc.URI.File())
 	} else {
 		rel = filepath.Base(loc.URI.File())
@@ -54,6 +54,7 @@ func newModule(name string, dir string, lang uniast.Language) *uniast.Module {
 }
 
 func (c *Collector) Export(ctx context.Context) (*uniast.Repository, error) {
+	log.Info("Collector.Export() started")
 	// recursively read all go files in repo
 	repo := uniast.NewRepository(c.repo)
 	modules, err := c.spec.WorkSpace(c.repo)
@@ -73,6 +74,7 @@ func (c *Collector) Export(ctx context.Context) (*uniast.Repository, error) {
 	// not allow local symbols inside another symbol
 	c.filterLocalSymbols()
 
+	log.Info("exporting symbols (tot=%d)...", len(c.syms))
 	// export symbols
 	visited := make(map[*DocumentSymbol]*uniast.Identity)
 	for _, symbol := range c.syms {
@@ -108,7 +110,7 @@ func (c *Collector) Export(ctx context.Context) (*uniast.Repository, error) {
 		}
 		f.Package = pkgpath
 	}
-
+	log.Info("Collector.Export() done")
 	return &repo, nil
 }
 
@@ -120,15 +122,18 @@ var (
 // NOTICE: for rust and golang, each entity has separate location
 // TODO: some language may allow local symbols inside another symbol,
 func (c *Collector) filterLocalSymbols() {
-	// filter symbols
-	for loc1 := range c.syms {
-		for loc2 := range c.syms {
-			if loc1 == loc2 {
-				continue
-			}
-			if loc2.Include(loc1) {
-				delete(c.syms, loc1)
-				break
+	for _, fileSyms := range c.perFileSyms {
+		for _, loc1 := range fileSyms {
+			for _, loc2 := range fileSyms {
+				if loc1 == loc2 {
+					continue
+				}
+				if loc2.Location.Include(loc1.Location) {
+					// This invalidates perFileSyms by making it inconsistent with syms.
+					// It's OK since perFileSyms is no longer used after this function.
+					delete(c.syms, loc1.Location)
+					break
+				}
 			}
 		}
 	}
@@ -162,7 +167,7 @@ func (c *Collector) exportSymbol(repo *uniast.Repository, symbol *DocumentSymbol
 	}
 
 	// Load external symbol on demands
-	if !c.LoadExternalSymbol && (!c.internal(symbol.Location) || symbol.Kind == SKUnknown) {
+	if !c.LoadExternalSymbol && (!c.isLocationInRepo(symbol.Location) || symbol.Kind == SKUnknown) {
 		e = ErrExternalSymbol
 		return
 	}
@@ -181,6 +186,7 @@ func (c *Collector) exportSymbol(repo *uniast.Repository, symbol *DocumentSymbol
 	id = &tmp
 	// Save to visited ONLY WHEN no errors occur
 	visited[symbol] = id
+	log.Info("  exporting symbols %d/%d...", len(visited), len(c.syms))
 
 	// Walk down from repo struct
 	if repo.Modules[mod] == nil {
@@ -222,7 +228,7 @@ func (c *Collector) exportSymbol(repo *uniast.Repository, symbol *DocumentSymbol
 		}
 		info := c.funcs[symbol]
 		obj.Signature = info.Signature
-		// NOTICE: type parames collect into types
+		// NOTICE: type params collect into types
 		if info.TypeParams != nil {
 			for _, input := range info.TypeParamsSorted {
 				tok, _ := c.cli.Locate(input.Location)

lang/collect/utils.go

@@ -0,0 +1,31 @@
+// Copyright 2025 CloudWeGo Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package collect
+
+import "github.com/cloudwego/abcoder/lang/lsp"
+
+func isFuncLike(sk lsp.SymbolKind) bool {
+	return sk == lsp.SKFunction || sk == lsp.SKMethod
+	// SKConstructor ?
+}
+
+func isTypeLike(sk lsp.SymbolKind) bool {
+	return sk == lsp.SKClass || sk == lsp.SKStruct || sk == lsp.SKInterface || sk == lsp.SKEnum
+}
+
+func isVarLike(sk lsp.SymbolKind) bool {
+	return sk == lsp.SKVariable || sk == lsp.SKConstant
+	// sk == lsp.SKField || sk == lsp.SKProperty ?
+}

lang/golang/parser/option.go

@@ -23,6 +23,7 @@ import (
 type Options struct {
 	ReferCodeDepth int
 	Excludes       []string
+	Includes       []string
 	CollectComment bool
 	NeedTest       bool
 	LoadByPackages bool

lang/golang/parser/parser.go

@@ -44,7 +44,8 @@ type GoParser struct {
 	interfaces  map[*types.Interface]Identity
 	types       map[types.Type]Identity
 	files       map[string][]byte
-	exclues     []*regexp.Regexp
+	excludes    []*regexp.Regexp
+	includes    []*regexp.Regexp
 }
 
 type moduleInfo struct {
@@ -82,7 +83,10 @@ func newGoParser(name string, homePageDir string, opts Options) *GoParser {
 	}
 
 	if opts.Excludes != nil {
-		p.exclues = compileExcludes(opts.Excludes)
+		p.excludes = compileExcludes(opts.Excludes)
+	}
+	if opts.Includes != nil {
+		p.includes = compileExcludes(opts.Includes)
 	}
 
 	if err := p.collectGoMods(p.homePageDir); err != nil {
@@ -180,7 +184,19 @@ func (p *GoParser) ParseModule(mod *Module, dir string) (err error) {
 			if e != nil || !info.IsDir() || shouldIgnoreDir(path) {
 				return nil
 			}
-			for _, exclude := range p.exclues {
+			if len(p.includes) > 0 {
+				included := false
+				for _, include := range p.includes {
+					if include.MatchString(path) {
+						included = true
+						break
+					}
+				}
+				if !included {
+					return nil
+				}
+			}
+			for _, exclude := range p.excludes {
 				if exclude.MatchString(path) {
 					return nil
 				}

lang/golang/parser/pkg.go

@@ -199,7 +199,7 @@ func (p *GoParser) loadPackages(mod *Module, dir string, pkgPath PkgPath) (err e
 				continue
 			}
 			filePath := pkg.GoFiles[idx]
-			for _, exclude := range p.exclues {
+			for _, exclude := range p.excludes {
 				if exclude.MatchString(filePath) {
 					fmt.Fprintf(os.Stderr, "skip file %s\n", filePath)
 					continue next_file

lang/lsp/cache.go

@@ -0,0 +1,127 @@
+// Copyright 2025 CloudWeGo Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package lsp
+
+import (
+	"context"
+	"encoding/json"
+	"os"
+	"sync"
+	"time"
+
+	"github.com/cloudwego/abcoder/lang/log"
+)
+
+type LSPRequestCache struct {
+	cachePath     string
+	cacheInterval int
+	mu            sync.Mutex
+	cache         map[string]map[string]json.RawMessage // method -> params -> result
+	cancel        context.CancelFunc
+}
+
+func NewLSPRequestCache(path string, interval int) *LSPRequestCache {
+	c := &LSPRequestCache{
+		cachePath:     path,
+		cacheInterval: interval,
+		cache:         make(map[string]map[string]json.RawMessage),
+	}
+	c.Init()
+	return c
+}
+
+func (c *LSPRequestCache) Init() {
+	if c.cachePath == "" {
+		return
+	}
+	if err := c.loadCacheFromDisk(); err != nil {
+		log.Error("failed to load LSP cache from disk: %v", err)
+	} else {
+		log.Info("LSP cache loaded from disk")
+	}
+	ctx, cancel := context.WithCancel(context.Background())
+	c.cancel = cancel
+	go c.PeriodicCacheSaver(ctx)
+}
+
+func (c *LSPRequestCache) Close() {
+	if c.cancel != nil {
+		c.cancel()
+	}
+}
+
+func (c *LSPRequestCache) saveCacheToDisk() error {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	data, err := json.Marshal(c.cache)
+	if err != nil {
+		return err
+	}
+	return os.WriteFile(c.cachePath, data, 0644)
+}
+
+func (c *LSPRequestCache) loadCacheFromDisk() error {
+	data, err := os.ReadFile(c.cachePath)
+	if err != nil {
+		return err
+	}
+	if err := json.Unmarshal(data, &c.cache); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (cli *LSPRequestCache) PeriodicCacheSaver(ctx context.Context) {
+	go func() {
+		ticker := time.NewTicker(time.Duration(cli.cacheInterval) * time.Second)
+		defer ticker.Stop()
+
+		for {
+			select {
+			case <-ticker.C:
+				if err := cli.saveCacheToDisk(); err != nil {
+					log.Error("failed to save LSP cache to disk: %v", err)
+				} else {
+					log.Info("LSP cache saved to disk")
+				}
+			case <-ctx.Done():
+				log.Info("LSP cache saver cancelled, shutting down.")
+				return
+			}
+		}
+	}()
+}
+
+func (cli *LSPRequestCache) Get(method, params string) (json.RawMessage, bool) {
+	cli.mu.Lock()
+	defer cli.mu.Unlock()
+	if methodCache, ok := cli.cache[method]; ok {
+		if result, ok := methodCache[params]; ok {
+			return result, true
+		}
+	}
+	return nil, false
+}
+
+func (cli *LSPRequestCache) Set(method, params string, result json.RawMessage) {
+	cli.mu.Lock()
+	defer cli.mu.Unlock()
+	methodCache, ok := cli.cache[method]
+	if !ok {
+		methodCache = make(map[string]json.RawMessage)
+		cli.cache[method] = methodCache
+	}
+	methodCache[params] = result
+}

lang/lsp/client.go

@@ -37,13 +37,17 @@ type LSPClient struct {
 	tokenModifiers         []string
 	hasSemanticTokensRange bool
 	files                  map[DocumentURI]*TextDocumentItem
+	cache                  *LSPRequestCache
 	ClientOptions
 }
 
 type ClientOptions struct {
 	Server string
 	uniast.Language
 	Verbose bool
+	// for lsp cache
+	LSPCachePath     string
+	LSPCacheInterval int
 }
 
 func NewLSPClient(repo string, openfile string, wait time.Duration, opts ClientOptions) (*LSPClient, error) {
@@ -60,6 +64,7 @@ func NewLSPClient(repo string, openfile string, wait time.Duration, opts ClientO
 
 	cli.ClientOptions = opts
 	cli.files = make(map[DocumentURI]*TextDocumentItem)
+	cli.cache = NewLSPRequestCache(opts.LSPCachePath, opts.LSPCacheInterval)
 
 	if openfile != "" {
 		_, err := cli.DidOpen(context.Background(), NewURI(openfile))
@@ -74,17 +79,28 @@ func NewLSPClient(repo string, openfile string, wait time.Duration, opts ClientO
 }
 
 func (c *LSPClient) Close() error {
+	c.cache.Close()
 	c.lspHandler.Close()
 	return c.Conn.Close()
 }
 
-// Extra wrapper around json rpc to
-// 1. implement a transparent, generic cache
+// By default all client requests are cached.
+// To use non-cached version, use `cli.Conn.Call` directly.
 func (cli *LSPClient) Call(ctx context.Context, method string, params, result interface{}, opts ...jsonrpc2.CallOption) error {
-	var raw json.RawMessage
-	if err := cli.Conn.Call(ctx, method, params, &raw); err != nil {
+	paramsMarshal, err := json.Marshal(params)
+	if err != nil {
+		log.Error("LSPClient.Call: marshal params error: %v", err)
 		return err
 	}
+	paramsStr := string(paramsMarshal)
+	var raw json.RawMessage
+	var ok bool
+	if raw, ok = cli.cache.Get(method, paramsStr); !ok {
+		if err = cli.Conn.Call(ctx, method, params, &raw); err != nil {
+			return err
+		}
+		cli.cache.Set(method, paramsStr, raw)
+	}
 	if err := json.Unmarshal(raw, result); err != nil {
 		return err
 	}

lang/lsp/lsp.go

@@ -114,6 +114,16 @@ type Location struct {
 	Range Range       `json:"range"`
 }
 
+func MakeLocation(uri DocumentURI, startLine, startChar, endLine, endChar int) Location {
+	return Location{
+		URI: uri,
+		Range: Range{
+			Start: Position{Line: startLine, Character: startChar},
+			End:   Position{Line: endLine, Character: endChar},
+		},
+	}
+}
+
 func (l Location) String() string {
 	return fmt.Sprintf("%s:%d:%d-%d:%d", l.URI, l.Range.Start.Line, l.Range.Start.Character, l.Range.End.Line, l.Range.End.Character)
 }