1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228
|
// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package cmd
import (
"bytes"
"context"
"flag"
"fmt"
"go/parser"
"go/token"
"io/ioutil"
"log"
"os"
"runtime"
"unicode/utf8"
"golang.org/x/tools/internal/lsp"
"golang.org/x/tools/internal/lsp/protocol"
"golang.org/x/tools/internal/lsp/source"
"golang.org/x/tools/internal/span"
)
// generate semantic tokens and interpolate them in the file
// The output is the input file decorated with comments showing the
// syntactic tokens. The comments are stylized:
// /*<arrow><length>,<token type>,[<modifiers]*/
// For most occurrences, the comment comes just before the token it
// describes, and arrow is a right arrow. If the token is inside a string
// the comment comes just after the string, and the arrow is a left arrow.
// <length> is the length of the token in runes, <token type> is one
// of the supported semantic token types, and <modifiers. is a
// (possibly empty) list of token type modifiers.
// There are 3 coordinate systems for lines and character offsets in lines
// LSP (what's returned from semanticTokens()):
// 0-based: the first line is line 0, the first character of a line
// is character 0, and characters are counted as UTF-16 code points
// gopls (and Go error messages):
// 1-based: the first line is line1, the first chararcter of a line
// is character 0, and characters are counted as bytes
// internal (as used in marks, and lines:=bytes.Split(buf, '\n'))
// 0-based: lines and character positions are 1 less than in
// the gopls coordinate system
type semtok struct {
app *Application
}
var colmap *protocol.ColumnMapper
func (c *semtok) Name() string { return "semtok" }
func (c *semtok) Usage() string { return "<filename>" }
func (c *semtok) ShortHelp() string { return "show semantic tokens for the specified file" }
func (c *semtok) DetailedHelp(f *flag.FlagSet) {
for i := 1; ; i++ {
_, f, l, ok := runtime.Caller(i)
if !ok {
break
}
log.Printf("%d: %s:%d", i, f, l)
}
fmt.Fprint(f.Output(), `
Example: show the semantic tokens for this file:
$ gopls semtok internal/lsp/cmd/semtok.go
gopls semtok flags are:
`)
f.PrintDefaults()
}
// Run performs the semtok on the files specified by args and prints the
// results to stdout in the format described above.
func (c *semtok) Run(ctx context.Context, args ...string) error {
if len(args) != 1 {
return fmt.Errorf("expected one file name, got %d", len(args))
}
// perhaps simpler if app had just had a FlagSet member
origOptions := c.app.options
c.app.options = func(opts *source.Options) {
origOptions(opts)
opts.SemanticTokens = true
}
conn, err := c.app.connect(ctx)
if err != nil {
return err
}
defer conn.terminate(ctx)
uri := span.URIFromPath(args[0])
file := conn.AddFile(ctx, uri)
if file.err != nil {
return file.err
}
resp, err := conn.semanticTokens(ctx, uri)
if err != nil {
return err
}
buf, err := ioutil.ReadFile(args[0])
if err != nil {
log.Fatal(err)
}
fset := token.NewFileSet()
f, err := parser.ParseFile(fset, args[0], buf, 0)
if err != nil {
log.Printf("parsing %s failed %v", args[0], err)
return err
}
tok := fset.File(f.Pos())
if tok == nil {
// can't happen; just parsed this file
return fmt.Errorf("can't find %s in fset", args[0])
}
tc := span.NewContentConverter(args[0], buf)
colmap = &protocol.ColumnMapper{
URI: span.URI(args[0]),
Content: buf,
Converter: tc,
}
err = decorate(file.uri.Filename(), resp.Data)
if err != nil {
return err
}
return nil
}
type mark struct {
line, offset int // 1-based, from RangeSpan
len int // bytes, not runes
typ string
mods []string
}
// prefixes for semantic token comments
const (
SemanticLeft = "/*⇐"
SemanticRight = "/*⇒"
)
func markLine(m mark, lines [][]byte) {
l := lines[m.line-1] // mx is 1-based
length := utf8.RuneCount(l[m.offset-1 : m.offset-1+m.len])
splitAt := m.offset - 1
insert := ""
if m.typ == "namespace" && m.offset-1+m.len < len(l) && l[m.offset-1+m.len] == '"' {
// it is the last component of an import spec
// cannot put a comment inside a string
insert = fmt.Sprintf("%s%d,namespace,[]*/", SemanticLeft, length)
splitAt = m.offset + m.len
} else {
// be careful not to generate //*
spacer := ""
if splitAt-1 >= 0 && l[splitAt-1] == '/' {
spacer = " "
}
insert = fmt.Sprintf("%s%s%d,%s,%v*/", spacer, SemanticRight, length, m.typ, m.mods)
}
x := append([]byte(insert), l[splitAt:]...)
l = append(l[:splitAt], x...)
lines[m.line-1] = l
}
func decorate(file string, result []float64) error {
buf, err := ioutil.ReadFile(file)
if err != nil {
return err
}
marks := newMarks(result)
if len(marks) == 0 {
return nil
}
lines := bytes.Split(buf, []byte{'\n'})
for i := len(marks) - 1; i >= 0; i-- {
mx := marks[i]
markLine(mx, lines)
}
os.Stdout.Write(bytes.Join(lines, []byte{'\n'}))
return nil
}
func newMarks(d []float64) []mark {
ans := []mark{}
// the following two loops could be merged, at the cost
// of making the logic slightly more complicated to understand
// first, convert from deltas to absolute, in LSP coordinates
lspLine := make([]float64, len(d)/5)
lspChar := make([]float64, len(d)/5)
line, char := 0.0, 0.0
for i := 0; 5*i < len(d); i++ {
lspLine[i] = line + d[5*i+0]
if d[5*i+0] > 0 {
char = 0
}
lspChar[i] = char + d[5*i+1]
char = lspChar[i]
line = lspLine[i]
}
// second, convert to gopls coordinates
for i := 0; 5*i < len(d); i++ {
pr := protocol.Range{
Start: protocol.Position{
Line: lspLine[i],
Character: lspChar[i],
},
End: protocol.Position{
Line: lspLine[i],
Character: lspChar[i] + d[5*i+2],
},
}
spn, err := colmap.RangeSpan(pr)
if err != nil {
log.Fatal(err)
}
m := mark{
line: spn.Start().Line(),
offset: spn.Start().Column(),
len: spn.End().Column() - spn.Start().Column(),
typ: lsp.SemType(int(d[5*i+3])),
mods: lsp.SemMods(int(d[5*i+4])),
}
ans = append(ans, m)
}
return ans
}
|