Skip to content

Commit cba2aac

Browse files
kyleconroykyleconroy
committed
feat(analyzer): Analyze queries using a running PostgreSQL database
94 of the open issues on sqlc are related to the analyzer. There are many cases where the current analyzer produces false positives or false negatives. sqlx and some other projects have proven that it's possible to extract query metadata from a running database. This approach is a bit different, in that the database analysis is layered on top of the existing query analyzer. We use the new analysis to provide better type information and support a wider set of cases when the existing analyzer fails.
1 parent 20fda73 commit cba2aac

27 files changed

Lines changed: 871 additions & 192 deletions

‎go.mod‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ require (
2929

3030
require (
3131
github.com/benbjohnson/clock v1.3.5 // indirect
32+
github.com/jackc/puddle/v2 v2.2.1 // indirect
3233
github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c // indirect
3334
github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect
3435
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect

‎go.sum‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -100,7 +100,10 @@ github.com/jackc/pgx/v5 v5.4.3/go.mod h1:Ig06C2Vu0t5qXC60W8sqIthScaEnFvojjj9dSlj
100100
github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
101101
github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
102102
github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
103+
github.com/jackc/puddle v1.3.0 h1:eHK/5clGOatcjX3oWGBO/MpxpbHzSwud5EWTSCI+MX0=
103104
github.com/jackc/puddle v1.3.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
105+
github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk=
106+
github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
104107
github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
105108
github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
106109
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=

‎internal/analyzer/analyzer.go‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
package analyzer
2+
3+
import (
4+
"context"
5+
6+
"github.com/sqlc-dev/sqlc/internal/sql/ast"
7+
"github.com/sqlc-dev/sqlc/internal/sql/named"
8+
)
9+
10+
type Column struct {
11+
Name string
12+
OriginalName string
13+
DataType string
14+
NotNull bool
15+
Unsigned bool
16+
IsArray bool
17+
ArrayDims int
18+
Comment string
19+
Length *int
20+
IsNamedParam bool
21+
IsFuncCall bool
22+
23+
// XXX: Figure out what PostgreSQL calls `foo.id`
24+
Scope string
25+
Table *ast.TableName
26+
TableAlias string
27+
Type *ast.TypeName
28+
EmbedTable *ast.TableName
29+
30+
IsSqlcSlice bool // is this sqlc.slice()
31+
}
32+
33+
type Parameter struct {
34+
Number int
35+
Column *Column
36+
}
37+
38+
type Analysis struct {
39+
Columns []Column
40+
Params []Parameter
41+
}
42+
43+
type Analyzer interface {
44+
Analyze(context.Context, ast.Node, string, []string, *named.ParamSet) (*Analysis, error)
45+
Close(context.Context) error
46+
}

‎internal/cmd/cmd.go‎

Lines changed: 19 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -197,7 +197,10 @@ var genCmd = &cobra.Command{
197197
defer trace.StartRegion(cmd.Context(), "generate").End()
198198
stderr := cmd.ErrOrStderr()
199199
dir, name := getConfigPath(stderr, cmd.Flag("file"))
200-
output, err := Generate(cmd.Context(), ParseEnv(cmd), dir, name, stderr)
200+
output, err := Generate(cmd.Context(), dir, name, &Options{
201+
Env: ParseEnv(cmd),
202+
Stderr: stderr,
203+
})
201204
if err != nil {
202205
os.Exit(1)
203206
}
@@ -219,7 +222,11 @@ var uploadCmd = &cobra.Command{
219222
RunE: func(cmd *cobra.Command, args []string) error {
220223
stderr := cmd.ErrOrStderr()
221224
dir, name := getConfigPath(stderr, cmd.Flag("file"))
222-
if err := createPkg(cmd.Context(), ParseEnv(cmd), dir, name, stderr); err != nil {
225+
opts := &Options{
226+
Env: ParseEnv(cmd),
227+
Stderr: stderr,
228+
}
229+
if err := createPkg(cmd.Context(), dir, name, opts); err != nil {
223230
fmt.Fprintf(stderr, "error uploading: %s\n", err)
224231
os.Exit(1)
225232
}
@@ -234,7 +241,11 @@ var checkCmd = &cobra.Command{
234241
defer trace.StartRegion(cmd.Context(), "compile").End()
235242
stderr := cmd.ErrOrStderr()
236243
dir, name := getConfigPath(stderr, cmd.Flag("file"))
237-
if _, err := Generate(cmd.Context(), ParseEnv(cmd), dir, name, stderr); err != nil {
244+
_, err := Generate(cmd.Context(), dir, name, &Options{
245+
Env: ParseEnv(cmd),
246+
Stderr: stderr,
247+
})
248+
if err != nil {
238249
os.Exit(1)
239250
}
240251
return nil
@@ -277,7 +288,11 @@ var diffCmd = &cobra.Command{
277288
defer trace.StartRegion(cmd.Context(), "diff").End()
278289
stderr := cmd.ErrOrStderr()
279290
dir, name := getConfigPath(stderr, cmd.Flag("file"))
280-
if err := Diff(cmd.Context(), ParseEnv(cmd), dir, name, stderr); err != nil {
291+
opts := &Options{
292+
Env: ParseEnv(cmd),
293+
Stderr: stderr,
294+
}
295+
if err := Diff(cmd.Context(), dir, name, opts); err != nil {
281296
os.Exit(1)
282297
}
283298
return nil

‎internal/cmd/diff.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@ import (
44
"context"
55
"errors"
66
"fmt"
7-
"io"
87
"os"
98
"runtime/trace"
109
"sort"
@@ -13,8 +12,9 @@ import (
1312
"github.com/cubicdaiya/gonp"
1413
)
1514

16-
func Diff(ctx context.Context, e Env, dir, name string, stderr io.Writer) error {
17-
output, err := Generate(ctx, e, dir, name, stderr)
15+
func Diff(ctx context.Context, dir, name string, opts *Options) error {
16+
stderr := opts.Stderr
17+
output, err := Generate(ctx, dir, name, opts)
1818
if err != nil {
1919
return err
2020
}

‎internal/cmd/generate.go‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -128,8 +128,11 @@ func readConfig(stderr io.Writer, dir, filename string) (string, *config.Config,
128128
return configPath, &conf, nil
129129
}
130130

131-
func Generate(ctx context.Context, e Env, dir, filename string, stderr io.Writer) (map[string]string, error) {
132-
configPath, conf, err := readConfig(stderr, dir, filename)
131+
func Generate(ctx context.Context, dir, filename string, o *Options) (map[string]string, error) {
132+
e := o.Env
133+
stderr := o.Stderr
134+
135+
configPath, conf, err := o.ReadConfig(dir, filename)
133136
if err != nil {
134137
return nil, err
135138
}
@@ -332,7 +335,12 @@ func remoteGenerate(ctx context.Context, configPath string, conf *config.Config,
332335

333336
func parse(ctx context.Context, name, dir string, sql config.SQL, combo config.CombinedSettings, parserOpts opts.Parser, stderr io.Writer) (*compiler.Result, bool) {
334337
defer trace.StartRegion(ctx, "parse").End()
335-
c := compiler.NewCompiler(sql, combo)
338+
c, err := compiler.NewCompiler(sql, combo)
339+
defer c.Close(ctx)
340+
if err != nil {
341+
fmt.Fprintf(stderr, "error creating compiler: %s\n", err)
342+
return nil, true
343+
}
336344
if err := c.ParseCatalog(sql.Schema); err != nil {
337345
fmt.Fprintf(stderr, "# package %s\n", name)
338346
if parserErr, ok := err.(*multierr.Error); ok {

‎internal/cmd/options.go‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
package cmd
2+
3+
import (
4+
"io"
5+
6+
"github.com/sqlc-dev/sqlc/internal/config"
7+
)
8+
9+
type Options struct {
10+
Env Env
11+
Stderr io.Writer
12+
MutateConfig func(*config.Config)
13+
}
14+
15+
func (o *Options) ReadConfig(dir, filename string) (string, *config.Config, error) {
16+
path, conf, err := readConfig(o.Stderr, dir, filename)
17+
if err != nil {
18+
return path, conf, err
19+
}
20+
if o.MutateConfig != nil {
21+
o.MutateConfig(conf)
22+
}
23+
return path, conf, nil
24+
}

‎internal/cmd/package.go‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,13 +2,14 @@ package cmd
22

33
import (
44
"context"
5-
"io"
65
"os"
76

87
"github.com/sqlc-dev/sqlc/internal/bundler"
98
)
109

11-
func createPkg(ctx context.Context, e Env, dir, filename string, stderr io.Writer) error {
10+
func createPkg(ctx context.Context, dir, filename string, opts *Options) error {
11+
e := opts.Env
12+
stderr := opts.Stderr
1213
configPath, conf, err := readConfig(stderr, dir, filename)
1314
if err != nil {
1415
return err
@@ -17,7 +18,7 @@ func createPkg(ctx context.Context, e Env, dir, filename string, stderr io.Write
1718
if err := up.Validate(); err != nil {
1819
return err
1920
}
20-
output, err := Generate(ctx, e, dir, filename, stderr)
21+
output, err := Generate(ctx, dir, filename, opts)
2122
if err != nil {
2223
os.Exit(1)
2324
}

‎internal/cmd/vet.go‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -47,8 +47,12 @@ func NewCmdVet() *cobra.Command {
4747
RunE: func(cmd *cobra.Command, args []string) error {
4848
defer trace.StartRegion(cmd.Context(), "vet").End()
4949
stderr := cmd.ErrOrStderr()
50+
opts := &Options{
51+
Env: ParseEnv(cmd),
52+
Stderr: stderr,
53+
}
5054
dir, name := getConfigPath(stderr, cmd.Flag("file"))
51-
if err := Vet(cmd.Context(), ParseEnv(cmd), dir, name, stderr); err != nil {
55+
if err := Vet(cmd.Context(), dir, name, opts); err != nil {
5256
if !errors.Is(err, ErrFailedChecks) {
5357
fmt.Fprintf(stderr, "%s\n", err)
5458
}
@@ -59,7 +63,9 @@ func NewCmdVet() *cobra.Command {
5963
}
6064
}
6165

62-
func Vet(ctx context.Context, e Env, dir, filename string, stderr io.Writer) error {
66+
func Vet(ctx context.Context, dir, filename string, opts *Options) error {
67+
e := opts.Env
68+
stderr := opts.Stderr
6369
configPath, conf, err := readConfig(stderr, dir, filename)
6470
if err != nil {
6571
return err

‎internal/codegen/golang/result.go‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -249,9 +249,7 @@ func buildQueries(req *plugin.CodeGenRequest, structs []Struct) ([]Query, error)
249249
if len(query.Columns) == 1 && query.Columns[0].EmbedTable == nil {
250250
c := query.Columns[0]
251251
name := columnName(c, 0)
252-
if c.IsFuncCall {
253-
name = strings.Replace(name, "$", "_", -1)
254-
}
252+
name = strings.Replace(name, "$", "_", -1)
255253
gq.Ret = QueryValue{
256254
Name: name,
257255
DBName: name,

0 commit comments

Comments
 (0)