Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions go/cloud-query/api/proto/cloudquery.proto
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,9 @@ message QueryInput {
message SchemaInput {
Connection connection = 1;
optional string table = 2;
// Exact table names to return schemas for. When non-empty, takes
// precedence over `table` and matches with table_name = ANY(tables).
repeated string tables = 3;
}

message ExtractInput {
Expand Down
22 changes: 20 additions & 2 deletions go/cloud-query/docs/api-reference.md
Original file line number Diff line number Diff line change
Expand Up @@ -186,9 +186,15 @@ The Schema method retrieves the schema information for cloud resources.
message SchemaInput {
Connection connection = 1;
optional string table = 2;
// Exact table names to return schemas for. When non-empty, takes
// precedence over `table` and matches with table_name = ANY(tables).
repeated string tables = 3;
}
```

- `table`: optional substring / LIKE filter (`%table%`). When omitted and `tables` is empty, returns schemas for all provider tables.
- `tables`: optional list of exact table names. When non-empty, takes precedence over `table`.

#### Response: SchemaOutput

```protobuf
Expand All @@ -212,7 +218,7 @@ message SchemaResult {
Using curl (with grpcurl):

```bash
# Using grpcurl to make a Schema request
# Using grpcurl to make a Schema request (LIKE filter)
grpcurl -d '{
"connection": {
"provider": "aws",
Expand All @@ -223,6 +229,18 @@ grpcurl -d '{
},
"table": "aws_ec2_%"
}' -plaintext localhost:9192 cloudquery.CloudQuery/Schema

# Exact schemas for multiple tables
grpcurl -d '{
"connection": {
"provider": "aws",
"aws": {
"access_key_id": "YOUR_ACCESS_KEY",
"secret_access_key": "YOUR_SECRET_KEY"
}
},
"tables": ["aws_vpc", "aws_ec2_instance", "aws_eks_cluster"]
}' -plaintext localhost:9192 cloudquery.CloudQuery/Schema
```

Using Postman:
Expand All @@ -242,7 +260,7 @@ Using Postman:
"secret_access_key": "YOUR_SECRET_KEY"
}
},
"table": "aws_ec2_%"
"tables": ["aws_vpc", "aws_ec2_instance"]
}
```

Expand Down
2 changes: 1 addition & 1 deletion go/cloud-query/internal/connection/connection.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ var defaultDataSource = common.DataSource(args.DatabaseHost(), args.DatabasePort

type Connection interface {
Configure() error
Schema(table string) ([]cloudquery.SchemaResult, error)
Schema(table string, tables []string) ([]cloudquery.SchemaResult, error)
Tables(table string) ([]string, error)
Query(q string, args ...any) (columns []string, rows [][]any, err error)
Exec(q string, args ...any) (sql.Result, error)
Expand Down
28 changes: 20 additions & 8 deletions go/cloud-query/internal/connection/schema.go
Original file line number Diff line number Diff line change
@@ -1,24 +1,21 @@
package connection

import (
"database/sql"
"fmt"

"github.com/lib/pq"
"github.com/samber/lo"
"k8s.io/klog/v2"

"github.com/pluralsh/console/go/cloud-query/internal/log"
"github.com/pluralsh/console/go/cloud-query/internal/proto/cloudquery"
)

func (in *connection) Schema(table string) ([]cloudquery.SchemaResult, error) {
klog.V(log.LogLevelDebug).InfoS("running schema query", "table", table)
func (in *connection) Schema(table string, tables []string) ([]cloudquery.SchemaResult, error) {
klog.V(log.LogLevelDebug).InfoS("running schema query", "table", table, "tables", tables)

prefix := fmt.Sprintf("%s_", in.provider())

qResponse, err := in.db.Query(`
SELECT table_name, column_name, data_type
FROM information_schema.columns
WHERE table_name LIKE $1;`, lo.Ternary(lo.IsEmpty(table), prefix+"%", "%"+table+"%"))
qResponse, err := in.querySchema(table, tables)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -55,6 +52,21 @@ func (in *connection) Schema(table string) ([]cloudquery.SchemaResult, error) {
return result, nil
}

func (in *connection) querySchema(table string, tables []string) (*sql.Rows, error) {
if len(tables) > 0 {
return in.db.Query(`
SELECT table_name, column_name, data_type
FROM information_schema.columns
WHERE table_name = ANY($1);`, pq.Array(tables))
}

prefix := fmt.Sprintf("%s_", in.provider())
return in.db.Query(`
SELECT table_name, column_name, data_type
FROM information_schema.columns
WHERE table_name LIKE $1;`, lo.Ternary(lo.IsEmpty(table), prefix+"%", "%"+table+"%"))
}

func (in *connection) Tables(table string) ([]string, error) {
klog.V(log.LogLevelDebug).InfoS("running tables query", "table", table)

Expand Down
23 changes: 17 additions & 6 deletions go/cloud-query/internal/proto/cloudquery/cloudquery.pb.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion go/cloud-query/internal/proto/toolquery/toolquery.pb.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

6 changes: 3 additions & 3 deletions go/cloud-query/internal/service/cloudquery_schema.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,11 @@ func (in *CloudQueryService) Schema(_ context.Context, input *cloudquery.SchemaI
return nil, err
}

return in.handleSchema(c, input.GetTable())
return in.handleSchema(c, input.GetTable(), input.GetTables())
}

func (in *CloudQueryService) handleSchema(c connection.Connection, table string) (*cloudquery.SchemaOutput, error) {
result, err := c.Schema(table)
func (in *CloudQueryService) handleSchema(c connection.Connection, table string, tables []string) (*cloudquery.SchemaOutput, error) {
result, err := c.Schema(table, tables)
if err != nil {
return nil, status.Errorf(codes.Internal, "failed to execute schema query '%s': %v", table, err)
}
Expand Down
1 change: 1 addition & 0 deletions lib/cloud_query/cloudquery.pb.ex
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,7 @@ defmodule Cloudquery.SchemaInput do

field :connection, 1, type: Cloudquery.Connection
field :table, 2, proto3_optional: true, type: :string
field :tables, 3, repeated: true, type: :string
end

defmodule Cloudquery.ExtractInput do
Expand Down
Loading