Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Use a string builder for queries instead of scanner #25

Merged
merged 2 commits into from
Oct 18, 2023
Merged

Use a string builder for queries instead of scanner #25

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
79f5382
Use a string builder for queries instead of scanner
nullism Oct 18, 2023
a36ecd1
Update coverage.svg and coverage.out
nullism Oct 18, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
165 changes: 73 additions & 92 deletions coverage.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,95 +50,76 @@ github.com/nullism/bqb/query.go:176.2,176.28 1 1
github.com/nullism/bqb/query.go:176.28,180.23 3 1
github.com/nullism/bqb/query.go:180.23,182.4 1 1
github.com/nullism/bqb/query.go:185.2,185.44 1 1
github.com/nullism/bqb/utils.go:13.80,20.17 2 1
github.com/nullism/bqb/utils.go:21.11,23.32 2 1
github.com/nullism/bqb/utils.go:23.32,25.18 2 1
github.com/nullism/bqb/utils.go:25.18,27.5 1 1
github.com/nullism/bqb/utils.go:28.4,28.15 1 1
github.com/nullism/bqb/utils.go:31.3,33.63 1 1
github.com/nullism/bqb/utils.go:33.63,33.81 1 1
github.com/nullism/bqb/utils.go:35.18,38.61 1 1
github.com/nullism/bqb/utils.go:38.61,38.84 1 1
github.com/nullism/bqb/utils.go:40.13,43.68 1 1
github.com/nullism/bqb/utils.go:43.68,43.91 1 1
github.com/nullism/bqb/utils.go:44.63,44.97 1 1
github.com/nullism/bqb/utils.go:46.10,48.18 1 1
github.com/nullism/bqb/utils.go:61.62,63.23 2 1
github.com/nullism/bqb/utils.go:63.23,67.3 3 1
github.com/nullism/bqb/utils.go:68.2,68.33 1 1
github.com/nullism/bqb/utils.go:71.77,75.85 3 1
github.com/nullism/bqb/utils.go:75.85,77.30 1 1
github.com/nullism/bqb/utils.go:77.30,79.4 1 1
github.com/nullism/bqb/utils.go:81.3,81.38 1 1
github.com/nullism/bqb/utils.go:82.15,83.40 1 1
github.com/nullism/bqb/utils.go:84.14,85.28 1 1
github.com/nullism/bqb/utils.go:90.3,90.12 1 1
github.com/nullism/bqb/utils.go:90.12,92.4 1 1
github.com/nullism/bqb/utils.go:94.3,94.9 1 1
github.com/nullism/bqb/utils.go:97.2,101.21 3 1
github.com/nullism/bqb/utils.go:101.21,102.37 1 1
github.com/nullism/bqb/utils.go:103.16,107.7 2 1
github.com/nullism/bqb/utils.go:109.11,112.30 1 1
github.com/nullism/bqb/utils.go:116.2,116.35 1 1
github.com/nullism/bqb/utils.go:119.64,123.25 3 1
github.com/nullism/bqb/utils.go:125.16,126.53 1 1
github.com/nullism/bqb/utils.go:128.21,131.17 3 1
github.com/nullism/bqb/utils.go:131.17,133.4 1 1
github.com/nullism/bqb/utils.go:133.9,135.4 1 1
github.com/nullism/bqb/utils.go:136.13,138.23 2 1
github.com/nullism/bqb/utils.go:138.23,141.4 2 1
github.com/nullism/bqb/utils.go:142.3,142.65 1 1
github.com/nullism/bqb/utils.go:144.14,146.23 2 1
github.com/nullism/bqb/utils.go:146.23,149.4 2 1
github.com/nullism/bqb/utils.go:150.3,150.21 1 1
github.com/nullism/bqb/utils.go:150.21,152.4 1 1
github.com/nullism/bqb/utils.go:152.9,155.4 2 1
github.com/nullism/bqb/utils.go:157.16,159.23 2 1
github.com/nullism/bqb/utils.go:159.23,162.4 2 1
github.com/nullism/bqb/utils.go:163.3,163.65 1 1
github.com/nullism/bqb/utils.go:165.17,167.23 2 1
github.com/nullism/bqb/utils.go:167.23,170.4 2 1
github.com/nullism/bqb/utils.go:171.3,171.21 1 1
github.com/nullism/bqb/utils.go:171.21,173.4 1 1
github.com/nullism/bqb/utils.go:173.9,176.4 2 1
github.com/nullism/bqb/utils.go:178.13,180.23 2 1
github.com/nullism/bqb/utils.go:180.23,183.4 2 1
github.com/nullism/bqb/utils.go:184.3,184.65 1 1
github.com/nullism/bqb/utils.go:186.14,187.15 1 1
github.com/nullism/bqb/utils.go:187.15,191.4 3 1
github.com/nullism/bqb/utils.go:192.3,194.17 3 1
github.com/nullism/bqb/utils.go:194.17,196.4 1 1
github.com/nullism/bqb/utils.go:196.9,198.4 1 1
github.com/nullism/bqb/utils.go:200.25,203.17 3 1
github.com/nullism/bqb/utils.go:203.17,205.4 1 1
github.com/nullism/bqb/utils.go:205.9,207.4 1 1
github.com/nullism/bqb/utils.go:209.27,212.17 3 1
github.com/nullism/bqb/utils.go:212.17,214.4 1 1
github.com/nullism/bqb/utils.go:214.9,216.4 1 1
github.com/nullism/bqb/utils.go:218.16,219.50 1 1
github.com/nullism/bqb/utils.go:221.10,223.31 2 1
github.com/nullism/bqb/utils.go:226.2,226.28 1 1
github.com/nullism/bqb/utils.go:229.64,231.20 2 1
github.com/nullism/bqb/utils.go:231.20,233.3 1 1
github.com/nullism/bqb/utils.go:235.2,236.28 2 1
github.com/nullism/bqb/utils.go:236.28,238.3 1 1
github.com/nullism/bqb/utils.go:239.2,239.12 1 1
github.com/nullism/bqb/utils.go:242.51,250.27 6 1
github.com/nullism/bqb/utils.go:250.27,252.23 2 1
github.com/nullism/bqb/utils.go:252.23,254.4 1 1
github.com/nullism/bqb/utils.go:255.3,256.17 2 1
github.com/nullism/bqb/utils.go:259.2,259.70 1 1
github.com/nullism/bqb/utils.go:259.70,261.3 1 1
github.com/nullism/bqb/utils.go:263.2,269.3 2 1
github.com/nullism/bqb/utils.go:272.44,273.27 1 1
github.com/nullism/bqb/utils.go:274.12,275.35 1 1
github.com/nullism/bqb/utils.go:277.33,278.35 1 1
github.com/nullism/bqb/utils.go:279.12,280.15 1 1
github.com/nullism/bqb/utils.go:280.15,282.4 1 1
github.com/nullism/bqb/utils.go:283.3,283.36 1 1
github.com/nullism/bqb/utils.go:284.14,285.37 1 1
github.com/nullism/bqb/utils.go:286.15,287.15 1 1
github.com/nullism/bqb/utils.go:287.15,289.4 1 1
github.com/nullism/bqb/utils.go:290.3,290.38 1 1
github.com/nullism/bqb/utils.go:291.11,292.21 1 1
github.com/nullism/bqb/utils.go:293.10,294.65 1 1
github.com/nullism/bqb/utils.go:11.80,18.17 2 1
github.com/nullism/bqb/utils.go:19.11,20.32 1 1
github.com/nullism/bqb/utils.go:20.32,22.18 2 1
github.com/nullism/bqb/utils.go:22.18,24.5 1 1
github.com/nullism/bqb/utils.go:25.4,25.45 1 1
github.com/nullism/bqb/utils.go:27.3,27.18 1 1
github.com/nullism/bqb/utils.go:28.18,29.61 1 1
github.com/nullism/bqb/utils.go:30.13,34.25 4 1
github.com/nullism/bqb/utils.go:34.25,36.4 1 1
github.com/nullism/bqb/utils.go:37.3,38.31 2 1
github.com/nullism/bqb/utils.go:39.10,41.18 1 1
github.com/nullism/bqb/utils.go:45.64,49.25 3 1
github.com/nullism/bqb/utils.go:51.16,52.53 1 1
github.com/nullism/bqb/utils.go:54.21,57.17 3 1
github.com/nullism/bqb/utils.go:57.17,59.4 1 1
github.com/nullism/bqb/utils.go:59.9,61.4 1 1
github.com/nullism/bqb/utils.go:62.13,64.23 2 1
github.com/nullism/bqb/utils.go:64.23,67.4 2 1
github.com/nullism/bqb/utils.go:68.3,68.65 1 1
github.com/nullism/bqb/utils.go:70.14,72.23 2 1
github.com/nullism/bqb/utils.go:72.23,75.4 2 1
github.com/nullism/bqb/utils.go:76.3,76.21 1 1
github.com/nullism/bqb/utils.go:76.21,78.4 1 1
github.com/nullism/bqb/utils.go:78.9,81.4 2 1
github.com/nullism/bqb/utils.go:83.16,85.23 2 1
github.com/nullism/bqb/utils.go:85.23,88.4 2 1
github.com/nullism/bqb/utils.go:89.3,89.65 1 1
github.com/nullism/bqb/utils.go:91.17,93.23 2 1
github.com/nullism/bqb/utils.go:93.23,96.4 2 1
github.com/nullism/bqb/utils.go:97.3,97.21 1 1
github.com/nullism/bqb/utils.go:97.21,99.4 1 1
github.com/nullism/bqb/utils.go:99.9,102.4 2 1
github.com/nullism/bqb/utils.go:104.13,106.23 2 1
github.com/nullism/bqb/utils.go:106.23,109.4 2 1
github.com/nullism/bqb/utils.go:110.3,110.65 1 1
github.com/nullism/bqb/utils.go:112.14,113.15 1 1
github.com/nullism/bqb/utils.go:113.15,117.4 3 1
github.com/nullism/bqb/utils.go:118.3,120.17 3 1
github.com/nullism/bqb/utils.go:120.17,122.4 1 1
github.com/nullism/bqb/utils.go:122.9,124.4 1 1
github.com/nullism/bqb/utils.go:126.25,129.17 3 1
github.com/nullism/bqb/utils.go:129.17,131.4 1 1
github.com/nullism/bqb/utils.go:131.9,133.4 1 1
github.com/nullism/bqb/utils.go:135.27,138.17 3 1
github.com/nullism/bqb/utils.go:138.17,140.4 1 1
github.com/nullism/bqb/utils.go:140.9,142.4 1 1
github.com/nullism/bqb/utils.go:144.16,145.50 1 1
github.com/nullism/bqb/utils.go:147.10,149.31 2 1
github.com/nullism/bqb/utils.go:152.2,152.28 1 1
github.com/nullism/bqb/utils.go:155.64,157.20 2 1
github.com/nullism/bqb/utils.go:157.20,159.3 1 1
github.com/nullism/bqb/utils.go:161.2,162.28 2 1
github.com/nullism/bqb/utils.go:162.28,164.3 1 1
github.com/nullism/bqb/utils.go:165.2,165.12 1 1
github.com/nullism/bqb/utils.go:168.51,176.27 6 1
github.com/nullism/bqb/utils.go:176.27,178.23 2 1
github.com/nullism/bqb/utils.go:178.23,180.4 1 1
github.com/nullism/bqb/utils.go:181.3,182.17 2 1
github.com/nullism/bqb/utils.go:185.2,185.70 1 1
github.com/nullism/bqb/utils.go:185.70,187.3 1 1
github.com/nullism/bqb/utils.go:189.2,195.3 2 1
github.com/nullism/bqb/utils.go:198.44,199.27 1 1
github.com/nullism/bqb/utils.go:200.12,201.35 1 1
github.com/nullism/bqb/utils.go:203.33,204.35 1 1
github.com/nullism/bqb/utils.go:205.12,206.15 1 1
github.com/nullism/bqb/utils.go:206.15,208.4 1 1
github.com/nullism/bqb/utils.go:209.3,209.36 1 1
github.com/nullism/bqb/utils.go:210.14,211.37 1 1
github.com/nullism/bqb/utils.go:212.15,213.15 1 1
github.com/nullism/bqb/utils.go:213.15,215.4 1 1
github.com/nullism/bqb/utils.go:216.3,216.38 1 1
github.com/nullism/bqb/utils.go:217.11,218.21 1 1
github.com/nullism/bqb/utils.go:219.10,220.65 1 1
41 changes: 39 additions & 2 deletions query_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -452,6 +452,23 @@ func TestQuery_ToPgsql(t *testing.T) {
if sql != want {
t.Errorf("got: %q, want: %q", sql, want)
}

ql := New("? ?? ? ? ? ?", "a", "b", "c", "d", "e")
sql, _, _ = ql.ToPgsql()
want = "$1 ? $2 $3 $4 $5"
if sql != want {
t.Errorf("got: %q, want: %q", sql, want)
}

want = "SELECT * FROM foo"
qno := New(want)
sql, params, _ = qno.ToPgsql()
if len(params) > 0 {
t.Errorf("expected no params, got: %v", params)
}
if sql != want {
t.Errorf("got: %q, want: %q", sql, want)
}
}

func TestQuery_ToRaw(t *testing.T) {
Expand Down Expand Up @@ -674,18 +691,38 @@ func Benchmark_ToPgsql_Params(b *testing.B) {
parts := []string{}
args := []any{}
for j := 0; j < 10; j++ {
for j := 0; j < 1000; j++ {
for j := 0; j < 400; j++ {
parts = append(parts, "?")
args = append(args, 1)
}
}

query := fmt.Sprintf("(%s)", strings.Join(parts, ","))

b.ResetTimer()
for i := 0; i < b.N; i++ {
_, _, err := New(query, args...).ToPgsql()
if err != nil {
b.Fatalf("failed to make benchmark sql: %v", err)
}
}
}

func Benchmark_ToMysql_Params(b *testing.B) {
parts := []string{}
args := []any{}
for j := 0; j < 10; j++ {
for j := 0; j < 1000; j++ {
parts = append(parts, "?")
args = append(args, 1)
}
}

query := fmt.Sprintf("(%s)", strings.Join(parts, ","))
b.ResetTimer()
for i := 0; i < b.N; i++ {
_, _, err := New(query, args...).ToMysql()
if err != nil {
b.Fatalf("failed to make benchmark sql: %v", err)
}
}
}
100 changes: 13 additions & 87 deletions utils.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,12 +1,10 @@
package bqb

import (
"bufio"
"bytes"
"database/sql/driver"
"encoding/json"
"errors"
"fmt"
"strconv"
"strings"
)

Expand All @@ -19,103 +17,31 @@ func dialectReplace(dialect Dialect, sql string, params []any) (string, error) {

switch dialect {
case RAW:
raws := make([]string, len(params))
for i, param := range params {
for _, param := range params {
p, err := paramToRaw(param)
if err != nil {
return "", err
}
raws[i] = p
sql = strings.Replace(sql, paramPh, p, 1)
}

return replaceWithScans(
sql,
scan{pattern: parameterPlaceholder, fn: func(i int) string { return raws[i] }},
)
return sql, nil
case MYSQL, SQL:
return replaceWithScans(
sql,
scan{pattern: parameterPlaceholder, fn: func(int) string { return questionMark }},
)
return strings.ReplaceAll(sql, paramPh, questionMark), nil
case PGSQL:
return replaceWithScans(
sql,
scan{pattern: doubleQuestionMarkDelimiter, fn: func(int) string { return questionMark }},
scan{pattern: parameterPlaceholder, fn: func(i int) string { return fmt.Sprintf("$%d", i+1) }},
)
sql = strings.ReplaceAll(sql, doubleQuestionMarkDelimiter, questionMark)
parts := strings.Split(sql, paramPh)
var builder strings.Builder
for i := range params {
_, _ = builder.WriteString(parts[i] + "$" + strconv.Itoa(i+1))
}
builder.WriteString(parts[len(parts)-1])
return builder.String(), nil
default:
// No replacement defined for dialect
return sql, nil
}
}

type replaceFn func(int) string

type scan struct {
pattern string
fn replaceFn
}

// replaceWithScans applies the given set of scanning arguments and joins their
// errors together.
func replaceWithScans(in string, ss ...scan) (string, error) {
errs := []error{}
for _, s := range ss {
out, err := scanReplace(in, s.pattern, s.fn)
errs = append(errs, err)
in = out
}
return in, errors.Join(errs...)
}

func scanReplace(stmt string, replace string, fn replaceFn) (string, error) {
// Build a scanner that will iterate based on the replace token
ph := []byte(replace)
scanner := bufio.NewScanner(bytes.NewBufferString(stmt))
scanner.Split(func(data []byte, atEOF bool) (advance int, token []byte, err error) {
// Return nothing if at end of file and no data passed
if atEOF && len(data) == 0 {
return 0, nil, nil
}

switch i := bytes.Index(data, ph); {
case i == 0:
return len(ph), data[0:len(ph)], nil
case i > 0:
return i, data[0:i], nil

}

// If at end of file with data return the data
if atEOF {
return len(data), data, nil
}

return
})

i := 0

// Scan replacing tokens with the value returned from delegate
sb := strings.Builder{}
for scanner.Scan() {
switch txt := scanner.Text(); txt {
case replace:
// String builder will always return nil for an err so it is thrown
// away.
_, _ = sb.WriteString(fn(i))
i++

default:
// String builder will always return nil for an err so it is thrown
// away.
_, _ = sb.WriteString(txt)
}
}

return sb.String(), scanner.Err()
}

func convertArg(text string, arg any) (string, []any, []error) {
var newArgs []any
var errs []error
Expand Down
Loading