diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 2a6900a5770..ed9275ebe14 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -317,6 +317,7 @@ /devenv/docker/blocks/mysql_opendata/ @grafana/data-sources-plugins /devenv/docker/blocks/mysql_tests/ @grafana/data-sources-plugins /devenv/docker/blocks/opentsdb/ @grafana/data-sources-plugins +/devenv/docker/blocks/pgvector/ @grafana/grafana-search-and-storage /devenv/docker/blocks/postgres/ @grafana/data-sources-plugins /devenv/docker/blocks/postgres_tests/ @grafana/data-sources-plugins /devenv/docker/blocks/prometheus/ @grafana/data-sources-plugins diff --git a/apps/dashvalidator/go.sum b/apps/dashvalidator/go.sum index 7385fbcf3a4..d9982ac73ab 100644 --- a/apps/dashvalidator/go.sum +++ b/apps/dashvalidator/go.sum @@ -888,6 +888,8 @@ github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0 github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= github.com/perimeterx/marshmallow v1.1.5 h1:a2LALqQ1BlHM8PZblsDdidgv1mWi1DgC2UmX50IvK2s= github.com/perimeterx/marshmallow v1.1.5/go.mod h1:dsXbUu8CRzfYP5a87xpp0xq9S3u0Vchtcl8we9tYaXw= +github.com/pgvector/pgvector-go v0.3.0 h1:Ij+Yt78R//uYqs3Zk35evZFvr+G0blW0OUN+Q2D1RWc= +github.com/pgvector/pgvector-go v0.3.0/go.mod h1:duFy+PXWfW7QQd5ibqutBO4GxLsUZ9RVXhFZGIBsWSA= github.com/pierrec/lz4/v4 v4.1.25 h1:kocOqRffaIbU5djlIBr7Wh+cx82C0vtFb0fOurZHqD0= github.com/pierrec/lz4/v4 v4.1.25/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= diff --git a/apps/quotas/go.mod b/apps/quotas/go.mod index d46bb3e1ac3..e77cc9595ab 100644 --- a/apps/quotas/go.mod +++ b/apps/quotas/go.mod @@ -313,6 +313,7 @@ require ( github.com/patrickmn/go-cache v2.1.0+incompatible // indirect github.com/pelletier/go-toml/v2 v2.2.4 // indirect github.com/perimeterx/marshmallow v1.1.5 // indirect + github.com/pgvector/pgvector-go v0.3.0 // indirect github.com/pierrec/lz4/v4 v4.1.25 // indirect github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect github.com/pkg/errors v0.9.1 // indirect diff --git a/apps/quotas/go.sum b/apps/quotas/go.sum index 60dd4f85760..91ad62ac47b 100644 --- a/apps/quotas/go.sum +++ b/apps/quotas/go.sum @@ -82,6 +82,8 @@ cuelang.org/go v0.11.1/go.mod h1:PBY6XvPUswPPJ2inpvUozP9mebDVTXaeehQikhZPBz0= dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +entgo.io/ent v0.14.3 h1:wokAV/kIlH9TeklJWGGS7AYJdVckr0DloWjIcO9iIIQ= +entgo.io/ent v0.14.3/go.mod h1:aDPE/OziPEu8+OWbzy4UlvWmD2/kbRuWfK2A40hcxJM= filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo= filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc= @@ -424,6 +426,10 @@ github.com/go-openapi/testify/v2 v2.4.1 h1:zB34HDKj4tHwyUQHrUkpV0Q0iXQ6dUCOQtIqn github.com/go-openapi/testify/v2 v2.4.1/go.mod h1:HCPmvFFnheKK2BuwSA0TbbdxJ3I16pjwMkYkP4Ywn54= github.com/go-openapi/validate v0.25.2 h1:12NsfLAwGegqbGWr2CnvT65X/Q2USJipmJ9b7xDJZz0= github.com/go-openapi/validate v0.25.2/go.mod h1:Pgl1LpPPGFnZ+ys4/hTlDiRYQdI1ocKypgE+8Q8BLfY= +github.com/go-pg/pg/v10 v10.11.0 h1:CMKJqLgTrfpE/aOVeLdybezR2om071Vh38OLZjsyMI0= +github.com/go-pg/pg/v10 v10.11.0/go.mod h1:4BpHRoxE61y4Onpof3x1a2SQvi9c+q1dJnrNdMjsroA= +github.com/go-pg/zerochecker v0.2.0 h1:pp7f72c3DobMWOb2ErtZsnrPaSvHd2W4o9//8HtF4mU= +github.com/go-pg/zerochecker v0.2.0/go.mod h1:NJZ4wKL0NmTtz0GKCoJ8kym6Xn/EQzXRl2OnAe7MmDo= github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= @@ -708,6 +714,10 @@ github.com/jhump/goprotoc v0.5.0/go.mod h1:VrbvcYrQOrTi3i0Vf+m+oqQWk9l72mjkJCYo7 github.com/jhump/protoreflect v1.11.0/go.mod h1:U7aMIjN0NWq9swDP7xDdoMfRHb35uiuTd3Z9nFXJf5E= github.com/jhump/protoreflect v1.17.0 h1:qOEr613fac2lOuTgWN4tPAtLL7fUSbuJL5X5XumQh94= github.com/jhump/protoreflect v1.17.0/go.mod h1:h9+vUUL38jiBzck8ck+6G/aeMX8Z4QUY/NiJPwPNi+8= +github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= +github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= +github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= +github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jmespath-community/go-jmespath v1.1.1 h1:bFikPhsi/FdmlZhVgSCd2jj1e7G/rw+zyQfyg5UF+L4= github.com/jmespath-community/go-jmespath v1.1.1/go.mod h1:4gOyFJsR/Gk+05RgTKYrifT7tBPWD8Lubtb5jRrfy9I= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= @@ -908,6 +918,8 @@ github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0 github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= github.com/perimeterx/marshmallow v1.1.5 h1:a2LALqQ1BlHM8PZblsDdidgv1mWi1DgC2UmX50IvK2s= github.com/perimeterx/marshmallow v1.1.5/go.mod h1:dsXbUu8CRzfYP5a87xpp0xq9S3u0Vchtcl8we9tYaXw= +github.com/pgvector/pgvector-go v0.3.0 h1:Ij+Yt78R//uYqs3Zk35evZFvr+G0blW0OUN+Q2D1RWc= +github.com/pgvector/pgvector-go v0.3.0/go.mod h1:duFy+PXWfW7QQd5ibqutBO4GxLsUZ9RVXhFZGIBsWSA= github.com/pierrec/lz4/v4 v4.1.25 h1:kocOqRffaIbU5djlIBr7Wh+cx82C0vtFb0fOurZHqD0= github.com/pierrec/lz4/v4 v4.1.25/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= @@ -1040,6 +1052,8 @@ github.com/tjhop/slog-gokit v0.1.6 h1:aRNB5omFUPs/5k40Ao1s7aBZHaKhNNu1JLc9ukF0qB github.com/tjhop/slog-gokit v0.1.6/go.mod h1:yA48zAHvV+Sg4z4VRyeFyFUNNXd3JY5Zg84u3USICq0= github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 h1:6fotK7otjonDflCTK0BCfls4SPy3NcCVb5dqqmbRknE= github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75/go.mod h1:KO6IkyS8Y3j8OdNO85qEYBsRPuteD+YciPomcXdrMnk= +github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc h1:9lRDQMhESg+zvGYmW5DyG0UqvY96Bu5QYsTLvCHdrgo= +github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc/go.mod h1:bciPuU6GHm1iF1pBvUfxfsH0Wmnc2VbpgvbI9ZWuIRs= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= github.com/uber/jaeger-client-go v2.30.0+incompatible h1:D6wyKGCecFaSRUpo8lCVbaOOb6ThwMmTEbhRwtKR97o= github.com/uber/jaeger-client-go v2.30.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= @@ -1047,6 +1061,20 @@ github.com/uber/jaeger-lib v2.4.1+incompatible h1:td4jdvLcExb4cBISKIpHuGoVXh+dVK github.com/uber/jaeger-lib v2.4.1+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU= github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= +github.com/uptrace/bun v1.1.12 h1:sOjDVHxNTuM6dNGaba0wUuz7KvDE1BmNu9Gqs2gJSXQ= +github.com/uptrace/bun v1.1.12/go.mod h1:NPG6JGULBeQ9IU6yHp7YGELRa5Agmd7ATZdz4tGZ6z0= +github.com/uptrace/bun/dialect/pgdialect v1.1.12 h1:m/CM1UfOkoBTglGO5CUTKnIKKOApOYxkcP2qn0F9tJk= +github.com/uptrace/bun/dialect/pgdialect v1.1.12/go.mod h1:Ij6WIxQILxLlL2frUBxUBOZJtLElD2QQNDcu/PWDHTc= +github.com/uptrace/bun/driver/pgdriver v1.1.12 h1:3rRWB1GK0psTJrHwxzNfEij2MLibggiLdTqjTtfHc1w= +github.com/uptrace/bun/driver/pgdriver v1.1.12/go.mod h1:ssYUP+qwSEgeDDS1xm2XBip9el1y9Mi5mTAvLoiADLM= +github.com/vmihailenco/bufpool v0.1.11 h1:gOq2WmBrq0i2yW5QJ16ykccQ4wH9UyEsgLm6czKAd94= +github.com/vmihailenco/bufpool v0.1.11/go.mod h1:AFf/MOy3l2CFTKbxwt0mp2MwnqjNEs5H/UxrkA5jxTQ= +github.com/vmihailenco/msgpack/v5 v5.4.1 h1:cQriyiUvjTwOHg8QZaPihLWeRAAVoCpE00IUPn0Bjt8= +github.com/vmihailenco/msgpack/v5 v5.4.1/go.mod h1:GaZTsDaehaPpQVyxrf5mtQlH+pc21PIudVV/E3rRQok= +github.com/vmihailenco/tagparser v0.1.2 h1:gnjoVuB/kljJ5wICEEOpx98oXMWPLj22G67Vbd1qPqc= +github.com/vmihailenco/tagparser v0.1.2/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= +github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g= +github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= github.com/wk8/go-ordered-map v1.0.0 h1:BV7z+2PaK8LTSd/mWgY12HyMAo5CEgkHqbkVq2thqr8= github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc= github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw= @@ -1717,6 +1745,10 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gorm.io/driver/postgres v1.5.4 h1:Iyrp9Meh3GmbSuyIAGyjkN+n9K+GHX9b9MqsTL4EJCo= +gorm.io/driver/postgres v1.5.4/go.mod h1:Bgo89+h0CRcdA33Y6frlaHHVuTdOf87pmyzwW9C/BH0= +gorm.io/gorm v1.25.5 h1:zR9lOiiYf09VNh5Q1gphfyia1JpiClIWG9hQaxB/mls= +gorm.io/gorm v1.25.5/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= @@ -1743,6 +1775,8 @@ k8s.io/kube-openapi v0.0.0-20260127142750-a19766b6e2d4 h1:HhDfevmPS+OalTjQRKbTHp k8s.io/kube-openapi v0.0.0-20260127142750-a19766b6e2d4/go.mod h1:kdmbQkyfwUagLfXIad1y2TdrjPFWp2Q89B3qkRwf/pQ= k8s.io/utils v0.0.0-20251002143259-bc988d571ff4 h1:SjGebBtkBqHFOli+05xYbK8YF1Dzkbzn+gDM4X9T4Ck= k8s.io/utils v0.0.0-20251002143259-bc988d571ff4/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +mellium.im/sasl v0.3.1 h1:wE0LW6g7U83vhvxjC1IY8DnXM+EU095yeo8XClvCdfo= +mellium.im/sasl v0.3.1/go.mod h1:xm59PUYpZHhgQ9ZqoJ5QaCqzWMi8IeS49dhp6plPCzw= modernc.org/cc/v4 v4.27.1 h1:9W30zRlYrefrDV2JE2O8VDtJ1yPGownxciz5rrbQZis= modernc.org/cc/v4 v4.27.1/go.mod h1:uVtb5OGqUKpoLWhqwNQo/8LwvoiEBLvZXIQ/SmO6mL0= modernc.org/ccgo/v4 v4.32.0 h1:hjG66bI/kqIPX1b2yT6fr/jt+QedtP2fqojG2VrFuVw= diff --git a/devenv/docker/blocks/pgvector/.env b/devenv/docker/blocks/pgvector/.env new file mode 100644 index 00000000000..634e7c653aa --- /dev/null +++ b/devenv/docker/blocks/pgvector/.env @@ -0,0 +1 @@ +pgvector_version=pg17 diff --git a/devenv/docker/blocks/pgvector/docker-compose.yaml b/devenv/docker/blocks/pgvector/docker-compose.yaml new file mode 100644 index 00000000000..b27320d0645 --- /dev/null +++ b/devenv/docker/blocks/pgvector/docker-compose.yaml @@ -0,0 +1,16 @@ + pgvector: + image: pgvector/pgvector:${pgvector_version} + environment: + POSTGRES_USER: grafana + POSTGRES_PASSWORD: password + POSTGRES_DB: grafana_vectors + ports: + - "5433:5432" + volumes: + - ./docker/blocks/pgvector/init.sql:/docker-entrypoint-initdb.d/init.sql + command: postgres -c log_connections=on -c log_disconnections=on -c log_destination=stderr + healthcheck: + test: [ "CMD", "pg_isready", "-q", "-d", "grafana_vectors", "-U", "grafana" ] + timeout: 45s + interval: 10s + retries: 10 diff --git a/devenv/docker/blocks/pgvector/init.sql b/devenv/docker/blocks/pgvector/init.sql new file mode 100644 index 00000000000..0aa0fc22558 --- /dev/null +++ b/devenv/docker/blocks/pgvector/init.sql @@ -0,0 +1 @@ +CREATE EXTENSION IF NOT EXISTS vector; diff --git a/go.mod b/go.mod index b8df86873aa..fab61f985d2 100644 --- a/go.mod +++ b/go.mod @@ -338,7 +338,7 @@ require ( github.com/ProtonMail/go-crypto v1.3.0 // indirect github.com/RoaringBitmap/roaring/v2 v2.4.5 // indirect github.com/Yiling-J/theine-go v0.6.2 // indirect - github.com/agext/levenshtein v1.2.1 // indirect + github.com/agext/levenshtein v1.2.3 // indirect github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b // indirect github.com/alicebob/gopher-json v0.0.0-20230218143504-906a9b012302 // indirect github.com/antlr4-go/antlr/v4 v4.13.1 // indirect @@ -698,6 +698,8 @@ require ( software.sslmate.com/src/go-pkcs12 v0.7.0 // indirect ) +require github.com/pgvector/pgvector-go v0.3.0 // @grafana/grafana-search-and-storage + replace ( // Use fork of crewjam/saml with fixes for some issues until changes get merged into upstream github.com/crewjam/saml => github.com/grafana/saml v0.4.15-0.20240917091248-ae3bbdad8a56 diff --git a/go.sum b/go.sum index 3f516146ca1..c508e7c1d05 100644 --- a/go.sum +++ b/go.sum @@ -636,6 +636,8 @@ cuelang.org/go v0.11.1/go.mod h1:PBY6XvPUswPPJ2inpvUozP9mebDVTXaeehQikhZPBz0= dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +entgo.io/ent v0.14.3 h1:wokAV/kIlH9TeklJWGGS7AYJdVckr0DloWjIcO9iIIQ= +entgo.io/ent v0.14.3/go.mod h1:aDPE/OziPEu8+OWbzy4UlvWmD2/kbRuWfK2A40hcxJM= filippo.io/age v1.2.1 h1:X0TZjehAZylOIj4DubWYU1vWQxv9bJpo+Uu2/LGhi1o= filippo.io/age v1.2.1/go.mod h1:JL9ew2lTN+Pyft4RiNGguFfOpewKwSHm5ayKD/A4004= gioui.org v0.0.0-20210308172011-57750fc8a0a6/go.mod h1:RSH6KIUZ0p2xy5zHDxgAM4zumjgTw83q2ge/PI+yyw8= @@ -770,8 +772,8 @@ github.com/Workiva/go-datastructures v1.1.5 h1:5YfhQ4ry7bZc2Mc7R0YZyYwpf5c6t1cEF github.com/Workiva/go-datastructures v1.1.5/go.mod h1:1yZL+zfsztete+ePzZz/Zb1/t5BnDuE2Ya2MMGhzP6A= github.com/Yiling-J/theine-go v0.6.2 h1:1GeoXeQ0O0AUkiwj2S9Jc0Mzx+hpqzmqsJ4kIC4M9AY= github.com/Yiling-J/theine-go v0.6.2/go.mod h1:08QpMa5JZ2pKN+UJCRrCasWYO1IKCdl54Xa836rpmDU= -github.com/agext/levenshtein v1.2.1 h1:QmvMAjj2aEICytGiWzmxoE0x2KZvE0fvmqMOfy2tjT8= -github.com/agext/levenshtein v1.2.1/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558= +github.com/agext/levenshtein v1.2.3 h1:YB2fHEn0UJagG8T1rrWknE3ZQzWM06O8AMAatNn7lmo= +github.com/agext/levenshtein v1.2.3/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558= github.com/ajstarks/deck v0.0.0-20200831202436-30c9fc6549a9/go.mod h1:JynElWSGnm/4RlzPXRlREEwqTHAN3T56Bv2ITsFT3gY= github.com/ajstarks/deck/generate v0.0.0-20210309230005-c3f852c02e19/go.mod h1:T13YZdzov6OU0A1+RfKZiZN9ca6VeKdBdyDV+BY97Tk= github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= @@ -1313,6 +1315,10 @@ github.com/go-openapi/validate v0.25.2 h1:12NsfLAwGegqbGWr2CnvT65X/Q2USJipmJ9b7x github.com/go-openapi/validate v0.25.2/go.mod h1:Pgl1LpPPGFnZ+ys4/hTlDiRYQdI1ocKypgE+8Q8BLfY= github.com/go-pdf/fpdf v0.5.0/go.mod h1:HzcnA+A23uwogo0tp9yU+l3V+KXhiESpt1PMayhOh5M= github.com/go-pdf/fpdf v0.6.0/go.mod h1:HzcnA+A23uwogo0tp9yU+l3V+KXhiESpt1PMayhOh5M= +github.com/go-pg/pg/v10 v10.11.0 h1:CMKJqLgTrfpE/aOVeLdybezR2om071Vh38OLZjsyMI0= +github.com/go-pg/pg/v10 v10.11.0/go.mod h1:4BpHRoxE61y4Onpof3x1a2SQvi9c+q1dJnrNdMjsroA= +github.com/go-pg/zerochecker v0.2.0 h1:pp7f72c3DobMWOb2ErtZsnrPaSvHd2W4o9//8HtF4mU= +github.com/go-pg/zerochecker v0.2.0/go.mod h1:NJZ4wKL0NmTtz0GKCoJ8kym6Xn/EQzXRl2OnAe7MmDo= github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= @@ -1796,6 +1802,10 @@ github.com/jhump/goprotoc v0.5.0/go.mod h1:VrbvcYrQOrTi3i0Vf+m+oqQWk9l72mjkJCYo7 github.com/jhump/protoreflect v1.11.0/go.mod h1:U7aMIjN0NWq9swDP7xDdoMfRHb35uiuTd3Z9nFXJf5E= github.com/jhump/protoreflect v1.17.0 h1:qOEr613fac2lOuTgWN4tPAtLL7fUSbuJL5X5XumQh94= github.com/jhump/protoreflect v1.17.0/go.mod h1:h9+vUUL38jiBzck8ck+6G/aeMX8Z4QUY/NiJPwPNi+8= +github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= +github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= +github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= +github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jmespath-community/go-jmespath v1.1.1 h1:bFikPhsi/FdmlZhVgSCd2jj1e7G/rw+zyQfyg5UF+L4= github.com/jmespath-community/go-jmespath v1.1.1/go.mod h1:4gOyFJsR/Gk+05RgTKYrifT7tBPWD8Lubtb5jRrfy9I= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= @@ -2142,6 +2152,8 @@ github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0 github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= github.com/perimeterx/marshmallow v1.1.5 h1:a2LALqQ1BlHM8PZblsDdidgv1mWi1DgC2UmX50IvK2s= github.com/perimeterx/marshmallow v1.1.5/go.mod h1:dsXbUu8CRzfYP5a87xpp0xq9S3u0Vchtcl8we9tYaXw= +github.com/pgvector/pgvector-go v0.3.0 h1:Ij+Yt78R//uYqs3Zk35evZFvr+G0blW0OUN+Q2D1RWc= +github.com/pgvector/pgvector-go v0.3.0/go.mod h1:duFy+PXWfW7QQd5ibqutBO4GxLsUZ9RVXhFZGIBsWSA= github.com/philhofer/fwd v1.1.2/go.mod h1:qkPdfjR2SIEbspLqpe1tO4n5yICnr2DY7mqEx2tUTP0= github.com/phpdave11/gofpdf v1.4.2/go.mod h1:zpO6xFn9yxo3YLyMvW8HcKWVdbNqgIfOOp2dXMnm1mY= github.com/phpdave11/gofpdi v1.0.12/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI= @@ -2423,6 +2435,8 @@ github.com/tklauser/numcpus v0.11.0 h1:nSTwhKH5e1dMNsCdVBukSZrURJRoHbSEQjdEbY+9R github.com/tklauser/numcpus v0.11.0/go.mod h1:z+LwcLq54uWZTX0u/bGobaV34u6V7KNlTZejzM6/3MQ= github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 h1:6fotK7otjonDflCTK0BCfls4SPy3NcCVb5dqqmbRknE= github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75/go.mod h1:KO6IkyS8Y3j8OdNO85qEYBsRPuteD+YciPomcXdrMnk= +github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc h1:9lRDQMhESg+zvGYmW5DyG0UqvY96Bu5QYsTLvCHdrgo= +github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc/go.mod h1:bciPuU6GHm1iF1pBvUfxfsH0Wmnc2VbpgvbI9ZWuIRs= github.com/trivago/tgo v1.0.7/go.mod h1:w4dpD+3tzNIIiIfkWWa85w5/B77tlvdZckQ+6PkFnhc= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= github.com/ua-parser/uap-go v0.0.0-20250213224047-9c035f085b90 h1:rB0J+hLNltG1Qv+UF+MkdFz89XMps5BOAFJN4xWjc+s= @@ -2440,6 +2454,12 @@ github.com/unknwon/com v1.0.1/go.mod h1:tOOxU81rwgoCLoOVVPHb6T/wt8HZygqH5id+GNnl github.com/unknwon/log v0.0.0-20150304194804-e617c87089d3/go.mod h1:1xEUf2abjfP92w2GZTV+GgaRxXErwRXcClbUwrNJffU= github.com/unknwon/log v0.0.0-20200308114134-929b1006e34a h1:vcrhXnj9g9PIE+cmZgaPSwOyJ8MAQTRmsgGrB0x5rF4= github.com/unknwon/log v0.0.0-20200308114134-929b1006e34a/go.mod h1:1xEUf2abjfP92w2GZTV+GgaRxXErwRXcClbUwrNJffU= +github.com/uptrace/bun v1.1.12 h1:sOjDVHxNTuM6dNGaba0wUuz7KvDE1BmNu9Gqs2gJSXQ= +github.com/uptrace/bun v1.1.12/go.mod h1:NPG6JGULBeQ9IU6yHp7YGELRa5Agmd7ATZdz4tGZ6z0= +github.com/uptrace/bun/dialect/pgdialect v1.1.12 h1:m/CM1UfOkoBTglGO5CUTKnIKKOApOYxkcP2qn0F9tJk= +github.com/uptrace/bun/dialect/pgdialect v1.1.12/go.mod h1:Ij6WIxQILxLlL2frUBxUBOZJtLElD2QQNDcu/PWDHTc= +github.com/uptrace/bun/driver/pgdriver v1.1.12 h1:3rRWB1GK0psTJrHwxzNfEij2MLibggiLdTqjTtfHc1w= +github.com/uptrace/bun/driver/pgdriver v1.1.12/go.mod h1:ssYUP+qwSEgeDDS1xm2XBip9el1y9Mi5mTAvLoiADLM= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli v1.22.17 h1:SYzXoiPfQjHBbkYxbew5prZHS1TOLT3ierW8SYLqtVQ= github.com/urfave/cli v1.22.17/go.mod h1:b0ht0aqgH/6pBYzzxURyrM4xXNgsoT/n2ZzwQiEhNVo= @@ -2447,6 +2467,14 @@ github.com/urfave/cli/v2 v2.27.7 h1:bH59vdhbjLv3LAvIu6gd0usJHgoTTPhCFib8qqOwXYU= github.com/urfave/cli/v2 v2.27.7/go.mod h1:CyNAG/xg+iAOg0N4MPGZqVmv2rCoP267496AOXUZjA4= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/vmihailenco/bufpool v0.1.11 h1:gOq2WmBrq0i2yW5QJ16ykccQ4wH9UyEsgLm6czKAd94= +github.com/vmihailenco/bufpool v0.1.11/go.mod h1:AFf/MOy3l2CFTKbxwt0mp2MwnqjNEs5H/UxrkA5jxTQ= +github.com/vmihailenco/msgpack/v5 v5.4.1 h1:cQriyiUvjTwOHg8QZaPihLWeRAAVoCpE00IUPn0Bjt8= +github.com/vmihailenco/msgpack/v5 v5.4.1/go.mod h1:GaZTsDaehaPpQVyxrf5mtQlH+pc21PIudVV/E3rRQok= +github.com/vmihailenco/tagparser v0.1.2 h1:gnjoVuB/kljJ5wICEEOpx98oXMWPLj22G67Vbd1qPqc= +github.com/vmihailenco/tagparser v0.1.2/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= +github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g= +github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= github.com/vultr/govultr/v2 v2.17.2 h1:gej/rwr91Puc/tgh+j33p/BLR16UrIPnSr+AIwYWZQs= github.com/vultr/govultr/v2 v2.17.2/go.mod h1:ZFOKGWmgjytfyjeyAdhQlSWwTjh2ig+X49cAp50dzXI= github.com/wk8/go-ordered-map v1.0.0 h1:BV7z+2PaK8LTSd/mWgY12HyMAo5CEgkHqbkVq2thqr8= @@ -3538,6 +3566,10 @@ gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gorm.io/driver/postgres v1.5.4 h1:Iyrp9Meh3GmbSuyIAGyjkN+n9K+GHX9b9MqsTL4EJCo= +gorm.io/driver/postgres v1.5.4/go.mod h1:Bgo89+h0CRcdA33Y6frlaHHVuTdOf87pmyzwW9C/BH0= +gorm.io/gorm v1.25.5 h1:zR9lOiiYf09VNh5Q1gphfyia1JpiClIWG9hQaxB/mls= +gorm.io/gorm v1.25.5/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q= @@ -3585,6 +3617,8 @@ k8s.io/utils v0.0.0-20251002143259-bc988d571ff4 h1:SjGebBtkBqHFOli+05xYbK8YF1Dzk k8s.io/utils v0.0.0-20251002143259-bc988d571ff4/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= lukechampine.com/uint128 v1.1.1/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk= lukechampine.com/uint128 v1.2.0/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk= +mellium.im/sasl v0.3.1 h1:wE0LW6g7U83vhvxjC1IY8DnXM+EU095yeo8XClvCdfo= +mellium.im/sasl v0.3.1/go.mod h1:xm59PUYpZHhgQ9ZqoJ5QaCqzWMi8IeS49dhp6plPCzw= modernc.org/cc/v3 v3.36.0/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI= modernc.org/cc/v3 v3.36.2/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI= modernc.org/cc/v3 v3.36.3/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI= diff --git a/go.work.sum b/go.work.sum index af233fd98ba..a1c5fd67f7b 100644 --- a/go.work.sum +++ b/go.work.sum @@ -1,3 +1,5 @@ +ariga.io/atlas v0.32.0 h1:y+77nueMrExLiKlz1CcPKh/nU7VSlWfBbwCShsJyvCw= +ariga.io/atlas v0.32.0/go.mod h1:Oe1xWPuu5q9LzyrWfbZmEZxFYeu4BHTyzfjeW2aZp/w= atomicgo.dev/assert v0.0.2/go.mod h1:ut4NcI3QDdJtlmAxQULOmA13Gz6e2DWbSAS8RUOmNYQ= atomicgo.dev/cursor v0.2.0 h1:H6XN5alUJ52FZZUkI7AlJbUc1aW38GWZalpYRPpoPOw= atomicgo.dev/cursor v0.2.0/go.mod h1:Lr4ZJB3U7DfPPOkbH7/6TOtJ4vFGHlgj1nc+n900IpU= @@ -274,6 +276,8 @@ contrib.go.opencensus.io/exporter/ocagent v0.6.0 h1:Z1n6UAyr0QwM284yUuh5Zd8JlvxU contrib.go.opencensus.io/exporter/stackdriver v0.13.15-0.20230702191903-2de6d2748484 h1:xRc46S76eyn4ZF3jWX8I+aUSKVLw5EQ1aDvHwfV5W1o= contrib.go.opencensus.io/exporter/stackdriver v0.13.15-0.20230702191903-2de6d2748484/go.mod h1:uxw+4/0SiKbbVSD/F2tk5pJTdVcfIBBcsQ8gwcu4X+E= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9 h1:VpgP7xuJadIUuKccphEpTJnWhS2jkQyMt6Y7pJCD7fY= +entgo.io/ent v0.14.3 h1:wokAV/kIlH9TeklJWGGS7AYJdVckr0DloWjIcO9iIIQ= +entgo.io/ent v0.14.3/go.mod h1:aDPE/OziPEu8+OWbzy4UlvWmD2/kbRuWfK2A40hcxJM= gioui.org v0.0.0-20210308172011-57750fc8a0a6 h1:K72hopUosKG3ntOPNG4OzzbuhxGuVf06fa2la1/H/Ho= gioui.org v0.2.0 h1:RbzDn1h/pCVf/q44ImQSa/J3MIFpY3OWphzT/Tyei+w= gioui.org v0.2.0/go.mod h1:1H72sKEk/fNFV+l0JNeM2Dt3co3Y4uaQcD+I+/GQ0e4= @@ -394,6 +398,8 @@ github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kk github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA= github.com/andybalholm/stroke v0.0.0-20221221101821-bd29b49d73f0 h1:uF5Q/hWnDU1XZeT6CsrRSxHLroUSEYYO3kgES+yd+So= github.com/andybalholm/stroke v0.0.0-20221221101821-bd29b49d73f0/go.mod h1:ccdDYaY5+gO+cbnQdFxEXqfy0RkoV25H3jLXUDNM3wg= +github.com/ankane/disco-go v0.1.2 h1:Amm1UV3oLttAJM18MW7zofTDAXbuE9BEQUt3YPF6pVI= +github.com/ankane/disco-go v0.1.2/go.mod h1:nkR7DLW+KkXeRRAsWk6poMTpTOWp9/4iKYGDwg8dSS0= github.com/antchfx/xmlquery v1.4.4 h1:mxMEkdYP3pjKSftxss4nUHfjBhnMk4imGoR96FRY2dg= github.com/antchfx/xmlquery v1.4.4/go.mod h1:AEPEEPYE9GnA2mj5Ur2L5Q5/2PycJ0N9Fusrx9b12fc= github.com/antchfx/xpath v1.3.4 h1:1ixrW1VnXd4HurCj7qnqnR0jo14g8JMe20Fshg1Vgz4= @@ -478,6 +484,8 @@ github.com/blevesearch/snowball v0.6.1/go.mod h1:ZF0IBg5vgpeoUhnMza2v0A/z8m1cWPl github.com/blevesearch/stempel v0.2.0 h1:CYzVPaScODMvgE9o+kf6D4RJ/VRomyi9uHF+PtB+Afc= github.com/blevesearch/stempel v0.2.0/go.mod h1:wjeTHqQv+nQdbPuJ/YcvOjTInA2EIc6Ks1FoSUzSLvc= github.com/bmatcuk/doublestar v1.1.1 h1:YroD6BJCZBYx06yYFEWvUuKVWQn3vLLQAVmDmvTSaiQ= +github.com/bmatcuk/doublestar v1.3.4 h1:gPypJ5xD31uhX6Tf54sDPUOBXTqKH4c9aPY66CyQrS0= +github.com/bmatcuk/doublestar v1.3.4/go.mod h1:wiQtGV+rzVYxB7WIlirSN++5HPtPlXEo9MEoZQC/PmE= github.com/bmatcuk/doublestar/v4 v4.8.1 h1:54Bopc5c2cAvhLRAzqOGCYHYyhcDHsFF4wWIR5wKP38= github.com/bmatcuk/doublestar/v4 v4.8.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY= @@ -700,6 +708,8 @@ github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbV github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-openapi/analysis v0.23.0/go.mod h1:9mz9ZWaSlV8TvjQHLl2mUW2PbZtemkE8yA5v22ohupo= github.com/go-openapi/errors v0.22.2/go.mod h1:+n/5UdIqdVnLIJ6Q9Se8HNGUXYaY6CN8ImWzfi/Gzp0= +github.com/go-openapi/inflect v0.21.0 h1:FoBjBTQEcbg2cJUWX6uwL9OyIW8eqc9k4KhN4lfbeYk= +github.com/go-openapi/inflect v0.21.0/go.mod h1:INezMuUu7SJQc2AyR3WO0DqqYUJSj8Kb4hBd7WtjlAw= github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= github.com/go-openapi/jsonreference v0.21.0/go.mod h1:LmZmgsrTkVg9LG4EaHeY8cBDslNPMo06cago5JNLkm4= github.com/go-openapi/loads v0.22.0/go.mod h1:yLsaTCS92mnSAZX5WWoxszLj0u+Ojl+Zs5Stn1oF+rs= @@ -712,6 +722,10 @@ github.com/go-openapi/testify/enable/yaml/v2 v2.4.0/go.mod h1:14iV8jyyQlinc9StD7 github.com/go-openapi/testify/v2 v2.4.0/go.mod h1:HCPmvFFnheKK2BuwSA0TbbdxJ3I16pjwMkYkP4Ywn54= github.com/go-openapi/validate v0.24.0/go.mod h1:iyeX1sEufmv3nPbBdX3ieNviWnOZaJ1+zquzJEf2BAQ= github.com/go-pdf/fpdf v0.6.0 h1:MlgtGIfsdMEEQJr2le6b/HNr1ZlQwxyWr77r2aj2U/8= +github.com/go-pg/pg/v10 v10.11.0 h1:CMKJqLgTrfpE/aOVeLdybezR2om071Vh38OLZjsyMI0= +github.com/go-pg/pg/v10 v10.11.0/go.mod h1:4BpHRoxE61y4Onpof3x1a2SQvi9c+q1dJnrNdMjsroA= +github.com/go-pg/zerochecker v0.2.0 h1:pp7f72c3DobMWOb2ErtZsnrPaSvHd2W4o9//8HtF4mU= +github.com/go-pg/zerochecker v0.2.0/go.mod h1:NJZ4wKL0NmTtz0GKCoJ8kym6Xn/EQzXRl2OnAe7MmDo= github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A= github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= @@ -853,6 +867,10 @@ github.com/jedib0t/go-pretty/v6 v6.6.7 h1:m+LbHpm0aIAPLzLbMfn8dc3Ht8MW7lsSO4MPIt github.com/jedib0t/go-pretty/v6 v6.6.7/go.mod h1:YwC5CE4fJ1HFUDeivSV1r//AmANFHyqczZk+U6BDALU= github.com/jhump/gopoet v0.1.0 h1:gYjOPnzHd2nzB37xYQZxj4EIQNpBrBskRqQQ3q4ZgSg= github.com/jhump/goprotoc v0.5.0 h1:Y1UgUX+txUznfqcGdDef8ZOVlyQvnV0pKWZH08RmZuo= +github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= +github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= +github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= +github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/joncrlsn/dque v0.0.0-20211108142734-c2ef48c5192a h1:sfe532Ipn7GX0V6mHdynBk393rDmqgI0QmjLK7ct7TU= @@ -1241,6 +1259,8 @@ github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhso github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= github.com/tinylib/msgp v1.1.8 h1:FCXC1xanKO4I8plpHGH2P7koL/RzZs12l/+r7vakfm0= +github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc h1:9lRDQMhESg+zvGYmW5DyG0UqvY96Bu5QYsTLvCHdrgo= +github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc/go.mod h1:bciPuU6GHm1iF1pBvUfxfsH0Wmnc2VbpgvbI9ZWuIRs= github.com/trivago/tgo v1.0.7 h1:uaWH/XIy9aWYWpjm2CU3RpcqZXmX2ysQ9/Go+d9gyrM= github.com/tursodatabase/libsql-client-go v0.0.0-20240902231107-85af5b9d094d h1:dOMI4+zEbDI37KGb0TI44GUAwxHF9cMsIoDTJ7UmgfU= github.com/tursodatabase/libsql-client-go v0.0.0-20240902231107-85af5b9d094d/go.mod h1:l8xTsYB90uaVdMHXMCxKKLSgw5wLYBwBKKefNIUnm9s= @@ -1269,14 +1289,24 @@ github.com/uber/jaeger-client-go v2.28.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMW github.com/uber/jaeger-lib v2.2.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= github.com/urfave/cli/v3 v3.6.2 h1:lQuqiPrZ1cIz8hz+HcrG0TNZFxU70dPZ3Yl+pSrH9A8= github.com/urfave/cli/v3 v3.6.2/go.mod h1:ysVLtOEmg2tOy6PknnYVhDoouyC/6N42TMeoMzskhso= +github.com/uptrace/bun v1.1.12 h1:sOjDVHxNTuM6dNGaba0wUuz7KvDE1BmNu9Gqs2gJSXQ= +github.com/uptrace/bun v1.1.12/go.mod h1:NPG6JGULBeQ9IU6yHp7YGELRa5Agmd7ATZdz4tGZ6z0= +github.com/uptrace/bun/dialect/pgdialect v1.1.12 h1:m/CM1UfOkoBTglGO5CUTKnIKKOApOYxkcP2qn0F9tJk= +github.com/uptrace/bun/dialect/pgdialect v1.1.12/go.mod h1:Ij6WIxQILxLlL2frUBxUBOZJtLElD2QQNDcu/PWDHTc= +github.com/uptrace/bun/driver/pgdriver v1.1.12 h1:3rRWB1GK0psTJrHwxzNfEij2MLibggiLdTqjTtfHc1w= +github.com/uptrace/bun/driver/pgdriver v1.1.12/go.mod h1:ssYUP+qwSEgeDDS1xm2XBip9el1y9Mi5mTAvLoiADLM= github.com/valyala/fasttemplate v1.2.2 h1:lxLXG0uE3Qnshl9QyaK6XJxMXlQZELvChBOCmQD0Loo= github.com/valyala/fasttemplate v1.2.2/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= github.com/vertica/vertica-sql-go v1.3.3 h1:fL+FKEAEy5ONmsvya2WH5T8bhkvY27y/Ik3ReR2T+Qw= github.com/vertica/vertica-sql-go v1.3.3/go.mod h1:jnn2GFuv+O2Jcjktb7zyc4Utlbu9YVqpHH/lx63+1M4= github.com/vertica/vertica-sql-go v1.3.5 h1:IrfH2WIgzZ45yDHyjVFrXU2LuKNIjF5Nwi90a6cfgUI= github.com/vertica/vertica-sql-go v1.3.5/go.mod h1:jnn2GFuv+O2Jcjktb7zyc4Utlbu9YVqpHH/lx63+1M4= +github.com/vmihailenco/bufpool v0.1.11 h1:gOq2WmBrq0i2yW5QJ16ykccQ4wH9UyEsgLm6czKAd94= +github.com/vmihailenco/bufpool v0.1.11/go.mod h1:AFf/MOy3l2CFTKbxwt0mp2MwnqjNEs5H/UxrkA5jxTQ= github.com/vmihailenco/msgpack/v5 v5.4.1 h1:cQriyiUvjTwOHg8QZaPihLWeRAAVoCpE00IUPn0Bjt8= github.com/vmihailenco/msgpack/v5 v5.4.1/go.mod h1:GaZTsDaehaPpQVyxrf5mtQlH+pc21PIudVV/E3rRQok= +github.com/vmihailenco/tagparser v0.1.2 h1:gnjoVuB/kljJ5wICEEOpx98oXMWPLj22G67Vbd1qPqc= +github.com/vmihailenco/tagparser v0.1.2/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g= github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= github.com/willf/bitset v1.1.11 h1:N7Z7E9UvjW+sGsEl7k/SJrvY2reP1A07MrGuCjIOjRE= @@ -1317,6 +1347,8 @@ github.com/yosssi/ace v0.0.5 h1:tUkIP/BLdKqrlrPwcmH0shwEEhTRHoGnc1wFIWmaBUA= github.com/yosssi/ace v0.0.5/go.mod h1:ALfIzm2vT7t5ZE7uoIZqF3TQ7SAOyupFZnkrF5id+K0= github.com/yuin/goldmark v1.7.13 h1:GPddIs617DnBLFFVJFgpo1aBfe/4xcvMc3SB5t/D0pA= github.com/yuin/goldmark v1.7.13/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg= +github.com/zclconf/go-cty-yaml v1.1.0 h1:nP+jp0qPHv2IhUVqmQSzjvqAWcObN0KBkUl2rWBdig0= +github.com/zclconf/go-cty-yaml v1.1.0/go.mod h1:9YLUH4g7lOhVWqUbctnVlZ5KLpg7JAprQNgxSZ1Gyxs= github.com/zeebo/errs v1.4.0 h1:XNdoD/RRMKP7HD0UhJnIzUy74ISdGGxURlYG8HSWSfM= github.com/zeebo/errs v1.4.0/go.mod h1:sgbWHsvVuTPHcqJJGQ1WhI5KbWlHYz+2+2C/LSEtCw4= github.com/zenazn/goji v1.0.1 h1:4lbD8Mx2h7IvloP7r2C0D6ltZP6Ufip8Hn0wmSK5LR8= @@ -1788,6 +1820,10 @@ gopkg.in/resty.v1 v1.12.0 h1:CuXP0Pjfw9rOuY6EP+UvtNvt5DSqHpIxILZKT/quCZI= gopkg.in/src-d/go-billy.v4 v4.3.2 h1:0SQA1pRztfTFx2miS8sA97XvooFeNOmvUenF4o0EcVg= gopkg.in/src-d/go-billy.v4 v4.3.2/go.mod h1:nDjArDMp+XMs1aFAESLRjfGSgfvoYN0hDfzEk0GjC98= gopkg.in/vmihailenco/msgpack.v2 v2.9.2 h1:gjPqo9orRVlSAH/065qw3MsFCDpH7fa1KpiizXyllY4= +gorm.io/driver/postgres v1.5.4 h1:Iyrp9Meh3GmbSuyIAGyjkN+n9K+GHX9b9MqsTL4EJCo= +gorm.io/driver/postgres v1.5.4/go.mod h1:Bgo89+h0CRcdA33Y6frlaHHVuTdOf87pmyzwW9C/BH0= +gorm.io/gorm v1.25.5 h1:zR9lOiiYf09VNh5Q1gphfyia1JpiClIWG9hQaxB/mls= +gorm.io/gorm v1.25.5/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= honnef.co/go/tools v0.3.2 h1:ytYb4rOqyp1TSa2EPvNVwtPQJctSELKaMyLfqNP4+34= honnef.co/go/tools v0.3.2/go.mod h1:jzwdWgg7Jdq75wlfblQxO4neNaFFSvgc1tD5Wv8U0Yw= howett.net/plist v1.0.1 h1:37GdZ8tP09Q35o9ych3ehygcsL+HqKSwzctveSlarvM= @@ -1802,6 +1838,8 @@ k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= k8s.io/klog/v2 v2.80.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/utils v0.0.0-20240711033017-18e509b52bc8/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= lukechampine.com/uint128 v1.2.0 h1:mBi/5l91vocEN8otkC5bDLhi2KdCticRiwbdB0O+rjI= +mellium.im/sasl v0.3.1 h1:wE0LW6g7U83vhvxjC1IY8DnXM+EU095yeo8XClvCdfo= +mellium.im/sasl v0.3.1/go.mod h1:xm59PUYpZHhgQ9ZqoJ5QaCqzWMi8IeS49dhp6plPCzw= modernc.org/cc/v3 v3.36.3 h1:uISP3F66UlixxWEcKuIWERa4TwrZENHSL8tWxZz8bHg= modernc.org/cc/v3 v3.41.0/go.mod h1:Ni4zjJYJ04CDOhG7dn640WGfwBzfE0ecX8TyMB0Fv0Y= modernc.org/ccgo/v3 v3.16.9 h1:AXquSwg7GuMk11pIdw7fmO1Y/ybgazVkMhsZWCV0mHM= diff --git a/pkg/modules/dependencies.go b/pkg/modules/dependencies.go index c2193b1c06e..82f6cd1d112 100644 --- a/pkg/modules/dependencies.go +++ b/pkg/modules/dependencies.go @@ -16,6 +16,7 @@ const ( InstrumentationServer string = "instrumentation-server" GRPCServer string = "grpc-server" UnifiedBackend string = "unified-backend" + UnifiedVectorBackend string = "unified-vector-backend" FrontendServer string = "frontend-server" OperatorServer string = "operator" ) @@ -26,8 +27,8 @@ var dependencyMap = map[string][]string{ GrafanaAPIServer: {InstrumentationServer}, // TODO: remove SearchServerRing once we only use sharding in SearchServer - StorageServer: {UnifiedBackend, InstrumentationServer, GRPCServer, SearchServerRing}, - SearchServer: {UnifiedBackend, InstrumentationServer, GRPCServer, SearchServerRing}, + StorageServer: {UnifiedBackend, UnifiedVectorBackend, InstrumentationServer, GRPCServer, SearchServerRing}, + SearchServer: {UnifiedBackend, UnifiedVectorBackend, InstrumentationServer, GRPCServer, SearchServerRing}, ZanzanaServer: {InstrumentationServer}, AuthnServer: {InstrumentationServer}, diff --git a/pkg/server/module_server.go b/pkg/server/module_server.go index d1753101e38..05bc2536a07 100644 --- a/pkg/server/module_server.go +++ b/pkg/server/module_server.go @@ -34,6 +34,7 @@ import ( "github.com/grafana/grafana/pkg/services/licensing" "github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/storage/unified/resource" + "github.com/grafana/grafana/pkg/storage/unified/search/vector" "github.com/grafana/grafana/pkg/storage/unified/sql" "go.opentelemetry.io/otel" ) @@ -138,6 +139,7 @@ type ModuleServer struct { isInitialized bool mtx sync.Mutex storageBackend resource.StorageBackend + vectorBackend vector.VectorBackend searchClient resourcepb.ResourceIndexClient storageMetrics *resource.StorageMetrics indexMetrics *resource.BleveIndexMetrics @@ -243,6 +245,24 @@ func (s *ModuleServer) Run() error { return services.NewIdleService(nil, nil).WithName(modules.UnifiedBackend), nil }) + m.RegisterInvisibleModule(modules.UnifiedVectorBackend, func() (services.Service, error) { + // StorageServer owns the schema and runs the promoter; other targets + // get a read-only backend. + ownsSchema := m.IsModuleEnabled(modules.StorageServer) + if s.vectorBackend == nil { + var err error + s.vectorBackend, err = vector.InitVectorBackend(context.Background(), s.cfg, ownsSchema) + if err != nil { + return nil, err + } + } + if s.vectorBackend != nil && ownsSchema { + runFn := func(ctx context.Context) error { return s.vectorBackend.Run(ctx) } + return services.NewBasicService(nil, runFn, nil).WithName(modules.UnifiedVectorBackend), nil + } + return services.NewIdleService(nil, nil).WithName(modules.UnifiedVectorBackend), nil + }) + m.RegisterModule(modules.MemberlistKV, s.initMemberlistKV) m.RegisterModule(modules.SearchServerRing, s.initSearchServerRing) m.RegisterModule(modules.SearchServerDistributor, func() (services.Service, error) { @@ -286,7 +306,7 @@ func (s *ModuleServer) Run() error { } indexMetrics = s.indexMetrics } - svc, err := sql.ProvideUnifiedStorageGrpcService(s.cfg, s.features, s.log, s.registerer, docBuilders, s.storageMetrics, indexMetrics, s.searchServerRing, s.MemberlistKVConfig, s.httpServerRouter, s.storageBackend, s.searchClient, s.grpcService, s.StorageServiceOptions...) + svc, err := sql.ProvideUnifiedStorageGrpcService(s.cfg, s.features, s.log, s.registerer, docBuilders, s.storageMetrics, indexMetrics, s.searchServerRing, s.MemberlistKVConfig, s.httpServerRouter, s.storageBackend, s.vectorBackend, s.searchClient, s.grpcService, s.StorageServiceOptions...) if err != nil { return nil, err } @@ -317,7 +337,7 @@ func (s *ModuleServer) Run() error { if err != nil { return nil, err } - svc, err := sql.ProvideSearchGRPCService(s.cfg, s.features, s.log, s.registerer, docBuilders, s.indexMetrics, s.searchServerRing, s.MemberlistKVConfig, s.httpServerRouter, s.storageBackend, s.grpcService, s.StorageServiceOptions...) + svc, err := sql.ProvideSearchGRPCService(s.cfg, s.features, s.log, s.registerer, docBuilders, s.indexMetrics, s.searchServerRing, s.MemberlistKVConfig, s.httpServerRouter, s.storageBackend, s.vectorBackend, s.grpcService, s.StorageServiceOptions...) if err != nil { return nil, err } diff --git a/pkg/server/wire_gen.go b/pkg/server/wire_gen.go index 036a84fdbf0..124b449194b 100644 --- a/pkg/server/wire_gen.go +++ b/pkg/server/wire_gen.go @@ -276,6 +276,7 @@ import ( "github.com/grafana/grafana/pkg/storage/unified/resource" "github.com/grafana/grafana/pkg/storage/unified/search" "github.com/grafana/grafana/pkg/storage/unified/search/builders" + "github.com/grafana/grafana/pkg/storage/unified/search/vector" "github.com/grafana/grafana/pkg/storage/unified/sql" "github.com/grafana/grafana/pkg/tsdb/azuremonitor" "github.com/grafana/grafana/pkg/tsdb/cloud-monitoring" @@ -558,15 +559,20 @@ func Initialize(ctx context.Context, cfg *setting.Cfg, opts Options, apiOpts api if err != nil { return nil, err } + vectorBackend, err := vector.ProvideVectorBackend(cfg) + if err != nil { + return nil, err + } options := &unified.Options{ - Cfg: cfg, - Features: featureToggles, - DB: sqlStore, - Tracer: tracingService, - Reg: registerer, - Authzc: accessClient, - Docs: documentBuilderSupplier, - SecureValues: inlineSecureValueSupport, + Cfg: cfg, + Features: featureToggles, + DB: sqlStore, + Tracer: tracingService, + Reg: registerer, + Authzc: accessClient, + Docs: documentBuilderSupplier, + SecureValues: inlineSecureValueSupport, + VectorBackend: vectorBackend, } storageMetrics := resource.ProvideStorageMetrics(registerer) bleveIndexMetrics := resource.ProvideIndexMetrics(registerer) @@ -1256,15 +1262,20 @@ func InitializeForTest(ctx context.Context, t sqlutil.ITestDB, testingT interfac if err != nil { return nil, err } + vectorBackend, err := vector.ProvideVectorBackend(cfg) + if err != nil { + return nil, err + } options := &unified.Options{ - Cfg: cfg, - Features: featureToggles, - DB: sqlStore, - Tracer: tracingService, - Reg: registerer, - Authzc: accessClient, - Docs: documentBuilderSupplier, - SecureValues: inlineSecureValueSupport, + Cfg: cfg, + Features: featureToggles, + DB: sqlStore, + Tracer: tracingService, + Reg: registerer, + Authzc: accessClient, + Docs: documentBuilderSupplier, + SecureValues: inlineSecureValueSupport, + VectorBackend: vectorBackend, } storageMetrics := resource.ProvideStorageMetrics(registerer) bleveIndexMetrics := resource.ProvideIndexMetrics(registerer) diff --git a/pkg/server/wireexts_oss.go b/pkg/server/wireexts_oss.go index 600904cafb5..474eeec433d 100644 --- a/pkg/server/wireexts_oss.go +++ b/pkg/server/wireexts_oss.go @@ -71,6 +71,7 @@ import ( "github.com/grafana/grafana/pkg/storage/unified/resource" search2 "github.com/grafana/grafana/pkg/storage/unified/search" "github.com/grafana/grafana/pkg/storage/unified/search/builders" + "github.com/grafana/grafana/pkg/storage/unified/search/vector" "github.com/grafana/grafana/pkg/storage/unified/sql" ) @@ -154,6 +155,7 @@ var wireExtsBasicSet = wire.NewSet( wire.Struct(new(unified.Options), "*"), unified.ProvideUnifiedStorageClient, sql.ProvideStorageBackend, + vector.ProvideVectorBackend, builder.ProvideDefaultBuildHandlerChainFuncFromBuilders, aggregatorrunner.ProvideNoopAggregatorConfigurator, apisregistry.WireSetExts, diff --git a/pkg/setting/setting.go b/pkg/setting/setting.go index bc1fbefee1d..cef99465e0d 100644 --- a/pkg/setting/setting.go +++ b/pkg/setting/setting.go @@ -682,19 +682,29 @@ type Cfg struct { SearchInjectFailuresPercent int EnableSearch bool EnableSearchClient bool - OverridesFilePath string - OverridesReloadInterval time.Duration - EnforcedQuotaResources []string - QuotasErrorMessageSupportInfo string - EnableSQLKVBackend bool - EnableSQLKVCompatibilityMode bool - EnableGarbageCollection bool - GarbageCollectionDryRun bool - GarbageCollectionInterval time.Duration - GarbageCollectionBatchSize int - GarbageCollectionBatchWait time.Duration - GarbageCollectionMaxAge time.Duration - DashboardsGarbageCollectionMaxAge time.Duration + // Vector storage (separate pgvector database) + EnableVectorBackend bool + VectorDBHost string + VectorDBPort string + VectorDBName string + VectorDBUser string + VectorDBPassword string + VectorDBSSLMode string + VectorPromotionThreshold int // row count per tenant to trigger leaf promotion + VectorPromoterInterval time.Duration // promoter tick interval; 0 disables + OverridesFilePath string + OverridesReloadInterval time.Duration + EnforcedQuotaResources []string + QuotasErrorMessageSupportInfo string + EnableSQLKVBackend bool + EnableSQLKVCompatibilityMode bool + EnableGarbageCollection bool + GarbageCollectionDryRun bool + GarbageCollectionInterval time.Duration + GarbageCollectionBatchSize int + GarbageCollectionBatchWait time.Duration + GarbageCollectionMaxAge time.Duration + DashboardsGarbageCollectionMaxAge time.Duration // StorageModeCacheTTL is the TTL for caching statusReader results in the dynamic dualwrite service. // Default: 5 seconds, 0 or negative means no expiration. StorageModeCacheTTL time.Duration diff --git a/pkg/setting/setting_unified_storage.go b/pkg/setting/setting_unified_storage.go index f1c60a72566..9bf33e0c526 100644 --- a/pkg/setting/setting_unified_storage.go +++ b/pkg/setting/setting_unified_storage.go @@ -174,6 +174,7 @@ func (cfg *Cfg) setUnifiedStorageConfig() { cfg.SearchInjectFailuresPercent = 100 } cfg.EnableSearch = section.Key("enable_search").MustBool(true) + cfg.EnableVectorBackend = section.Key("vector_backend").MustBool(false) cfg.applyMigrationEnforcements() cfg.EnableSearchClient = section.Key("enable_search_client").MustBool(false) cfg.MaxPageSizeBytes = section.Key("max_page_size_bytes").MustInt(0) @@ -262,6 +263,17 @@ func (cfg *Cfg) setUnifiedStorageConfig() { cfg.Logger.Warn("index_snapshot_max_age is smaller than max_file_index_age, overriding", "configured", cfg.IndexSnapshotMaxAge, "max_file_index_age", cfg.MaxFileIndexAge) cfg.IndexSnapshotMaxAge = cfg.MaxFileIndexAge } + + // Vector storage (separate pgvector database) + vectorSection := cfg.Raw.Section("database_vector") + cfg.VectorDBHost = vectorSection.Key("db_host").String() + cfg.VectorDBPort = vectorSection.Key("db_port").MustString("5432") + cfg.VectorDBName = vectorSection.Key("db_name").String() + cfg.VectorDBUser = vectorSection.Key("db_user").String() + cfg.VectorDBPassword = vectorSection.Key("db_password").String() + cfg.VectorDBSSLMode = vectorSection.Key("db_sslmode").MustString("disable") + cfg.VectorPromotionThreshold = vectorSection.Key("promotion_threshold").MustInt(9999999) // effectively disabled by default + cfg.VectorPromoterInterval = vectorSection.Key("promoter_interval").MustDuration(1 * time.Hour) } // applyMigrationEnforcements enforces unified storage migration configs when migrations should run, diff --git a/pkg/storage/unified/client.go b/pkg/storage/unified/client.go index 2ade67380e5..b04a05b2cf1 100644 --- a/pkg/storage/unified/client.go +++ b/pkg/storage/unified/client.go @@ -33,19 +33,21 @@ import ( "github.com/grafana/grafana/pkg/storage/unified/federated" "github.com/grafana/grafana/pkg/storage/unified/resource" "github.com/grafana/grafana/pkg/storage/unified/search" + "github.com/grafana/grafana/pkg/storage/unified/search/vector" "github.com/grafana/grafana/pkg/storage/unified/sql" "github.com/grafana/grafana/pkg/util/scheduler" ) type Options struct { - Cfg *setting.Cfg - Features featuremgmt.FeatureToggles - DB infraDB.DB - Tracer tracing.Tracer - Reg prometheus.Registerer - Authzc types.AccessClient - Docs resource.DocumentBuilderSupplier - SecureValues secrets.InlineSecureValueSupport + Cfg *setting.Cfg + Features featuremgmt.FeatureToggles + DB infraDB.DB + Tracer tracing.Tracer + Reg prometheus.Registerer + Authzc types.AccessClient + Docs resource.DocumentBuilderSupplier + SecureValues secrets.InlineSecureValueSupport + VectorBackend vector.VectorBackend } type clientMetrics struct { @@ -67,7 +69,7 @@ func ProvideUnifiedStorageClient(opts *Options, BlobStoreURL: apiserverCfg.Key("blob_url").MustString(""), BlobThresholdBytes: apiserverCfg.Key("blob_threshold_bytes").MustInt(options.BlobThresholdDefault), GrpcClientKeepaliveTime: apiserverCfg.Key("grpc_client_keepalive_time").MustDuration(0), - }, opts.Cfg, opts.Features, opts.DB, opts.Tracer, opts.Reg, opts.Authzc, opts.Docs, storageMetrics, indexMetrics, opts.SecureValues) + }, opts.Cfg, opts.Features, opts.DB, opts.Tracer, opts.Reg, opts.Authzc, opts.Docs, storageMetrics, indexMetrics, opts.SecureValues, opts.VectorBackend) if err == nil { // Used to get the folder stats // Pass cfg directly so the federated client reads the current dual-writer mode @@ -94,6 +96,7 @@ func newClient(opts options.StorageOptions, storageMetrics *resource.StorageMetrics, indexMetrics *resource.BleveIndexMetrics, secure secrets.InlineSecureValueSupport, + vectorBackend vector.VectorBackend, ) (resource.ResourceClient, error) { ctx := context.Background() @@ -164,6 +167,7 @@ func newClient(opts options.StorageOptions, serverOptions := sql.ServerOptions{ Backend: backend, + VectorBackend: vectorBackend, Cfg: cfg, Tracer: tracer, Reg: reg, diff --git a/pkg/storage/unified/client_test.go b/pkg/storage/unified/client_test.go index d79dbe67d2b..2b9586f986b 100644 --- a/pkg/storage/unified/client_test.go +++ b/pkg/storage/unified/client_test.go @@ -47,6 +47,7 @@ func TestUnifiedStorageClient(t *testing.T) { nil, nil, nil, + nil, ) require.NoError(t, err) @@ -81,6 +82,7 @@ func TestUnifiedStorageClient(t *testing.T) { nil, nil, nil, + nil, ) require.NoError(t, err) diff --git a/pkg/storage/unified/resource/search.go b/pkg/storage/unified/resource/search.go index 04417706dfd..1870cd00477 100644 --- a/pkg/storage/unified/resource/search.go +++ b/pkg/storage/unified/resource/search.go @@ -28,6 +28,7 @@ import ( "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/infra/tracing" "github.com/grafana/grafana/pkg/storage/unified/resourcepb" + "github.com/grafana/grafana/pkg/storage/unified/search/vector" "github.com/grafana/grafana/pkg/util/debouncer" ) @@ -136,14 +137,15 @@ type SearchBackend interface { // searchServer supports indexing+search regardless of implementation. type searchServer struct { - log log.Logger - storage StorageBackend - search SearchBackend - indexMetrics *BleveIndexMetrics - access types.AccessClient - builders *builderCache - initWorkers int - initMinSize int + log log.Logger + storage StorageBackend + vectorBackend vector.VectorBackend + search SearchBackend + indexMetrics *BleveIndexMetrics + access types.AccessClient + builders *builderCache + initWorkers int + initMinSize int ownsIndexFn func(key NamespacedResource) (bool, error) @@ -185,7 +187,7 @@ var ( ) // newSearchServer creates a new search server implementation. -func newSearchServer(opts SearchOptions, storage StorageBackend, access types.AccessClient, blob BlobSupport, indexMetrics *BleveIndexMetrics, ownsIndexFn func(key NamespacedResource) (bool, error)) (*searchServer, error) { +func newSearchServer(opts SearchOptions, storage StorageBackend, vectorBackend vector.VectorBackend, access types.AccessClient, blob BlobSupport, indexMetrics *BleveIndexMetrics, ownsIndexFn func(key NamespacedResource) (bool, error)) (*searchServer, error) { // No backend search support if opts.Backend == nil { return nil, nil @@ -208,6 +210,7 @@ func newSearchServer(opts SearchOptions, storage StorageBackend, access types.Ac s := &searchServer{ access: access, storage: storage, + vectorBackend: vectorBackend, search: opts.Backend, log: log.New("resource-search"), initWorkers: opts.InitWorkerThreads, diff --git a/pkg/storage/unified/resource/search_test.go b/pkg/storage/unified/resource/search_test.go index 1e5ba4d7bcb..c7d49208119 100644 --- a/pkg/storage/unified/resource/search_test.go +++ b/pkg/storage/unified/resource/search_test.go @@ -205,7 +205,7 @@ func TestSearchGetOrCreateIndex(t *testing.T) { InitMinCount: 1, // set min count to default for this test } - support, err := newSearchServer(opts, storage, nil, nil, nil, nil) + support, err := newSearchServer(opts, storage, nil, nil, nil, nil, nil) require.NoError(t, err) require.NotNil(t, support) @@ -261,7 +261,7 @@ func TestSearchGetOrCreateIndexWithIndexUpdate(t *testing.T) { } // Enable searchAfterWrite - support, err := newSearchServer(opts, storage, nil, nil, nil, nil) + support, err := newSearchServer(opts, storage, nil, nil, nil, nil, nil) require.NoError(t, err) require.NotNil(t, support) @@ -311,7 +311,7 @@ func TestSearchGetOrCreateIndexWithCancellation(t *testing.T) { InitMinCount: 1, // set min count to default for this test } - support, err := newSearchServer(opts, storage, nil, nil, nil, nil) + support, err := newSearchServer(opts, storage, nil, nil, nil, nil, nil) require.NoError(t, err) require.NotNil(t, support) @@ -662,7 +662,7 @@ func TestFindIndexesForRebuild(t *testing.T) { BuildVersion: semver.MustParse("6.5.0"), // Running version } - support, err := newSearchServer(opts, storage, nil, nil, nil, nil) + support, err := newSearchServer(opts, storage, nil, nil, nil, nil, nil) require.NoError(t, err) require.NotNil(t, support) @@ -739,7 +739,7 @@ func TestRebuildIndexes(t *testing.T) { Resources: supplier, } - support, err := newSearchServer(opts, storage, nil, nil, nil, nil) + support, err := newSearchServer(opts, storage, nil, nil, nil, nil, nil) require.NoError(t, err) require.NotNil(t, support) @@ -831,7 +831,7 @@ func TestRebuildIndexes(t *testing.T) { InitMinCount: 1, } - support, err := newSearchServer(opts, storage, nil, nil, nil, nil) + support, err := newSearchServer(opts, storage, nil, nil, nil, nil, nil) require.NoError(t, err) require.NotNil(t, support) @@ -944,7 +944,7 @@ func TestRebuildIndexesForResource(t *testing.T) { InitMinCount: 1, } - support, err := newSearchServer(opts, storage, nil, nil, nil, nil) + support, err := newSearchServer(opts, storage, nil, nil, nil, nil, nil) require.NoError(t, err) require.NotNil(t, support) @@ -1025,7 +1025,7 @@ func TestSearchValidatesNegativeLimitAndOffset(t *testing.T) { InitMinCount: 1, } - support, err := newSearchServer(opts, nil, nil, nil, nil, nil) + support, err := newSearchServer(opts, nil, nil, nil, nil, nil, nil) require.NoError(t, err) require.NotNil(t, support) @@ -1134,7 +1134,7 @@ func TestFindIndexesToRebuildWithJitter(t *testing.T) { MinBuildVersion: semver.MustParse("5.0.0"), } - support, err := newSearchServer(opts, storage, nil, nil, nil, nil) + support, err := newSearchServer(opts, storage, nil, nil, nil, nil, nil) require.NoError(t, err) require.NotNil(t, support) @@ -1145,7 +1145,7 @@ func TestFindIndexesToRebuildWithJitter(t *testing.T) { require.Equal(t, numIndexes, len(chsNoJitter)) // Create a second server with the same config to get a fresh rebuild queue. - support2, err := newSearchServer(opts, storage, nil, nil, nil, nil) + support2, err := newSearchServer(opts, storage, nil, nil, nil, nil, nil) require.NoError(t, err) // With jitter: some indexes get extra tolerance, so fewer should be queued. diff --git a/pkg/storage/unified/resource/server.go b/pkg/storage/unified/resource/server.go index 99e849b9fbe..4fad3778d66 100644 --- a/pkg/storage/unified/resource/server.go +++ b/pkg/storage/unified/resource/server.go @@ -33,6 +33,7 @@ import ( "github.com/grafana/grafana/pkg/infra/log" secrets "github.com/grafana/grafana/pkg/registry/apis/secret/contracts" "github.com/grafana/grafana/pkg/storage/unified/resourcepb" + "github.com/grafana/grafana/pkg/storage/unified/search/vector" "github.com/grafana/grafana/pkg/util/scheduler" ) @@ -337,6 +338,12 @@ type ResourceServerOptions struct { // BookmarkFrequency controls how often periodic bookmark events are sent to // Watch clients that set AllowWatchBookmarks. Zero defaults to defaultBookmarkFrequency. BookmarkFrequency time.Duration + + // VectorBackend is the optional pgvector-backed store for semantic search. + // nil when the [unified_storage] vector_backend flag is off. When present, + // the resource and search servers hold a reference for use by future + // write and query paths. + VectorBackend vector.VectorBackend } func (opts ResourceServerOptions) bulkBatchOptions() BulkBatchOptions { @@ -365,7 +372,7 @@ func NewUninitializedSearchServer(opts ResourceServerOptions) (SearchServer, err } // Create the search server using the search.go factory - searchServer, err := newSearchServer(opts.Search, opts.Backend, opts.AccessClient, blobstore, opts.IndexMetrics, opts.OwnsIndexFn) + searchServer, err := newSearchServer(opts.Search, opts.Backend, opts.VectorBackend, opts.AccessClient, blobstore, opts.IndexMetrics, opts.OwnsIndexFn) if err != nil || searchServer == nil { return nil, fmt.Errorf("search server could not be created: %w", err) } @@ -449,6 +456,7 @@ func NewUninitializedResourceServer(opts ResourceServerOptions) (*server, error) s := &server{ log: logger, backend: opts.Backend, + vectorBackend: opts.VectorBackend, bulkBatchOptions: opts.bulkBatchOptions(), blob: blobstore, diagnostics: opts.Diagnostics, @@ -473,7 +481,7 @@ func NewUninitializedResourceServer(opts ResourceServerOptions) (*server, error) if opts.Search.Resources != nil { var err error - s.search, err = newSearchServer(opts.Search, s.backend, s.access, s.blob, opts.IndexMetrics, opts.OwnsIndexFn) + s.search, err = newSearchServer(opts.Search, s.backend, opts.VectorBackend, s.access, s.blob, opts.IndexMetrics, opts.OwnsIndexFn) if err != nil { return nil, err } @@ -524,6 +532,7 @@ var _ ResourceServer = &server{} type server struct { log log.Logger backend StorageBackend + vectorBackend vector.VectorBackend bulkBatchOptions BulkBatchOptions blob BlobSupport secure secrets.InlineSecureValueSupport diff --git a/pkg/storage/unified/search/vector/data/vector_collection_delete.sql b/pkg/storage/unified/search/vector/data/vector_collection_delete.sql new file mode 100644 index 00000000000..7517f554070 --- /dev/null +++ b/pkg/storage/unified/search/vector/data/vector_collection_delete.sql @@ -0,0 +1,6 @@ +DELETE FROM embeddings + WHERE {{ .Ident "resource" }} = {{ .Arg .Resource }} + AND {{ .Ident "namespace" }} = {{ .Arg .Namespace }} + AND {{ .Ident "model" }} = {{ .Arg .Model }} + AND {{ .Ident "uid" }} = {{ .Arg .UID }} +; diff --git a/pkg/storage/unified/search/vector/data/vector_collection_delete_subresources.sql b/pkg/storage/unified/search/vector/data/vector_collection_delete_subresources.sql new file mode 100644 index 00000000000..c785837684a --- /dev/null +++ b/pkg/storage/unified/search/vector/data/vector_collection_delete_subresources.sql @@ -0,0 +1,7 @@ +DELETE FROM embeddings + WHERE {{ .Ident "resource" }} = {{ .Arg .Resource }} + AND {{ .Ident "namespace" }} = {{ .Arg .Namespace }} + AND {{ .Ident "model" }} = {{ .Arg .Model }} + AND {{ .Ident "uid" }} = {{ .Arg .UID }} + AND {{ .Ident "subresource" }} IN ({{ .ArgList .SubresourcesSlice }}) +; diff --git a/pkg/storage/unified/search/vector/data/vector_collection_get_content.sql b/pkg/storage/unified/search/vector/data/vector_collection_get_content.sql new file mode 100644 index 00000000000..cb361f3b49d --- /dev/null +++ b/pkg/storage/unified/search/vector/data/vector_collection_get_content.sql @@ -0,0 +1,9 @@ +SELECT + {{ .Ident "subresource" | .Into .Response.Subresource }}, + {{ .Ident "content" | .Into .Response.Content }} + FROM embeddings + WHERE {{ .Ident "resource" }} = {{ .Arg .Resource }} + AND {{ .Ident "namespace" }} = {{ .Arg .Namespace }} + AND {{ .Ident "model" }} = {{ .Arg .Model }} + AND {{ .Ident "uid" }} = {{ .Arg .UID }} +; diff --git a/pkg/storage/unified/search/vector/data/vector_collection_search.sql b/pkg/storage/unified/search/vector/data/vector_collection_search.sql new file mode 100644 index 00000000000..9c506096cc5 --- /dev/null +++ b/pkg/storage/unified/search/vector/data/vector_collection_search.sql @@ -0,0 +1,24 @@ +SELECT + {{ .Ident "uid" | .Into .Response.UID }}, + {{ .Ident "title" | .Into .Response.Title }}, + {{ .Ident "subresource" | .Into .Response.Subresource }}, + {{ .Ident "content" | .Into .Response.Content }}, + {{ .Ident "embedding" }} <=> {{ .Arg .QueryEmbedding }} AS {{ .Ident "score" | .Into .Response.Score }}, + {{ .Ident "folder" | .Into .Response.Folder }}, + {{ .Ident "metadata" | .Into .Response.Metadata }} + FROM embeddings + WHERE {{ .Ident "resource" }} = {{ .Arg .Resource }} + AND {{ .Ident "namespace" }} = {{ .Arg .Namespace }} + AND {{ .Ident "model" }} = {{ .Arg .Model }} + {{ if .UIDFilter }} + AND {{ .Ident "uid" }} IN ({{ .ArgList .UIDFilterSlice }}) + {{ end }} + {{ if .FolderFilter }} + AND {{ .Ident "folder" }} IN ({{ .ArgList .FolderFilterSlice }}) + {{ end }} + {{ range .MetadataFilters }} + AND {{ $.Ident "metadata" }} @> {{ $.Arg .JSON }} + {{ end }} + ORDER BY {{ .Ident "embedding" }} <=> {{ .Arg .QueryEmbedding }} + LIMIT {{ .Arg .Limit }} +; diff --git a/pkg/storage/unified/search/vector/data/vector_collection_upsert.sql b/pkg/storage/unified/search/vector/data/vector_collection_upsert.sql new file mode 100644 index 00000000000..49d630bedff --- /dev/null +++ b/pkg/storage/unified/search/vector/data/vector_collection_upsert.sql @@ -0,0 +1,32 @@ +INSERT INTO embeddings ( + {{ .Ident "resource" }}, + {{ .Ident "namespace" }}, + {{ .Ident "model" }}, + {{ .Ident "uid" }}, + {{ .Ident "title" }}, + {{ .Ident "subresource" }}, + {{ .Ident "folder" }}, + {{ .Ident "content" }}, + {{ .Ident "metadata" }}, + {{ .Ident "embedding" }} +) +VALUES ( + {{ .Arg .Resource }}, + {{ .Arg .Vector.Namespace }}, + {{ .Arg .Vector.Model }}, + {{ .Arg .Vector.UID }}, + {{ .Arg .Vector.Title }}, + {{ .Arg .Vector.Subresource }}, + {{ .Arg .Vector.Folder }}, + {{ .Arg .Vector.Content }}, + {{ .Arg .Vector.Metadata }}, + {{ .Arg .Embedding }} +) +ON CONFLICT ({{ .Ident "resource" }}, {{ .Ident "namespace" }}, {{ .Ident "model" }}, {{ .Ident "uid" }}, {{ .Ident "subresource" }}) +DO UPDATE SET + {{ .Ident "title" }} = {{ .Arg .Vector.Title }}, + {{ .Ident "folder" }} = {{ .Arg .Vector.Folder }}, + {{ .Ident "content" }} = {{ .Arg .Vector.Content }}, + {{ .Ident "metadata" }} = {{ .Arg .Vector.Metadata }}, + {{ .Ident "embedding" }} = {{ .Arg .Embedding }} +; diff --git a/pkg/storage/unified/search/vector/integration_test.go b/pkg/storage/unified/search/vector/integration_test.go new file mode 100644 index 00000000000..c6c12f8599a --- /dev/null +++ b/pkg/storage/unified/search/vector/integration_test.go @@ -0,0 +1,287 @@ +package vector + +import ( + "context" + "encoding/json" + "fmt" + "os" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/grafana/grafana/pkg/setting" + "github.com/grafana/grafana/pkg/storage/unified/sql/db/dbimpl" + "github.com/grafana/grafana/pkg/util/testutil" + "github.com/grafana/grafana/pkg/util/xorm" +) + +const testModel = "test-model" +const testResource = "dashboards" + +var testSubtree = subtreeName(testResource) + +// To run: start the pgvector devenv (localhost:5433) and +// +// PGVECTOR_TEST_DB="host=localhost port=5433 dbname=grafana_vectors user=grafana password=password sslmode=disable" \ +// go test -run TestIntegration ./pkg/storage/unified/search/vector/... -v -count=1 +func setupIntegrationTest(t *testing.T) (VectorBackend, *xorm.Engine, context.Context) { + t.Helper() + testutil.SkipIntegrationTestInShortMode(t) + + connStr := os.Getenv("PGVECTOR_TEST_DB") + if connStr == "" { + t.Skip("PGVECTOR_TEST_DB not set, skipping pgvector integration test") + } + + ctx := context.Background() + + engine, err := xorm.NewEngine("postgres", connStr) + require.NoError(t, err) + t.Cleanup(func() { + if err := engine.Close(); err != nil { + t.Logf("closing xorm engine: %v", err) + } + }) + + cfg := setting.NewCfg() + err = MigrateVectorStore(ctx, engine, cfg) + require.NoError(t, err) + + database := dbimpl.NewDB(engine.DB().DB, engine.Dialect().DriverName()) + // interval=0 keeps Run idle; promotion tests call Promote(ctx) directly. + backend := NewPgvectorBackend(database, 1000, 0) + + cleanIntegrationState(t, engine) + + return backend, engine, ctx +} + +// cleanIntegrationState drops any `integration-test*` partial indexes, clears +// rows, and resets the checkpoint. +func cleanIntegrationState(t *testing.T, engine *xorm.Engine) { + t.Helper() + ctx := context.Background() + + indexPrefix := fmt.Sprintf("%s_integration_test", testResource) + rows, err := engine.DB().QueryContext(ctx, ` + SELECT c.relname FROM pg_class c + JOIN pg_index i ON i.indexrelid = c.oid + JOIN pg_class t ON t.oid = i.indrelid + WHERE t.relname = $1 AND c.relkind = 'i' AND c.relname LIKE $2`, + testSubtree, indexPrefix+"%") + require.NoError(t, err) + var indexes []string + for rows.Next() { + var n string + require.NoError(t, rows.Scan(&n)) + indexes = append(indexes, n) + } + _ = rows.Close() + + for _, idx := range indexes { + _, _ = engine.DB().ExecContext(ctx, fmt.Sprintf(`DROP INDEX IF EXISTS %s`, idx)) + } + + _, _ = engine.DB().ExecContext(ctx, + fmt.Sprintf(`DELETE FROM %s WHERE namespace LIKE 'integration-test%%'`, testSubtree)) + _, _ = engine.DB().ExecContext(ctx, + `DELETE FROM vector_promoted WHERE namespace LIKE 'integration-test%'`) + _, _ = engine.DB().ExecContext(ctx, + `UPDATE vector_latest_rv SET latest_rv = 0 WHERE id = 1`) +} + +func TestIntegrationVectorUpsertAndSearch(t *testing.T) { + backend, _, ctx := setupIntegrationTest(t) + + vectors := []Vector{ + { + Namespace: "integration-test", Resource: testResource, UID: "dash-1", Title: "CPU Dashboard", Subresource: "panel/1", + ResourceVersion: 10, Folder: "folder-a", + Content: "CPU usage over time for production servers", + Metadata: json.RawMessage(`{"datasource_uids":["prom-1"],"query_languages":["promql"]}`), + Embedding: makeEmbedding(0.9, 0.1), Model: testModel, + }, + { + Namespace: "integration-test", Resource: testResource, UID: "dash-1", Title: "CPU Dashboard", Subresource: "panel/2", + ResourceVersion: 10, Folder: "folder-a", + Content: "Memory usage alerts dashboard", + Metadata: json.RawMessage(`{"datasource_uids":["prom-1"],"query_languages":["promql"]}`), + Embedding: makeEmbedding(0.1, 0.9), Model: testModel, + }, + { + Namespace: "integration-test", Resource: testResource, UID: "dash-2", Title: "Logs Dashboard", Subresource: "panel/1", + ResourceVersion: 20, Folder: "folder-b", + Content: "Log volume by service", + Metadata: json.RawMessage(`{"datasource_uids":["loki-1"],"query_languages":["logql"]}`), + Embedding: makeEmbedding(0.5, 0.5), Model: testModel, + }, + } + + require.NoError(t, backend.Upsert(ctx, vectors)) + + results, err := backend.Search(ctx, "integration-test", testModel, testResource, makeEmbedding(0.85, 0.15), 10) + require.NoError(t, err) + require.GreaterOrEqual(t, len(results), 3) + assert.Equal(t, "dash-1", results[0].UID) + assert.Equal(t, "CPU Dashboard", results[0].Title) + assert.Equal(t, "panel/1", results[0].Subresource) + + results, err = backend.Search(ctx, "integration-test", testModel, testResource, makeEmbedding(0.5, 0.5), 10, + SearchFilter{Field: "uid", Values: []string{"dash-2"}}) + require.NoError(t, err) + require.Len(t, results, 1) + assert.Equal(t, "dash-2", results[0].UID) + + results, err = backend.Search(ctx, "integration-test", testModel, testResource, makeEmbedding(0.5, 0.5), 10, + SearchFilter{Field: "folder", Values: []string{"folder-a"}}) + require.NoError(t, err) + require.Len(t, results, 2) + + results, err = backend.Search(ctx, "integration-test", testModel, testResource, makeEmbedding(0.5, 0.5), 10, + SearchFilter{Field: "query_languages", Values: []string{"logql"}}) + require.NoError(t, err) + require.Len(t, results, 1) + assert.Equal(t, "dash-2", results[0].UID) +} + +func TestIntegrationVectorDeleteSubresources(t *testing.T) { + backend, _, ctx := setupIntegrationTest(t) + + err := backend.Upsert(ctx, []Vector{ + {Namespace: "integration-test", Resource: testResource, UID: "dash", Title: "Dash", Subresource: "panel/1", + ResourceVersion: 10, Content: "panel one", Metadata: json.RawMessage(`{}`), + Embedding: makeEmbedding(0.5, 0.5), Model: testModel}, + {Namespace: "integration-test", Resource: testResource, UID: "dash", Title: "Dash", Subresource: "panel/2", + ResourceVersion: 10, Content: "panel two", Metadata: json.RawMessage(`{}`), + Embedding: makeEmbedding(0.5, 0.5), Model: testModel}, + {Namespace: "integration-test", Resource: testResource, UID: "dash", Title: "Dash", Subresource: "panel/3", + ResourceVersion: 10, Content: "panel three", Metadata: json.RawMessage(`{}`), + Embedding: makeEmbedding(0.5, 0.5), Model: testModel}, + }) + require.NoError(t, err) + + stored, err := backend.GetSubresourceContent(ctx, "integration-test", testModel, testResource, "dash") + require.NoError(t, err) + require.Len(t, stored, 3) + + current := map[string]string{"panel/1": "panel one"} + var toDelete []string + for sub := range stored { + if _, ok := current[sub]; !ok { + toDelete = append(toDelete, sub) + } + } + require.ElementsMatch(t, []string{"panel/2", "panel/3"}, toDelete) + + err = backend.DeleteSubresources(ctx, "integration-test", testModel, testResource, "dash", toDelete) + require.NoError(t, err) + + results, err := backend.Search(ctx, "integration-test", testModel, testResource, makeEmbedding(0.5, 0.5), 10) + require.NoError(t, err) + require.Len(t, results, 1) + assert.Equal(t, "panel/1", results[0].Subresource) + + err = backend.Delete(ctx, "integration-test", testModel, testResource, "dash") + require.NoError(t, err) +} + +func TestIntegrationVectorGetLatestRV(t *testing.T) { + backend, _, ctx := setupIntegrationTest(t) + + rv, err := backend.GetLatestRV(ctx) + require.NoError(t, err) + assert.Equal(t, int64(0), rv) + + err = backend.Upsert(ctx, []Vector{ + {Namespace: "integration-test", Resource: testResource, UID: "rv-test", Title: "RV Test", Subresource: "x", + ResourceVersion: 42, Content: "test content", Metadata: json.RawMessage(`{}`), + Embedding: makeEmbedding(0.5, 0.5), Model: testModel}, + }) + require.NoError(t, err) + + rv, err = backend.GetLatestRV(ctx) + require.NoError(t, err) + assert.Equal(t, int64(42), rv) +} + +func TestIntegrationPromoterPromotesLargeTenant(t *testing.T) { + backend, engine, ctx := setupIntegrationTest(t) + + const ns = "integration-test-big" + const threshold = 50 + const nRows = threshold + 10 + + vectors := make([]Vector, 0, nRows) + for i := 0; i < nRows; i++ { + vectors = append(vectors, Vector{ + Namespace: ns, Resource: testResource, UID: "dash", Title: "Dash", Subresource: fmt.Sprintf("panel/%d", i), + ResourceVersion: int64(i + 1), Content: fmt.Sprintf("content %d", i), + Metadata: json.RawMessage(`{}`), Embedding: makeEmbedding(0.5, 0.5), Model: testModel, + }) + } + require.NoError(t, backend.Upsert(ctx, vectors)) + + idxName := partialHNSWName(testResource, ns) + require.False(t, indexExists(t, engine, idxName)) + require.Equal(t, nRows, countRowsIn(t, engine, testSubtree, ns)) + + database := dbimpl.NewDB(engine.DB().DB, engine.Dialect().DriverName()) + promoter := NewPromoter(database, threshold, 0) + require.NoError(t, promoter.Promote(ctx)) + + require.True(t, indexExists(t, engine, idxName)) + require.Equal(t, nRows, countRowsIn(t, engine, testSubtree, ns)) + + results, err := backend.Search(ctx, ns, testModel, testResource, makeEmbedding(0.5, 0.5), 5) + require.NoError(t, err) + assert.Len(t, results, 5) +} + +func TestIntegrationPromoterSkipsSmallTenant(t *testing.T) { + backend, engine, ctx := setupIntegrationTest(t) + + const ns = "integration-test-small" + require.NoError(t, backend.Upsert(ctx, []Vector{ + {Namespace: ns, Resource: testResource, UID: "dash", Title: "Dash", Subresource: "panel/1", + ResourceVersion: 1, Content: "only row", Metadata: json.RawMessage(`{}`), + Embedding: makeEmbedding(0.1, 0.1), Model: testModel}, + })) + + database := dbimpl.NewDB(engine.DB().DB, engine.Dialect().DriverName()) + promoter := NewPromoter(database, 100, 0) + require.NoError(t, promoter.Promote(ctx)) + + idxName := partialHNSWName(testResource, ns) + require.False(t, indexExists(t, engine, idxName), + "small tenant should not be promoted") +} + +func indexExists(t *testing.T, engine *xorm.Engine, idxName string) bool { + t.Helper() + var exists bool + err := engine.DB().QueryRowContext(context.Background(), ` + SELECT EXISTS ( + SELECT 1 FROM pg_class c + JOIN pg_index i ON i.indexrelid = c.oid + WHERE c.relname = $1 AND c.relkind = 'i' AND i.indisvalid + )`, idxName).Scan(&exists) + require.NoError(t, err) + return exists +} + +func countRowsIn(t *testing.T, engine *xorm.Engine, table, ns string) int { + t.Helper() + var n int + require.NoError(t, engine.DB().QueryRowContext(context.Background(), + fmt.Sprintf(`SELECT COUNT(*) FROM %s WHERE namespace = $1`, table), ns).Scan(&n)) + return n +} + +// makeEmbedding builds a 1024-dim halfvec with the first two values set. +func makeEmbedding(a, b float32) []float32 { + e := make([]float32, 1024) + e[0] = a + e[1] = b + return e +} diff --git a/pkg/storage/unified/search/vector/pgvector.go b/pkg/storage/unified/search/vector/pgvector.go new file mode 100644 index 00000000000..7b8d8561083 --- /dev/null +++ b/pkg/storage/unified/search/vector/pgvector.go @@ -0,0 +1,227 @@ +package vector + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "time" + + pgvector "github.com/pgvector/pgvector-go" + + "github.com/grafana/grafana/pkg/infra/log" + "github.com/grafana/grafana/pkg/storage/unified/sql/db" + "github.com/grafana/grafana/pkg/storage/unified/sql/dbutil" + "github.com/grafana/grafana/pkg/storage/unified/sql/sqltemplate" +) + +var _ VectorBackend = (*pgvectorBackend)(nil) + +type pgvectorBackend struct { + db db.DB + dialect sqltemplate.Dialect + log log.Logger + promoter *Promoter +} + +// NewPgvectorBackend builds a VectorBackend. `promoterInterval=0` leaves the +// promoter idle — safe on any target; schema-owning targets call Run. +func NewPgvectorBackend(database db.DB, promotionThreshold int, promoterInterval time.Duration) *pgvectorBackend { + return &pgvectorBackend{ + db: database, + dialect: sqltemplate.PostgreSQL, + log: log.New("vector-pgvector"), + promoter: NewPromoter(database, promotionThreshold, promoterInterval), + } +} + +func (b *pgvectorBackend) Run(ctx context.Context) error { + return b.promoter.Run(ctx) +} + +// validateResource rejects resources that don't have a sub-tree attached. +// Adding a resource means attaching an `embeddings_` sub-tree. +// TODO dynamically add new partition if for resource if one doesnt exist +func validateResource(resource string) error { + if resource != "dashboards" { + return fmt.Errorf("unsupported resource %q (no embeddings sub-tree provisioned)", resource) + } + return nil +} + +func (b *pgvectorBackend) Upsert(ctx context.Context, vectors []Vector) error { + if len(vectors) == 0 { + return nil + } + + for i := range vectors { + if err := vectors[i].Validate(); err != nil { + return fmt.Errorf("vector[%d]: %w", i, err) + } + } + + // track max rv so we can update global db rv + var batchMaxRV int64 + for i := range vectors { + if vectors[i].ResourceVersion > batchMaxRV { + batchMaxRV = vectors[i].ResourceVersion + } + } + + return b.db.WithTx(ctx, nil, func(ctx context.Context, tx db.Tx) error { + for i := range vectors { + if err := validateResource(vectors[i].Resource); err != nil { + return fmt.Errorf("vector[%d]: %w", i, err) + } + req := &sqlVectorCollectionUpsertRequest{ + SQLTemplate: sqltemplate.New(b.dialect), + Resource: vectors[i].Resource, + Vector: &vectors[i], + Embedding: pgvector.NewHalfVector(vectors[i].Embedding), + } + if _, err := dbutil.Exec(ctx, tx, sqlVectorCollectionUpsert, req); err != nil { + return fmt.Errorf("upsert vector %s/%s: %w", vectors[i].UID, vectors[i].Subresource, err) + } + } + + // WHERE clause keeps this monotonic under concurrent writers. + if batchMaxRV > 0 { + if _, err := tx.ExecContext(ctx, + `UPDATE vector_latest_rv SET latest_rv = $1 WHERE id = 1 AND latest_rv < $1`, + batchMaxRV, + ); err != nil { + return fmt.Errorf("bump vector_latest_rv: %w", err) + } + } + return nil + }) +} + +func (b *pgvectorBackend) Delete(ctx context.Context, namespace, model, resource, uid string) error { + if model == "" { + return fmt.Errorf("model must not be empty") + } + if err := validateResource(resource); err != nil { + return err + } + req := &sqlVectorCollectionDeleteRequest{ + SQLTemplate: sqltemplate.New(b.dialect), + Resource: resource, + Namespace: namespace, + Model: model, + UID: uid, + } + _, err := dbutil.Exec(ctx, b.db, sqlVectorCollectionDelete, req) + return err +} + +func (b *pgvectorBackend) DeleteSubresources(ctx context.Context, namespace, model, resource, uid string, subresources []string) error { + if model == "" { + return fmt.Errorf("model must not be empty") + } + if len(subresources) == 0 { + return nil + } + if err := validateResource(resource); err != nil { + return err + } + req := &sqlVectorCollectionDeleteSubresourcesRequest{ + SQLTemplate: sqltemplate.New(b.dialect), + Resource: resource, + Namespace: namespace, + Model: model, + UID: uid, + Subresources: subresources, + } + _, err := dbutil.Exec(ctx, b.db, sqlVectorCollectionDeleteSubresource, req) + return err +} + +func (b *pgvectorBackend) GetSubresourceContent(ctx context.Context, namespace, model, resource, uid string) (map[string]string, error) { + if err := validateResource(resource); err != nil { + return nil, err + } + req := &sqlVectorCollectionGetContentRequest{ + SQLTemplate: sqltemplate.New(b.dialect), + Resource: resource, + Namespace: namespace, + Model: model, + UID: uid, + Response: &sqlVectorCollectionGetContentResponse{}, + } + rows, err := dbutil.Query(ctx, b.db, sqlVectorCollectionGetContent, req) + if err != nil { + return nil, err + } + if len(rows) == 0 { + return nil, nil + } + out := make(map[string]string, len(rows)) + for _, r := range rows { + out[r.Subresource] = r.Content + } + return out, nil +} + +func (b *pgvectorBackend) Search(ctx context.Context, namespace, model, resource string, + embedding []float32, limit int, filters ...SearchFilter) ([]VectorSearchResult, error) { + if err := validateResource(resource); err != nil { + return nil, err + } + + // TODO Search is currently single resource but we will need to support cross-resource search eventually + req := &sqlVectorCollectionSearchRequest{ + SQLTemplate: sqltemplate.New(b.dialect), + Resource: resource, + Namespace: namespace, + Model: model, + QueryEmbedding: pgvector.NewHalfVector(embedding), + Limit: int64(limit), + Response: &sqlVectorCollectionSearchResponse{}, + } + + for _, f := range filters { + switch f.Field { + case "uid": + req.UIDValues = f.Values + case "folder": + req.FolderValues = f.Values + default: + // JSONB containment: metadata @> '{"field":["v1","v2"]}' + j, _ := json.Marshal(map[string][]string{f.Field: f.Values}) + req.MetadataFilters = append(req.MetadataFilters, MetadataFilterEntry{JSON: string(j)}) + } + } + + rows, err := dbutil.Query(ctx, b.db, sqlVectorCollectionSearch, req) + if err != nil { + return nil, err + } + + results := make([]VectorSearchResult, len(rows)) + for i, row := range rows { + results[i] = VectorSearchResult{ + UID: row.UID, + Title: row.Title, + Subresource: row.Subresource, + Content: row.Content, + Score: row.Score, + Folder: row.Folder, + Metadata: row.Metadata, + } + } + return results, nil +} + +func (b *pgvectorBackend) GetLatestRV(ctx context.Context) (int64, error) { + var rv int64 + row := b.db.QueryRowContext(ctx, `SELECT latest_rv FROM vector_latest_rv WHERE id = 1`) + if err := row.Scan(&rv); err != nil { + if errors.Is(err, sql.ErrNoRows) { + return 0, nil + } + return 0, fmt.Errorf("read vector_latest_rv: %w", err) + } + return rv, nil +} diff --git a/pkg/storage/unified/search/vector/pgvector_test.go b/pkg/storage/unified/search/vector/pgvector_test.go new file mode 100644 index 00000000000..872a4bf85b6 --- /dev/null +++ b/pkg/storage/unified/search/vector/pgvector_test.go @@ -0,0 +1,153 @@ +package vector + +import ( + "database/sql" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/grafana/grafana/pkg/storage/unified/sql/test" + "github.com/grafana/grafana/pkg/util/testutil" +) + +func TestVector_Validate(t *testing.T) { + ok := Vector{Namespace: "ns", Model: "m", Resource: "r", UID: "u", Title: "t"} + require.NoError(t, ok.Validate()) + + cases := []struct { + name string + mutate func(*Vector) + wantErr string + }{ + {"empty namespace", func(v *Vector) { v.Namespace = "" }, "namespace must not be empty"}, + {"empty model", func(v *Vector) { v.Model = "" }, "model must not be empty"}, + {"empty resource", func(v *Vector) { v.Resource = "" }, "resource must not be empty"}, + {"empty uid", func(v *Vector) { v.UID = "" }, "uid must not be empty"}, + {"empty title", func(v *Vector) { v.Title = "" }, "title must not be empty"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + v := ok + tc.mutate(&v) + err := v.Validate() + require.Error(t, err) + require.Contains(t, err.Error(), tc.wantErr) + }) + } +} + +func TestValidateResource(t *testing.T) { + cases := []struct { + in string + wantErr bool + }{ + {"dashboards", false}, + {"folders", true}, // not provisioned yet + {"", true}, + } + for _, tc := range cases { + t.Run(tc.in, func(t *testing.T) { + err := validateResource(tc.in) + if tc.wantErr { + require.Error(t, err) + return + } + require.NoError(t, err) + }) + } +} + +func TestPgvectorBackend_Upsert_EmptySlice(t *testing.T) { + rdb := test.NewDBProviderNopSQL(t) + backend := NewPgvectorBackend(rdb.DB, 1000, 0) + ctx := testutil.NewDefaultTestContext(t) + + require.NoError(t, backend.Upsert(ctx, nil)) + require.NoError(t, backend.Upsert(ctx, []Vector{})) + require.NoError(t, rdb.SQLMock.ExpectationsWereMet()) +} + +func TestPgvectorBackend_Upsert_InvalidVector_Rejected(t *testing.T) { + rdb := test.NewDBProviderNopSQL(t) + backend := NewPgvectorBackend(rdb.DB, 1000, 0) + ctx := testutil.NewDefaultTestContext(t) + + err := backend.Upsert(ctx, []Vector{ + {Namespace: "ns", Model: "m", Resource: "dashboards", UID: "", Content: "x", Embedding: []float32{0.1}}, + }) + require.Error(t, err) + require.Contains(t, err.Error(), "uid must not be empty") + require.NoError(t, rdb.SQLMock.ExpectationsWereMet()) +} + +func TestPgvectorBackend_Upsert_UnknownResource_Rejected(t *testing.T) { + // Unknown resource has no shared table; Upsert errors before any DB work. + rdb := test.NewDBProviderNopSQL(t) + backend := NewPgvectorBackend(rdb.DB, 1000, 0) + ctx := testutil.NewDefaultTestContext(t) + + rdb.SQLMock.ExpectBegin() + rdb.SQLMock.ExpectRollback() + + err := backend.Upsert(ctx, []Vector{ + {Namespace: "ns", Model: "m", Resource: "folders", UID: "x", Title: "t", Embedding: []float32{0.1}}, + }) + require.Error(t, err) + require.Contains(t, err.Error(), "unsupported resource") + require.NoError(t, rdb.SQLMock.ExpectationsWereMet()) +} + +func TestPgvectorBackend_Delete_EmptyModel_Rejected(t *testing.T) { + rdb := test.NewDBProviderNopSQL(t) + backend := NewPgvectorBackend(rdb.DB, 1000, 0) + ctx := testutil.NewDefaultTestContext(t) + + err := backend.Delete(ctx, "ns", "", "dashboards", "dash-1") + require.Error(t, err) + require.Contains(t, err.Error(), "model must not be empty") + require.NoError(t, rdb.SQLMock.ExpectationsWereMet()) +} + +func TestPgvectorBackend_DeleteSubresources_EmptySlice_NoOp(t *testing.T) { + rdb := test.NewDBProviderNopSQL(t) + backend := NewPgvectorBackend(rdb.DB, 1000, 0) + ctx := testutil.NewDefaultTestContext(t) + + require.NoError(t, backend.DeleteSubresources(ctx, "ns", "m", "dashboards", "dash-1", nil)) + require.NoError(t, backend.DeleteSubresources(ctx, "ns", "m", "dashboards", "dash-1", []string{})) + require.NoError(t, rdb.SQLMock.ExpectationsWereMet()) +} + +func TestPgvectorBackend_GetLatestRV(t *testing.T) { + rdb := test.NewDBProviderNopSQL(t) + backend := NewPgvectorBackend(rdb.DB, 1000, 0) + ctx := testutil.NewDefaultTestContext(t) + + rdb.SQLMock.ExpectQuery("SELECT latest_rv FROM vector_latest_rv"). + WillReturnRows(rdb.SQLMock.NewRows([]string{"latest_rv"}).AddRow(int64(42))) + + rv, err := backend.GetLatestRV(ctx) + require.NoError(t, err) + require.Equal(t, int64(42), rv) + require.NoError(t, rdb.SQLMock.ExpectationsWereMet()) +} + +func TestPgvectorBackend_GetLatestRV_SeedRowMissing(t *testing.T) { + rdb := test.NewDBProviderNopSQL(t) + backend := NewPgvectorBackend(rdb.DB, 1000, 0) + ctx := testutil.NewDefaultTestContext(t) + + rdb.SQLMock.ExpectQuery("SELECT latest_rv FROM vector_latest_rv"). + WillReturnError(sql.ErrNoRows) + + rv, err := backend.GetLatestRV(ctx) + require.NoError(t, err) + require.Equal(t, int64(0), rv) + require.NoError(t, rdb.SQLMock.ExpectationsWereMet()) +} + +func TestPartialHNSWName(t *testing.T) { + require.Equal(t, "dashboards_stacks_123_hnsw", partialHNSWName("dashboards", "stacks-123")) + require.Equal(t, "dashboards_weird__name_hnsw", partialHNSWName("dashboards", "weird!!name")) + require.Equal(t, "dashboards_upper_ns_hnsw", partialHNSWName("dashboards", "UPPER-NS")) +} diff --git a/pkg/storage/unified/search/vector/promoter.go b/pkg/storage/unified/search/vector/promoter.go new file mode 100644 index 00000000000..f2fa341bf4d --- /dev/null +++ b/pkg/storage/unified/search/vector/promoter.go @@ -0,0 +1,263 @@ +package vector + +import ( + "context" + "database/sql" + "errors" + "fmt" + "regexp" + "strings" + "time" + + "github.com/grafana/grafana/pkg/infra/log" + "github.com/grafana/grafana/pkg/storage/unified/sql/db" +) + +const unifiedParent = "embeddings" + +var sanitizeRe = regexp.MustCompile(`[^a-zA-Z0-9]`) + +// sanitizeIdentifier folds non-alphanumerics to underscore and lowercases — +// stable across Postgres's case-folding. +func sanitizeIdentifier(s string) string { + return strings.ToLower(sanitizeRe.ReplaceAllString(s, "_")) +} + +// subtreeName is the per-resource leaf table: `embeddings_`. +func subtreeName(resource string) string { + return fmt.Sprintf("%s_%s", unifiedParent, resource) +} + +// partialHNSWName builds the per-tenant partial HNSW index name. Format is +// `__hnsw`. Postgres caps identifiers at 63 +// chars: with `stacks_` namespaces (max 26 chars), resource names up +// to ~31 chars are safe. +func partialHNSWName(resource, namespace string) string { + return fmt.Sprintf("%s_%s_hnsw", resource, sanitizeIdentifier(namespace)) +} + +// Promoter creates per-tenant partial HNSW indexes on each resource sub-tree +// for namespaces over the row-count threshold. Timer-driven; off the write +// path. +type Promoter struct { + db db.DB + threshold int + interval time.Duration + log log.Logger +} + +func NewPromoter(database db.DB, threshold int, interval time.Duration) *Promoter { + return &Promoter{ + db: database, + threshold: threshold, + interval: interval, + log: log.New("vector-promoter"), + } +} + +// Run ticks every s.interval until ctx is cancelled. Per-iteration errors +// are logged; the loop keeps running. +func (s *Promoter) Run(ctx context.Context) error { + if s.interval <= 0 { + s.log.Info("vector promoter disabled (interval <= 0)") + <-ctx.Done() + return nil + } + s.log.Info("vector promoter starting", "interval", s.interval, "threshold", s.threshold) + t := time.NewTicker(s.interval) + defer t.Stop() + for { + select { + case <-ctx.Done(): + return nil + case <-t.C: + if err := s.Promote(ctx); err != nil { + s.log.Warn("vector promoter iteration failed", "err", err) + } + } + } +} + +// Promote runs one pass over every resource sub-tree. Exported for tests. +func (s *Promoter) Promote(ctx context.Context) error { + resources, err := s.discoverResources(ctx) + if err != nil { + return fmt.Errorf("discover resource sub-trees: %w", err) + } + for _, resource := range resources { + if err := s.promoteResource(ctx, resource); err != nil { + return fmt.Errorf("promote %s: %w", resource, err) + } + } + return nil +} + +// discoverResources returns resource names from leaf tables attached to the +// unified parent. Filters relkind='r' (regular leaf tables). +func (s *Promoter) discoverResources(ctx context.Context) ([]string, error) { + rows, err := s.db.QueryContext(ctx, ` + SELECT c.relname FROM pg_inherits i + JOIN pg_class c ON c.oid = i.inhrelid + JOIN pg_class p ON p.oid = i.inhparent + WHERE p.relname = $1 AND c.relkind = 'r' + `, unifiedParent) + if err != nil { + return nil, err + } + defer func() { _ = rows.Close() }() + + prefix := unifiedParent + "_" + var resources []string + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + return nil, err + } + resources = append(resources, strings.TrimPrefix(name, prefix)) + } + return resources, rows.Err() +} + +func (s *Promoter) promoteResource(ctx context.Context, resource string) error { + subtree := subtreeName(resource) + + if err := s.dropInvalidIndexes(ctx, subtree); err != nil { + return err + } + + rows, err := s.db.QueryContext(ctx, fmt.Sprintf( + `SELECT namespace FROM %s GROUP BY namespace HAVING COUNT(*) > $1`, subtree), + s.threshold) + if err != nil { + return fmt.Errorf("enumerate tenants in %s: %w", subtree, err) + } + var namespaces []string + for rows.Next() { + var ns string + if err := rows.Scan(&ns); err != nil { + _ = rows.Close() + return err + } + namespaces = append(namespaces, ns) + } + if err := rows.Err(); err != nil { + _ = rows.Close() + return err + } + if err := rows.Close(); err != nil { + return err + } + + for _, ns := range namespaces { + if err := ctx.Err(); err != nil { + return err + } + if err := s.promote(ctx, resource, ns); err != nil { + // One bad tenant shouldn't stop the pass. + s.log.Warn("promote failed", "resource", resource, "namespace", ns, "err", err) + } + } + return nil +} + +// promote builds a partial HNSW index scoped to (resource, namespace). +// CREATE INDEX CONCURRENTLY can't run in a transaction. Idempotent — skips +// when a valid index already exists. +// +// Concurrency: CREATE INDEX CONCURRENTLY takes only SHARE UPDATE EXCLUSIVE +// on the table, so reads and writes proceed during the build. Build cost +// scales with rows matching the predicate, not the whole table. +func (s *Promoter) promote(ctx context.Context, resource, namespace string) error { + subtree := subtreeName(resource) + idxName := partialHNSWName(resource, namespace) + + valid, err := s.indexExistsValid(ctx, idxName) + if err != nil { + return err + } + if valid { + return nil + } + + nsLit := "'" + strings.ReplaceAll(namespace, "'", "''") + "'" + ddl := fmt.Sprintf( + `CREATE INDEX CONCURRENTLY IF NOT EXISTS %s + ON %s USING hnsw (embedding halfvec_cosine_ops) + WITH (m = 16, ef_construction = 64) + WHERE namespace = %s`, + idxName, subtree, nsLit, + ) + + s.log.Info("promoting tenant", "resource", resource, "namespace", namespace, "index", idxName) + if _, err := s.db.ExecContext(ctx, ddl); err != nil { + return fmt.Errorf("create partial index %s: %w", idxName, err) + } + + // ANALYZE so the planner's row estimate for the partial index reflects + // reality. + if _, err := s.db.ExecContext(ctx, fmt.Sprintf(`ANALYZE %s`, subtree)); err != nil { + s.log.Warn("ANALYZE after promote failed", "subtree", subtree, "err", err) + } + + // Best-effort observability — pg_indexes is the source of truth. + if _, err := s.db.ExecContext(ctx, + `INSERT INTO vector_promoted (namespace, resource, promoted_at) + VALUES ($1, $2, CURRENT_TIMESTAMP) + ON CONFLICT (namespace, resource) DO UPDATE SET promoted_at = EXCLUDED.promoted_at`, + namespace, resource, + ); err != nil { + s.log.Warn("vector_promoted upsert failed", "err", err) + } + return nil +} + +func (s *Promoter) indexExistsValid(ctx context.Context, idxName string) (bool, error) { + var valid bool + err := s.db.QueryRowContext(ctx, ` + SELECT i.indisvalid FROM pg_class c + JOIN pg_index i ON i.indexrelid = c.oid + WHERE c.relname = $1 AND c.relkind = 'i' + `, idxName).Scan(&valid) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + return false, nil + } + return false, err + } + return valid, nil +} + +// dropInvalidIndexes drops INVALID indexes on the resource sub-tree — +// leftovers from a prior CREATE INDEX CONCURRENTLY that failed mid-build. +func (s *Promoter) dropInvalidIndexes(ctx context.Context, subtree string) error { + rows, err := s.db.QueryContext(ctx, ` + SELECT c.relname FROM pg_class c + JOIN pg_index i ON i.indexrelid = c.oid + JOIN pg_class t ON t.oid = i.indrelid + WHERE t.relname = $1 AND c.relkind = 'i' AND NOT i.indisvalid + `, subtree) + if err != nil { + return fmt.Errorf("list invalid indexes on %s: %w", subtree, err) + } + var names []string + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + _ = rows.Close() + return err + } + names = append(names, name) + } + if err := rows.Err(); err != nil { + _ = rows.Close() + return err + } + _ = rows.Close() + for _, n := range names { + s.log.Warn("dropping invalid index", "index", n) + if _, err := s.db.ExecContext(ctx, fmt.Sprintf(`DROP INDEX CONCURRENTLY IF EXISTS %s`, n)); err != nil { + s.log.Warn("DROP INDEX failed", "index", n, "err", err) + } + } + return nil +} diff --git a/pkg/storage/unified/search/vector/provider.go b/pkg/storage/unified/search/vector/provider.go new file mode 100644 index 00000000000..164bc4561ab --- /dev/null +++ b/pkg/storage/unified/search/vector/provider.go @@ -0,0 +1,51 @@ +package vector + +import ( + "context" + "fmt" + + "github.com/grafana/grafana/pkg/infra/log" + "github.com/grafana/grafana/pkg/setting" + "github.com/grafana/grafana/pkg/storage/unified/sql/db/dbimpl" + "github.com/grafana/grafana/pkg/util/xorm" +) + +func ProvideVectorBackend(cfg *setting.Cfg) (VectorBackend, error) { + return InitVectorBackend(context.Background(), cfg, true) +} + +// InitVectorBackend opens the [database_vector] connection and returns a +// pgvectorBackend. runMigrations=true applies schema migrations before +// returning. Returns (nil, nil) when EnableVectorBackend is false. +// Caller starts the promoter via backend.Run(ctx); gate on schema ownership. +func InitVectorBackend(ctx context.Context, cfg *setting.Cfg, runMigrations bool) (VectorBackend, error) { + if !cfg.EnableVectorBackend { + return nil, nil + } + if cfg.VectorDBHost == "" { + return nil, fmt.Errorf("vector backend is enabled but [database_vector] db_host is not set") + } + + logger := log.New("vector-db") + + connStr := fmt.Sprintf("host=%s port=%s dbname=%s user=%s password=%s sslmode=%s", + cfg.VectorDBHost, cfg.VectorDBPort, cfg.VectorDBName, cfg.VectorDBUser, cfg.VectorDBPassword, cfg.VectorDBSSLMode, + ) + + engine, err := xorm.NewEngine("postgres", connStr) + if err != nil { + return nil, fmt.Errorf("open vector database: %w", err) + } + + if runMigrations { + logger.Info("Running vector database migrations") + if err := MigrateVectorStore(ctx, engine, cfg); err != nil { + return nil, fmt.Errorf("migrate vector database: %w", err) + } + } else { + logger.Info("Skipping vector database migrations") + } + + database := dbimpl.NewDB(engine.DB().DB, engine.Dialect().DriverName()) + return NewPgvectorBackend(database, cfg.VectorPromotionThreshold, cfg.VectorPromoterInterval), nil +} diff --git a/pkg/storage/unified/search/vector/queries.go b/pkg/storage/unified/search/vector/queries.go new file mode 100644 index 00000000000..8b411d2433d --- /dev/null +++ b/pkg/storage/unified/search/vector/queries.go @@ -0,0 +1,180 @@ +package vector + +import ( + "embed" + "encoding/json" + "fmt" + "reflect" + "text/template" + + "github.com/grafana/grafana/pkg/storage/unified/sql/sqltemplate" +) + +//go:embed data/*.sql +var sqlTemplatesFS embed.FS + +var sqlTemplates = template.Must( + template.New("sql").ParseFS(sqlTemplatesFS, `data/*.sql`), +) + +func mustTemplate(filename string) *template.Template { + if t := sqlTemplates.Lookup(filename); t != nil { + return t + } + panic(fmt.Sprintf("template file not found: %s", filename)) +} + +var ( + sqlVectorCollectionUpsert = mustTemplate("vector_collection_upsert.sql") + sqlVectorCollectionDelete = mustTemplate("vector_collection_delete.sql") + sqlVectorCollectionDeleteSubresource = mustTemplate("vector_collection_delete_subresources.sql") + sqlVectorCollectionGetContent = mustTemplate("vector_collection_get_content.sql") + sqlVectorCollectionSearch = mustTemplate("vector_collection_search.sql") +) + +// All queries target `embeddings` and include `resource = $1 AND +// namespace = $2` so nested partition pruning routes to one leaf. + +type sqlVectorCollectionUpsertRequest struct { + sqltemplate.SQLTemplate + Resource string + Vector *Vector + Embedding any // pgvector.HalfVector +} + +func (r *sqlVectorCollectionUpsertRequest) Validate() error { + if r.Resource == "" { + return fmt.Errorf("missing resource") + } + if r.Vector == nil { + return fmt.Errorf("missing vector") + } + return nil +} + +type sqlVectorCollectionDeleteRequest struct { + sqltemplate.SQLTemplate + Resource string + Namespace string + Model string + UID string +} + +func (r *sqlVectorCollectionDeleteRequest) Validate() error { + if r.Resource == "" || r.Namespace == "" || r.Model == "" || r.UID == "" { + return fmt.Errorf("missing required fields") + } + return nil +} + +type sqlVectorCollectionDeleteSubresourcesRequest struct { + sqltemplate.SQLTemplate + Resource string + Namespace string + Model string + UID string + Subresources []string +} + +func (r *sqlVectorCollectionDeleteSubresourcesRequest) Validate() error { + if r.Resource == "" || r.Namespace == "" || r.Model == "" || r.UID == "" { + return fmt.Errorf("missing required fields") + } + if len(r.Subresources) == 0 { + return fmt.Errorf("subresources must not be empty") + } + return nil +} + +func (r *sqlVectorCollectionDeleteSubresourcesRequest) SubresourcesSlice() reflect.Value { + return reflect.ValueOf(r.Subresources) +} + +type sqlVectorCollectionGetContentResponse struct { + Subresource string + Content string +} + +type sqlVectorCollectionGetContentRequest struct { + sqltemplate.SQLTemplate + Resource string + Namespace string + Model string + UID string + Response *sqlVectorCollectionGetContentResponse +} + +func (r *sqlVectorCollectionGetContentRequest) Validate() error { + if r.Resource == "" || r.Namespace == "" || r.Model == "" || r.UID == "" { + return fmt.Errorf("missing required fields") + } + return nil +} + +func (r *sqlVectorCollectionGetContentRequest) Results() (*sqlVectorCollectionGetContentResponse, error) { + cp := *r.Response + return &cp, nil +} + +type sqlVectorCollectionSearchResponse struct { + UID string + Title string + Subresource string + Content string + Score float64 + Folder string + Metadata json.RawMessage +} + +// MetadataFilterEntry is a pre-built JSONB containment filter. +type MetadataFilterEntry struct { + JSON string // e.g. `{"datasource_uids":["ds1"]}` +} + +type sqlVectorCollectionSearchRequest struct { + sqltemplate.SQLTemplate + Resource string + Namespace string + Model string + QueryEmbedding any // pgvector.HalfVector + Limit int64 + Response *sqlVectorCollectionSearchResponse + + // nil/empty means no filter on that field. + UIDValues []string + FolderValues []string + MetadataFilters []MetadataFilterEntry +} + +func (r *sqlVectorCollectionSearchRequest) Validate() error { + if r.Resource == "" || r.Namespace == "" || r.Model == "" { + return fmt.Errorf("missing required fields") + } + if r.Limit <= 0 { + return fmt.Errorf("limit must be positive") + } + return nil +} + +func (r *sqlVectorCollectionSearchRequest) Results() (*sqlVectorCollectionSearchResponse, error) { + // Response is reused across Scan calls; shallow copy is safe because Scan + // allocates a fresh []byte for Metadata. + cp := *r.Response + return &cp, nil +} + +func (r *sqlVectorCollectionSearchRequest) UIDFilter() bool { + return len(r.UIDValues) > 0 +} + +func (r *sqlVectorCollectionSearchRequest) UIDFilterSlice() reflect.Value { + return reflect.ValueOf(r.UIDValues) +} + +func (r *sqlVectorCollectionSearchRequest) FolderFilter() bool { + return len(r.FolderValues) > 0 +} + +func (r *sqlVectorCollectionSearchRequest) FolderFilterSlice() reflect.Value { + return reflect.ValueOf(r.FolderValues) +} diff --git a/pkg/storage/unified/search/vector/queries_test.go b/pkg/storage/unified/search/vector/queries_test.go new file mode 100644 index 00000000000..4f880781c09 --- /dev/null +++ b/pkg/storage/unified/search/vector/queries_test.go @@ -0,0 +1,128 @@ +package vector + +import ( + "encoding/json" + "testing" + "text/template" + + pgvector "github.com/pgvector/pgvector-go" + + "github.com/grafana/grafana/pkg/storage/unified/sql/sqltemplate" + "github.com/grafana/grafana/pkg/storage/unified/sql/sqltemplate/mocks" +) + +func TestVectorQueries(t *testing.T) { + mocks.CheckQuerySnapshots(t, mocks.TemplateTestSetup{ + RootDir: "testdata", + SQLTemplatesFS: sqlTemplatesFS, + Dialects: []sqltemplate.Dialect{sqltemplate.PostgreSQL}, + Templates: map[*template.Template][]mocks.TemplateTestCase{ + sqlVectorCollectionUpsert: { + { + Name: "simple", + Data: &sqlVectorCollectionUpsertRequest{ + SQLTemplate: mocks.NewTestingSQLTemplate(), + Resource: "dashboards", + Vector: &Vector{ + Namespace: "stacks-123", + Resource: "dashboards", + UID: "abc-uid", + Title: "CPU Dashboard", + Subresource: "panel/5", + ResourceVersion: 42, + Folder: "folder-uid", + Content: "panel title with queries", + Metadata: json.RawMessage(`{"datasource_uids":["ds1"]}`), + Embedding: []float32{0.1, 0.2, 0.3}, + Model: "text-embedding-005", + }, + Embedding: pgvector.NewHalfVector([]float32{0.1, 0.2, 0.3}), + }, + }, + }, + sqlVectorCollectionDelete: { + { + Name: "simple", + Data: &sqlVectorCollectionDeleteRequest{ + SQLTemplate: mocks.NewTestingSQLTemplate(), + Resource: "dashboards", + Namespace: "stacks-123", + Model: "text-embedding-005", + UID: "abc-uid", + }, + }, + }, + sqlVectorCollectionDeleteSubresource: { + { + Name: "simple", + Data: &sqlVectorCollectionDeleteSubresourcesRequest{ + SQLTemplate: mocks.NewTestingSQLTemplate(), + Resource: "dashboards", + Namespace: "stacks-123", + Model: "text-embedding-005", + UID: "abc-uid", + Subresources: []string{"panel/1", "panel/2"}, + }, + }, + }, + sqlVectorCollectionGetContent: { + { + Name: "simple", + Data: &sqlVectorCollectionGetContentRequest{ + SQLTemplate: mocks.NewTestingSQLTemplate(), + Resource: "dashboards", + Namespace: "stacks-123", + Model: "text-embedding-005", + UID: "abc-uid", + Response: &sqlVectorCollectionGetContentResponse{}, + }, + }, + }, + sqlVectorCollectionSearch: { + { + Name: "no filters", + Data: &sqlVectorCollectionSearchRequest{ + SQLTemplate: mocks.NewTestingSQLTemplate(), + Resource: "dashboards", + Namespace: "stacks-123", + Model: "text-embedding-005", + QueryEmbedding: []float32{0.1, 0.2, 0.3}, + Limit: 10, + Response: &sqlVectorCollectionSearchResponse{}, + }, + }, + { + Name: "with uid filter", + Data: &sqlVectorCollectionSearchRequest{ + SQLTemplate: mocks.NewTestingSQLTemplate(), + Resource: "dashboards", + Namespace: "stacks-123", + Model: "text-embedding-005", + QueryEmbedding: []float32{0.1, 0.2, 0.3}, + Limit: 10, + UIDValues: []string{"dash-1", "dash-2"}, + Response: &sqlVectorCollectionSearchResponse{}, + }, + }, + { + Name: "with all filters", + Data: &sqlVectorCollectionSearchRequest{ + SQLTemplate: mocks.NewTestingSQLTemplate(), + Resource: "dashboards", + Namespace: "stacks-123", + Model: "text-embedding-005", + QueryEmbedding: []float32{0.1, 0.2, 0.3}, + Limit: 5, + UIDValues: []string{"dash-1"}, + FolderValues: []string{"folder-a", "folder-b"}, + MetadataFilters: []MetadataFilterEntry{ + {JSON: `{"datasource_uids":["ds-uid-1"]}`}, + {JSON: `{"query_languages":["promql"]}`}, + }, + Response: &sqlVectorCollectionSearchResponse{}, + }, + }, + }, + }, + }) +} diff --git a/pkg/storage/unified/search/vector/schema.go b/pkg/storage/unified/search/vector/schema.go new file mode 100644 index 00000000000..9db21d69015 --- /dev/null +++ b/pkg/storage/unified/search/vector/schema.go @@ -0,0 +1,100 @@ +package vector + +import ( + "context" + + "github.com/grafana/grafana/pkg/services/sqlstore/migrator" + "github.com/grafana/grafana/pkg/setting" + "github.com/grafana/grafana/pkg/util/xorm" +) + +// MigrateVectorStore runs migrations for the vector database. +// +// LIST partitioning by resource. Each resource gets a leaf table: +// +// embeddings PARTITION BY LIST (resource) +// └── embeddings_dashboards leaf — all dashboard rows live here +// + partial HNSW per namespace for promoted (big) tenants +// + base GIN(metadata) +// +// Future resource subtrees will be attached at runtime via advisory-lock. +func MigrateVectorStore(ctx context.Context, engine *xorm.Engine, cfg *setting.Cfg) error { + mg := migrator.NewScopedMigrator(engine, cfg, "vector") + mg.AddCreateMigration() + + initVectorTables(mg) + + sec := cfg.Raw.Section("database_vector") + return mg.RunMigrations( + ctx, + sec.Key("migration_locking").MustBool(true), + sec.Key("locking_attempt_timeout_sec").MustInt(), + ) +} + +func initVectorTables(mg *migrator.Migrator) { + mg.AddMigration("create pgvector extension", + migrator.NewRawSQLMigration("").Postgres(`CREATE EXTENSION IF NOT EXISTS vector;`)) + + // (resource, namespace) lead the PK so partition pruning can use it. + // halfvec + nested partitioning aren't expressible via xorm, so raw SQL. + mg.AddMigration("create embeddings parent", + migrator.NewRawSQLMigration("").Postgres(` + CREATE TABLE IF NOT EXISTS embeddings ( + id BIGSERIAL, + resource VARCHAR(256) NOT NULL, + namespace VARCHAR(256) NOT NULL, + model VARCHAR(256) NOT NULL, + uid VARCHAR(256) NOT NULL, + title VARCHAR(1024) NOT NULL, + subresource VARCHAR(256) NOT NULL DEFAULT '', + folder VARCHAR(256), + content TEXT NOT NULL, + metadata JSONB, + embedding halfvec(1024) NOT NULL, + PRIMARY KEY (resource, namespace, model, uid, subresource) + ) PARTITION BY LIST (resource); + `)) + + mg.AddMigration("create embeddings_dashboards leaf", + migrator.NewRawSQLMigration("").Postgres(` + CREATE TABLE IF NOT EXISTS embeddings_dashboards + PARTITION OF embeddings + FOR VALUES IN ('dashboards'); + `)) + + mg.AddMigration("create metadata GIN on embeddings_dashboards", + migrator.NewRawSQLMigration("").Postgres( + `CREATE INDEX IF NOT EXISTS embeddings_dashboards_metadata_idx + ON embeddings_dashboards USING GIN (metadata);`, + )) + + // Observability log. pg_inherits is the source of truth for leaf existence. + vectorPromoted := migrator.Table{ + Name: "vector_promoted", + Columns: []*migrator.Column{ + {Name: "namespace", Type: migrator.DB_Varchar, Length: 256, Nullable: false, IsPrimaryKey: true}, + {Name: "resource", Type: migrator.DB_Varchar, Length: 256, Nullable: false, IsPrimaryKey: true}, + {Name: "promoted_at", Type: migrator.DB_TimeStampz, Nullable: false, Default: "CURRENT_TIMESTAMP"}, + }, + } + mg.AddMigration("create vector_promoted table", + migrator.NewAddTableMigration(vectorPromoted)) + + // Single-row global checkpoint for startup recovery. Retired once we + // have a persistent queue. + vectorLatestRV := migrator.Table{ + Name: "vector_latest_rv", + Columns: []*migrator.Column{ + {Name: "id", Type: migrator.DB_Int, IsPrimaryKey: true, Nullable: false}, + {Name: "latest_rv", Type: migrator.DB_BigInt, Nullable: false, Default: "0"}, + }, + } + mg.AddMigration("create vector_latest_rv table", + migrator.NewAddTableMigration(vectorLatestRV)) + mg.AddMigration("enforce single-row and seed vector_latest_rv", + migrator.NewRawSQLMigration("").Postgres(` + ALTER TABLE vector_latest_rv ADD CONSTRAINT vector_latest_rv_single CHECK (id = 1); + INSERT INTO vector_latest_rv (id, latest_rv) VALUES (1, 0) ON CONFLICT DO NOTHING; + `)) +} diff --git a/pkg/storage/unified/search/vector/store.go b/pkg/storage/unified/search/vector/store.go new file mode 100644 index 00000000000..ee8bb9dc821 --- /dev/null +++ b/pkg/storage/unified/search/vector/store.go @@ -0,0 +1,84 @@ +package vector + +import ( + "context" + "encoding/json" + "errors" +) + +// VectorBackend is vector storage isolated per (namespace, model) so an HNSW +// never mixes embeddings from different vector spaces. +type VectorBackend interface { + // Search returns top-N nearest neighbors by cosine distance. Query + // embedding must come from the same model as stored vectors. + Search(ctx context.Context, namespace, model, resource string, + embedding []float32, limit int, filters ...SearchFilter) ([]VectorSearchResult, error) + + Upsert(ctx context.Context, vectors []Vector) error + + // Delete removes every resource and subresource under `uid`. model must be non-empty. + Delete(ctx context.Context, namespace, model, resource, uid string) error + + // DeleteSubresources removes specific subresources under `uid`. Empty + // slice is a no-op. model must be non-empty. + DeleteSubresources(ctx context.Context, namespace, model, resource, uid string, subresources []string) error + + // GetSubresourceContent returns subresource → stored content. Callers + // diff against candidate content to skip re-embedding unchanged rows. + // Used for deleting stale subresource embeddings. + GetSubresourceContent(ctx context.Context, namespace, model, resource, uid string) (map[string]string, error) + + // GetLatestRV is the global write-pipeline checkpoint. 0 if empty. + GetLatestRV(ctx context.Context) (int64, error) + + // Run starts background maintenance (promoter). Gate on schema ownership. + Run(ctx context.Context) error +} + +// Vector is one embeddable subresource (e.g. a dashboard panel). +type Vector struct { + Namespace string + Resource string // e.g. "dashboards" + UID string // stable resource identifier (e.g. dashboard UID) + Title string // human-readable title for search results + Subresource string // e.g. "panel/5" + ResourceVersion int64 // feeds the global checkpoint; not stored per-row + Folder string // folder UID for authz filtering + Content string // text that was embedded + Metadata json.RawMessage + Embedding []float32 + Model string +} + +func (v *Vector) Validate() error { + switch { + case v.Namespace == "": + return errors.New("namespace must not be empty") + case v.Model == "": + return errors.New("model must not be empty") + case v.Resource == "": + return errors.New("resource must not be empty") + case v.UID == "": + return errors.New("uid must not be empty") + case v.Title == "": + return errors.New("title must not be empty") + } + return nil +} + +type VectorSearchResult struct { + UID string + Title string + Subresource string + Content string + Score float64 + Folder string + Metadata json.RawMessage +} + +// SearchFilter constrains results. Field is a top-level column +// ("uid", "folder") or a JSONB metadata key. +type SearchFilter struct { + Field string + Values []string +} diff --git a/pkg/storage/unified/search/vector/testdata/postgres--vector_collection_delete-simple.sql b/pkg/storage/unified/search/vector/testdata/postgres--vector_collection_delete-simple.sql new file mode 100755 index 00000000000..e56885ae3a7 --- /dev/null +++ b/pkg/storage/unified/search/vector/testdata/postgres--vector_collection_delete-simple.sql @@ -0,0 +1,6 @@ +DELETE FROM embeddings + WHERE "resource" = 'dashboards' + AND "namespace" = 'stacks-123' + AND "model" = 'text-embedding-005' + AND "uid" = 'abc-uid' +; diff --git a/pkg/storage/unified/search/vector/testdata/postgres--vector_collection_delete_subresources-simple.sql b/pkg/storage/unified/search/vector/testdata/postgres--vector_collection_delete_subresources-simple.sql new file mode 100755 index 00000000000..6fea6d0682e --- /dev/null +++ b/pkg/storage/unified/search/vector/testdata/postgres--vector_collection_delete_subresources-simple.sql @@ -0,0 +1,7 @@ +DELETE FROM embeddings + WHERE "resource" = 'dashboards' + AND "namespace" = 'stacks-123' + AND "model" = 'text-embedding-005' + AND "uid" = 'abc-uid' + AND "subresource" IN ('panel/1', 'panel/2') +; diff --git a/pkg/storage/unified/search/vector/testdata/postgres--vector_collection_get_content-simple.sql b/pkg/storage/unified/search/vector/testdata/postgres--vector_collection_get_content-simple.sql new file mode 100755 index 00000000000..be2af206ab3 --- /dev/null +++ b/pkg/storage/unified/search/vector/testdata/postgres--vector_collection_get_content-simple.sql @@ -0,0 +1,9 @@ +SELECT + "subresource", + "content" + FROM embeddings + WHERE "resource" = 'dashboards' + AND "namespace" = 'stacks-123' + AND "model" = 'text-embedding-005' + AND "uid" = 'abc-uid' +; diff --git a/pkg/storage/unified/search/vector/testdata/postgres--vector_collection_search-no filters.sql b/pkg/storage/unified/search/vector/testdata/postgres--vector_collection_search-no filters.sql new file mode 100755 index 00000000000..77d68b5655e --- /dev/null +++ b/pkg/storage/unified/search/vector/testdata/postgres--vector_collection_search-no filters.sql @@ -0,0 +1,15 @@ +SELECT + "uid", + "title", + "subresource", + "content", + "embedding" <=> '[0.1 0.2 0.3]' AS "score", + "folder", + "metadata" + FROM embeddings + WHERE "resource" = 'dashboards' + AND "namespace" = 'stacks-123' + AND "model" = 'text-embedding-005' + ORDER BY "embedding" <=> '[0.1 0.2 0.3]' + LIMIT 10 +; diff --git a/pkg/storage/unified/search/vector/testdata/postgres--vector_collection_search-with all filters.sql b/pkg/storage/unified/search/vector/testdata/postgres--vector_collection_search-with all filters.sql new file mode 100755 index 00000000000..a2829e1255e --- /dev/null +++ b/pkg/storage/unified/search/vector/testdata/postgres--vector_collection_search-with all filters.sql @@ -0,0 +1,19 @@ +SELECT + "uid", + "title", + "subresource", + "content", + "embedding" <=> '[0.1 0.2 0.3]' AS "score", + "folder", + "metadata" + FROM embeddings + WHERE "resource" = 'dashboards' + AND "namespace" = 'stacks-123' + AND "model" = 'text-embedding-005' + AND "uid" IN ('dash-1') + AND "folder" IN ('folder-a', 'folder-b') + AND "metadata" @> '{"datasource_uids":["ds-uid-1"]}' + AND "metadata" @> '{"query_languages":["promql"]}' + ORDER BY "embedding" <=> '[0.1 0.2 0.3]' + LIMIT 5 +; diff --git a/pkg/storage/unified/search/vector/testdata/postgres--vector_collection_search-with uid filter.sql b/pkg/storage/unified/search/vector/testdata/postgres--vector_collection_search-with uid filter.sql new file mode 100755 index 00000000000..22f7e5375ba --- /dev/null +++ b/pkg/storage/unified/search/vector/testdata/postgres--vector_collection_search-with uid filter.sql @@ -0,0 +1,16 @@ +SELECT + "uid", + "title", + "subresource", + "content", + "embedding" <=> '[0.1 0.2 0.3]' AS "score", + "folder", + "metadata" + FROM embeddings + WHERE "resource" = 'dashboards' + AND "namespace" = 'stacks-123' + AND "model" = 'text-embedding-005' + AND "uid" IN ('dash-1', 'dash-2') + ORDER BY "embedding" <=> '[0.1 0.2 0.3]' + LIMIT 10 +; diff --git a/pkg/storage/unified/search/vector/testdata/postgres--vector_collection_upsert-simple.sql b/pkg/storage/unified/search/vector/testdata/postgres--vector_collection_upsert-simple.sql new file mode 100755 index 00000000000..bd3d6d95c8f --- /dev/null +++ b/pkg/storage/unified/search/vector/testdata/postgres--vector_collection_upsert-simple.sql @@ -0,0 +1,32 @@ +INSERT INTO embeddings ( + "resource", + "namespace", + "model", + "uid", + "title", + "subresource", + "folder", + "content", + "metadata", + "embedding" +) +VALUES ( + 'dashboards', + 'stacks-123', + 'text-embedding-005', + 'abc-uid', + 'CPU Dashboard', + 'panel/5', + 'folder-uid', + 'panel title with queries', + '[123 34 100 97 116 97 115 111 117 114 99 101 95 117 105 100 115 34 58 91 34 100 115 49 34 93 125]', + '[0.1,0.2,0.3]' +) +ON CONFLICT ("resource", "namespace", "model", "uid", "subresource") +DO UPDATE SET + "title" = 'CPU Dashboard', + "folder" = 'folder-uid', + "content" = 'panel title with queries', + "metadata" = '[123 34 100 97 116 97 115 111 117 114 99 101 95 117 105 100 115 34 58 91 34 100 115 49 34 93 125]', + "embedding" = '[0.1,0.2,0.3]' +; diff --git a/pkg/storage/unified/sql/server.go b/pkg/storage/unified/sql/server.go index 1a6ad1bb210..5f8ee7f1659 100644 --- a/pkg/storage/unified/sql/server.go +++ b/pkg/storage/unified/sql/server.go @@ -18,6 +18,7 @@ import ( "github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/storage/unified/resource" "github.com/grafana/grafana/pkg/storage/unified/resourcepb" + "github.com/grafana/grafana/pkg/storage/unified/search/vector" ) type QOSEnqueueDequeuer interface { @@ -29,6 +30,7 @@ type QOSEnqueueDequeuer interface { // ServerOptions contains the options for creating a new ResourceServer type ServerOptions struct { Backend resource.StorageBackend + VectorBackend vector.VectorBackend OverridesService *resource.OverridesService Cfg *setting.Cfg Tracer trace.Tracer @@ -59,6 +61,7 @@ func NewUninitializedResourceServer(opts ServerOptions) (resource.ResourceServer withAccessClient, withMaxPageSizeBytes, withBackend, + withVectorBackend, withQOSQueue, withOverridesService, withSearch, @@ -92,6 +95,7 @@ func NewUninitializedSearchServer(opts ServerOptions) (resource.SearchServer, er withBlobConfig, withAccessClient, withBackend, + withVectorBackend, withSearch, ) if err != nil { @@ -181,6 +185,13 @@ func withBackend(opts *ServerOptions, resourceOpts *resource.ResourceServerOptio return nil } +// withVectorBackend propagates the optional VectorBackend through. nil is +// allowed; callers fall back to non-vector search paths when it's absent. +func withVectorBackend(opts *ServerOptions, resourceOpts *resource.ResourceServerOptions) error { + resourceOpts.VectorBackend = opts.VectorBackend + return nil +} + func withSearchClient(opts *ServerOptions, resourceOpts *resource.ResourceServerOptions) error { resourceOpts.SearchClient = opts.SearchClient return nil diff --git a/pkg/storage/unified/sql/service.go b/pkg/storage/unified/sql/service.go index 66303d29c4a..c79e8f40122 100644 --- a/pkg/storage/unified/sql/service.go +++ b/pkg/storage/unified/sql/service.go @@ -31,6 +31,7 @@ import ( "github.com/grafana/grafana/pkg/storage/unified/resource" "github.com/grafana/grafana/pkg/storage/unified/resourcepb" "github.com/grafana/grafana/pkg/storage/unified/search" + "github.com/grafana/grafana/pkg/storage/unified/search/vector" "github.com/grafana/grafana/pkg/util/scheduler" ) @@ -49,6 +50,7 @@ type service struct { // -- Shared Components backend resource.StorageBackend + vectorBackend vector.VectorBackend serverStopper resource.ResourceServerStopper cfg *setting.Cfg features featuremgmt.FeatureToggles @@ -97,10 +99,11 @@ func ProvideSearchGRPCService(cfg *setting.Cfg, memberlistKVConfig kv.Config, httpServerRouter *mux.Router, backend resource.StorageBackend, + vectorBackend vector.VectorBackend, provider grpcserver.Provider, opts ...ServiceOption, ) (resource.UnifiedStorageGrpcService, error) { - s := newService(cfg, features, log, reg, otel.Tracer("unified-storage"), docBuilders, nil, indexMetrics, searchRing, backend, nil) + s := newService(cfg, features, log, reg, otel.Tracer("unified-storage"), docBuilders, nil, indexMetrics, searchRing, backend, vectorBackend, nil) for _, opt := range opts { opt(s) } @@ -135,11 +138,12 @@ func ProvideUnifiedStorageGrpcService(cfg *setting.Cfg, memberlistKVConfig kv.Config, httpServerRouter *mux.Router, backend resource.StorageBackend, + vectorBackend vector.VectorBackend, searchClient resourcepb.ResourceIndexClient, provider grpcserver.Provider, opts ...ServiceOption, ) (resource.UnifiedStorageGrpcService, error) { - s := newService(cfg, features, log, reg, otel.Tracer("unified-storage"), docBuilders, storageMetrics, indexMetrics, searchRing, backend, searchClient) + s := newService(cfg, features, log, reg, otel.Tracer("unified-storage"), docBuilders, storageMetrics, indexMetrics, searchRing, backend, vectorBackend, searchClient) for _, opt := range opts { opt(s) } @@ -195,12 +199,14 @@ func newService( indexMetrics *resource.BleveIndexMetrics, searchRing *ring.Ring, backend resource.StorageBackend, + vectorBackend vector.VectorBackend, searchClient resourcepb.ResourceIndexClient, ) *service { authn := grpcutils.NewAuthenticator(ReadGrpcServerConfig(cfg), tracer) return &service{ backend: backend, + vectorBackend: vectorBackend, cfg: cfg, features: features, authenticator: authn, diff --git a/pkg/storage/unified/sql/sqltemplate/sqltemplate.go b/pkg/storage/unified/sql/sqltemplate/sqltemplate.go index 895a0fcd654..0acf1623b46 100644 --- a/pkg/storage/unified/sql/sqltemplate/sqltemplate.go +++ b/pkg/storage/unified/sql/sqltemplate/sqltemplate.go @@ -143,7 +143,7 @@ type reFormatting struct { var formatREs = []reFormatting{ {re: regexp.MustCompile(`\s+`), replacement: " "}, - {re: regexp.MustCompile(` ?([+-/*=<>%!~]+) ?`), replacement: " $1 "}, + {re: regexp.MustCompile(` ?([+-/*=<>%!~@#]+) ?`), replacement: " $1 "}, {re: regexp.MustCompile(`([([{]) `), replacement: "$1"}, {re: regexp.MustCompile(` ([)\]}])`), replacement: "$1"}, {re: regexp.MustCompile(` ?, ?`), replacement: ", "}, diff --git a/pkg/storage/unified/sql/sqltemplate/sqltemplate_test.go b/pkg/storage/unified/sql/sqltemplate/sqltemplate_test.go index c46fc2d7cca..7e5b01b3987 100644 --- a/pkg/storage/unified/sql/sqltemplate/sqltemplate_test.go +++ b/pkg/storage/unified/sql/sqltemplate/sqltemplate_test.go @@ -89,3 +89,20 @@ func TestFormatSQL(t *testing.T) { got) } } + +func TestFormatSQL_PreservesJSONBOperators(t *testing.T) { + t.Parallel() + + cases := map[string]string{ + `"m" @> $1`: `"m" @> $1`, + `"m" <@ $1`: `"m" <@ $1`, + `"m" #> $1`: `"m" #> $1`, + `"m" #>> $1`: `"m" #>> $1`, + `"m" @@ $1`: `"m" @@ $1`, + } + for input, expected := range cases { + if got := FormatSQL(input); got != expected { + t.Errorf("input %q: expected %q, got %q", input, expected, got) + } + } +} diff --git a/pkg/storage/unified/sql/test/integration_test.go b/pkg/storage/unified/sql/test/integration_test.go index ede129bd8e0..633bdced462 100644 --- a/pkg/storage/unified/sql/test/integration_test.go +++ b/pkg/storage/unified/sql/test/integration_test.go @@ -233,7 +233,7 @@ func TestClientServer(t *testing.T) { _ = services.StopAndAwaitTerminated(ctx, grpcService) }) - svc, err := sql.ProvideUnifiedStorageGrpcService(cfg, features, nil, registerer, nil, nil, nil, nil, kv.Config{}, nil, backend, nil, grpcService, + svc, err := sql.ProvideUnifiedStorageGrpcService(cfg, features, nil, registerer, nil, nil, nil, nil, kv.Config{}, nil, backend, nil, nil, grpcService, sql.WithAuthenticator(func(ctx context.Context) (context.Context, error) { auth := grpcUtils.Authenticator{Tracer: otel.Tracer("test")} return auth.Authenticate(ctx) @@ -342,7 +342,7 @@ func TestIntegrationSearchClientServer(t *testing.T) { grpcService, err := grpcserver.ProvideDSKitService(cfg, otel.Tracer("test-grpc-server"), prometheus.NewPedanticRegistry(), "test-grpc-server") require.NoError(t, err) - svc, err := sql.ProvideSearchGRPCService(cfg, features, log.New("test"), registerer, docBuilders, nil, nil, kv.Config{}, nil, backend, grpcService, + svc, err := sql.ProvideSearchGRPCService(cfg, features, log.New("test"), registerer, docBuilders, nil, nil, kv.Config{}, nil, backend, nil, grpcService, sql.WithAuthenticator(func(ctx context.Context) (context.Context, error) { auth := grpcUtils.Authenticator{Tracer: otel.Tracer("test")} return auth.Authenticate(ctx) diff --git a/pkg/storage/unified/testing/storage_backend_test.go b/pkg/storage/unified/testing/storage_backend_test.go index c5d1ec150c7..f29fb5165ef 100644 --- a/pkg/storage/unified/testing/storage_backend_test.go +++ b/pkg/storage/unified/testing/storage_backend_test.go @@ -199,7 +199,7 @@ func newRemoteClient(t *testing.T, backend resource.KVBackend) resource.Resource grpcService, err := grpcserver.ProvideDSKitService(cfg, otel.Tracer("test"), prometheus.NewPedanticRegistry(), "test") require.NoError(t, err) - svc, err := sql.ProvideUnifiedStorageGrpcService(cfg, features, log.NewNopLogger(), reg, nil, nil, nil, nil, kv.Config{}, nil, backend, nil, grpcService, + svc, err := sql.ProvideUnifiedStorageGrpcService(cfg, features, log.NewNopLogger(), reg, nil, nil, nil, nil, kv.Config{}, nil, backend, nil, nil, grpcService, sql.WithAuthenticator(func(ctx context.Context) (context.Context, error) { auth := grpcUtils.Authenticator{Tracer: otel.Tracer("test")} return auth.Authenticate(ctx) diff --git a/pkg/tests/testinfra/testinfra.go b/pkg/tests/testinfra/testinfra.go index cff21f0a3b9..b1ca31f12e5 100644 --- a/pkg/tests/testinfra/testinfra.go +++ b/pkg/tests/testinfra/testinfra.go @@ -181,7 +181,7 @@ func StartGrafanaEnvWithManualCleanup(t *testing.T, grafDir, cfgPath string) (st require.NoError(t, services.StartAndAwaitRunning(context.Background(), backendService)) storage, err = sql.ProvideUnifiedStorageGrpcService(env.Cfg, env.FeatureToggles, - env.Cfg.Logger, registerer, nil, nil, nil, nil, kv.Config{}, nil, storageBackend, nil, grpcService) + env.Cfg.Logger, registerer, nil, nil, nil, nil, kv.Config{}, nil, storageBackend, nil, nil, grpcService) require.NoError(t, err) err = grpcService.StartAsync(ctx) require.NoError(t, err)