diff --git a/Dockerfile b/Dockerfile index 06d15a9288c..31cdb0e6c65 100644 --- a/Dockerfile +++ b/Dockerfile @@ -140,6 +140,8 @@ ENV BUILD_BRANCH=${BUILD_BRANCH} RUN make build-go GO_BUILD_TAGS=${GO_BUILD_TAGS} WIRE_TAGS=${WIRE_TAGS} +RUN mkdir -p data/plugins-bundled + # From-tarball build stage FROM ${BASE_IMAGE} AS tgz-builder @@ -152,6 +154,8 @@ COPY ${GRAFANA_TGZ} /tmp/grafana.tar.gz # add -v to make tar print every file it extracts RUN tar x -z -f /tmp/grafana.tar.gz --strip-components=1 +RUN mkdir -p data/plugins-bundled + # helpers for COPY --from FROM ${GO_SRC} AS go-src FROM ${JS_SRC} AS js-src @@ -239,6 +243,7 @@ RUN if [ ! $(getent group "$GF_GID") ]; then \ COPY --from=go-src /tmp/grafana/bin/grafana* /tmp/grafana/bin/*/grafana* ./bin/ COPY --from=js-src /tmp/grafana/public ./public COPY --from=js-src /tmp/grafana/LICENSE ./ +COPY --from=go-src /tmp/grafana/data/plugins-bundled ./data/plugins-bundled RUN grafana server -v | sed -e 's/Version //' > /.grafana-version RUN chmod 644 /.grafana-version diff --git a/pkg/build/daggerbuild/arguments/catalog_plugins.go b/pkg/build/daggerbuild/arguments/catalog_plugins.go new file mode 100644 index 00000000000..2ca6ce36d45 --- /dev/null +++ b/pkg/build/daggerbuild/arguments/catalog_plugins.go @@ -0,0 +1,213 @@ +package arguments + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/grafana/grafana/pkg/build/daggerbuild/pipeline" + "github.com/urfave/cli/v2" +) + +// CatalogPluginSpec defines a plugin to download from the Grafana catalog. +type CatalogPluginSpec struct { + ID string `json:"id"` + Version string `json:"version"` + Checksum string `json:"checksum,omitempty"` // Optional SHA256 checksum for verification +} + +// CatalogPluginsManifest is the JSON structure for the manifest file. +type CatalogPluginsManifest struct { + Plugins []CatalogPluginSpec `json:"plugins"` +} + +var flagBundleCatalogPlugins = &cli.StringSliceFlag{ + Name: "bundle-catalog-plugins", + Usage: "Plugins to download from grafana.com catalog (format: id or id:version, version optional). Supports comma-separated and repeated flags.", +} + +var flagBundleCatalogPluginsFile = &cli.StringFlag{ + Name: "bundle-catalog-plugins-file", + Usage: "Path to JSON manifest file containing catalog plugins to bundle", +} + +var CatalogPluginsFlags = []cli.Flag{ + flagBundleCatalogPlugins, + flagBundleCatalogPluginsFile, +} + +// CatalogPlugins is the argument that provides the list of catalog plugins to bundle. +var CatalogPlugins = pipeline.Argument{ + Name: "catalog-plugins", + Description: "List of plugins to download from the Grafana catalog", + Flags: CatalogPluginsFlags, + ValueFunc: catalogPluginsValueFunc, +} + +func catalogPluginsValueFunc(ctx context.Context, opts *pipeline.ArgumentOpts) (any, error) { + // StringSlice handles both comma-separated and repeated flags + pluginsList := opts.CLIContext.StringSlice("bundle-catalog-plugins") + manifestFile := opts.CLIContext.String("bundle-catalog-plugins-file") + + var plugins []CatalogPluginSpec + + // Parse plugin specs from CLI flags + for _, item := range pluginsList { + parsed, err := ParseCatalogPluginsList(item) + if err != nil { + return nil, fmt.Errorf("failed to parse --bundle-catalog-plugins: %w", err) + } + plugins = append(plugins, parsed...) + } + + // Parse manifest file if provided + if manifestFile != "" { + parsed, err := ParseCatalogPluginsFile(manifestFile) + if err != nil { + return nil, fmt.Errorf("failed to parse --bundle-catalog-plugins-file: %w", err) + } + plugins = append(plugins, parsed...) + } + + merged, err := MergeCatalogPluginSpecs(plugins) + if err != nil { + return nil, err + } + + return merged, nil +} + +// ParseCatalogPluginsList parses a comma-separated list of plugins. +// Format: "id" or "id:version" - version is optional and will resolve to latest compatible if omitted. +func ParseCatalogPluginsList(list string) ([]CatalogPluginSpec, error) { + if list == "" { + return nil, nil + } + + items := strings.Split(list, ",") + plugins := make([]CatalogPluginSpec, 0, len(items)) + + for _, item := range items { + item = strings.TrimSpace(item) + if item == "" { + continue + } + + parts := strings.SplitN(item, ":", 2) + id := strings.TrimSpace(parts[0]) + if id == "" { + return nil, fmt.Errorf("invalid plugin format %q, id cannot be empty", item) + } + + version := "" + if len(parts) == 2 { + version = strings.TrimSpace(parts[1]) + } + + plugins = append(plugins, CatalogPluginSpec{ + ID: id, + Version: version, + }) + } + + return plugins, nil +} + +// ParseCatalogPluginsFile parses a JSON manifest file containing plugins to bundle. +func ParseCatalogPluginsFile(path string) ([]CatalogPluginSpec, error) { + data, err := os.ReadFile(filepath.Clean(path)) + if err != nil { + return nil, fmt.Errorf("failed to read manifest file: %w", err) + } + + var manifest CatalogPluginsManifest + if err := json.Unmarshal(data, &manifest); err != nil { + return nil, fmt.Errorf("failed to parse manifest JSON: %w", err) + } + + // Validate all plugins have required fields + // Version is optional - empty version means "latest compatible" + for i, plugin := range manifest.Plugins { + if plugin.ID == "" { + return nil, fmt.Errorf("plugin at index %d is missing required 'id' field", i) + } + } + + return manifest.Plugins, nil +} + +// HasCatalogPlugins returns true if any catalog plugins were specified via CLI flags. +func HasCatalogPlugins(ctx context.Context, opts *pipeline.ArgumentOpts) bool { + pluginsList := opts.CLIContext.StringSlice("bundle-catalog-plugins") + manifestFile := opts.CLIContext.String("bundle-catalog-plugins-file") + return len(pluginsList) > 0 || manifestFile != "" +} + +// GetCatalogPlugins retrieves the catalog plugins from the argument state. +func GetCatalogPlugins(ctx context.Context, state pipeline.StateHandler) ([]CatalogPluginSpec, error) { + v, ok := pipeline.UnwrapState(state) + if !ok { + return nil, fmt.Errorf("state is not backed by *pipeline.State (got %T)", state) + } + + if val, ok := v.Data.Load(CatalogPlugins.Name); ok { + plugins, ok := val.([]CatalogPluginSpec) + if !ok { + return nil, fmt.Errorf("unexpected type for catalog plugins") + } + return plugins, nil + } + + // If not in state, compute it + opts := v.ArgumentOpts() + result, err := catalogPluginsValueFunc(ctx, opts) + if err != nil { + return nil, err + } + + plugins := result.([]CatalogPluginSpec) + v.Data.Store(CatalogPlugins.Name, plugins) + return plugins, nil +} + +// MergeCatalogPluginSpecs deduplicates plugin specs by id and rejects conflicts. +// If the same plugin appears multiple times with identical version/checksum, it is collapsed into one entry. +func MergeCatalogPluginSpecs(plugins []CatalogPluginSpec) ([]CatalogPluginSpec, error) { + if len(plugins) == 0 { + return nil, nil + } + + seen := make(map[string]int, len(plugins)) + merged := make([]CatalogPluginSpec, 0, len(plugins)) + + for _, plugin := range plugins { + idx, ok := seen[plugin.ID] + if !ok { + seen[plugin.ID] = len(merged) + merged = append(merged, plugin) + continue + } + + existing := merged[idx] + if existing.Version != plugin.Version { + return nil, fmt.Errorf("conflicting versions for plugin %q: %q vs %q", plugin.ID, existing.Version, plugin.Version) + } + + switch { + case existing.Checksum == plugin.Checksum: + // Identical duplicate, no-op. + case existing.Checksum == "": + existing.Checksum = plugin.Checksum + merged[idx] = existing + case plugin.Checksum == "": + // Keep existing checksum. + default: + return nil, fmt.Errorf("conflicting checksums for plugin %q", plugin.ID) + } + } + + return merged, nil +} diff --git a/pkg/build/daggerbuild/arguments/catalog_plugins_test.go b/pkg/build/daggerbuild/arguments/catalog_plugins_test.go new file mode 100644 index 00000000000..5a707a1b132 --- /dev/null +++ b/pkg/build/daggerbuild/arguments/catalog_plugins_test.go @@ -0,0 +1,364 @@ +package arguments + +import ( + "context" + "io" + "log/slog" + "os" + "path/filepath" + "reflect" + "testing" + + "github.com/grafana/grafana/pkg/build/daggerbuild/pipeline" +) + +func TestParseCatalogPluginsList(t *testing.T) { + tests := []struct { + name string + input string + want []CatalogPluginSpec + wantErr bool + }{ + { + name: "empty string returns nil", + input: "", + want: nil, + }, + { + name: "single plugin with version", + input: "grafana-clock-panel:1.3.1", + want: []CatalogPluginSpec{ + {ID: "grafana-clock-panel", Version: "1.3.1"}, + }, + }, + { + name: "single plugin without version (latest compatible)", + input: "grafana-lokiexplore-app", + want: []CatalogPluginSpec{ + {ID: "grafana-lokiexplore-app", Version: ""}, + }, + }, + { + name: "multiple plugins with versions", + input: "grafana-clock-panel:1.3.1,grafana-worldmap-panel:1.0.6", + want: []CatalogPluginSpec{ + {ID: "grafana-clock-panel", Version: "1.3.1"}, + {ID: "grafana-worldmap-panel", Version: "1.0.6"}, + }, + }, + { + name: "multiple plugins without versions (latest compatible)", + input: "grafana-lokiexplore-app,grafana-pyroscope-app", + want: []CatalogPluginSpec{ + {ID: "grafana-lokiexplore-app", Version: ""}, + {ID: "grafana-pyroscope-app", Version: ""}, + }, + }, + { + name: "mixed - some with version, some without", + input: "grafana-lokiexplore-app,grafana-pyroscope-app:1.5.0,grafana-exploretraces-app", + want: []CatalogPluginSpec{ + {ID: "grafana-lokiexplore-app", Version: ""}, + {ID: "grafana-pyroscope-app", Version: "1.5.0"}, + {ID: "grafana-exploretraces-app", Version: ""}, + }, + }, + { + name: "plugins with spaces", + input: "grafana-clock-panel:1.3.1 , grafana-worldmap-panel:1.0.6", + want: []CatalogPluginSpec{ + {ID: "grafana-clock-panel", Version: "1.3.1"}, + {ID: "grafana-worldmap-panel", Version: "1.0.6"}, + }, + }, + { + name: "trailing comma is ignored", + input: "grafana-clock-panel:1.3.1,", + want: []CatalogPluginSpec{ + {ID: "grafana-clock-panel", Version: "1.3.1"}, + }, + }, + { + name: "empty id with colon", + input: ":1.3.1", + wantErr: true, + }, + { + name: "id with empty version after colon", + input: "grafana-clock-panel:", + want: []CatalogPluginSpec{ + {ID: "grafana-clock-panel", Version: ""}, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := ParseCatalogPluginsList(tt.input) + if (err != nil) != tt.wantErr { + t.Errorf("ParseCatalogPluginsList() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !tt.wantErr { + if len(got) != len(tt.want) { + t.Errorf("ParseCatalogPluginsList() returned %d plugins, want %d", len(got), len(tt.want)) + return + } + for i, plugin := range got { + if plugin.ID != tt.want[i].ID || plugin.Version != tt.want[i].Version { + t.Errorf("ParseCatalogPluginsList()[%d] = %v, want %v", i, plugin, tt.want[i]) + } + } + } + }) + } +} + +func TestParseCatalogPluginsFile(t *testing.T) { + tests := []struct { + name string + content string + want []CatalogPluginSpec + wantErr bool + }{ + { + name: "empty plugins array", + content: `{"plugins": []}`, + want: []CatalogPluginSpec{}, + }, + { + name: "single plugin with version", + content: `{ + "plugins": [ + {"id": "grafana-clock-panel", "version": "1.3.1"} + ] + }`, + want: []CatalogPluginSpec{ + {ID: "grafana-clock-panel", Version: "1.3.1"}, + }, + }, + { + name: "single plugin without version (latest compatible)", + content: `{ + "plugins": [ + {"id": "grafana-lokiexplore-app"} + ] + }`, + want: []CatalogPluginSpec{ + {ID: "grafana-lokiexplore-app", Version: ""}, + }, + }, + { + name: "multiple plugins with checksum", + content: `{ + "plugins": [ + {"id": "grafana-clock-panel", "version": "1.3.1"}, + {"id": "grafana-worldmap-panel", "version": "1.0.6", "checksum": "sha256:abc123"} + ] + }`, + want: []CatalogPluginSpec{ + {ID: "grafana-clock-panel", Version: "1.3.1"}, + {ID: "grafana-worldmap-panel", Version: "1.0.6", Checksum: "sha256:abc123"}, + }, + }, + { + name: "mixed - some with version, some without", + content: `{ + "plugins": [ + {"id": "grafana-lokiexplore-app"}, + {"id": "grafana-pyroscope-app", "version": "1.5.0"}, + {"id": "grafana-exploretraces-app", "checksum": "sha256:def456"} + ] + }`, + want: []CatalogPluginSpec{ + {ID: "grafana-lokiexplore-app", Version: ""}, + {ID: "grafana-pyroscope-app", Version: "1.5.0"}, + {ID: "grafana-exploretraces-app", Version: "", Checksum: "sha256:def456"}, + }, + }, + { + name: "missing id", + content: `{ + "plugins": [ + {"version": "1.3.1"} + ] + }`, + wantErr: true, + }, + { + name: "invalid json", + content: `{invalid}`, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Create a temporary file + tmpDir := t.TempDir() + tmpFile := filepath.Join(tmpDir, "plugins.json") + if err := os.WriteFile(tmpFile, []byte(tt.content), 0644); err != nil { + t.Fatalf("Failed to write temp file: %v", err) + } + + got, err := ParseCatalogPluginsFile(tmpFile) + if (err != nil) != tt.wantErr { + t.Errorf("ParseCatalogPluginsFile() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !tt.wantErr { + if len(got) != len(tt.want) { + t.Errorf("ParseCatalogPluginsFile() returned %d plugins, want %d", len(got), len(tt.want)) + return + } + for i, plugin := range got { + if plugin.ID != tt.want[i].ID || plugin.Version != tt.want[i].Version || plugin.Checksum != tt.want[i].Checksum { + t.Errorf("ParseCatalogPluginsFile()[%d] = %v, want %v", i, plugin, tt.want[i]) + } + } + } + }) + } +} + +func TestParseCatalogPluginsFile_FileNotFound(t *testing.T) { + _, err := ParseCatalogPluginsFile("/nonexistent/path/plugins.json") + if err == nil { + t.Error("ParseCatalogPluginsFile() should return error for non-existent file") + } +} + +func TestMergeCatalogPluginSpecs(t *testing.T) { + tests := []struct { + name string + input []CatalogPluginSpec + want []CatalogPluginSpec + wantErr bool + }{ + { + name: "dedupe identical entries", + input: []CatalogPluginSpec{ + {ID: "grafana-clock-panel", Version: "1.3.1"}, + {ID: "grafana-clock-panel", Version: "1.3.1"}, + }, + want: []CatalogPluginSpec{ + {ID: "grafana-clock-panel", Version: "1.3.1"}, + }, + }, + { + name: "merge checksum enrichment", + input: []CatalogPluginSpec{ + {ID: "grafana-clock-panel", Version: "1.3.1"}, + {ID: "grafana-clock-panel", Version: "1.3.1", Checksum: "sha256:abc123"}, + }, + want: []CatalogPluginSpec{ + {ID: "grafana-clock-panel", Version: "1.3.1", Checksum: "sha256:abc123"}, + }, + }, + { + name: "conflicting versions fail", + input: []CatalogPluginSpec{ + {ID: "grafana-clock-panel", Version: "1.3.1"}, + {ID: "grafana-clock-panel", Version: "1.4.0"}, + }, + wantErr: true, + }, + { + name: "conflicting checksums fail", + input: []CatalogPluginSpec{ + {ID: "grafana-clock-panel", Version: "1.3.1", Checksum: "sha256:abc123"}, + {ID: "grafana-clock-panel", Version: "1.3.1", Checksum: "sha256:def456"}, + }, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := MergeCatalogPluginSpecs(tt.input) + if (err != nil) != tt.wantErr { + t.Fatalf("MergeCatalogPluginSpecs() error = %v, wantErr %v", err, tt.wantErr) + } + if tt.wantErr { + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Fatalf("MergeCatalogPluginSpecs() = %#v, want %#v", got, tt.want) + } + }) + } +} + +func TestGetCatalogPlugins_WithWrappedState(t *testing.T) { + state := &pipeline.State{ + CLIContext: &fakeCLIContext{ + stringSliceValues: map[string][]string{ + "bundle-catalog-plugins": {"grafana-clock-panel:1.3.1,grafana-clock-panel:1.3.1"}, + }, + }, + } + + wrapped := pipeline.StateWithLogger( + slog.New(slog.NewTextHandler(io.Discard, nil)), + state, + ) + + got, err := GetCatalogPlugins(context.Background(), wrapped) + if err != nil { + t.Fatalf("GetCatalogPlugins() error = %v", err) + } + + want := []CatalogPluginSpec{ + {ID: "grafana-clock-panel", Version: "1.3.1"}, + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("GetCatalogPlugins() = %#v, want %#v", got, want) + } +} + +type fakeCLIContext struct { + boolValues map[string]bool + stringValues map[string]string + stringSliceValues map[string][]string + int64Values map[string]int64 +} + +func (f *fakeCLIContext) Bool(name string) bool { + if f.boolValues == nil { + return false + } + return f.boolValues[name] +} + +func (f *fakeCLIContext) String(name string) string { + if f.stringValues == nil { + return "" + } + return f.stringValues[name] +} + +func (f *fakeCLIContext) Set(name, value string) error { + if f.stringValues == nil { + f.stringValues = map[string]string{} + } + f.stringValues[name] = value + return nil +} + +func (f *fakeCLIContext) StringSlice(name string) []string { + if f.stringSliceValues == nil { + return nil + } + return f.stringSliceValues[name] +} + +func (f *fakeCLIContext) Path(name string) string { + return f.String(name) +} + +func (f *fakeCLIContext) Int64(name string) int64 { + if f.int64Values == nil { + return 0 + } + return f.int64Values[name] +} diff --git a/pkg/build/daggerbuild/artifacts/package_targz.go b/pkg/build/daggerbuild/artifacts/package_targz.go index d16e97d21bf..f9cd3d0a5fc 100644 --- a/pkg/build/daggerbuild/artifacts/package_targz.go +++ b/pkg/build/daggerbuild/artifacts/package_targz.go @@ -30,6 +30,9 @@ var ( arguments.GoVersion, arguments.ViceroyVersion, arguments.YarnCacheDirectory, + + // Optional catalog plugins to bundle + arguments.CatalogPlugins, } TargzFlags = flags.JoinFlags( flags.StdPackageFlags(), @@ -56,6 +59,7 @@ type Tarball struct { Backend *pipeline.Artifact Frontend *pipeline.Artifact BundledPlugins *pipeline.Artifact + CatalogPlugins *pipeline.Artifact // Optional: plugins downloaded from grafana.com catalog } func NewTarballFromString(ctx context.Context, log *slog.Logger, artifact string, state pipeline.StateHandler) (*pipeline.Artifact, error) { @@ -124,7 +128,14 @@ func NewTarballFromString(ctx context.Context, log *slog.Logger, artifact string return nil, err } cgoEnabled := !cgoDisabled - return NewTarball(ctx, log, artifact, p.Distribution, p.Enterprise, p.Name, p.Version, p.BuildID, src, yarnCache, goModCache, goBuildCache, static, wireTag, tags, goVersion, viceroyVersion, experiments, cgoEnabled) + + // Get catalog plugins (optional) + catalogPlugins, err := arguments.GetCatalogPlugins(ctx, state) + if err != nil { + return nil, err + } + + return NewTarball(ctx, log, artifact, p.Distribution, p.Enterprise, p.Name, p.Version, p.BuildID, src, yarnCache, goModCache, goBuildCache, static, wireTag, tags, goVersion, viceroyVersion, experiments, cgoEnabled, catalogPlugins) } // NewTarball returns a properly initialized Tarball artifact. @@ -149,6 +160,7 @@ func NewTarball( viceroyVersion string, experiments []string, cgoEnabled bool, + catalogPlugins []arguments.CatalogPluginSpec, ) (*pipeline.Artifact, error) { backendArtifact, err := NewBackend(ctx, log, artifact, &NewBackendOpts{ Name: name, @@ -179,6 +191,16 @@ func NewTarball( return nil, err } + // Create catalog plugins artifact if plugins were specified + var catalogPluginsArtifact *pipeline.Artifact + if len(catalogPlugins) > 0 { + log.Info("Creating catalog plugins artifact", "plugins", len(catalogPlugins)) + catalogPluginsArtifact, err = NewCatalogPlugins(ctx, log, artifact, catalogPlugins, distro, version) + if err != nil { + return nil, err + } + } + tarball := &Tarball{ Name: name, Distribution: distro, @@ -192,6 +214,7 @@ func NewTarball( Backend: backendArtifact, Frontend: frontendArtifact, BundledPlugins: bundledPluginsArtifact, + CatalogPlugins: catalogPluginsArtifact, } return pipeline.ArtifactWithLogging(ctx, log, &pipeline.Artifact{ @@ -264,6 +287,14 @@ func (t *Tarball) BuildFile(ctx context.Context, b *dagger.Container, opts *pipe targz.NewMappedDir("plugins-bundled", pluginsDir), } + if t.CatalogPlugins != nil { + catalogDir, err := opts.Store.Directory(ctx, t.CatalogPlugins) + if err != nil { + return nil, err + } + directories = append(directories, targz.NewMappedDir("data/plugins-bundled", catalogDir)) + } + root := fmt.Sprintf("grafana-%s", version) return targz.Build( @@ -308,11 +339,17 @@ func (t *Tarball) VerifyDirectory(ctx context.Context, client *dagger.Client, di } func (t *Tarball) Dependencies(ctx context.Context) ([]*pipeline.Artifact, error) { - return []*pipeline.Artifact{ + deps := []*pipeline.Artifact{ t.Backend, t.Frontend, t.BundledPlugins, - }, nil + } + + if t.CatalogPlugins != nil { + deps = append(deps, t.CatalogPlugins) + } + + return deps, nil } func (t *Tarball) Filename(ctx context.Context) (string, error) { diff --git a/pkg/build/daggerbuild/artifacts/plugins_catalog.go b/pkg/build/daggerbuild/artifacts/plugins_catalog.go new file mode 100644 index 00000000000..fb60b89b9d3 --- /dev/null +++ b/pkg/build/daggerbuild/artifacts/plugins_catalog.go @@ -0,0 +1,175 @@ +package artifacts + +import ( + "context" + "fmt" + "log/slog" + "path" + "strings" + + "dagger.io/dagger" + "github.com/grafana/grafana/pkg/build/daggerbuild/arguments" + "github.com/grafana/grafana/pkg/build/daggerbuild/backend" + "github.com/grafana/grafana/pkg/build/daggerbuild/flags" + "github.com/grafana/grafana/pkg/build/daggerbuild/pipeline" + "github.com/grafana/grafana/pkg/build/daggerbuild/plugins" +) + +var ( + CatalogPluginsFlags = flags.JoinFlags(flags.PackageNameFlags, flags.DistroFlags()) + CatalogPluginsArguments = []pipeline.Argument{ + arguments.CatalogPlugins, + arguments.Version, // Used for compatibility headers when downloading plugins + } +) + +var CatalogPluginsInitializer = Initializer{ + InitializerFunc: NewCatalogPluginsFromString, + Arguments: CatalogPluginsArguments, +} + +// CatalogPlugins downloads plugins from the Grafana catalog (grafana.com). +type CatalogPlugins struct { + // ResolvedPlugins contains plugins with resolved versions (after calling ResolvePluginVersions) + ResolvedPlugins []plugins.ResolvedPlugin + Distribution backend.Distribution + GrafanaVersion string // Optional: used for API compatibility headers +} + +// Dependencies returns nil as catalog plugins have no dependencies. +func (c *CatalogPlugins) Dependencies(ctx context.Context) ([]*pipeline.Artifact, error) { + return nil, nil +} + +// Builder creates the container that will download the plugins. +func (c *CatalogPlugins) Builder(ctx context.Context, opts *pipeline.ArtifactContainerOpts) (*dagger.Container, error) { + return opts.Client.Container().From(plugins.AlpineImage). + WithExec([]string{"apk", "add", "--no-cache", "curl", "unzip"}), nil +} + +// BuildFile is not implemented as CatalogPlugins returns a directory. +func (c *CatalogPlugins) BuildFile(ctx context.Context, builder *dagger.Container, opts *pipeline.ArtifactContainerOpts) (*dagger.File, error) { + panic("not implemented") // CatalogPlugins doesn't return a file +} + +// BuildDir downloads the plugins and returns a directory containing them. +func (c *CatalogPlugins) BuildDir(ctx context.Context, builder *dagger.Container, opts *pipeline.ArtifactContainerOpts) (*dagger.Directory, error) { + return plugins.DownloadPlugins(opts.Client, &plugins.DownloadOpts{ + Distribution: c.Distribution, + GrafanaVersion: c.GrafanaVersion, + }, c.ResolvedPlugins), nil +} + +// Publisher is not implemented. +func (c *CatalogPlugins) Publisher(ctx context.Context, opts *pipeline.ArtifactContainerOpts) (*dagger.Container, error) { + return nil, nil +} + +// PublishFile is not implemented. +func (c *CatalogPlugins) PublishFile(ctx context.Context, opts *pipeline.ArtifactPublishFileOpts) error { + panic("not implemented") +} + +// PublishDir is not implemented. +func (c *CatalogPlugins) PublishDir(ctx context.Context, opts *pipeline.ArtifactPublishDirOpts) error { + return nil +} + +// VerifyFile is not implemented as CatalogPlugins returns a directory. +func (c *CatalogPlugins) VerifyFile(ctx context.Context, client *dagger.Client, file *dagger.File) error { + return nil +} + +// VerifyDirectory verifies the downloaded plugins directory. +func (c *CatalogPlugins) VerifyDirectory(ctx context.Context, client *dagger.Client, dir *dagger.Directory) error { + entries, err := dir.Entries(ctx) + if err != nil { + return fmt.Errorf("failed to list plugin directory entries: %w", err) + } + entrySet := make(map[string]struct{}, len(entries)) + for _, entry := range entries { + entrySet[entry] = struct{}{} + } + + // Verify that each expected plugin directory exists + for _, plugin := range c.ResolvedPlugins { + if _, ok := entrySet[plugin.ID]; !ok { + return fmt.Errorf("plugin %s not found in downloaded plugins", plugin.ID) + } + } + return nil +} + +// String returns the name of this artifact handler. +func (c *CatalogPlugins) String() string { + return "catalog-plugins" +} + +// Filename returns a deterministic path for caching purposes. +func (c *CatalogPlugins) Filename(ctx context.Context) (string, error) { + // Create a unique filename based on plugins and distribution + pluginIDs := make([]string, 0, len(c.ResolvedPlugins)) + for _, p := range c.ResolvedPlugins { + pluginIDs = append(pluginIDs, fmt.Sprintf("%s-%s", p.ID, p.Version)) + } + + os, arch := backend.OSAndArch(c.Distribution) + return path.Join("bin", "catalog-plugins", os, arch, strings.Join(pluginIDs, "_")), nil +} + +// NewCatalogPluginsFromString creates a CatalogPlugins artifact from an artifact string. +func NewCatalogPluginsFromString(ctx context.Context, log *slog.Logger, artifact string, state pipeline.StateHandler) (*pipeline.Artifact, error) { + options, err := pipeline.ParseFlags(artifact, CatalogPluginsFlags) + if err != nil { + return nil, err + } + + distro, err := options.String(flags.Distribution) + if err != nil { + return nil, err + } + + pluginSpecs, err := arguments.GetCatalogPlugins(ctx, state) + if err != nil { + return nil, err + } + + // Get Grafana version for API compatibility headers (optional, may fail if not available) + version, _ := state.String(ctx, arguments.Version) + + if len(pluginSpecs) == 0 { + log.Info("No catalog plugins specified, returning empty artifact") + } else { + log.Info("Creating catalog plugins artifact", "plugins", len(pluginSpecs), "distribution", distro, "grafanaVersion", version) + } + + return NewCatalogPlugins(ctx, log, artifact, pluginSpecs, backend.Distribution(distro), version) +} + +// NewCatalogPlugins creates a new CatalogPlugins artifact. +// This resolves plugin versions for plugins that don't have a specific version specified. +func NewCatalogPlugins( + ctx context.Context, + log *slog.Logger, + artifact string, + pluginSpecs []arguments.CatalogPluginSpec, + distro backend.Distribution, + grafanaVersion string, +) (*pipeline.Artifact, error) { + // Resolve versions for plugins that don't have a specific version specified + resolvedPlugins, err := plugins.ResolvePluginVersions(ctx, log, pluginSpecs, grafanaVersion, distro) + if err != nil { + return nil, fmt.Errorf("failed to resolve plugin versions: %w", err) + } + + return pipeline.ArtifactWithLogging(ctx, log, &pipeline.Artifact{ + ArtifactString: artifact, + Type: pipeline.ArtifactTypeDirectory, + Flags: CatalogPluginsFlags, + Handler: &CatalogPlugins{ + ResolvedPlugins: resolvedPlugins, + Distribution: distro, + GrafanaVersion: grafanaVersion, + }, + }) +} diff --git a/pkg/build/daggerbuild/cmd/artifacts.go b/pkg/build/daggerbuild/cmd/artifacts.go index 57a04f5652a..475ade65bc7 100644 --- a/pkg/build/daggerbuild/cmd/artifacts.go +++ b/pkg/build/daggerbuild/cmd/artifacts.go @@ -18,4 +18,5 @@ var Artifacts = map[string]artifacts.Initializer{ "storybook": artifacts.StorybookInitializer, "msi": artifacts.MSIInitializer, "version": artifacts.VersionInitializer, + "catalog-plugins": artifacts.CatalogPluginsInitializer, } diff --git a/pkg/build/daggerbuild/pipeline/state.go b/pkg/build/daggerbuild/pipeline/state.go index 2e7d096169b..7af167d9090 100644 --- a/pkg/build/daggerbuild/pipeline/state.go +++ b/pkg/build/daggerbuild/pipeline/state.go @@ -160,3 +160,15 @@ func (s *State) CacheVolume(ctx context.Context, arg Argument) (*dagger.CacheVol s.Data.Store(arg.Name, dir) return dir, nil } + +// UnwrapState returns the underlying *State when the handler is wrapped (for example by StateLogger). +func UnwrapState(handler StateHandler) (*State, bool) { + switch v := handler.(type) { + case *State: + return v, true + case *StateLogger: + return UnwrapState(v.Handler) + default: + return nil, false + } +} diff --git a/pkg/build/daggerbuild/plugins/downloader.go b/pkg/build/daggerbuild/plugins/downloader.go new file mode 100644 index 00000000000..6af3e8dd338 --- /dev/null +++ b/pkg/build/daggerbuild/plugins/downloader.go @@ -0,0 +1,216 @@ +package plugins + +import ( + "context" + "fmt" + "log/slog" + "strings" + + "dagger.io/dagger" + "github.com/grafana/grafana/pkg/build/daggerbuild/arguments" + "github.com/grafana/grafana/pkg/build/daggerbuild/backend" + "github.com/grafana/grafana/pkg/plugins/repo" +) + +const ( + // GrafanaComAPIURL is the base URL for grafana.com API (same as used in pkg/plugins/repo) + GrafanaComAPIURL = "https://grafana.com/api/plugins" + // AlpineImage is the Alpine image used for downloading plugins (includes curl and unzip) + AlpineImage = "alpine/curl" + // ExtractPluginScriptPath is the script path used for deterministic plugin extraction. + ExtractPluginScriptPath = "/usr/local/bin/extract-plugin" +) + +const extractPluginScript = `#!/bin/sh +set -eu + +extract_dir="$1" +plugin_dir="$2" + +mkdir -p "$plugin_dir" + +dir_count="$(find "$extract_dir" -mindepth 1 -maxdepth 1 -type d | wc -l | tr -d ' ')" +file_count="$(find "$extract_dir" -mindepth 1 -maxdepth 1 ! -type d | wc -l | tr -d ' ')" + +if [ "$dir_count" -eq 1 ] && [ "$file_count" -eq 0 ]; then + root_dir="$(find "$extract_dir" -mindepth 1 -maxdepth 1 -type d | head -n 1)" + cp -a "$root_dir"/. "$plugin_dir"/ +else + cp -a "$extract_dir"/. "$plugin_dir"/ +fi +` + +// DownloadOpts contains options for downloading plugins. +type DownloadOpts struct { + // Plugins is the list of plugins to download + Plugins []arguments.CatalogPluginSpec + // Distribution is the target OS/arch for the plugins + Distribution backend.Distribution + // GrafanaVersion is the target Grafana version for compatibility checking (optional) + GrafanaVersion string +} + +// ResolvedPlugin contains a plugin with resolved version information. +type ResolvedPlugin struct { + ID string + Version string + Checksum string + URL string +} + +// ResolvePluginVersions resolves versions for plugins that don't have a specific version specified. +// For plugins without a version, it queries grafana.com API to find the latest compatible version. +// This happens in Go code before the Dagger container starts. +func ResolvePluginVersions(ctx context.Context, log *slog.Logger, plugins []arguments.CatalogPluginSpec, grafanaVersion string, distro backend.Distribution) ([]ResolvedPlugin, error) { + if len(plugins) == 0 { + return nil, nil + } + + os, arch := backend.OSAndArch(distro) + repoManager := repo.NewManager(repo.ManagerCfg{ + BaseURL: GrafanaComAPIURL, + }) + + resolved := make([]ResolvedPlugin, 0, len(plugins)) + + for _, plugin := range plugins { + if plugin.Version != "" { + // Version is specified, use it directly + log.Info("Using specified plugin version", "plugin", plugin.ID, "version", plugin.Version) + resolved = append(resolved, ResolvedPlugin{ + ID: plugin.ID, + Version: plugin.Version, + Checksum: plugin.Checksum, + URL: BuildPluginDownloadURL(plugin.ID, plugin.Version), + }) + continue + } + + // Version not specified, resolve to latest compatible + log.Info("Resolving latest compatible version", "plugin", plugin.ID, "grafanaVersion", grafanaVersion, "os", os, "arch", arch) + + compatOpts := NewCompatOpts(grafanaVersion, os, arch) + archiveInfo, err := repoManager.GetPluginArchiveInfo(ctx, plugin.ID, "", compatOpts) + if err != nil { + return nil, fmt.Errorf("failed to resolve version for plugin %s: %w", plugin.ID, err) + } + + log.Info("Resolved plugin version", "plugin", plugin.ID, "version", archiveInfo.Version, "checksum", archiveInfo.Checksum) + + checksum := plugin.Checksum + if checksum == "" { + checksum = archiveInfo.Checksum + } + + resolved = append(resolved, ResolvedPlugin{ + ID: plugin.ID, + Version: archiveInfo.Version, + Checksum: checksum, + URL: archiveInfo.URL, + }) + } + + return resolved, nil +} + +// DownloadPlugins creates a Dagger container that downloads the specified plugins +// and returns a directory containing the extracted plugins. +// This reuses URL construction logic compatible with pkg/plugins/repo. +func DownloadPlugins(client *dagger.Client, opts *DownloadOpts, resolvedPlugins []ResolvedPlugin) *dagger.Directory { + if len(resolvedPlugins) == 0 { + // Return an empty directory if no plugins specified + return client.Directory() + } + + os, arch := backend.OSAndArch(opts.Distribution) + + container := client.Container(). + From(AlpineImage). + WithNewFile(ExtractPluginScriptPath, extractPluginScript, dagger.ContainerWithNewFileOpts{ + Permissions: 0o755, + }). + WithWorkdir("/plugins") + + for idx, plugin := range resolvedPlugins { + zipFile := fmt.Sprintf("/tmp/plugin-%d.zip", idx) + extractDir := fmt.Sprintf("/tmp/extract-%d", idx) + pluginDir := fmt.Sprintf("/plugins/%s", plugin.ID) + + // Build curl command with proper headers (matching pkg/plugins/repo/client.go) + // These headers help the API return the correct platform-specific plugin archive + curlCmd := []string{ + "curl", "-fSL", + "-H", fmt.Sprintf("grafana-os: %s", os), + "-H", fmt.Sprintf("grafana-arch: %s", arch), + "-H", "User-Agent: grafana-build", + } + + // Add Grafana version header if specified (for compatibility selection) + if opts.GrafanaVersion != "" { + curlCmd = append(curlCmd, "-H", fmt.Sprintf("grafana-version: %s", opts.GrafanaVersion)) + } + + curlCmd = append(curlCmd, "-o", zipFile, plugin.URL) + + container = container. + WithExec([]string{"rm", "-rf", pluginDir, extractDir}). + WithExec([]string{"mkdir", "-p", pluginDir, extractDir}) + + // Download the plugin zip + container = container.WithExec(curlCmd) + + // Verify checksum if provided + if plugin.Checksum != "" { + checksum := strings.TrimPrefix(plugin.Checksum, "sha256:") + container = container.WithExec([]string{ + "/bin/sh", "-c", + fmt.Sprintf(`echo "%s %s" | sha256sum -c -`, checksum, zipFile), + }) + } + + // Create plugin directory and extract + // This mirrors the extraction logic in pkg/plugins/storage/fs.go + // but adapted for execution in a container. + container = container. + WithExec([]string{"unzip", "-q", "-o", zipFile, "-d", extractDir}). + WithExec([]string{ExtractPluginScriptPath, extractDir, pluginDir}). + WithExec([]string{"rm", "-rf", extractDir, zipFile}) + } + + return container.Directory("/plugins") +} + +// BuildPluginDownloadURL constructs the URL for downloading a plugin from grafana.com. +// This uses the same URL format as pkg/plugins/repo/service.go:downloadURL +func BuildPluginDownloadURL(pluginID, version string) string { + return fmt.Sprintf("%s/%s/versions/%s/download", GrafanaComAPIURL, pluginID, version) +} + +// NewCompatOpts creates repo.CompatOpts for API requests. +// This allows reusing the version selection logic from pkg/plugins/repo if needed. +func NewCompatOpts(grafanaVersion, os, arch string) repo.CompatOpts { + if grafanaVersion != "" { + return repo.NewCompatOpts(grafanaVersion, os, arch) + } + return repo.NewSystemCompatOpts(os, arch) +} + +// PluginDownloadInfo contains information about a plugin download for logging. +type PluginDownloadInfo struct { + ID string + Version string + URL string +} + +// GetPluginDownloadInfos returns download information for the specified plugins. +func GetPluginDownloadInfos(resolved []ResolvedPlugin) []PluginDownloadInfo { + infos := make([]PluginDownloadInfo, len(resolved)) + for i, plugin := range resolved { + infos[i] = PluginDownloadInfo{ + ID: plugin.ID, + Version: plugin.Version, + URL: plugin.URL, + } + } + return infos +}