From 043f2251378fc2b6774402556528866d1ba1353b Mon Sep 17 00:00:00 2001 From: "Philip K. Warren" Date: Thu, 16 Apr 2026 15:17:00 -0500 Subject: [PATCH 1/3] Create a 'make package' makefile target Add a `make package` makefile target which packages up a plugin in a .zip file similar to the official release. Refactor the release command to move the .zip file creation to a package shared by the release and the new package command. --- .golangci.yml | 8 +++ Makefile | 9 ++- internal/cmd/package/main.go | 81 ++++++++++++++++++++++++ internal/cmd/release/main.go | 100 +++++------------------------ internal/docker/build.go | 12 +++- internal/pluginzip/pluginzip.go | 108 ++++++++++++++++++++++++++++++++ 6 files changed, 229 insertions(+), 89 deletions(-) create mode 100644 internal/cmd/package/main.go create mode 100644 internal/pluginzip/pluginzip.go diff --git a/.golangci.yml b/.golangci.yml index 6e60fcd33..235a4ec58 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -66,6 +66,14 @@ linters: - linters: - err113 text: do not define dynamic errors + # G703: Path traversal via taint analysis (very noisy) + - linters: + - gosec + text: G703 + # G706: Log injection via taint analysis (very noisy) + - linters: + - gosec + text: G706 # dupword reports several errors in .proto test fixtures # gosec reports a few minor issues in tests # prealloc is not important in tests diff --git a/Makefile b/Makefile index 298ddd48d..6ab1dc9f4 100644 --- a/Makefile +++ b/Makefile @@ -8,8 +8,10 @@ DOCKER_BUILD_EXTRA_ARGS ?= DOCKER_BUILDER := bufbuild-plugins DOCKER_CACHE_DIR ?= $(TMP)/dockercache GO ?= go -GOLANGCI_LINT_VERSION ?= v2.9.0 +GOLANGCI_LINT_VERSION ?= v2.11.4 GOLANGCI_LINT := $(TMP)/golangci-lint-$(GOLANGCI_LINT_VERSION) +# OUTPUT_DIR is the directory where plugin zip files are written by `make package`. +OUTPUT_DIR ?= $(TMP)/plugins GO_TEST_FLAGS ?= -race -count=1 @@ -49,7 +51,6 @@ lintfix: $(GOLANGCI_LINT) $(GOLANGCI_LINT) run --timeout=5m --fix $(GOLANGCI_LINT) fmt - $(GOLANGCI_LINT): GOBIN=$(abspath $(TMP)) $(GO) install -ldflags="-s -w" github.com/golangci/golangci-lint/v2/cmd/golangci-lint@$(GOLANGCI_LINT_VERSION) ln -sf $(abspath $(TMP))/golangci-lint $@ @@ -77,6 +78,10 @@ push: build $(BUF) beta registry plugin push $${plugin_dir} $(BUF_PLUGIN_PUSH_ARGS) --image $(DOCKER_ORG)/plugins-$${PLUGIN_OWNER}-$${PLUGIN_NAME}:$${PLUGIN_VERSION} || exit 1; \ done +.PHONY: package +package: build + $(GO) run ./internal/cmd/package --dir . --org "$(DOCKER_ORG)" --output-dir "$(OUTPUT_DIR)" + .PHONY: clean clean: rm -rf $(TMP) diff --git a/internal/cmd/package/main.go b/internal/cmd/package/main.go new file mode 100644 index 000000000..09f6732ce --- /dev/null +++ b/internal/cmd/package/main.go @@ -0,0 +1,81 @@ +// Package main implements the "package" helper: for each plugin selected via +// the PLUGINS env var, it produces a distributable zip archive at +// /--.zip. The plugin's Docker image must +// already be available to the local Docker daemon (typically via "make build"). +package main + +import ( + "context" + "fmt" + "log/slog" + "os" + + "buf.build/go/app/appcmd" + "buf.build/go/app/appext" + "github.com/spf13/pflag" + + "github.com/bufbuild/plugins/internal/docker" + "github.com/bufbuild/plugins/internal/plugin" + "github.com/bufbuild/plugins/internal/pluginzip" +) + +func main() { + appcmd.Main(context.Background(), newRootCommand("package")) +} + +func newRootCommand(name string) *appcmd.Command { + builder := appext.NewBuilder(name) + f := &flags{} + return &appcmd.Command{ + Use: name, + Short: "Creates distributable plugin zips from locally-built Docker images.", + Args: appcmd.NoArgs, + Run: builder.NewRunFunc(func(ctx context.Context, container appext.Container) error { + return run(ctx, container.Logger(), f) + }), + BindFlags: f.Bind, + BindPersistentFlags: builder.BindRoot, + } +} + +type flags struct { + pluginsDir string + dockerOrg string + outputDir string +} + +func (f *flags) Bind(flagSet *pflag.FlagSet) { + flagSet.StringVar(&f.pluginsDir, "dir", ".", "directory path to plugins") + flagSet.StringVar(&f.dockerOrg, "org", "bufbuild", "Docker organization used to name locally-built images") + flagSet.StringVar(&f.outputDir, "output-dir", "", "directory to write plugin zip files") + _ = appcmd.MarkFlagRequired(flagSet, "output-dir") +} + +func run(ctx context.Context, logger *slog.Logger, f *flags) error { + if err := os.MkdirAll(f.outputDir, 0755); err != nil { + return fmt.Errorf("create output dir %q: %w", f.outputDir, err) + } + plugins, err := plugin.FindAll(f.pluginsDir) + if err != nil { + return fmt.Errorf("find plugins: %w", err) + } + selected, err := plugin.FilterByPluginsEnv(plugins, os.Getenv("PLUGINS")) + if err != nil { + return fmt.Errorf("filter plugins by PLUGINS env var: %w", err) + } + if len(selected) == 0 { + logger.InfoContext(ctx, "no plugins selected") + return nil + } + for _, p := range selected { + imageRef := docker.ImageName(p, f.dockerOrg) + logger.InfoContext(ctx, "packaging plugin", + slog.String("plugin", p.String()), + slog.String("image", imageRef), + ) + if _, err := pluginzip.Create(ctx, logger, p, imageRef, f.outputDir); err != nil { + return fmt.Errorf("package %s: %w", p, err) + } + } + return nil +} diff --git a/internal/cmd/release/main.go b/internal/cmd/release/main.go index 177705599..de23a294a 100644 --- a/internal/cmd/release/main.go +++ b/internal/cmd/release/main.go @@ -1,14 +1,11 @@ package main import ( - "archive/zip" "cmp" - "compress/flate" "context" "encoding/json" "errors" "fmt" - "io" "io/fs" "log/slog" "os" @@ -30,6 +27,7 @@ import ( "golang.org/x/mod/semver" "github.com/bufbuild/plugins/internal/plugin" + "github.com/bufbuild/plugins/internal/pluginzip" "github.com/bufbuild/plugins/internal/release" ) @@ -319,7 +317,7 @@ func sortPluginsByNameVersion(plugins []release.PluginRelease) { } func (c *command) createRelease(ctx context.Context, client *release.Client, releaseName string, plugins []release.PluginRelease, tmpDir string, privateKey minisign.PrivateKey) error { - releaseBody, err := c.createReleaseBody(releaseName, plugins, privateKey) //nolint:staticcheck // SA4006 false positive with Go 1.26 new(value) + releaseBody, err := c.createReleaseBody(releaseName, plugins, privateKey) if err != nil { return err } @@ -366,7 +364,7 @@ func (c *command) createReleaseBody(name string, plugins []release.PluginRelease pluginsByStatus[p.Status] = append(pluginsByStatus[p.Status], p) } - sb.WriteString(fmt.Sprintf("# Buf Remote Plugins Release %s\n\n", name)) + fmt.Fprintf(&sb, "# Buf Remote Plugins Release %s\n\n", name) if newPlugins := pluginsByStatus[release.StatusNew]; len(newPlugins) > 0 { sb.WriteString(`## New Plugins @@ -375,7 +373,7 @@ func (c *command) createReleaseBody(name string, plugins []release.PluginRelease |--------|---------|------| `) for _, p := range newPlugins { - sb.WriteString(fmt.Sprintf("| %s | %s | [Download](%s) |\n", p.PluginName, p.PluginVersion, p.URL)) + fmt.Fprintf(&sb, "| %s | %s | [Download](%s) |\n", p.PluginName, p.PluginVersion, p.URL) } sb.WriteString("\n") } @@ -387,14 +385,14 @@ func (c *command) createReleaseBody(name string, plugins []release.PluginRelease |--------|---------|------| `) for _, p := range updatedPlugins { - sb.WriteString(fmt.Sprintf("| %s | %s | [Download](%s) |\n", p.PluginName, p.PluginVersion, p.URL)) + fmt.Fprintf(&sb, "| %s | %s | [Download](%s) |\n", p.PluginName, p.PluginVersion, p.URL) } sb.WriteString("\n") } if existingPlugins := pluginsByStatus[release.StatusExisting]; len(existingPlugins) > 0 { sb.WriteString("## Previously Released Plugins\n\n") - sb.WriteString(fmt.Sprintf("A complete list of previously released plugins can be found in the [plugin-releases.json](%s) file.\n", c.pluginReleasesURL(name))) + fmt.Fprintf(&sb, "A complete list of previously released plugins can be found in the [plugin-releases.json](%s) file.\n", c.pluginReleasesURL(name)) } if !privateKey.Equal(minisign.PrivateKey{}) { @@ -404,12 +402,12 @@ func (c *command) createReleaseBody(name string, plugins []release.PluginRelease } sb.WriteString("## Verifying a release\n\n") sb.WriteString("Releases are signed using our [minisign](https://github.com/jedisct1/minisign) public key:\n\n") - sb.WriteString(fmt.Sprintf("```\n%s\n```\n\n", publicKey.String())) + fmt.Fprintf(&sb, "```\n%s\n```\n\n", publicKey.String()) sb.WriteString("The release assets can be verified using this command (assuming that minisign is installed):\n\n") releasesFile := fmt.Sprintf("https://github.com/%s/%s/releases/download/%s/%s", c.githubReleaseOwner, release.GithubRepoPlugins, name, release.PluginReleasesFile) - sb.WriteString(fmt.Sprintf("```\ncurl -OL %s && \\\n", releasesFile)) - sb.WriteString(fmt.Sprintf(" curl -OL %s && \\\n", releasesFile+".minisig")) - sb.WriteString(fmt.Sprintf(" minisign -Vm %s -P %s\n```\n", release.PluginReleasesFile, publicKey.String())) + fmt.Fprintf(&sb, "```\ncurl -OL %s && \\\n", releasesFile) + fmt.Fprintf(&sb, " curl -OL %s && \\\n", releasesFile+".minisig") + fmt.Fprintf(&sb, " minisign -Vm %s -P %s\n```\n", release.PluginReleasesFile, publicKey.String()) } return sb.String(), nil } @@ -442,75 +440,11 @@ func createPluginZip( if err := pullImage(ctx, logger, registryImage); err != nil { return "", err } - zipName := pluginZipName(plugin) - pluginTempDir, err := os.MkdirTemp(basedir, strings.TrimSuffix(zipName, filepath.Ext(zipName))) + zipPath, err := pluginzip.Create(ctx, logger, plugin, registryImage, basedir) if err != nil { return "", err } - defer func() { - if err := os.RemoveAll(pluginTempDir); err != nil { - logger.WarnContext(ctx, "failed to remove tmp dir", slog.String("dir", pluginTempDir), slog.Any("error", err)) - } - }() - if err := saveImageToDir(ctx, registryImage, pluginTempDir); err != nil { - return "", err - } - logger.InfoContext(ctx, "creating zip", slog.String("name", zipName)) - zipFile := filepath.Join(basedir, zipName) - zf, err := os.OpenFile(zipFile, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0644) - if err != nil { - return "", err - } - defer func() { - if err := zf.Close(); err != nil && !errors.Is(err, os.ErrClosed) { - logger.WarnContext(ctx, "failed to close zip file", slog.Any("error", err)) - } - }() - zw := zip.NewWriter(zf) - zw.RegisterCompressor(zip.Deflate, func(w io.Writer) (io.WriteCloser, error) { - return flate.NewWriter(w, flate.BestCompression) - }) - if err := addFileToZip(zw, plugin.Path); err != nil { - return "", err - } - if err := addFileToZip(zw, filepath.Join(pluginTempDir, "image.tar")); err != nil { - return "", err - } - if err := zw.Close(); err != nil { - return "", err - } - if err := zf.Close(); err != nil { - return "", err - } - digest, err := release.CalculateDigest(zipFile) - if err != nil { - return "", err - } - return digest, nil -} - -func addFileToZip(zipWriter *zip.Writer, path string) (retErr error) { - w, err := zipWriter.Create(filepath.Base(path)) - if err != nil { - return err - } - r, err := os.Open(path) - if err != nil { - return err - } - defer func() { - retErr = errors.Join(retErr, r.Close()) - }() - if _, err := io.Copy(w, r); err != nil { - return err - } - return nil -} - -func saveImageToDir(ctx context.Context, imageRef string, dir string) error { - cmd := dockerCmd(ctx, "save", imageRef, "-o", "image.tar") - cmd.Dir = dir - return cmd.Run() + return release.CalculateDigest(zipPath) } func createPluginReleases(dir string, plugins []release.PluginRelease) (retErr error) { @@ -562,8 +496,9 @@ func calculateNextRelease(now time.Time, latestRelease *github.RepositoryRelease } func (c *command) pluginDownloadURL(plugin *plugin.Plugin, releaseName string) string { - zipName := pluginZipName(plugin) - return fmt.Sprintf("https://github.com/%s/%s/releases/download/%s/%s", c.githubReleaseOwner, release.GithubRepoPlugins, releaseName, zipName) + return fmt.Sprintf("https://github.com/%s/%s/releases/download/%s/%s", + c.githubReleaseOwner, release.GithubRepoPlugins, releaseName, pluginzip.Name(plugin), + ) } func (c *command) pluginReleasesURL(releaseName string) string { @@ -576,11 +511,6 @@ func (c *command) pluginReleasesURL(releaseName string) string { ) } -func pluginZipName(plugin *plugin.Plugin) string { - identity := plugin.Identity - return fmt.Sprintf("%s-%s-%s.zip", identity.Owner(), identity.Plugin(), plugin.PluginVersion) -} - func fetchRegistryImageAndImageID(plugin *plugin.Plugin) (string, string, error) { identity := plugin.Identity imageName := fmt.Sprintf("ghcr.io/%s/plugins-%s-%s:%s", release.GithubOwnerBufbuild, identity.Owner(), identity.Plugin(), plugin.PluginVersion) diff --git a/internal/docker/build.go b/internal/docker/build.go index 3472f5a0c..c5776337e 100644 --- a/internal/docker/build.go +++ b/internal/docker/build.go @@ -2,6 +2,7 @@ package docker import ( "context" + "errors" "fmt" "io/fs" "os" @@ -22,7 +23,7 @@ func Build( imageName string, cachePath string, args []string, -) ([]byte, error) { +) (_ []byte, retErr error) { identity := plugin.Identity commonArgs := []string{ "buildx", @@ -74,12 +75,19 @@ func Build( }) cmd := exec.CommandContext(ctx, "docker", buildArgs...) // Set file modification times to bust Docker cache for local files + root, err := os.OpenRoot(filepath.Dir(plugin.Path)) + if err != nil { + return nil, err + } + defer func() { + retErr = errors.Join(retErr, root.Close()) + }() if err := filepath.WalkDir(filepath.Dir(plugin.Path), func(path string, d fs.DirEntry, err error) error { if err != nil { return err } if !d.IsDir() { - if err := os.Chtimes(path, time.Time{}, time.Now().UTC()); err != nil { + if err := root.Chtimes(path, time.Now(), time.Now()); err != nil { return fmt.Errorf("failed to set mtime for %q: %w", path, err) } } diff --git a/internal/pluginzip/pluginzip.go b/internal/pluginzip/pluginzip.go new file mode 100644 index 000000000..fd5b3ae04 --- /dev/null +++ b/internal/pluginzip/pluginzip.go @@ -0,0 +1,108 @@ +// Package pluginzip creates distributable zip archives for plugins. Each +// archive contains the plugin's buf.plugin.yaml and an image.tar produced by +// "docker save" of a locally available image. +package pluginzip + +import ( + "archive/zip" + "compress/flate" + "context" + "errors" + "fmt" + "io" + "log/slog" + "os" + "os/exec" + "path/filepath" + + "github.com/bufbuild/plugins/internal/plugin" +) + +// Name returns the zip filename for a plugin: "--.zip". +func Name(p *plugin.Plugin) string { + identity := p.Identity + return fmt.Sprintf("%s-%s-%s.zip", identity.Owner(), identity.Plugin(), p.PluginVersion) +} + +// Create writes / containing the plugin's buf.plugin.yaml +// and an image.tar produced by "docker save imageRef". The image must already +// be available to the local Docker daemon. Returns the path to the zip file. +func Create( + ctx context.Context, + logger *slog.Logger, + p *plugin.Plugin, + imageRef string, + outputDir string, +) (string, error) { + stagingDir, err := os.MkdirTemp(outputDir, ".pluginzip-") + if err != nil { + return "", fmt.Errorf("create staging dir: %w", err) + } + defer func() { + if err := os.RemoveAll(stagingDir); err != nil { + logger.WarnContext(ctx, "failed to remove staging dir", + slog.String("dir", stagingDir), + slog.Any("error", err), + ) + } + }() + imageTar := filepath.Join(stagingDir, "image.tar") + if err := saveImage(ctx, imageRef, imageTar); err != nil { + return "", fmt.Errorf("docker save %q: %w", imageRef, err) + } + zipPath := filepath.Join(outputDir, Name(p)) + logger.InfoContext(ctx, "creating zip", slog.String("path", zipPath)) + if err := writeZip(zipPath, p.Path, imageTar); err != nil { + return "", fmt.Errorf("write zip: %w", err) + } + return zipPath, nil +} + +func saveImage(ctx context.Context, imageRef, outputPath string) error { + cmd := exec.CommandContext(ctx, "docker", "save", imageRef, "-o", outputPath) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + return cmd.Run() +} + +func writeZip(zipPath, pluginYAMLPath, imageTarPath string) (retErr error) { + zf, err := os.OpenFile(zipPath, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0644) + if err != nil { + return err + } + defer func() { + retErr = errors.Join(retErr, zf.Close()) + if retErr != nil { + if removeErr := os.Remove(zipPath); removeErr != nil && !errors.Is(retErr, os.ErrNotExist) { + retErr = errors.Join(retErr, fmt.Errorf("failed to remove zip: %w", removeErr)) + } + } + }() + zw := zip.NewWriter(zf) + defer func() { + retErr = errors.Join(retErr, zw.Close()) + }() + zw.RegisterCompressor(zip.Deflate, func(w io.Writer) (io.WriteCloser, error) { + return flate.NewWriter(w, flate.BestCompression) + }) + if err := addFileToZip(zw, pluginYAMLPath); err != nil { + return err + } + return addFileToZip(zw, imageTarPath) +} + +func addFileToZip(zw *zip.Writer, path string) (retErr error) { + w, err := zw.Create(filepath.Base(path)) + if err != nil { + return err + } + r, err := os.Open(path) + if err != nil { + return err + } + defer func() { + retErr = errors.Join(retErr, r.Close()) + }() + _, err = io.Copy(w, r) + return err +} From 099ebb8e5636759fe456ba5a4708ada50045c31a Mon Sep 17 00:00:00 2001 From: "Philip K. Warren" Date: Fri, 17 Apr 2026 10:23:37 -0500 Subject: [PATCH 2/3] Update --- internal/docker/build.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/docker/build.go b/internal/docker/build.go index c5776337e..e8ea7eace 100644 --- a/internal/docker/build.go +++ b/internal/docker/build.go @@ -87,7 +87,7 @@ func Build( return err } if !d.IsDir() { - if err := root.Chtimes(path, time.Now(), time.Now()); err != nil { + if err := root.Chtimes(path, time.Now(), time.Now().UTC()); err != nil { return fmt.Errorf("failed to set mtime for %q: %w", path, err) } } From eacebf00c56eaf08217796f5201fff0aa28af0c4 Mon Sep 17 00:00:00 2001 From: "Philip K. Warren" Date: Fri, 17 Apr 2026 10:31:33 -0500 Subject: [PATCH 3/3] Fix chtimes regression --- internal/docker/build.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/docker/build.go b/internal/docker/build.go index e8ea7eace..fa0f8233c 100644 --- a/internal/docker/build.go +++ b/internal/docker/build.go @@ -82,12 +82,12 @@ func Build( defer func() { retErr = errors.Join(retErr, root.Close()) }() - if err := filepath.WalkDir(filepath.Dir(plugin.Path), func(path string, d fs.DirEntry, err error) error { + if err := fs.WalkDir(root.FS(), ".", func(path string, d fs.DirEntry, err error) error { if err != nil { return err } if !d.IsDir() { - if err := root.Chtimes(path, time.Now(), time.Now().UTC()); err != nil { + if err := root.Chtimes(path, time.Time{}, time.Now().UTC()); err != nil { return fmt.Errorf("failed to set mtime for %q: %w", path, err) } }