Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions .golangci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,14 @@ linters:
- linters:
- err113
text: do not define dynamic errors
# G703: Path traversal via taint analysis (very noisy)
- linters:
- gosec
text: G703
# G706: Log injection via taint analysis (very noisy)
- linters:
- gosec
text: G706
# dupword reports several errors in .proto test fixtures
# gosec reports a few minor issues in tests
# prealloc is not important in tests
Expand Down
9 changes: 7 additions & 2 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,10 @@ DOCKER_BUILD_EXTRA_ARGS ?=
DOCKER_BUILDER := bufbuild-plugins
DOCKER_CACHE_DIR ?= $(TMP)/dockercache
GO ?= go
GOLANGCI_LINT_VERSION ?= v2.9.0
GOLANGCI_LINT_VERSION ?= v2.11.4
GOLANGCI_LINT := $(TMP)/golangci-lint-$(GOLANGCI_LINT_VERSION)
# OUTPUT_DIR is the directory where plugin zip files are written by `make package`.
OUTPUT_DIR ?= $(TMP)/plugins

GO_TEST_FLAGS ?= -race -count=1

Expand Down Expand Up @@ -49,7 +51,6 @@ lintfix: $(GOLANGCI_LINT)
$(GOLANGCI_LINT) run --timeout=5m --fix
$(GOLANGCI_LINT) fmt


$(GOLANGCI_LINT):
GOBIN=$(abspath $(TMP)) $(GO) install -ldflags="-s -w" github.com/golangci/golangci-lint/v2/cmd/golangci-lint@$(GOLANGCI_LINT_VERSION)
ln -sf $(abspath $(TMP))/golangci-lint $@
Expand Down Expand Up @@ -77,6 +78,10 @@ push: build
$(BUF) beta registry plugin push $${plugin_dir} $(BUF_PLUGIN_PUSH_ARGS) --image $(DOCKER_ORG)/plugins-$${PLUGIN_OWNER}-$${PLUGIN_NAME}:$${PLUGIN_VERSION} || exit 1; \
done

.PHONY: package
package: build
$(GO) run ./internal/cmd/package --dir . --org "$(DOCKER_ORG)" --output-dir "$(OUTPUT_DIR)"

.PHONY: clean
clean:
rm -rf $(TMP)
81 changes: 81 additions & 0 deletions internal/cmd/package/main.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
// Package main implements the "package" helper: for each plugin selected via
// the PLUGINS env var, it produces a distributable zip archive at
// <output-dir>/<owner>-<name>-<version>.zip. The plugin's Docker image must
// already be available to the local Docker daemon (typically via "make build").
package main

import (
"context"
"fmt"
"log/slog"
"os"

"buf.build/go/app/appcmd"
"buf.build/go/app/appext"
"github.com/spf13/pflag"

"github.com/bufbuild/plugins/internal/docker"
"github.com/bufbuild/plugins/internal/plugin"
"github.com/bufbuild/plugins/internal/pluginzip"
)

func main() {
appcmd.Main(context.Background(), newRootCommand("package"))
}

func newRootCommand(name string) *appcmd.Command {
builder := appext.NewBuilder(name)
f := &flags{}
return &appcmd.Command{
Use: name,
Short: "Creates distributable plugin zips from locally-built Docker images.",
Args: appcmd.NoArgs,
Run: builder.NewRunFunc(func(ctx context.Context, container appext.Container) error {
return run(ctx, container.Logger(), f)
}),
BindFlags: f.Bind,
BindPersistentFlags: builder.BindRoot,
}
}

type flags struct {
pluginsDir string
dockerOrg string
outputDir string
}

func (f *flags) Bind(flagSet *pflag.FlagSet) {
flagSet.StringVar(&f.pluginsDir, "dir", ".", "directory path to plugins")
flagSet.StringVar(&f.dockerOrg, "org", "bufbuild", "Docker organization used to name locally-built images")
flagSet.StringVar(&f.outputDir, "output-dir", "", "directory to write plugin zip files")
_ = appcmd.MarkFlagRequired(flagSet, "output-dir")
}

func run(ctx context.Context, logger *slog.Logger, f *flags) error {
if err := os.MkdirAll(f.outputDir, 0755); err != nil {
return fmt.Errorf("create output dir %q: %w", f.outputDir, err)
}
plugins, err := plugin.FindAll(f.pluginsDir)
if err != nil {
return fmt.Errorf("find plugins: %w", err)
}
selected, err := plugin.FilterByPluginsEnv(plugins, os.Getenv("PLUGINS"))
if err != nil {
return fmt.Errorf("filter plugins by PLUGINS env var: %w", err)
}
if len(selected) == 0 {
logger.InfoContext(ctx, "no plugins selected")
return nil
}
for _, p := range selected {
imageRef := docker.ImageName(p, f.dockerOrg)
logger.InfoContext(ctx, "packaging plugin",
slog.String("plugin", p.String()),
slog.String("image", imageRef),
)
if _, err := pluginzip.Create(ctx, logger, p, imageRef, f.outputDir); err != nil {
return fmt.Errorf("package %s: %w", p, err)
}
}
return nil
}
100 changes: 15 additions & 85 deletions internal/cmd/release/main.go
Original file line number Diff line number Diff line change
@@ -1,14 +1,11 @@
package main

import (
"archive/zip"
"cmp"
"compress/flate"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"io/fs"
"log/slog"
"os"
Expand All @@ -30,6 +27,7 @@ import (
"golang.org/x/mod/semver"

"github.com/bufbuild/plugins/internal/plugin"
"github.com/bufbuild/plugins/internal/pluginzip"
"github.com/bufbuild/plugins/internal/release"
)

Expand Down Expand Up @@ -319,7 +317,7 @@ func sortPluginsByNameVersion(plugins []release.PluginRelease) {
}

func (c *command) createRelease(ctx context.Context, client *release.Client, releaseName string, plugins []release.PluginRelease, tmpDir string, privateKey minisign.PrivateKey) error {
releaseBody, err := c.createReleaseBody(releaseName, plugins, privateKey) //nolint:staticcheck // SA4006 false positive with Go 1.26 new(value)
releaseBody, err := c.createReleaseBody(releaseName, plugins, privateKey)
if err != nil {
return err
}
Expand Down Expand Up @@ -366,7 +364,7 @@ func (c *command) createReleaseBody(name string, plugins []release.PluginRelease
pluginsByStatus[p.Status] = append(pluginsByStatus[p.Status], p)
}

sb.WriteString(fmt.Sprintf("# Buf Remote Plugins Release %s\n\n", name))
fmt.Fprintf(&sb, "# Buf Remote Plugins Release %s\n\n", name)

if newPlugins := pluginsByStatus[release.StatusNew]; len(newPlugins) > 0 {
sb.WriteString(`## New Plugins
Expand All @@ -375,7 +373,7 @@ func (c *command) createReleaseBody(name string, plugins []release.PluginRelease
|--------|---------|------|
`)
for _, p := range newPlugins {
sb.WriteString(fmt.Sprintf("| %s | %s | [Download](%s) |\n", p.PluginName, p.PluginVersion, p.URL))
fmt.Fprintf(&sb, "| %s | %s | [Download](%s) |\n", p.PluginName, p.PluginVersion, p.URL)
}
sb.WriteString("\n")
}
Expand All @@ -387,14 +385,14 @@ func (c *command) createReleaseBody(name string, plugins []release.PluginRelease
|--------|---------|------|
`)
for _, p := range updatedPlugins {
sb.WriteString(fmt.Sprintf("| %s | %s | [Download](%s) |\n", p.PluginName, p.PluginVersion, p.URL))
fmt.Fprintf(&sb, "| %s | %s | [Download](%s) |\n", p.PluginName, p.PluginVersion, p.URL)
}
sb.WriteString("\n")
}

if existingPlugins := pluginsByStatus[release.StatusExisting]; len(existingPlugins) > 0 {
sb.WriteString("## Previously Released Plugins\n\n")
sb.WriteString(fmt.Sprintf("A complete list of previously released plugins can be found in the [plugin-releases.json](%s) file.\n", c.pluginReleasesURL(name)))
fmt.Fprintf(&sb, "A complete list of previously released plugins can be found in the [plugin-releases.json](%s) file.\n", c.pluginReleasesURL(name))
}

if !privateKey.Equal(minisign.PrivateKey{}) {
Expand All @@ -404,12 +402,12 @@ func (c *command) createReleaseBody(name string, plugins []release.PluginRelease
}
sb.WriteString("## Verifying a release\n\n")
sb.WriteString("Releases are signed using our [minisign](https://github.com/jedisct1/minisign) public key:\n\n")
sb.WriteString(fmt.Sprintf("```\n%s\n```\n\n", publicKey.String()))
fmt.Fprintf(&sb, "```\n%s\n```\n\n", publicKey.String())
sb.WriteString("The release assets can be verified using this command (assuming that minisign is installed):\n\n")
releasesFile := fmt.Sprintf("https://github.com/%s/%s/releases/download/%s/%s", c.githubReleaseOwner, release.GithubRepoPlugins, name, release.PluginReleasesFile)
sb.WriteString(fmt.Sprintf("```\ncurl -OL %s && \\\n", releasesFile))
sb.WriteString(fmt.Sprintf(" curl -OL %s && \\\n", releasesFile+".minisig"))
sb.WriteString(fmt.Sprintf(" minisign -Vm %s -P %s\n```\n", release.PluginReleasesFile, publicKey.String()))
fmt.Fprintf(&sb, "```\ncurl -OL %s && \\\n", releasesFile)
fmt.Fprintf(&sb, " curl -OL %s && \\\n", releasesFile+".minisig")
fmt.Fprintf(&sb, " minisign -Vm %s -P %s\n```\n", release.PluginReleasesFile, publicKey.String())
}
return sb.String(), nil
}
Expand Down Expand Up @@ -442,75 +440,11 @@ func createPluginZip(
if err := pullImage(ctx, logger, registryImage); err != nil {
return "", err
}
zipName := pluginZipName(plugin)
pluginTempDir, err := os.MkdirTemp(basedir, strings.TrimSuffix(zipName, filepath.Ext(zipName)))
zipPath, err := pluginzip.Create(ctx, logger, plugin, registryImage, basedir)
if err != nil {
return "", err
}
defer func() {
if err := os.RemoveAll(pluginTempDir); err != nil {
logger.WarnContext(ctx, "failed to remove tmp dir", slog.String("dir", pluginTempDir), slog.Any("error", err))
}
}()
if err := saveImageToDir(ctx, registryImage, pluginTempDir); err != nil {
return "", err
}
logger.InfoContext(ctx, "creating zip", slog.String("name", zipName))
zipFile := filepath.Join(basedir, zipName)
zf, err := os.OpenFile(zipFile, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0644)
if err != nil {
return "", err
}
defer func() {
if err := zf.Close(); err != nil && !errors.Is(err, os.ErrClosed) {
logger.WarnContext(ctx, "failed to close zip file", slog.Any("error", err))
}
}()
zw := zip.NewWriter(zf)
zw.RegisterCompressor(zip.Deflate, func(w io.Writer) (io.WriteCloser, error) {
return flate.NewWriter(w, flate.BestCompression)
})
if err := addFileToZip(zw, plugin.Path); err != nil {
return "", err
}
if err := addFileToZip(zw, filepath.Join(pluginTempDir, "image.tar")); err != nil {
return "", err
}
if err := zw.Close(); err != nil {
return "", err
}
if err := zf.Close(); err != nil {
return "", err
}
digest, err := release.CalculateDigest(zipFile)
if err != nil {
return "", err
}
return digest, nil
}

func addFileToZip(zipWriter *zip.Writer, path string) (retErr error) {
w, err := zipWriter.Create(filepath.Base(path))
if err != nil {
return err
}
r, err := os.Open(path)
if err != nil {
return err
}
defer func() {
retErr = errors.Join(retErr, r.Close())
}()
if _, err := io.Copy(w, r); err != nil {
return err
}
return nil
}

func saveImageToDir(ctx context.Context, imageRef string, dir string) error {
cmd := dockerCmd(ctx, "save", imageRef, "-o", "image.tar")
cmd.Dir = dir
return cmd.Run()
return release.CalculateDigest(zipPath)
}

func createPluginReleases(dir string, plugins []release.PluginRelease) (retErr error) {
Expand Down Expand Up @@ -562,8 +496,9 @@ func calculateNextRelease(now time.Time, latestRelease *github.RepositoryRelease
}

func (c *command) pluginDownloadURL(plugin *plugin.Plugin, releaseName string) string {
zipName := pluginZipName(plugin)
return fmt.Sprintf("https://github.com/%s/%s/releases/download/%s/%s", c.githubReleaseOwner, release.GithubRepoPlugins, releaseName, zipName)
return fmt.Sprintf("https://github.com/%s/%s/releases/download/%s/%s",
c.githubReleaseOwner, release.GithubRepoPlugins, releaseName, pluginzip.Name(plugin),
)
}

func (c *command) pluginReleasesURL(releaseName string) string {
Expand All @@ -576,11 +511,6 @@ func (c *command) pluginReleasesURL(releaseName string) string {
)
}

func pluginZipName(plugin *plugin.Plugin) string {
identity := plugin.Identity
return fmt.Sprintf("%s-%s-%s.zip", identity.Owner(), identity.Plugin(), plugin.PluginVersion)
}

func fetchRegistryImageAndImageID(plugin *plugin.Plugin) (string, string, error) {
identity := plugin.Identity
imageName := fmt.Sprintf("ghcr.io/%s/plugins-%s-%s:%s", release.GithubOwnerBufbuild, identity.Owner(), identity.Plugin(), plugin.PluginVersion)
Expand Down
12 changes: 10 additions & 2 deletions internal/docker/build.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package docker

import (
"context"
"errors"
"fmt"
"io/fs"
"os"
Expand All @@ -22,7 +23,7 @@ func Build(
imageName string,
cachePath string,
args []string,
) ([]byte, error) {
) (_ []byte, retErr error) {
identity := plugin.Identity
commonArgs := []string{
"buildx",
Expand Down Expand Up @@ -74,12 +75,19 @@ func Build(
})
cmd := exec.CommandContext(ctx, "docker", buildArgs...)
// Set file modification times to bust Docker cache for local files
root, err := os.OpenRoot(filepath.Dir(plugin.Path))
if err != nil {
return nil, err
}
defer func() {
retErr = errors.Join(retErr, root.Close())
}()
if err := filepath.WalkDir(filepath.Dir(plugin.Path), func(path string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
if !d.IsDir() {
if err := os.Chtimes(path, time.Time{}, time.Now().UTC()); err != nil {
if err := root.Chtimes(path, time.Now(), time.Now()); err != nil {
Comment thread
pkwarren marked this conversation as resolved.
Outdated
return fmt.Errorf("failed to set mtime for %q: %w", path, err)
}
}
Expand Down
Loading
Loading