Mise github token (#159)

* ability to pass a github token to the mise install command

* buildkit frontend docs

Author: coffee-cup

buildkit/build.go

@@ -34,6 +34,7 @@ type BuildWithBuildkitClientOptions struct {
 	ImportCache  string
 	ExportCache  string
 	CacheKey     string
+	GitHubToken  string
 }
 
 func BuildWithBuildkitClient(appDir string, plan *plan.BuildPlan, opts BuildWithBuildkitClientOptions) error {
@@ -73,6 +74,7 @@ func BuildWithBuildkitClient(appDir string, plan *plan.BuildPlan, opts BuildWith
 		BuildPlatform: buildPlatform,
 		SecretsHash:   opts.SecretsHash,
 		CacheKey:      opts.CacheKey,
+		GitHubToken:   opts.GitHubToken,
 	})
 	if err != nil {
 		return fmt.Errorf("error converting plan to LLB: %w", err)

buildkit/build_llb/build_graph.go

@@ -12,6 +12,7 @@ import (
 	"github.com/moby/buildkit/util/system"
 	specs "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/railwayapp/railpack/buildkit/graph"
+	"github.com/railwayapp/railpack/core/generate"
 	"github.com/railwayapp/railpack/core/plan"
 )
 
@@ -22,6 +23,7 @@ type BuildGraph struct {
 	Platform   *specs.Platform
 	LocalState *llb.State
 
+	githubToken     string
 	secretsFile     *llb.State
 	usedSecretsBase *llb.State
 }
@@ -31,7 +33,7 @@ type BuildGraphOutput struct {
 	GraphEnv BuildEnvironment
 }
 
-func NewBuildGraph(plan *plan.BuildPlan, localState *llb.State, cacheStore *BuildKitCacheStore, secretsHash string, platform *specs.Platform) (*BuildGraph, error) {
+func NewBuildGraph(plan *plan.BuildPlan, localState *llb.State, cacheStore *BuildKitCacheStore, secretsHash string, platform *specs.Platform, githubToken string) (*BuildGraph, error) {
 	var secretsFile *llb.State
 	if secretsHash != "" {
 		st := llb.Scratch().File(llb.Mkfile("/secrets-hash", 0644, []byte(secretsHash)), llb.WithCustomName("[railpack] secrets hash"))
@@ -46,6 +48,7 @@ func NewBuildGraph(plan *plan.BuildPlan, localState *llb.State, cacheStore *Buil
 		Platform:   platform,
 		LocalState: localState,
 
+		githubToken:     githubToken,
 		secretsFile:     secretsFile,
 		usedSecretsBase: &usedSecretsBase,
 	}
@@ -271,7 +274,14 @@ func (g *BuildGraph) convertExecCommandToLLB(node *StepNode, cmd plan.ExecComman
 		opts = append(opts, cacheOpts...)
 	}
 
+	// If we have a GitHub token AND we are installing mise packages, we want to pass the GitHub token
+	// It is important to validate the command so that we don't pass the token to other commands
+	if g.githubToken != "" && cmd.Cmd == generate.MiseInstallCommand {
+		opts = append(opts, llb.AddEnv("GITHUB_TOKEN", g.githubToken))
+	}
+
 	s := state.Run(opts...).Root()
+
 	return s, nil
 }
 

buildkit/convert.go

@@ -15,9 +15,18 @@ import (
 
 type ConvertPlanOptions struct {
 	BuildPlatform BuildPlatform
-	SecretsHash   string
-	CacheKey      string
-	SessionID     string
+
+	// Hash of all the secrets values that can be used to invalidate the layer cache when a secret changes
+	SecretsHash string
+
+	// Unique value prepended to all cache mount keys
+	CacheKey string
+
+	// BuildKit session ID
+	SessionID string
+
+	// Token used to make authenticated API requests to GitHub to increase rate limits
+	GitHubToken string
 }
 
 const (
@@ -35,7 +44,7 @@ func ConvertPlanToLLB(plan *p.BuildPlan, opts ConvertPlanOptions) (*llb.State, *
 	)
 
 	cacheStore := build_llb.NewBuildKitCacheStore(opts.CacheKey)
-	graph, err := build_llb.NewBuildGraph(plan, &localState, cacheStore, opts.SecretsHash, &platform)
+	graph, err := build_llb.NewBuildGraph(plan, &localState, cacheStore, opts.SecretsHash, &platform, opts.GitHubToken)
 	if err != nil {
 		return nil, nil, err
 	}

buildkit/frontend.go

@@ -28,6 +28,8 @@ const (
 	secretsHash = "secrets-hash"
 
 	cacheKey = "cache-key"
+
+	githubToken = "github-token"
 )
 
 func StartFrontend() {
@@ -46,6 +48,7 @@ func Build(ctx context.Context, c client.Client) (*client.Result, error) {
 
 	cacheKey := buildArgs[cacheKey]
 	secretsHash := buildArgs[secretsHash]
+	githubToken := buildArgs[githubToken]
 
 	// TODO: Support building for multiple platforms
 	buildPlatform, err := validatePlatform(opts)
@@ -68,6 +71,7 @@ func Build(ctx context.Context, c client.Client) (*client.Result, error) {
 		SecretsHash:   secretsHash,
 		CacheKey:      cacheKey,
 		SessionID:     c.BuildOpts().SessionID,
+		GitHubToken:   githubToken,
 	})
 	if err != nil {
 		return nil, fmt.Errorf("error converting plan to LLB: %w", err)

cli/build.go

@@ -61,12 +61,12 @@ var BuildCommand = &cli.Command{
 			return nil
 		}
 
-		serializedPlan, err := json.MarshalIndent(buildResult.Plan, "", "  ")
-		if err != nil {
-			return cli.Exit(err, 1)
-		}
-
 		if cmd.Bool("show-plan") {
+			serializedPlan, err := json.MarshalIndent(buildResult.Plan, "", "  ")
+			if err != nil {
+				return cli.Exit(err, 1)
+			}
+
 			fmt.Println(string(serializedPlan))
 		}
 
@@ -91,6 +91,7 @@ var BuildCommand = &cli.Command{
 			SecretsHash:  secretsHash,
 			Secrets:      env.Variables,
 			Platform:     platform,
+			GitHubToken:  os.Getenv("GITHUB_TOKEN"),
 		})
 		if err != nil {
 			return cli.Exit(err, 1)

core/generate/mise_step_builder.go

@@ -14,6 +14,7 @@ import (
 
 const (
 	MisePackageStepName = "packages:mise"
+	MiseInstallCommand  = "sh -c 'mise trust -a && mise install'"
 )
 
 type MiseStepBuilder struct {
@@ -174,7 +175,7 @@ func (b *MiseStepBuilder) Build(p *plan.BuildPlan, options *BuildStepOptions) er
 			plan.NewFileCommand("/etc/mise/config.toml", "mise.toml", plan.FileOptions{
 				CustomName: "create mise config",
 			}),
-			plan.NewExecCommand("sh -c 'mise trust -a && mise install'", plan.ExecOptions{
+			plan.NewExecCommand(MiseInstallCommand, plan.ExecOptions{
 				CustomName: "install mise packages: " + strings.Join(pkgNames, ", "),
 			}),
 		})

core/mise/mise.go

@@ -20,8 +20,9 @@ const (
 )
 
 type Mise struct {
-	binaryPath string
-	cacheDir   string
+	binaryPath  string
+	cacheDir    string
+	githubToken string
 }
 
 const (
@@ -34,9 +35,12 @@ func New(cacheDir string) (*Mise, error) {
 		return nil, fmt.Errorf("failed to ensure mise is installed: %w", err)
 	}
 
+	githubToken := os.Getenv("GITHUB_TOKEN")
+
 	return &Mise{
-		binaryPath: binaryPath,
-		cacheDir:   cacheDir,
+		binaryPath:  binaryPath,
+		cacheDir:    cacheDir,
+		githubToken: githubToken,
 	}, nil
 }
 
@@ -114,6 +118,10 @@ func (m *Mise) runCmd(args ...string) (string, error) {
 		fmt.Sprintf("PATH=%s", os.Getenv("PATH")),
 	)
 
+	if m.githubToken != "" {
+		cmd.Env = append(cmd.Env, fmt.Sprintf("GITHUB_TOKEN=%s", m.githubToken))
+	}
+
 	if err := cmd.Run(); err != nil {
 		return "", fmt.Errorf("failed to run mise command '%s': %w\n%s\n\n%s",
 			strings.Join(append([]string{m.binaryPath}, args...), " "),

docs/astro.config.mjs

@@ -38,14 +38,6 @@ export default defineConfig({
         {
           label: "Guides",
           items: [
-            // {
-            //   label: "Building with CLI and BuildKit",
-            //   link: "/guides/building-with-cli",
-            // },
-            // {
-            //   label: "Building with a Custom Frontend",
-            //   link: "/guides/custom-frontend",
-            // },
             {
               label: "Installing Additional Packages",
               link: "/guides/installing-packages",
@@ -92,7 +84,10 @@ export default defineConfig({
         },
         {
           label: "Reference",
-          items: [{ label: "CLI Commands", link: "/reference/cli" }],
+          items: [
+            { label: "CLI Commands", link: "/reference/cli" },
+            { label: "BuildKit Frontend", link: "/reference/frontend" },
+          ],
         },
         {
           label: "Architecture",

docs/src/content/docs/reference/frontend.md

@@ -0,0 +1,130 @@
+---
+title: BuildKit Frontend Reference
+description: Complete reference for the Railpack BuildKit frontend
+---
+
+Complete reference documentation for the Railpack BuildKit frontend. For a full example of how to use the frontend for a production build, see the [running railpack in production](../guides/running-railpack-in-production) guide.
+
+## Expected
+
+To use the Railpack BuildKit frontend, you must provide a build plan file (e.g.
+`railpack-plan.json`) generated by the `railpack prepare` command. This file
+describes how your app should be built and must be accessible to the build
+process.
+
+**You must specify the path to the build plan file:**
+
+For **Docker** (with BuildKit enabled):
+
+```sh
+docker buildx build \
+  --build-arg BUILDKIT_SYNTAX="ghcr.io/railwayapp/railpack-frontend" \
+  -f /path/to/railpack-plan.json \
+  /path/to/app/to/build
+```
+
+For **BuildKit** directly:
+
+```sh
+buildctl build \
+  --local context=/path/to/app/to/build \
+  --local dockerfile=/path/to/dir/containing/railpack-plan.json \
+  --frontend=gateway.v0 \
+  --opt source=ghcr.io/railwayapp/railpack:railpack-frontend \
+  --output type=docker,name=test
+```
+
+The build plan file does not need to be in the same directory as your app, but
+you must reference it correctly with the `-f` flag (Docker) or `--local
+dockerfile` (BuildKit).
+
+## Configuration
+
+You can pass advanced options to the frontend using the `--opt` flag (for BuildKit) or as `--build-arg` (for Docker). The following options are supported:
+
+| Flag             | Description                                                                            | Default |
+| ---------------- | -------------------------------------------------------------------------------------- | ------- |
+| `--cache-key`    | Unique ID to prefix to cache keys for cache invalidation.                              |         |
+| `--secrets-hash` | Hash of all secret values, used to invalidate cache when secrets change.               |         |
+| `--github-token` | GitHub token to increase API rate limits for private repositories or package installs. |         |
+
+### Example
+
+**Docker:**
+
+```sh
+docker buildx build \
+  --build-arg BUILDKIT_SYNTAX="ghcr.io/railwayapp/railpack-frontend" \
+  --build-arg cache-key=my-key \
+  --build-arg secrets-hash=abc123 \
+  --build-arg github-token=ghp_xxx \
+  --build-arg FOO=bar \
+  -f /path/to/railpack-plan.json \
+  /path/to/app/to/build
+```
+
+**BuildKit:**
+
+```sh
+buildctl build \
+  --frontend=railwayapp/railpack \
+  --opt cache-key=my-key \
+  --opt secrets-hash=abc123 \
+  --opt github-token=ghp_xxx \
+  --opt build-arg:FOO=bar
+```
+
+## Secrets
+
+To use secrets in your build, you must:
+
+1. Pass secret names to `railpack prepare` (so they are included in the build
+   plan):
+
+```sh
+railpack prepare /dir/to/build --env STRIPE_LIVE_KEY=sk_live_asdf
+```
+
+2. Pass secret values to the build using Docker or BuildKit:
+
+**Docker:**
+
+```sh
+STRIPE_LIVE_KEY=asdfasdf docker buildx build \
+  --build-arg BUILDKIT_SYNTAX="ghcr.io/railwayapp/railpack-frontend" \
+  -f /path/to/railpack-plan.json \
+  --secret id=STRIPE_LIVE_KEY,env=STRIPE_LIVE_KEY \
+  /path/to/app/to/build
+```
+
+**BuildKit:**
+
+```sh
+buildctl build \
+  --local context=/path/to/app/to/build \
+  --local dockerfile=/path/to/dir/containing/railpack-plan.json \
+  --frontend=gateway.v0 \
+  --opt source=ghcr.io/railwayapp/railpack:railpack-frontend \
+  --secret id=STRIPE_LIVE_KEY,env=STRIPE_LIVE_KEY \
+  --output type=docker,name=test
+```
+
+## Layer Invalidation
+
+To ensure build layers are invalidated when secret values change, compute a hash
+of your secret values and pass it as a build argument:
+
+```sh
+--build-arg secrets-hash=$(echo -n "STRIPE_LIVE_KEY=sk_live_asdf" | sha256sum | awk '{print $1}')
+```
+
+This ensures the build cache is properly invalidated when secrets change.
+
+## GitHub Token
+
+If provided, the GitHub token is passed to Mise as the `GITHUB_TOKEN`
+([Docs](https://mise.jdx.dev/getting-started.html#github-api-rate-limiting)).
+This increases the rate limits when fetching info from the GitHub API.
+
+Once passed, this token will be accessible to the build context, so you should
+not pass a token that the owner of the build code should not have access to.