fix(op-deployer): avoid embedded artifacts recompilation

.circleci/config.yml

@@ -578,7 +578,7 @@ jobs:
           working_directory: packages/contracts-bedrock
       - run:
           name: Build contracts
-          command: forge build <<parameters.build_args>>
+          command: just forge-build <<parameters.build_args>>
           environment:
             FOUNDRY_PROFILE: <<parameters.profile>>
           working_directory: packages/contracts-bedrock
@@ -614,7 +614,7 @@ jobs:
           working_directory: packages/contracts-bedrock
       - run:
           name: Build Kontrol summary files
-          command: forge build ./test/kontrol/proofs
+          command: just forge-build ./test/kontrol/proofs
           working_directory: packages/contracts-bedrock
       - notify-failures-on-develop
 

op-deployer/justfile

@@ -9,8 +9,13 @@ build-contracts:
   just ../packages/contracts-bedrock/forge-build
 
 copy-contract-artifacts:
-    rm -f ./pkg/deployer/artifacts/forge-artifacts/artifacts.tgz
-    tar -cvzf ./pkg/deployer/artifacts/forge-artifacts/artifacts.tgz -C ../packages/contracts-bedrock/forge-artifacts --exclude="*.t.sol" .
+    #!/bin/bash
+    set -euo pipefail
+
+    rm -f ./pkg/deployer/artifacts/forge-artifacts/artifacts.tzst
+    go run ./pkg/deployer/artifacts/cmd/mktar \
+        -base ../packages/contracts-bedrock \
+        -out ./pkg/deployer/artifacts/forge-artifacts/artifacts.tzst
 
 _LDFLAGSSTRING := "'" + trim(
     "-X main.GitCommit=" + GITCOMMIT + " " + \

op-deployer/pkg/deployer/artifacts/cmd/mktar/main.go

@@ -0,0 +1,175 @@
+package main
+
+import (
+	"archive/tar"
+	"flag"
+	"io"
+	"log"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/klauspost/compress/zstd"
+)
+
+var (
+	baseDir = flag.String("base", "", "directory to archive")
+	outFile = flag.String("out", "", "path to output tzst")
+)
+
+// mktar creates a zstd-compressed tarball of the given base directory.
+// It excludes certain directories and files that are not needed for the
+// forge client.
+//
+// Usage: mktar -base DIR -out FILE
+//
+// Example: mktar -base ../packages/contracts-bedrock -out ./pkg/deployer/artifacts/forge-artifacts/artifacts.tzst
+//
+// The output file will be a zstd-compressed tarball of the given base directory.
+// Do not confuse this script with the ops/publish-artifacts.sh script, which is
+// used to publish the tarball to GCS.
+func main() {
+	flag.Parse()
+
+	if *baseDir == "" || *outFile == "" {
+		log.Fatalf("usage: mktar -base DIR -out FILE")
+	}
+
+	absBase, err := filepath.Abs(*baseDir)
+	if err != nil {
+		log.Fatalf("resolve base: %v", err)
+	}
+
+	info, err := os.Stat(absBase)
+	if err != nil {
+		log.Fatalf("stat base: %v", err)
+	}
+	if !info.IsDir() {
+		log.Fatalf("base must be a directory: %s", absBase)
+	}
+
+	if err := os.MkdirAll(filepath.Dir(*outFile), 0o755); err != nil {
+		log.Fatalf("create output directory: %v", err)
+	}
+
+	f, err := os.Create(*outFile)
+	if err != nil {
+		log.Fatalf("create output file: %v", err)
+	}
+	defer f.Close()
+
+	gz, err := zstd.NewWriter(f, zstd.WithEncoderLevel(zstd.SpeedBestCompression))
+	if err != nil {
+		log.Fatalf("create zstd writer: %v", err)
+	}
+	defer func() {
+		if err := gz.Close(); err != nil {
+			log.Fatalf("close zstd: %v", err)
+		}
+	}()
+
+	tw := tar.NewWriter(gz)
+	defer func() {
+		if err := tw.Close(); err != nil {
+			log.Fatalf("close tar: %v", err)
+		}
+	}()
+
+	if err := filepath.WalkDir(absBase, func(path string, d os.DirEntry, walkErr error) error {
+		if walkErr != nil {
+			return walkErr
+		}
+
+		rel, err := filepath.Rel(absBase, path)
+		if err != nil {
+			return err
+		}
+
+		if shouldExclude(rel, d) {
+			if d.IsDir() {
+				return filepath.SkipDir
+			}
+			return nil
+		}
+
+		if rel == "." {
+			return nil
+		}
+
+		info, err := d.Info()
+		if err != nil {
+			return err
+		}
+
+		hdr, err := tar.FileInfoHeader(info, linkTarget(path, info))
+		if err != nil {
+			return err
+		}
+
+		hdr.Name = filepath.ToSlash(rel)
+		hdr.ModTime = info.ModTime()
+		hdr.AccessTime = info.ModTime()
+
+		// tar-like progress output
+		log.Printf("a %s", hdr.Name)
+		if err := tw.WriteHeader(hdr); err != nil {
+			return err
+		}
+
+		if info.Mode().IsRegular() {
+			file, err := os.Open(path)
+			if err != nil {
+				return err
+			}
+			defer file.Close()
+
+			if _, err := io.Copy(tw, file); err != nil {
+				return err
+			}
+		}
+
+		return nil
+	}); err != nil {
+		log.Fatalf("walk: %v", err)
+	}
+
+	if err := tw.Flush(); err != nil {
+		log.Fatalf("flush tar: %v", err)
+	}
+
+	log.Printf("wrote %s", *outFile)
+}
+
+func shouldExclude(rel string, d os.DirEntry) bool {
+	if rel == "." {
+		return false
+	}
+
+	rel = filepath.ToSlash(rel)
+
+	if strings.HasPrefix(rel, "book/") || rel == "book" {
+		return true
+	}
+	if strings.HasPrefix(rel, "snapshots/") || rel == "snapshots" {
+		return true
+	}
+
+	if !d.IsDir() {
+		if strings.HasSuffix(d.Name(), ".t.sol") {
+			return true
+		}
+	}
+
+	return false
+}
+
+func linkTarget(path string, info os.FileInfo) string {
+	if info.Mode()&os.ModeSymlink == 0 {
+		return ""
+	}
+	target, err := os.Readlink(path)
+	if err != nil {
+		log.Fatalf("readlink %s: %v", path, err)
+	}
+	return target
+}

op-deployer/pkg/deployer/artifacts/embedded.go

@@ -5,8 +5,12 @@ import (
 	"compress/gzip"
 	"embed"
 	"fmt"
+	"io"
 	"os"
 	"path/filepath"
+	"strings"
+
+	"github.com/klauspost/compress/zstd"
 
 	"github.com/ethereum-optimism/optimism/op-chain-ops/foundry"
 	"github.com/ethereum-optimism/optimism/op-service/ioutil"
@@ -15,25 +19,110 @@ import (
 //go:embed forge-artifacts
 var embedDir embed.FS
 
-const embeddedArtifactsFile = "artifacts.tgz"
+// Primary filenames for embedded artifacts. Prefer zstd (.tzst); support legacy gzip (.tgz).
+const embeddedArtifactsZstdShort = "artifacts.tzst"
+const embeddedArtifactsGzip = "artifacts.tgz"
+
+func ExtractEmbedded(destDir string) (foundry.StatDirFs, error) {
+	var (
+		f    io.ReadCloser
+		err  error
+		comp string
+	)
+	// Prefer zstd, fall back to gzip for legacy bundles
+	if rf, openErr := embedDir.Open(filepath.Join("forge-artifacts", embeddedArtifactsZstdShort)); openErr == nil {
+		f = rf
+		comp = "zstd"
+	} else if rf, openErr2 := embedDir.Open(filepath.Join("forge-artifacts", embeddedArtifactsGzip)); openErr2 == nil {
+		f = rf
+		comp = "gzip"
+	} else {
+		return nil, fmt.Errorf("could not open embedded artifacts: tried %q, %q", embeddedArtifactsZstdShort, embeddedArtifactsGzip)
+	}
+	defer f.Close()
+
+	var reader io.ReadCloser
+	switch comp {
+	case "zstd":
+		zr, zerr := zstd.NewReader(f)
+		if zerr != nil {
+			return nil, fmt.Errorf("could not create zstd reader: %w", zerr)
+		}
+		reader = io.NopCloser(zr)
+		defer zr.Close()
+	default:
+		gzr, gerr := gzip.NewReader(f)
+		if gerr != nil {
+			return nil, fmt.Errorf("could not create gzip reader: %w", gerr)
+		}
+		reader = gzr
+		defer gzr.Close()
+	}
+
+	// Untar into a unique subdirectory to avoid collisions with pre-existing paths
+	if err := os.MkdirAll(destDir, 0o755); err != nil {
+		return nil, fmt.Errorf("failed to ensure destination dir: %w", err)
+	}
+	untarPath, err := os.MkdirTemp(destDir, "bundle-*")
+	if err != nil {
+		return nil, fmt.Errorf("failed to create temp untar dir: %w", err)
+	}
+
+	tr := tar.NewReader(reader)
+	if err := ioutil.Untar(untarPath, tr); err != nil {
+		return nil, fmt.Errorf("failed to untar embedded artifacts: %w", err)
+	}
+
+	forgeArtifactsDir := filepath.Join(untarPath, "forge-artifacts")
+	if _, err := os.Stat(forgeArtifactsDir); err != nil {
+		return nil, fmt.Errorf("forge-artifacts directory not found within embedded artifacts: %w", err)
+	}
+
+	return os.DirFS(forgeArtifactsDir).(foundry.StatDirFs), nil
+}
 
-func ExtractEmbedded(dir string) (foundry.StatDirFs, error) {
-	f, err := embedDir.Open(filepath.Join("forge-artifacts", embeddedArtifactsFile))
+func ExtractFromFile(destDir string, tarFilePath string) (foundry.StatDirFs, error) {
+	f, err := os.Open(tarFilePath)
 	if err != nil {
-		return nil, fmt.Errorf("could not open embedded artifacts: %w", err)
+		return nil, fmt.Errorf("could not open tar file: %w", err)
 	}
 	defer f.Close()
 
-	gzr, err := gzip.NewReader(f)
+	var reader io.ReadCloser
+	if strings.HasSuffix(tarFilePath, ".tar.zst") || strings.HasSuffix(tarFilePath, ".tzst") || strings.HasSuffix(tarFilePath, ".zst") {
+		zr, zerr := zstd.NewReader(f)
+		if zerr != nil {
+			return nil, fmt.Errorf("could not create zstd reader: %w", zerr)
+		}
+		reader = io.NopCloser(zr)
+		defer zr.Close()
+	} else {
+		gzr, gerr := gzip.NewReader(f)
+		if gerr != nil {
+			return nil, fmt.Errorf("could not create gzip reader: %w", gerr)
+		}
+		reader = gzr
+		defer gzr.Close()
+	}
+
+	// Untar into a unique subdirectory to avoid collisions with pre-existing paths
+	if err := os.MkdirAll(destDir, 0o755); err != nil {
+		return nil, fmt.Errorf("failed to ensure destination dir: %w", err)
+	}
+	untarPath, err := os.MkdirTemp(destDir, "bundle-*")
 	if err != nil {
-		return nil, fmt.Errorf("could not create gzip reader: %w", err)
+		return nil, fmt.Errorf("failed to create temp untar dir: %w", err)
 	}
-	defer gzr.Close()
 
-	tr := tar.NewReader(gzr)
-	if err := ioutil.Untar(dir, tr); err != nil {
+	tr := tar.NewReader(reader)
+	if err := ioutil.Untar(untarPath, tr); err != nil {
 		return nil, fmt.Errorf("failed to untar embedded artifacts: %w", err)
 	}
 
-	return os.DirFS(dir).(foundry.StatDirFs), nil
+	forgeArtifactsDir := filepath.Join(untarPath, "out")
+	if _, err := os.Stat(forgeArtifactsDir); err != nil {
+		return nil, fmt.Errorf("forge-artifacts directory not found within embedded artifacts: %w", err)
+	}
+
+	return os.DirFS(forgeArtifactsDir).(foundry.StatDirFs), nil
 }

op-deployer/pkg/deployer/forge/client.go

@@ -8,6 +8,7 @@ import (
 	"io"
 	"os"
 	"os/exec"
+	"path/filepath"
 	"regexp"
 	"strings"
 )
@@ -29,6 +30,22 @@ type Client struct {
 	Wd     string
 }
 
+func NewStandardClient(workdir string) (*Client, error) {
+	forgeBinary, err := NewStandardBinary()
+	if err != nil {
+		return nil, fmt.Errorf("failed to initialize forge binary: %w", err)
+	}
+	if err := forgeBinary.Ensure(context.Background()); err != nil {
+		return nil, fmt.Errorf("failed to ensure forge binary: %w", err)
+	}
+
+	forgeClient := NewClient(forgeBinary)
+	forgeClient.Wd = filepath.Dir(workdir)
+	fmt.Printf("Forge client working directory: %s\n", forgeClient.Wd)
+
+	return forgeClient, nil
+}
+
 func NewClient(binary Binary) *Client {
 	return &Client{
 		Binary: binary,