From 3ae70269f8152930c5212c8513eaa1a7f7350011 Mon Sep 17 00:00:00 2001 From: Matthew Zipkin Date: Tue, 24 Sep 2024 13:11:53 -0400 Subject: [PATCH 01/13] scenarios: remove unecessary import logic --- resources/scenarios/ln_init.py | 7 +------ resources/scenarios/miner_std.py | 7 +------ resources/scenarios/reconnaissance.py | 7 +------ resources/scenarios/signet_miner.py | 6 +----- resources/scenarios/tx_flood.py | 8 ++------ 5 files changed, 6 insertions(+), 29 deletions(-) diff --git a/resources/scenarios/ln_init.py b/resources/scenarios/ln_init.py index 59df5e38e..391246fd0 100644 --- a/resources/scenarios/ln_init.py +++ b/resources/scenarios/ln_init.py @@ -1,12 +1,7 @@ #!/usr/bin/env python3 from time import sleep - -# The base class exists inside the commander container -try: - from commander import Commander -except ImportError: - from resources.scenarios.commander import Commander +from commander import Commander class LNInit(Commander): diff --git a/resources/scenarios/miner_std.py b/resources/scenarios/miner_std.py index 5aa368d40..764210efc 100755 --- a/resources/scenarios/miner_std.py +++ b/resources/scenarios/miner_std.py @@ -1,12 +1,7 @@ #!/usr/bin/env python3 from time import sleep - -# The base class exists inside the commander container -try: - from commander import Commander -except ImportError: - from resources.scenarios.commander import Commander +from commander import Commander class Miner: diff --git a/resources/scenarios/reconnaissance.py b/resources/scenarios/reconnaissance.py index 3fc2269e4..2d500e274 100755 --- a/resources/scenarios/reconnaissance.py +++ b/resources/scenarios/reconnaissance.py @@ -2,12 +2,7 @@ import socket -# The base class exists inside the commander container when deployed, -# but requires a relative path inside the python source code for other functions. -try: - from commander import Commander -except ImportError: - from resources.scenarios.commander import Commander +from commander import Commander # The entire Bitcoin Core test_framework directory is available as a library from test_framework.messages import MSG_TX, CInv, hash256, msg_getdata diff --git a/resources/scenarios/signet_miner.py b/resources/scenarios/signet_miner.py index 0edc635e3..7c4652240 100644 --- a/resources/scenarios/signet_miner.py +++ b/resources/scenarios/signet_miner.py @@ -11,11 +11,7 @@ # we use the authproxy from the test framework. ### -# The base class exists inside the commander container -try: - from commander import Commander -except ImportError: - from resources.scenarios.commander import Commander +from commander import Commander import json import logging diff --git a/resources/scenarios/tx_flood.py b/resources/scenarios/tx_flood.py index a4896e958..996b88de5 100755 --- a/resources/scenarios/tx_flood.py +++ b/resources/scenarios/tx_flood.py @@ -1,13 +1,9 @@ #!/usr/bin/env python3 + import threading from random import choice, randrange from time import sleep - -# The base class exists inside the commander container -try: - from commander import Commander -except ImportError: - from resources.scenarios.commander import Commander +from commander import Commander class TXFlood(Commander): From 8ac9219264eac00b877d15ee21fa5bea1b129cd1 Mon Sep 17 00:00:00 2001 From: Matthew Zipkin Date: Tue, 24 Sep 2024 14:37:13 -0400 Subject: [PATCH 02/13] scenarios: move test_framework in here --- pyproject.toml | 2 +- {src => resources/scenarios}/test_framework/__init__.py | 0 {src => resources/scenarios}/test_framework/address.py | 0 {src => resources/scenarios}/test_framework/authproxy.py | 0 {src => resources/scenarios}/test_framework/bdb.py | 0 .../scenarios}/test_framework/bip340_test_vectors.csv | 0 {src => resources/scenarios}/test_framework/blockfilter.py | 0 {src => resources/scenarios}/test_framework/blocktools.py | 0 {src => resources/scenarios}/test_framework/coverage.py | 0 {src => resources/scenarios}/test_framework/descriptors.py | 0 {src => resources/scenarios}/test_framework/ellswift.py | 0 .../scenarios}/test_framework/ellswift_decode_test_vectors.csv | 0 {src => resources/scenarios}/test_framework/key.py | 0 {src => resources/scenarios}/test_framework/messages.py | 0 {src => resources/scenarios}/test_framework/muhash.py | 0 {src => resources/scenarios}/test_framework/netutil.py | 0 {src => resources/scenarios}/test_framework/p2p.py | 0 {src => resources/scenarios}/test_framework/psbt.py | 0 {src => resources/scenarios}/test_framework/ripemd160.py | 0 {src => resources/scenarios}/test_framework/script.py | 0 {src => resources/scenarios}/test_framework/script_util.py | 0 {src => resources/scenarios}/test_framework/secp256k1.py | 0 {src => resources/scenarios}/test_framework/segwit_addr.py | 0 {src => resources/scenarios}/test_framework/siphash.py | 0 {src => resources/scenarios}/test_framework/socks5.py | 0 {src => resources/scenarios}/test_framework/test_framework.py | 0 {src => resources/scenarios}/test_framework/test_node.py | 0 {src => resources/scenarios}/test_framework/test_shell.py | 0 {src => resources/scenarios}/test_framework/util.py | 0 {src => resources/scenarios}/test_framework/wallet.py | 0 {src => resources/scenarios}/test_framework/wallet_util.py | 0 .../scenarios}/test_framework/xswiftec_inv_test_vectors.csv | 0 ruff.toml | 2 +- 33 files changed, 2 insertions(+), 2 deletions(-) rename {src => resources/scenarios}/test_framework/__init__.py (100%) rename {src => resources/scenarios}/test_framework/address.py (100%) rename {src => resources/scenarios}/test_framework/authproxy.py (100%) rename {src => resources/scenarios}/test_framework/bdb.py (100%) rename {src => resources/scenarios}/test_framework/bip340_test_vectors.csv (100%) rename {src => resources/scenarios}/test_framework/blockfilter.py (100%) rename {src => resources/scenarios}/test_framework/blocktools.py (100%) rename {src => resources/scenarios}/test_framework/coverage.py (100%) rename {src => resources/scenarios}/test_framework/descriptors.py (100%) rename {src => resources/scenarios}/test_framework/ellswift.py (100%) rename {src => resources/scenarios}/test_framework/ellswift_decode_test_vectors.csv (100%) rename {src => resources/scenarios}/test_framework/key.py (100%) rename {src => resources/scenarios}/test_framework/messages.py (100%) rename {src => resources/scenarios}/test_framework/muhash.py (100%) rename {src => resources/scenarios}/test_framework/netutil.py (100%) rename {src => resources/scenarios}/test_framework/p2p.py (100%) rename {src => resources/scenarios}/test_framework/psbt.py (100%) rename {src => resources/scenarios}/test_framework/ripemd160.py (100%) rename {src => resources/scenarios}/test_framework/script.py (100%) rename {src => resources/scenarios}/test_framework/script_util.py (100%) rename {src => resources/scenarios}/test_framework/secp256k1.py (100%) rename {src => resources/scenarios}/test_framework/segwit_addr.py (100%) rename {src => resources/scenarios}/test_framework/siphash.py (100%) rename {src => resources/scenarios}/test_framework/socks5.py (100%) rename {src => resources/scenarios}/test_framework/test_framework.py (100%) rename {src => resources/scenarios}/test_framework/test_node.py (100%) rename {src => resources/scenarios}/test_framework/test_shell.py (100%) rename {src => resources/scenarios}/test_framework/util.py (100%) rename {src => resources/scenarios}/test_framework/wallet.py (100%) rename {src => resources/scenarios}/test_framework/wallet_util.py (100%) rename {src => resources/scenarios}/test_framework/xswiftec_inv_test_vectors.csv (100%) diff --git a/pyproject.toml b/pyproject.toml index 73f2876d5..5fe46468b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,7 +51,7 @@ build-backend = "setuptools.build_meta" include-package-data = true [tool.setuptools.packages.find] -where = ["src", "."] +where = ["src", ".", "resources/scenarios"] include = ["warnet*", "test_framework*", "resources*"] [tool.setuptools.package-data] diff --git a/src/test_framework/__init__.py b/resources/scenarios/test_framework/__init__.py similarity index 100% rename from src/test_framework/__init__.py rename to resources/scenarios/test_framework/__init__.py diff --git a/src/test_framework/address.py b/resources/scenarios/test_framework/address.py similarity index 100% rename from src/test_framework/address.py rename to resources/scenarios/test_framework/address.py diff --git a/src/test_framework/authproxy.py b/resources/scenarios/test_framework/authproxy.py similarity index 100% rename from src/test_framework/authproxy.py rename to resources/scenarios/test_framework/authproxy.py diff --git a/src/test_framework/bdb.py b/resources/scenarios/test_framework/bdb.py similarity index 100% rename from src/test_framework/bdb.py rename to resources/scenarios/test_framework/bdb.py diff --git a/src/test_framework/bip340_test_vectors.csv b/resources/scenarios/test_framework/bip340_test_vectors.csv similarity index 100% rename from src/test_framework/bip340_test_vectors.csv rename to resources/scenarios/test_framework/bip340_test_vectors.csv diff --git a/src/test_framework/blockfilter.py b/resources/scenarios/test_framework/blockfilter.py similarity index 100% rename from src/test_framework/blockfilter.py rename to resources/scenarios/test_framework/blockfilter.py diff --git a/src/test_framework/blocktools.py b/resources/scenarios/test_framework/blocktools.py similarity index 100% rename from src/test_framework/blocktools.py rename to resources/scenarios/test_framework/blocktools.py diff --git a/src/test_framework/coverage.py b/resources/scenarios/test_framework/coverage.py similarity index 100% rename from src/test_framework/coverage.py rename to resources/scenarios/test_framework/coverage.py diff --git a/src/test_framework/descriptors.py b/resources/scenarios/test_framework/descriptors.py similarity index 100% rename from src/test_framework/descriptors.py rename to resources/scenarios/test_framework/descriptors.py diff --git a/src/test_framework/ellswift.py b/resources/scenarios/test_framework/ellswift.py similarity index 100% rename from src/test_framework/ellswift.py rename to resources/scenarios/test_framework/ellswift.py diff --git a/src/test_framework/ellswift_decode_test_vectors.csv b/resources/scenarios/test_framework/ellswift_decode_test_vectors.csv similarity index 100% rename from src/test_framework/ellswift_decode_test_vectors.csv rename to resources/scenarios/test_framework/ellswift_decode_test_vectors.csv diff --git a/src/test_framework/key.py b/resources/scenarios/test_framework/key.py similarity index 100% rename from src/test_framework/key.py rename to resources/scenarios/test_framework/key.py diff --git a/src/test_framework/messages.py b/resources/scenarios/test_framework/messages.py similarity index 100% rename from src/test_framework/messages.py rename to resources/scenarios/test_framework/messages.py diff --git a/src/test_framework/muhash.py b/resources/scenarios/test_framework/muhash.py similarity index 100% rename from src/test_framework/muhash.py rename to resources/scenarios/test_framework/muhash.py diff --git a/src/test_framework/netutil.py b/resources/scenarios/test_framework/netutil.py similarity index 100% rename from src/test_framework/netutil.py rename to resources/scenarios/test_framework/netutil.py diff --git a/src/test_framework/p2p.py b/resources/scenarios/test_framework/p2p.py similarity index 100% rename from src/test_framework/p2p.py rename to resources/scenarios/test_framework/p2p.py diff --git a/src/test_framework/psbt.py b/resources/scenarios/test_framework/psbt.py similarity index 100% rename from src/test_framework/psbt.py rename to resources/scenarios/test_framework/psbt.py diff --git a/src/test_framework/ripemd160.py b/resources/scenarios/test_framework/ripemd160.py similarity index 100% rename from src/test_framework/ripemd160.py rename to resources/scenarios/test_framework/ripemd160.py diff --git a/src/test_framework/script.py b/resources/scenarios/test_framework/script.py similarity index 100% rename from src/test_framework/script.py rename to resources/scenarios/test_framework/script.py diff --git a/src/test_framework/script_util.py b/resources/scenarios/test_framework/script_util.py similarity index 100% rename from src/test_framework/script_util.py rename to resources/scenarios/test_framework/script_util.py diff --git a/src/test_framework/secp256k1.py b/resources/scenarios/test_framework/secp256k1.py similarity index 100% rename from src/test_framework/secp256k1.py rename to resources/scenarios/test_framework/secp256k1.py diff --git a/src/test_framework/segwit_addr.py b/resources/scenarios/test_framework/segwit_addr.py similarity index 100% rename from src/test_framework/segwit_addr.py rename to resources/scenarios/test_framework/segwit_addr.py diff --git a/src/test_framework/siphash.py b/resources/scenarios/test_framework/siphash.py similarity index 100% rename from src/test_framework/siphash.py rename to resources/scenarios/test_framework/siphash.py diff --git a/src/test_framework/socks5.py b/resources/scenarios/test_framework/socks5.py similarity index 100% rename from src/test_framework/socks5.py rename to resources/scenarios/test_framework/socks5.py diff --git a/src/test_framework/test_framework.py b/resources/scenarios/test_framework/test_framework.py similarity index 100% rename from src/test_framework/test_framework.py rename to resources/scenarios/test_framework/test_framework.py diff --git a/src/test_framework/test_node.py b/resources/scenarios/test_framework/test_node.py similarity index 100% rename from src/test_framework/test_node.py rename to resources/scenarios/test_framework/test_node.py diff --git a/src/test_framework/test_shell.py b/resources/scenarios/test_framework/test_shell.py similarity index 100% rename from src/test_framework/test_shell.py rename to resources/scenarios/test_framework/test_shell.py diff --git a/src/test_framework/util.py b/resources/scenarios/test_framework/util.py similarity index 100% rename from src/test_framework/util.py rename to resources/scenarios/test_framework/util.py diff --git a/src/test_framework/wallet.py b/resources/scenarios/test_framework/wallet.py similarity index 100% rename from src/test_framework/wallet.py rename to resources/scenarios/test_framework/wallet.py diff --git a/src/test_framework/wallet_util.py b/resources/scenarios/test_framework/wallet_util.py similarity index 100% rename from src/test_framework/wallet_util.py rename to resources/scenarios/test_framework/wallet_util.py diff --git a/src/test_framework/xswiftec_inv_test_vectors.csv b/resources/scenarios/test_framework/xswiftec_inv_test_vectors.csv similarity index 100% rename from src/test_framework/xswiftec_inv_test_vectors.csv rename to resources/scenarios/test_framework/xswiftec_inv_test_vectors.csv diff --git a/ruff.toml b/ruff.toml index 1e17fe2d6..0d6fc35bd 100644 --- a/ruff.toml +++ b/ruff.toml @@ -1,5 +1,5 @@ extend-exclude = [ - "resources/images/commander/src/test_framework", + "resources/scenarios/test_framework", "resources/images/exporter/authproxy.py", "resources/scenarios/signet_miner.py", "src/test_framework/*", From 076346d313d0c46d48a6683727dcbcfa5a153fa9 Mon Sep 17 00:00:00 2001 From: Matthew Zipkin Date: Tue, 24 Sep 2024 14:42:24 -0400 Subject: [PATCH 03/13] scenarios: get -- --help before using any helm or k8s commands --- src/warnet/control.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/warnet/control.py b/src/warnet/control.py index 36601a0dd..6e973477d 100644 --- a/src/warnet/control.py +++ b/src/warnet/control.py @@ -178,6 +178,9 @@ def run(scenario_file: str, debug: bool, additional_args: tuple[str]): scenario_path = Path(scenario_file).resolve() scenario_name = scenario_path.stem + if additional_args and ("--help" in additional_args or "-h" in additional_args): + return subprocess.run([sys.executable, scenario_path, "--help"]) + with open(scenario_path, "rb") as file: scenario_data = base64.b64encode(file.read()).decode() @@ -219,8 +222,6 @@ def run(scenario_file: str, debug: bool, additional_args: tuple[str]): # Add additional arguments if additional_args: helm_command.extend(["--set", f"args={' '.join(additional_args)}"]) - if "--help" in additional_args or "-h" in additional_args: - return subprocess.run([sys.executable, scenario_path, "--help"]) helm_command.extend([name, COMMANDER_CHART]) From b5eeea15a11796f5bad88933f9c4419b177db3a6 Mon Sep 17 00:00:00 2001 From: Matthew Zipkin Date: Wed, 25 Sep 2024 10:09:53 -0400 Subject: [PATCH 04/13] scenarios: add main() to all files scenarios will need a callable entrypoint for the pyz archive --- resources/scenarios/ln_init.py | 2 +- resources/scenarios/miner_std.py | 2 +- resources/scenarios/reconnaissance.py | 2 +- resources/scenarios/signet_miner.py | 2 +- resources/scenarios/tx_flood.py | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/resources/scenarios/ln_init.py b/resources/scenarios/ln_init.py index 391246fd0..051c8d1ab 100644 --- a/resources/scenarios/ln_init.py +++ b/resources/scenarios/ln_init.py @@ -180,5 +180,5 @@ def funded_lnnodes(): ) -if __name__ == "__main__": +def main(): LNInit().main() diff --git a/resources/scenarios/miner_std.py b/resources/scenarios/miner_std.py index 764210efc..1411bd666 100755 --- a/resources/scenarios/miner_std.py +++ b/resources/scenarios/miner_std.py @@ -67,5 +67,5 @@ def run_test(self): sleep(self.options.interval) -if __name__ == "__main__": +def main(): MinerStd().main() diff --git a/resources/scenarios/reconnaissance.py b/resources/scenarios/reconnaissance.py index 2d500e274..1c539f5f7 100755 --- a/resources/scenarios/reconnaissance.py +++ b/resources/scenarios/reconnaissance.py @@ -80,5 +80,5 @@ def run_test(self): self.log.info(f"Got notfound message from {dstaddr}:{dstport}") -if __name__ == "__main__": +def main(): Reconnaissance().main() diff --git a/resources/scenarios/signet_miner.py b/resources/scenarios/signet_miner.py index 7c4652240..9a20ecc97 100644 --- a/resources/scenarios/signet_miner.py +++ b/resources/scenarios/signet_miner.py @@ -562,5 +562,5 @@ def get_args(parser): return args -if __name__ == "__main__": +def main(): SignetMinerScenario().main() diff --git a/resources/scenarios/tx_flood.py b/resources/scenarios/tx_flood.py index 996b88de5..aca5573bb 100755 --- a/resources/scenarios/tx_flood.py +++ b/resources/scenarios/tx_flood.py @@ -66,5 +66,5 @@ def run_test(self): sleep(30) -if __name__ == "__main__": +def main(): TXFlood().main() From 8d8ae247d2b0168c4534a69ef7eee85c7cefaa38 Mon Sep 17 00:00:00 2001 From: Matthew Zipkin Date: Wed, 25 Sep 2024 10:13:36 -0400 Subject: [PATCH 05/13] charts: plain python image for scenarios --- resources/charts/commander/templates/configmap.yaml | 9 --------- resources/charts/commander/templates/pod.yaml | 12 +++--------- resources/charts/commander/values.yaml | 8 -------- resources/images/commander/Dockerfile | 11 ----------- 4 files changed, 3 insertions(+), 37 deletions(-) delete mode 100644 resources/images/commander/Dockerfile diff --git a/resources/charts/commander/templates/configmap.yaml b/resources/charts/commander/templates/configmap.yaml index 9c45ea0d2..27cf7d9b5 100644 --- a/resources/charts/commander/templates/configmap.yaml +++ b/resources/charts/commander/templates/configmap.yaml @@ -1,14 +1,5 @@ apiVersion: v1 kind: ConfigMap -metadata: - name: {{ include "commander.fullname" . }}-scenario - labels: - {{- include "commander.labels" . | nindent 4 }} -binaryData: - scenario.py: {{ .Values.scenario }} ---- -apiVersion: v1 -kind: ConfigMap metadata: name: {{ include "commander.fullname" . }}-warnet labels: diff --git a/resources/charts/commander/templates/pod.yaml b/resources/charts/commander/templates/pod.yaml index 94c79205f..6b385285e 100644 --- a/resources/charts/commander/templates/pod.yaml +++ b/resources/charts/commander/templates/pod.yaml @@ -10,23 +10,17 @@ spec: restartPolicy: {{ .Values.restartPolicy }} containers: - name: {{ .Chart.Name }} - image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} + image: python:3.12-slim + imagePullPolicy: IfNotPresent command: ["/bin/sh", "-c"] args: - | - python3 /scenario.py {{ .Values.args }} + python3 /archive.pyz {{ .Values.args }} volumeMounts: - - name: scenario - mountPath: /scenario.py - subPath: scenario.py - name: warnet mountPath: /warnet.json subPath: warnet.json volumes: - - name: scenario - configMap: - name: {{ include "commander.fullname" . }}-scenario - name: warnet configMap: name: {{ include "commander.fullname" . }}-warnet diff --git a/resources/charts/commander/values.yaml b/resources/charts/commander/values.yaml index fc7e8233d..8f3efc4f0 100644 --- a/resources/charts/commander/values.yaml +++ b/resources/charts/commander/values.yaml @@ -5,12 +5,6 @@ namespace: warnet restartPolicy: Never -image: - repository: bitcoindevproject/warnet-commander - pullPolicy: IfNotPresent - # Overrides the image tag whose default is the chart appVersion. - tag: "latest" - imagePullSecrets: [] nameOverride: "" fullnameOverride: "" @@ -71,8 +65,6 @@ volumeMounts: [] port: -scenario: "" - warnet: "" args: "" diff --git a/resources/images/commander/Dockerfile b/resources/images/commander/Dockerfile deleted file mode 100644 index 3a8314c21..000000000 --- a/resources/images/commander/Dockerfile +++ /dev/null @@ -1,11 +0,0 @@ -# Use an official Python runtime as the base image -FROM python:3.12-slim - -# Python dependencies -#RUN pip install --no-cache-dir prometheus_client - -COPY resources/scenarios/commander.py / -COPY src/test_framework /test_framework - -# -u: force the stdout and stderr streams to be unbuffered -ENTRYPOINT ["python", "-u", "/scenario.py"] From b72904ec1b2e1076e2e43660c34a739df49f78c6 Mon Sep 17 00:00:00 2001 From: Matthew Zipkin Date: Wed, 25 Sep 2024 10:24:05 -0400 Subject: [PATCH 06/13] control: use zipapp to create archive --- resources/scenarios/commander.py | 2 +- src/warnet/control.py | 23 +++++++++++++++++------ 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/resources/scenarios/commander.py b/resources/scenarios/commander.py index 1ecf0b6c4..ffc9961a5 100644 --- a/resources/scenarios/commander.py +++ b/resources/scenarios/commander.py @@ -21,7 +21,7 @@ from test_framework.test_node import TestNode from test_framework.util import PortSeed, get_rpc_proxy -WARNET_FILE = Path(os.path.dirname(__file__)) / "warnet.json" +WARNET_FILE = Path("/warnet.json") try: with open(WARNET_FILE) as file: diff --git a/src/warnet/control.py b/src/warnet/control.py index 6e973477d..b0a5a450e 100644 --- a/src/warnet/control.py +++ b/src/warnet/control.py @@ -1,9 +1,11 @@ import base64 +import io import json import os import subprocess import sys import time +import zipapp from concurrent.futures import ThreadPoolExecutor, as_completed from pathlib import Path @@ -176,14 +178,13 @@ def run(scenario_file: str, debug: bool, additional_args: tuple[str]): Pass `-- --help` to get individual scenario help """ scenario_path = Path(scenario_file).resolve() + scenario_dir = scenario_path.parent scenario_name = scenario_path.stem if additional_args and ("--help" in additional_args or "-h" in additional_args): return subprocess.run([sys.executable, scenario_path, "--help"]) - with open(scenario_path, "rb") as file: - scenario_data = base64.b64encode(file.read()).decode() - + # Collect tank data for warnet.json name = f"commander-{scenario_name.replace('_', '')}-{int(time.time())}" namespace = get_default_namespace() tankpods = get_mission("tank") @@ -200,9 +201,21 @@ def run(scenario_file: str, debug: bool, additional_args: tuple[str]): for tank in tankpods ] - # Encode warnet data + # Encode tank data for warnet.json warnet_data = base64.b64encode(json.dumps(tanks).encode()).decode() + # Create in-memory buffer to store python archive instead of writing to disk + archive_buffer = io.BytesIO() + + # Compile python archive + zipapp.create_archive( + source=scenario_dir, target=archive_buffer, main=f"{scenario_name}:main", compressed=True + ) + + # Encode the binary data as Base64 + archive_buffer.seek(0) + archive_data = base64.b64encode(archive_buffer.read()).decode() + try: # Construct Helm command helm_command = [ @@ -214,8 +227,6 @@ def run(scenario_file: str, debug: bool, additional_args: tuple[str]): "--set", f"fullnameOverride={name}", "--set", - f"scenario={scenario_data}", - "--set", f"warnet={warnet_data}", ] From 1b37969c09f46eac70a675e36f1e09a177124bf2 Mon Sep 17 00:00:00 2001 From: Matthew Zipkin Date: Wed, 25 Sep 2024 14:08:39 -0400 Subject: [PATCH 07/13] test: move test scenarios to resources/scenarios also filter out unused files from the archive sent to the commander --- .../scenarios/test_buggy_failure.py | 2 +- .../scenarios/test_connect_dag.py | 2 +- .../scenarios/test_p2p_interface.py | 2 +- src/warnet/control.py | 20 ++++++++++++++++++- test/dag_connection_test.py | 2 +- test/scenarios_test.py | 4 ++-- 6 files changed, 25 insertions(+), 7 deletions(-) rename test/data/scenario_buggy_failure.py => resources/scenarios/test_buggy_failure.py (95%) rename test/data/scenario_connect_dag.py => resources/scenarios/test_connect_dag.py (99%) rename test/data/scenario_p2p_interface.py => resources/scenarios/test_p2p_interface.py (98%) diff --git a/test/data/scenario_buggy_failure.py b/resources/scenarios/test_buggy_failure.py similarity index 95% rename from test/data/scenario_buggy_failure.py rename to resources/scenarios/test_buggy_failure.py index 0867218d0..700f78ea1 100644 --- a/test/data/scenario_buggy_failure.py +++ b/resources/scenarios/test_buggy_failure.py @@ -20,5 +20,5 @@ def run_test(self): raise Exception("Failed execution!") -if __name__ == "__main__": +def main(): Failure().main() diff --git a/test/data/scenario_connect_dag.py b/resources/scenarios/test_connect_dag.py similarity index 99% rename from test/data/scenario_connect_dag.py rename to resources/scenarios/test_connect_dag.py index 95e50ea28..4019e8944 100644 --- a/test/data/scenario_connect_dag.py +++ b/resources/scenarios/test_connect_dag.py @@ -117,5 +117,5 @@ def assert_connection(self, connector, connectee_index, connection_type: Connect raise ValueError("ConnectionType must be of type DNS or IP") -if __name__ == "__main__": +def main(): ConnectDag().main() diff --git a/test/data/scenario_p2p_interface.py b/resources/scenarios/test_p2p_interface.py similarity index 98% rename from test/data/scenario_p2p_interface.py rename to resources/scenarios/test_p2p_interface.py index b9d0ff65f..47eee9006 100644 --- a/test/data/scenario_p2p_interface.py +++ b/resources/scenarios/test_p2p_interface.py @@ -52,5 +52,5 @@ def run_test(self): p2p_block_store.wait_until(lambda: p2p_block_store.blocks[best_block] == 1) -if __name__ == "__main__": +def main(): GetdataTest().main() diff --git a/src/warnet/control.py b/src/warnet/control.py index b0a5a450e..cce22728c 100644 --- a/src/warnet/control.py +++ b/src/warnet/control.py @@ -207,9 +207,27 @@ def run(scenario_file: str, debug: bool, additional_args: tuple[str]): # Create in-memory buffer to store python archive instead of writing to disk archive_buffer = io.BytesIO() + # No need to copy the entire scenarios/ directory into the archive + def filter(path): + if any( + needle in str(path) for needle in [ + ".pyc", + ".csv", + ".DS_Store" + ] + ): + return False + return any( + needle in str(path) for needle in [ + "commander.py", + "test_framework", + scenario_name + ] + ) + # Compile python archive zipapp.create_archive( - source=scenario_dir, target=archive_buffer, main=f"{scenario_name}:main", compressed=True + source=scenario_dir, target=archive_buffer, main=f"{scenario_name}:main", compressed=True, filter=filter ) # Encode the binary data as Base64 diff --git a/test/dag_connection_test.py b/test/dag_connection_test.py index 258052fc4..8dfc9fde7 100755 --- a/test/dag_connection_test.py +++ b/test/dag_connection_test.py @@ -26,7 +26,7 @@ def setup_network(self): def run_connect_dag_scenario(self): self.log.info("Running connect_dag scenario") - self.warnet("run test/data/scenario_connect_dag.py") + self.warnet("run resources/scenarios/test_connect_dag.py") self.wait_for_all_scenarios() diff --git a/test/scenarios_test.py b/test/scenarios_test.py index 867d5107f..f3187dba0 100755 --- a/test/scenarios_test.py +++ b/test/scenarios_test.py @@ -82,7 +82,7 @@ def run_and_check_miner_scenario_from_file(self): self.stop_scenario() def run_and_check_scenario_from_file(self): - scenario_file = "test/data/scenario_p2p_interface.py" + scenario_file = "resources/scenarios/test_p2p_interface.py" self.log.info(f"Running scenario from: {scenario_file}") self.warnet(f"run {scenario_file}") self.wait_for_predicate(self.check_scenario_clean_exit) @@ -94,7 +94,7 @@ def check_regtest_recon(self): self.wait_for_predicate(self.check_scenario_clean_exit) def check_active_count(self): - scenario_file = "test/data/scenario_buggy_failure.py" + scenario_file = "resources/scenarios/test_buggy_failure.py" self.log.info(f"Running scenario from: {scenario_file}") self.warnet(f"run {scenario_file}") From 3e60889383b04673fc947a4943de660643fb4c7a Mon Sep 17 00:00:00 2001 From: Matthew Zipkin Date: Wed, 25 Sep 2024 14:57:03 -0400 Subject: [PATCH 08/13] project: exclude test scenarios --- .../networks/6_node_bitcoin/network.yaml | 2 -- .../{ => fork_observer}/node-defaults.yaml | 0 src/warnet/graph.py | 2 +- src/warnet/network.py | 22 ++++++++----------- test/dag_connection_test.py | 2 +- test/scenarios_test.py | 4 ++-- 6 files changed, 13 insertions(+), 19 deletions(-) rename resources/networks/{ => fork_observer}/node-defaults.yaml (100%) diff --git a/resources/networks/6_node_bitcoin/network.yaml b/resources/networks/6_node_bitcoin/network.yaml index 21b05875d..86c9a27ec 100644 --- a/resources/networks/6_node_bitcoin/network.yaml +++ b/resources/networks/6_node_bitcoin/network.yaml @@ -28,7 +28,5 @@ nodes: connect: - tank-0006 - name: tank-0006 -fork_observer: - enabled: true caddy: enabled: true diff --git a/resources/networks/node-defaults.yaml b/resources/networks/fork_observer/node-defaults.yaml similarity index 100% rename from resources/networks/node-defaults.yaml rename to resources/networks/fork_observer/node-defaults.yaml diff --git a/src/warnet/graph.py b/src/warnet/graph.py index 6e5b3fd6b..691aa4ec3 100644 --- a/src/warnet/graph.py +++ b/src/warnet/graph.py @@ -74,7 +74,7 @@ def custom_graph( yaml.dump(network_yaml_data, f, default_flow_style=False) # Generate node-defaults.yaml - default_yaml_path = files("resources.networks").joinpath("node-defaults.yaml") + default_yaml_path = files("resources.networks").joinpath("fork_observer").joinpath("node-defaults.yaml") with open(str(default_yaml_path)) as f: defaults_yaml_content = yaml.safe_load(f) diff --git a/src/warnet/network.py b/src/warnet/network.py index 18a064210..0cfa5be5c 100644 --- a/src/warnet/network.py +++ b/src/warnet/network.py @@ -1,4 +1,5 @@ import json +import re import shutil from pathlib import Path @@ -18,17 +19,12 @@ def copy_defaults(directory: Path, target_subdir: str, source_path: Path, exclud target_dir.mkdir(parents=True, exist_ok=True) print(f"Creating directory: {target_dir}") - def should_copy(item: Path) -> bool: - return item.name not in exclude_list - - for item in source_path.iterdir(): - if should_copy(item): - if item.is_file(): - shutil.copy2(item, target_dir) - print(f"Copied file: {item.name}") - elif item.is_dir(): - shutil.copytree(item, target_dir / item.name, dirs_exist_ok=True) - print(f"Copied directory: {item.name}") + shutil.copytree( + src=source_path, + dst=target_dir, + dirs_exist_ok=True, + ignore=shutil.ignore_patterns(*exclude_list) + ) print(f"Finished copying files to {target_dir}") @@ -39,7 +35,7 @@ def copy_network_defaults(directory: Path): directory, NETWORK_DIR.name, NETWORK_DIR, - ["node-defaults.yaml", "__pycache__", "__init__.py"], + ["__pycache__", "__init__.py"], ) @@ -49,7 +45,7 @@ def copy_scenario_defaults(directory: Path): directory, SCENARIOS_DIR.name, SCENARIOS_DIR, - ["__init__.py", "__pycache__", "commander.py"], + ["__pycache__", "TEST_*.py"], ) diff --git a/test/dag_connection_test.py b/test/dag_connection_test.py index 8dfc9fde7..349e97449 100755 --- a/test/dag_connection_test.py +++ b/test/dag_connection_test.py @@ -26,7 +26,7 @@ def setup_network(self): def run_connect_dag_scenario(self): self.log.info("Running connect_dag scenario") - self.warnet("run resources/scenarios/test_connect_dag.py") + self.warnet("run resources/scenarios/TEST_connect_dag.py") self.wait_for_all_scenarios() diff --git a/test/scenarios_test.py b/test/scenarios_test.py index f3187dba0..a6bf6b41f 100755 --- a/test/scenarios_test.py +++ b/test/scenarios_test.py @@ -82,7 +82,7 @@ def run_and_check_miner_scenario_from_file(self): self.stop_scenario() def run_and_check_scenario_from_file(self): - scenario_file = "resources/scenarios/test_p2p_interface.py" + scenario_file = "resources/scenarios/TEST_p2p_interface.py" self.log.info(f"Running scenario from: {scenario_file}") self.warnet(f"run {scenario_file}") self.wait_for_predicate(self.check_scenario_clean_exit) @@ -94,7 +94,7 @@ def check_regtest_recon(self): self.wait_for_predicate(self.check_scenario_clean_exit) def check_active_count(self): - scenario_file = "resources/scenarios/test_buggy_failure.py" + scenario_file = "resources/scenarios/TEST_buggy_failure.py" self.log.info(f"Running scenario from: {scenario_file}") self.warnet(f"run {scenario_file}") From 77e932ae0b1f51c793ad16cd443d59c1021dcfdb Mon Sep 17 00:00:00 2001 From: Matthew Zipkin Date: Wed, 25 Sep 2024 14:57:35 -0400 Subject: [PATCH 09/13] lint --- resources/scenarios/ln_init.py | 1 + resources/scenarios/miner_std.py | 1 + resources/scenarios/tx_flood.py | 1 + src/warnet/bitcoin.py | 3 +-- src/warnet/control.py | 20 +++++++------------- src/warnet/graph.py | 4 +++- src/warnet/network.py | 3 +-- 7 files changed, 15 insertions(+), 18 deletions(-) diff --git a/resources/scenarios/ln_init.py b/resources/scenarios/ln_init.py index 051c8d1ab..9f24e5040 100644 --- a/resources/scenarios/ln_init.py +++ b/resources/scenarios/ln_init.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 from time import sleep + from commander import Commander diff --git a/resources/scenarios/miner_std.py b/resources/scenarios/miner_std.py index 1411bd666..d91736d82 100755 --- a/resources/scenarios/miner_std.py +++ b/resources/scenarios/miner_std.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 from time import sleep + from commander import Commander diff --git a/resources/scenarios/tx_flood.py b/resources/scenarios/tx_flood.py index aca5573bb..1197fc8a6 100755 --- a/resources/scenarios/tx_flood.py +++ b/resources/scenarios/tx_flood.py @@ -3,6 +3,7 @@ import threading from random import choice, randrange from time import sleep + from commander import Commander diff --git a/src/warnet/bitcoin.py b/src/warnet/bitcoin.py index a27da3bc7..8942662e9 100644 --- a/src/warnet/bitcoin.py +++ b/src/warnet/bitcoin.py @@ -5,10 +5,9 @@ from io import BytesIO import click -from urllib3.exceptions import MaxRetryError - from test_framework.messages import ser_uint256 from test_framework.p2p import MESSAGEMAP +from urllib3.exceptions import MaxRetryError from .k8s import get_default_namespace, get_mission from .process import run_command diff --git a/src/warnet/control.py b/src/warnet/control.py index cce22728c..fa28a5481 100644 --- a/src/warnet/control.py +++ b/src/warnet/control.py @@ -209,25 +209,19 @@ def run(scenario_file: str, debug: bool, additional_args: tuple[str]): # No need to copy the entire scenarios/ directory into the archive def filter(path): - if any( - needle in str(path) for needle in [ - ".pyc", - ".csv", - ".DS_Store" - ] - ): + if any(needle in str(path) for needle in [".pyc", ".csv", ".DS_Store"]): return False return any( - needle in str(path) for needle in [ - "commander.py", - "test_framework", - scenario_name - ] + needle in str(path) for needle in ["commander.py", "test_framework", scenario_name] ) # Compile python archive zipapp.create_archive( - source=scenario_dir, target=archive_buffer, main=f"{scenario_name}:main", compressed=True, filter=filter + source=scenario_dir, + target=archive_buffer, + main=f"{scenario_name}:main", + compressed=True, + filter=filter, ) # Encode the binary data as Base64 diff --git a/src/warnet/graph.py b/src/warnet/graph.py index 691aa4ec3..0e418b8d3 100644 --- a/src/warnet/graph.py +++ b/src/warnet/graph.py @@ -74,7 +74,9 @@ def custom_graph( yaml.dump(network_yaml_data, f, default_flow_style=False) # Generate node-defaults.yaml - default_yaml_path = files("resources.networks").joinpath("fork_observer").joinpath("node-defaults.yaml") + default_yaml_path = ( + files("resources.networks").joinpath("fork_observer").joinpath("node-defaults.yaml") + ) with open(str(default_yaml_path)) as f: defaults_yaml_content = yaml.safe_load(f) diff --git a/src/warnet/network.py b/src/warnet/network.py index 0cfa5be5c..52b67b617 100644 --- a/src/warnet/network.py +++ b/src/warnet/network.py @@ -1,5 +1,4 @@ import json -import re import shutil from pathlib import Path @@ -23,7 +22,7 @@ def copy_defaults(directory: Path, target_subdir: str, source_path: Path, exclud src=source_path, dst=target_dir, dirs_exist_ok=True, - ignore=shutil.ignore_patterns(*exclude_list) + ignore=shutil.ignore_patterns(*exclude_list), ) print(f"Finished copying files to {target_dir}") From 2ab431be3da2e38bd5df2bd87b7044ea28884af2 Mon Sep 17 00:00:00 2001 From: Matthew Zipkin Date: Wed, 25 Sep 2024 15:22:59 -0400 Subject: [PATCH 10/13] remove commander image deployment from CI --- .github/workflows/deploy.yml | 61 ------------------------------------ .github/workflows/test.yml | 39 +---------------------- 2 files changed, 1 insertion(+), 99 deletions(-) delete mode 100644 .github/workflows/deploy.yml diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml deleted file mode 100644 index ac363ab0b..000000000 --- a/.github/workflows/deploy.yml +++ /dev/null @@ -1,61 +0,0 @@ -name: Publish Commander Docker image - -on: - push: - branches: - - main - paths: - - resources/images/commander/Dockerfile - - resources/scenarios/commander.py - tags-ignore: - - "*" - -jobs: - push_to_registry: - name: Push commander Docker image to Docker Hub - runs-on: ubuntu-latest - permissions: - packages: write - contents: read - attestations: write - id-token: write - steps: - - name: Check out the repo - uses: actions/checkout@v4 - - - name: Log in to Docker Hub - uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} - - - name: Docker meta - id: meta - uses: docker/metadata-action@v5 - with: - images: bitcoindevproject/warnet-commander - tags: | - type=ref,event=tag - type=ref,event=pr - type=raw,value=latest,enable={{is_default_branch}} - labels: | - maintainer=bitcoindevproject - org.opencontainers.image.title=warnet-commander - org.opencontainers.image.description=Warnet Commander - - - name: Build and push Docker image - id: push - uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671 - with: - context: . - file: resources/images/commander/Dockerfile - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - - - name: Generate artifact attestation - uses: actions/attest-build-provenance@v1 - with: - subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME}} - subject-digest: ${{ steps.push.outputs.digest }} - push-to-registry: true diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f3ff8f1e8..54d2e36df 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -30,34 +30,8 @@ jobs: enable-cache: true - run: uvx ruff format . --check - build-image: - needs: [ruff, ruff-format] - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Build and export - uses: docker/build-push-action@v5 - with: - file: resources/images/commander/Dockerfile - context: . - tags: bitcoindevproject/warnet-commander:latest - cache-from: type=gha - cache-to: type=gha,mode=max - outputs: type=docker,dest=/tmp/commander.tar - - - name: Upload artifact - uses: actions/upload-artifact@v4 - with: - name: commander - path: /tmp/commander.tar - test: - needs: [build-image] + needs: [ruff, ruff-format] runs-on: ubuntu-latest strategy: matrix: @@ -80,11 +54,6 @@ jobs: memory: 4000m - name: Start minikube's loadbalancer tunnel run: minikube tunnel &> /dev/null & - - name: Download commander artifact - uses: actions/download-artifact@v4 - with: - name: commander - path: /tmp - name: Install the latest version of uv uses: astral-sh/setup-uv@v2 with: @@ -94,12 +63,6 @@ jobs: run: uv python install $PYTHON_VERSION - name: Install project run: uv sync --all-extras --dev - - name: Install commander image - run: | - echo loading commander image into minikube docker - eval $(minikube -p minikube docker-env) - docker load --input /tmp/commander.tar - docker image ls -a - name: Run tests run: | source .venv/bin/activate From f2a322267d0a50dea968ebaa99cd55d884d5ca5d Mon Sep 17 00:00:00 2001 From: Matthew Zipkin Date: Thu, 26 Sep 2024 17:09:56 -0400 Subject: [PATCH 11/13] scenarios: use initContainer --- resources/charts/commander/templates/pod.yaml | 25 ++++++++--- resources/scenarios/commander.py | 3 +- ...ilure.py => testscenario_buggy_failure.py} | 0 ...ect_dag.py => testscenario_connect_dag.py} | 0 ...rface.py => testscenario_p2p_interface.py} | 0 src/warnet/control.py | 25 ++++++----- src/warnet/k8s.py | 43 +++++++++++++++++++ src/warnet/network.py | 2 +- test/dag_connection_test.py | 6 ++- test/scenarios_test.py | 9 ++-- 10 files changed, 87 insertions(+), 26 deletions(-) rename resources/scenarios/{test_buggy_failure.py => testscenario_buggy_failure.py} (100%) rename resources/scenarios/{test_connect_dag.py => testscenario_connect_dag.py} (100%) rename resources/scenarios/{test_p2p_interface.py => testscenario_p2p_interface.py} (100%) diff --git a/resources/charts/commander/templates/pod.yaml b/resources/charts/commander/templates/pod.yaml index 6b385285e..1a9bb9310 100644 --- a/resources/charts/commander/templates/pod.yaml +++ b/resources/charts/commander/templates/pod.yaml @@ -8,6 +8,19 @@ metadata: mission: commander spec: restartPolicy: {{ .Values.restartPolicy }} + initContainers: + - name: init + image: busybox + command: ["/bin/sh", "-c"] + args: + - | + while [ ! -f /shared/archive.pyz ]; do + echo "Waiting for /shared/archive.pyz to exist..." + sleep 2 + done + volumeMounts: + - name: shared-volume + mountPath: /shared containers: - name: {{ .Chart.Name }} image: python:3.12-slim @@ -15,12 +28,10 @@ spec: command: ["/bin/sh", "-c"] args: - | - python3 /archive.pyz {{ .Values.args }} + python3 /shared/archive.pyz {{ .Values.args }} volumeMounts: - - name: warnet - mountPath: /warnet.json - subPath: warnet.json + - name: shared-volume + mountPath: /shared volumes: - - name: warnet - configMap: - name: {{ include "commander.fullname" . }}-warnet + - name: shared-volume + emptyDir: {} diff --git a/resources/scenarios/commander.py b/resources/scenarios/commander.py index ffc9961a5..1f7d34a80 100644 --- a/resources/scenarios/commander.py +++ b/resources/scenarios/commander.py @@ -8,7 +8,6 @@ import signal import sys import tempfile -from pathlib import Path from typing import Dict from test_framework.authproxy import AuthServiceProxy @@ -21,7 +20,7 @@ from test_framework.test_node import TestNode from test_framework.util import PortSeed, get_rpc_proxy -WARNET_FILE = Path("/warnet.json") +WARNET_FILE = "/shared/warnet.json" try: with open(WARNET_FILE) as file: diff --git a/resources/scenarios/test_buggy_failure.py b/resources/scenarios/testscenario_buggy_failure.py similarity index 100% rename from resources/scenarios/test_buggy_failure.py rename to resources/scenarios/testscenario_buggy_failure.py diff --git a/resources/scenarios/test_connect_dag.py b/resources/scenarios/testscenario_connect_dag.py similarity index 100% rename from resources/scenarios/test_connect_dag.py rename to resources/scenarios/testscenario_connect_dag.py diff --git a/resources/scenarios/test_p2p_interface.py b/resources/scenarios/testscenario_p2p_interface.py similarity index 100% rename from resources/scenarios/test_p2p_interface.py rename to resources/scenarios/testscenario_p2p_interface.py diff --git a/src/warnet/control.py b/src/warnet/control.py index fa28a5481..52086d189 100644 --- a/src/warnet/control.py +++ b/src/warnet/control.py @@ -1,4 +1,3 @@ -import base64 import io import json import os @@ -26,6 +25,8 @@ pod_log, snapshot_bitcoin_datadir, wait_for_pod, + wait_for_init, + write_file_to_container, ) from .process import run_command, stream_command @@ -202,7 +203,7 @@ def run(scenario_file: str, debug: bool, additional_args: tuple[str]): ] # Encode tank data for warnet.json - warnet_data = base64.b64encode(json.dumps(tanks).encode()).decode() + warnet_data = json.dumps(tanks).encode() # Create in-memory buffer to store python archive instead of writing to disk archive_buffer = io.BytesIO() @@ -226,8 +227,9 @@ def filter(path): # Encode the binary data as Base64 archive_buffer.seek(0) - archive_data = base64.b64encode(archive_buffer.read()).decode() + archive_data = archive_buffer.read() + # Start the commander pod with python and init containers try: # Construct Helm command helm_command = [ @@ -238,8 +240,6 @@ def filter(path): namespace, "--set", f"fullnameOverride={name}", - "--set", - f"warnet={warnet_data}", ] # Add additional arguments @@ -252,19 +252,24 @@ def filter(path): result = subprocess.run(helm_command, check=True, capture_output=True, text=True) if result.returncode == 0: - print(f"Successfully started scenario: {scenario_name}") + print(f"Successfully deployed scenario commander: {scenario_name}") print(f"Commander pod name: {name}") else: - print(f"Failed to start scenario: {scenario_name}") + print(f"Failed to deploy scenario commander: {scenario_name}") print(f"Error: {result.stderr}") except subprocess.CalledProcessError as e: - print(f"Failed to start scenario: {scenario_name}") + print(f"Failed to deploy scenario commander: {scenario_name}") print(f"Error: {e.stderr}") + # upload scenario files and network data to the init container + wait_for_init(name) + if write_file_to_container( + name, "init", "/shared/warnet.json", warnet_data + ) and write_file_to_container(name, "init", "/shared/archive.pyz", archive_data): + print(f"Successfully uploaded scenario data to commander: {scenario_name}") + if debug: - print("Waiting for commander pod to start...") - wait_for_pod(name) _logs(pod_name=name, follow=True) print("Deleting pod...") delete_pod(name) diff --git a/src/warnet/k8s.py b/src/warnet/k8s.py index ffe61d067..1a690b4ed 100644 --- a/src/warnet/k8s.py +++ b/src/warnet/k8s.py @@ -264,6 +264,24 @@ def wait_for_pod_ready(name, namespace, timeout=300): return False +def wait_for_init(pod_name, timeout=300): + sclient = get_static_client() + namespace = get_default_namespace() + w = watch.Watch() + for event in w.stream( + sclient.list_namespaced_pod, namespace=namespace, timeout_seconds=timeout + ): + pod = event["object"] + if pod.metadata.name == pod_name: + for init_container_status in pod.status.init_container_statuses: + if init_container_status.state.running: + print(f"initContainer in pod {pod_name} is ready") + w.stop() + return True + print(f"Timeout waiting for initContainer in {pod_name} to be ready.") + return False + + def wait_for_ingress_controller(timeout=300): # get name of ingress controller pod sclient = get_static_client() @@ -308,3 +326,28 @@ def wait_for_pod(pod_name, timeout_seconds=10): return sleep(1) timeout_seconds -= 1 + + +def write_file_to_container(pod_name, container_name, dst_path, data): + sclient = get_static_client() + namespace = get_default_namespace() + exec_command = ["sh", "-c", f"cat > {dst_path}"] + try: + res = stream( + sclient.connect_get_namespaced_pod_exec, + pod_name, + namespace, + command=exec_command, + container=container_name, + stdin=True, + stderr=True, + stdout=True, + tty=False, + _preload_content=False, + ) + res.write_stdin(data) + res.close() + print(f"Successfully copied data to {pod_name}({container_name}):{dst_path}") + return True + except Exception as e: + print(f"Failed to copy data to {pod_name}({container_name}):{dst_path}:\n{e}") diff --git a/src/warnet/network.py b/src/warnet/network.py index 52b67b617..f78eda42b 100644 --- a/src/warnet/network.py +++ b/src/warnet/network.py @@ -44,7 +44,7 @@ def copy_scenario_defaults(directory: Path): directory, SCENARIOS_DIR.name, SCENARIOS_DIR, - ["__pycache__", "TEST_*.py"], + ["__pycache__", "testscenario_*.py"], ) diff --git a/test/dag_connection_test.py b/test/dag_connection_test.py index 349e97449..4d8d953eb 100755 --- a/test/dag_connection_test.py +++ b/test/dag_connection_test.py @@ -10,6 +10,7 @@ class DAGConnectionTest(TestBase): def __init__(self): super().__init__() self.network_dir = Path(os.path.dirname(__file__)) / "data" / "ten_semi_unconnected" + self.scen_dir = Path(os.path.dirname(__file__)).parent / "resources" / "scenarios" def run_test(self): try: @@ -25,8 +26,9 @@ def setup_network(self): self.wait_for_all_edges() def run_connect_dag_scenario(self): - self.log.info("Running connect_dag scenario") - self.warnet("run resources/scenarios/TEST_connect_dag.py") + scenario_file = self.scen_dir / "testscenario_connect_dag.py" + self.log.info(f"Running scenario from: {scenario_file}") + self.warnet(f"run {scenario_file}") self.wait_for_all_scenarios() diff --git a/test/scenarios_test.py b/test/scenarios_test.py index a6bf6b41f..835c273c5 100755 --- a/test/scenarios_test.py +++ b/test/scenarios_test.py @@ -14,6 +14,7 @@ class ScenariosTest(TestBase): def __init__(self): super().__init__() self.network_dir = Path(os.path.dirname(__file__)) / "data" / "12_node_ring" + self.scen_dir = Path(os.path.dirname(__file__)).parent / "resources" / "scenarios" def run_test(self): try: @@ -71,7 +72,7 @@ def check_blocks(self, target_blocks, start: int = 0): return count >= start + target_blocks def run_and_check_miner_scenario_from_file(self): - scenario_file = "resources/scenarios/miner_std.py" + scenario_file = self.scen_dir / "miner_std.py" self.log.info(f"Running scenario from file: {scenario_file}") self.warnet(f"run {scenario_file} --allnodes --interval=1") start = int(self.warnet("bitcoin rpc tank-0000 getblockcount")) @@ -82,19 +83,19 @@ def run_and_check_miner_scenario_from_file(self): self.stop_scenario() def run_and_check_scenario_from_file(self): - scenario_file = "resources/scenarios/TEST_p2p_interface.py" + scenario_file = self.scen_dir / "testscenario_p2p_interface.py" self.log.info(f"Running scenario from: {scenario_file}") self.warnet(f"run {scenario_file}") self.wait_for_predicate(self.check_scenario_clean_exit) def check_regtest_recon(self): - scenario_file = "resources/scenarios/reconnaissance.py" + scenario_file = self.scen_dir / "reconnaissance.py" self.log.info(f"Running scenario from file: {scenario_file}") self.warnet(f"run {scenario_file}") self.wait_for_predicate(self.check_scenario_clean_exit) def check_active_count(self): - scenario_file = "resources/scenarios/TEST_buggy_failure.py" + scenario_file = self.scen_dir / "testscenario_buggy_failure.py" self.log.info(f"Running scenario from: {scenario_file}") self.warnet(f"run {scenario_file}") From 371fc20ea614ba83f972badbccb793c675cacd13 Mon Sep 17 00:00:00 2001 From: Matthew Zipkin Date: Fri, 27 Sep 2024 20:11:08 -0400 Subject: [PATCH 12/13] move test scenarios to subdirectory and add optional --source_dir --- resources/scenarios/ln_init.py | 3 +++ resources/scenarios/miner_std.py | 3 +++ resources/scenarios/reconnaissance.py | 3 +++ resources/scenarios/signet_miner.py | 3 +++ .../scenarios/test_scenarios/__init__.py | 0 .../buggy_failure.py} | 3 +++ .../connect_dag.py} | 3 +++ .../p2p_interface.py} | 3 +++ resources/scenarios/tx_flood.py | 4 ++++ src/warnet/control.py | 24 +++++++++++++------ src/warnet/network.py | 2 +- test/dag_connection_test.py | 4 ++-- test/scenarios_test.py | 8 +++---- 13 files changed, 49 insertions(+), 14 deletions(-) create mode 100644 resources/scenarios/test_scenarios/__init__.py rename resources/scenarios/{testscenario_buggy_failure.py => test_scenarios/buggy_failure.py} (93%) rename resources/scenarios/{testscenario_connect_dag.py => test_scenarios/connect_dag.py} (99%) rename resources/scenarios/{testscenario_p2p_interface.py => test_scenarios/p2p_interface.py} (97%) diff --git a/resources/scenarios/ln_init.py b/resources/scenarios/ln_init.py index 9f24e5040..c2f3d3474 100644 --- a/resources/scenarios/ln_init.py +++ b/resources/scenarios/ln_init.py @@ -183,3 +183,6 @@ def funded_lnnodes(): def main(): LNInit().main() + +if __name__ == "__main__": + main() diff --git a/resources/scenarios/miner_std.py b/resources/scenarios/miner_std.py index d91736d82..ac507171b 100755 --- a/resources/scenarios/miner_std.py +++ b/resources/scenarios/miner_std.py @@ -70,3 +70,6 @@ def run_test(self): def main(): MinerStd().main() + +if __name__ == "__main__": + main() diff --git a/resources/scenarios/reconnaissance.py b/resources/scenarios/reconnaissance.py index 1c539f5f7..453d6fc4a 100755 --- a/resources/scenarios/reconnaissance.py +++ b/resources/scenarios/reconnaissance.py @@ -82,3 +82,6 @@ def run_test(self): def main(): Reconnaissance().main() + +if __name__ == "__main__": + main() diff --git a/resources/scenarios/signet_miner.py b/resources/scenarios/signet_miner.py index 9a20ecc97..e4375515b 100644 --- a/resources/scenarios/signet_miner.py +++ b/resources/scenarios/signet_miner.py @@ -564,3 +564,6 @@ def get_args(parser): def main(): SignetMinerScenario().main() + +if __name__ == "__main__": + main() diff --git a/resources/scenarios/test_scenarios/__init__.py b/resources/scenarios/test_scenarios/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/resources/scenarios/testscenario_buggy_failure.py b/resources/scenarios/test_scenarios/buggy_failure.py similarity index 93% rename from resources/scenarios/testscenario_buggy_failure.py rename to resources/scenarios/test_scenarios/buggy_failure.py index 700f78ea1..249001bd4 100644 --- a/resources/scenarios/testscenario_buggy_failure.py +++ b/resources/scenarios/test_scenarios/buggy_failure.py @@ -22,3 +22,6 @@ def run_test(self): def main(): Failure().main() + +if __name__ == "__main__": + main() diff --git a/resources/scenarios/testscenario_connect_dag.py b/resources/scenarios/test_scenarios/connect_dag.py similarity index 99% rename from resources/scenarios/testscenario_connect_dag.py rename to resources/scenarios/test_scenarios/connect_dag.py index 4019e8944..2df50bc1b 100644 --- a/resources/scenarios/testscenario_connect_dag.py +++ b/resources/scenarios/test_scenarios/connect_dag.py @@ -119,3 +119,6 @@ def assert_connection(self, connector, connectee_index, connection_type: Connect def main(): ConnectDag().main() + +if __name__ == "__main__": + main() diff --git a/resources/scenarios/testscenario_p2p_interface.py b/resources/scenarios/test_scenarios/p2p_interface.py similarity index 97% rename from resources/scenarios/testscenario_p2p_interface.py rename to resources/scenarios/test_scenarios/p2p_interface.py index 47eee9006..4f88f49cc 100644 --- a/resources/scenarios/testscenario_p2p_interface.py +++ b/resources/scenarios/test_scenarios/p2p_interface.py @@ -54,3 +54,6 @@ def run_test(self): def main(): GetdataTest().main() + +if __name__ == "__main__": + main() diff --git a/resources/scenarios/tx_flood.py b/resources/scenarios/tx_flood.py index 1197fc8a6..7a60bccc5 100755 --- a/resources/scenarios/tx_flood.py +++ b/resources/scenarios/tx_flood.py @@ -69,3 +69,7 @@ def run_test(self): def main(): TXFlood().main() + + +if __name__ == "__main__": + main() diff --git a/src/warnet/control.py b/src/warnet/control.py index 52086d189..8c5127a63 100644 --- a/src/warnet/control.py +++ b/src/warnet/control.py @@ -172,14 +172,15 @@ def get_active_network(namespace): default=False, help="Stream scenario output and delete container when stopped", ) +@click.option("--source_dir", type=click.Path(exists=True, file_okay=False, dir_okay=True), required=False) @click.argument("additional_args", nargs=-1, type=click.UNPROCESSED) -def run(scenario_file: str, debug: bool, additional_args: tuple[str]): +def run(scenario_file: str, source_dir, additional_args: tuple[str]): """ Run a scenario from a file. Pass `-- --help` to get individual scenario help """ scenario_path = Path(scenario_file).resolve() - scenario_dir = scenario_path.parent + scenario_dir = scenario_path.parent if not source_dir else Path(source_dir).resolve() scenario_name = scenario_path.stem if additional_args and ("--help" in additional_args or "-h" in additional_args): @@ -212,15 +213,24 @@ def run(scenario_file: str, debug: bool, additional_args: tuple[str]): def filter(path): if any(needle in str(path) for needle in [".pyc", ".csv", ".DS_Store"]): return False - return any( - needle in str(path) for needle in ["commander.py", "test_framework", scenario_name] - ) - + if any(needle in str(path) for needle in ["__init__.py", "commander.py", "test_framework", scenario_path.name]): + print(f"Including: {path}") + return True + return False + + # In case the scenario file is not in the root of the archive directory, + # we need to specify its relative path as a submodule + # First get the path of the file relative to the source directory + relative_path = scenario_path.relative_to(scenario_dir) + # Remove the '.py' extension + relative_name = relative_path.with_suffix("") + # Replace path separators with dots and pray the user included __init__.py + module_name = ".".join(relative_name.parts) # Compile python archive zipapp.create_archive( source=scenario_dir, target=archive_buffer, - main=f"{scenario_name}:main", + main=f"{module_name}:main", compressed=True, filter=filter, ) diff --git a/src/warnet/network.py b/src/warnet/network.py index f78eda42b..401ab5106 100644 --- a/src/warnet/network.py +++ b/src/warnet/network.py @@ -44,7 +44,7 @@ def copy_scenario_defaults(directory: Path): directory, SCENARIOS_DIR.name, SCENARIOS_DIR, - ["__pycache__", "testscenario_*.py"], + ["__pycache__", "test_scenarios"], ) diff --git a/test/dag_connection_test.py b/test/dag_connection_test.py index 4d8d953eb..dee38356a 100755 --- a/test/dag_connection_test.py +++ b/test/dag_connection_test.py @@ -26,9 +26,9 @@ def setup_network(self): self.wait_for_all_edges() def run_connect_dag_scenario(self): - scenario_file = self.scen_dir / "testscenario_connect_dag.py" + scenario_file = self.scen_dir / "test_scenarios" / "connect_dag.py" self.log.info(f"Running scenario from: {scenario_file}") - self.warnet(f"run {scenario_file}") + self.warnet(f"run {scenario_file} --source_dir={self.scen_dir}") self.wait_for_all_scenarios() diff --git a/test/scenarios_test.py b/test/scenarios_test.py index 835c273c5..0b8ba7a4a 100755 --- a/test/scenarios_test.py +++ b/test/scenarios_test.py @@ -83,9 +83,9 @@ def run_and_check_miner_scenario_from_file(self): self.stop_scenario() def run_and_check_scenario_from_file(self): - scenario_file = self.scen_dir / "testscenario_p2p_interface.py" + scenario_file = self.scen_dir / "test_scenarios" / "p2p_interface.py" self.log.info(f"Running scenario from: {scenario_file}") - self.warnet(f"run {scenario_file}") + self.warnet(f"run {scenario_file} --source_dir={self.scen_dir}") self.wait_for_predicate(self.check_scenario_clean_exit) def check_regtest_recon(self): @@ -95,9 +95,9 @@ def check_regtest_recon(self): self.wait_for_predicate(self.check_scenario_clean_exit) def check_active_count(self): - scenario_file = self.scen_dir / "testscenario_buggy_failure.py" + scenario_file = self.scen_dir / "test_scenarios" / "buggy_failure.py" self.log.info(f"Running scenario from: {scenario_file}") - self.warnet(f"run {scenario_file}") + self.warnet(f"run {scenario_file} --source_dir={self.scen_dir}") def two_pass_one_fail(): deployed = scenarios_deployed() From 5ee0c8b3e083c3030b275a7eddbc494dfe185a77 Mon Sep 17 00:00:00 2001 From: Matthew Zipkin Date: Fri, 27 Sep 2024 20:31:27 -0400 Subject: [PATCH 13/13] clean up rebase on main, lint --- resources/scenarios/ln_init.py | 1 + resources/scenarios/miner_std.py | 1 + resources/scenarios/reconnaissance.py | 1 + .../scenarios/test_scenarios/buggy_failure.py | 1 + resources/scenarios/test_scenarios/connect_dag.py | 1 + .../scenarios/test_scenarios/p2p_interface.py | 1 + src/warnet/control.py | 15 +++++++++++---- src/warnet/k8s.py | 2 +- 8 files changed, 18 insertions(+), 5 deletions(-) diff --git a/resources/scenarios/ln_init.py b/resources/scenarios/ln_init.py index c2f3d3474..82745a123 100644 --- a/resources/scenarios/ln_init.py +++ b/resources/scenarios/ln_init.py @@ -184,5 +184,6 @@ def funded_lnnodes(): def main(): LNInit().main() + if __name__ == "__main__": main() diff --git a/resources/scenarios/miner_std.py b/resources/scenarios/miner_std.py index ac507171b..568934e67 100755 --- a/resources/scenarios/miner_std.py +++ b/resources/scenarios/miner_std.py @@ -71,5 +71,6 @@ def run_test(self): def main(): MinerStd().main() + if __name__ == "__main__": main() diff --git a/resources/scenarios/reconnaissance.py b/resources/scenarios/reconnaissance.py index 453d6fc4a..8c3f683cb 100755 --- a/resources/scenarios/reconnaissance.py +++ b/resources/scenarios/reconnaissance.py @@ -83,5 +83,6 @@ def run_test(self): def main(): Reconnaissance().main() + if __name__ == "__main__": main() diff --git a/resources/scenarios/test_scenarios/buggy_failure.py b/resources/scenarios/test_scenarios/buggy_failure.py index 249001bd4..e982680d5 100644 --- a/resources/scenarios/test_scenarios/buggy_failure.py +++ b/resources/scenarios/test_scenarios/buggy_failure.py @@ -23,5 +23,6 @@ def run_test(self): def main(): Failure().main() + if __name__ == "__main__": main() diff --git a/resources/scenarios/test_scenarios/connect_dag.py b/resources/scenarios/test_scenarios/connect_dag.py index 2df50bc1b..5747291cb 100644 --- a/resources/scenarios/test_scenarios/connect_dag.py +++ b/resources/scenarios/test_scenarios/connect_dag.py @@ -120,5 +120,6 @@ def assert_connection(self, connector, connectee_index, connection_type: Connect def main(): ConnectDag().main() + if __name__ == "__main__": main() diff --git a/resources/scenarios/test_scenarios/p2p_interface.py b/resources/scenarios/test_scenarios/p2p_interface.py index 4f88f49cc..e3854658e 100644 --- a/resources/scenarios/test_scenarios/p2p_interface.py +++ b/resources/scenarios/test_scenarios/p2p_interface.py @@ -55,5 +55,6 @@ def run_test(self): def main(): GetdataTest().main() + if __name__ == "__main__": main() diff --git a/src/warnet/control.py b/src/warnet/control.py index 8c5127a63..cea7bc9a0 100644 --- a/src/warnet/control.py +++ b/src/warnet/control.py @@ -24,8 +24,8 @@ get_pods, pod_log, snapshot_bitcoin_datadir, - wait_for_pod, wait_for_init, + wait_for_pod, write_file_to_container, ) from .process import run_command, stream_command @@ -172,9 +172,11 @@ def get_active_network(namespace): default=False, help="Stream scenario output and delete container when stopped", ) -@click.option("--source_dir", type=click.Path(exists=True, file_okay=False, dir_okay=True), required=False) +@click.option( + "--source_dir", type=click.Path(exists=True, file_okay=False, dir_okay=True), required=False +) @click.argument("additional_args", nargs=-1, type=click.UNPROCESSED) -def run(scenario_file: str, source_dir, additional_args: tuple[str]): +def run(scenario_file: str, debug: bool, source_dir, additional_args: tuple[str]): """ Run a scenario from a file. Pass `-- --help` to get individual scenario help @@ -213,7 +215,10 @@ def run(scenario_file: str, source_dir, additional_args: tuple[str]): def filter(path): if any(needle in str(path) for needle in [".pyc", ".csv", ".DS_Store"]): return False - if any(needle in str(path) for needle in ["__init__.py", "commander.py", "test_framework", scenario_path.name]): + if any( + needle in str(path) + for needle in ["__init__.py", "commander.py", "test_framework", scenario_path.name] + ): print(f"Including: {path}") return True return False @@ -280,6 +285,8 @@ def filter(path): print(f"Successfully uploaded scenario data to commander: {scenario_name}") if debug: + print("Waiting for commander pod to start...") + wait_for_pod(name) _logs(pod_name=name, follow=True) print("Deleting pod...") delete_pod(name) diff --git a/src/warnet/k8s.py b/src/warnet/k8s.py index 1a690b4ed..9c18d095d 100644 --- a/src/warnet/k8s.py +++ b/src/warnet/k8s.py @@ -118,7 +118,7 @@ def delete_namespace(namespace: str) -> bool: def delete_pod(pod_name: str) -> bool: - command = f"kubectl delete pod {pod_name}" + command = f"kubectl -n {get_default_namespace()} delete pod {pod_name}" return stream_command(command)