From e25f126e34666b823bcad31ef1970ce43e1f6833 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 21 May 2025 13:17:43 +0200 Subject: [PATCH 001/156] PMM-7: Fix pmm server port --- pmm_qa/percona_server/percona-server-setup.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/percona_server/percona-server-setup.yml b/pmm_qa/percona_server/percona-server-setup.yml index f0602ca5..11db1f86 100644 --- a/pmm_qa/percona_server/percona-server-setup.yml +++ b/pmm_qa/percona_server/percona-server-setup.yml @@ -99,7 +99,7 @@ community.docker.docker_container_exec: container: "ps_pmm_{{ ps_version }}_1" command: > - sh -c 'curl --location --insecure -u"admin:{{ admin_password }}" -s --request GET "http://{{ pmm_server_ip }}:{{ '80' if pmm_server_ip is ansible.utils.ipv4 else '8080' }}/v1/management/services" | jq -r ".services[].service_name"' + sh -c 'curl --location --insecure -u"admin:{{ admin_password }}" -s --request GET "http://{{ pmm_server_ip }}:{{ '443' if pmm_server_ip is ansible.utils.ipv4 else '8443' }}/v1/management/services" | jq -r ".services[].service_name"' register: pmm_server_services - name: Display already connected services to pmm server From 9059f607bfbc7cb788e35bb0c853b92be20416a1 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 21 May 2025 19:43:11 +0200 Subject: [PATCH 002/156] PMM-7: Fix pmm server port --- pmm_qa/percona_server/percona-server-setup.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/percona_server/percona-server-setup.yml b/pmm_qa/percona_server/percona-server-setup.yml index 11db1f86..4af81dcd 100644 --- a/pmm_qa/percona_server/percona-server-setup.yml +++ b/pmm_qa/percona_server/percona-server-setup.yml @@ -99,7 +99,7 @@ community.docker.docker_container_exec: container: "ps_pmm_{{ ps_version }}_1" command: > - sh -c 'curl --location --insecure -u"admin:{{ admin_password }}" -s --request GET "http://{{ pmm_server_ip }}:{{ '443' if pmm_server_ip is ansible.utils.ipv4 else '8443' }}/v1/management/services" | jq -r ".services[].service_name"' + sh -c 'curl --location --insecure -u"admin:{{ admin_password }}" -s --request GET "https://{{ pmm_server_ip }}:{{ '443' if pmm_server_ip is ansible.utils.ipv4 else '8443' }}/v1/management/services" | jq -r ".services[].service_name"' register: pmm_server_services - name: Display already connected services to pmm server From 010823d54569dc4a3da7cd42a4cea0b41dd6d170 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Fri, 23 May 2025 12:34:17 +0200 Subject: [PATCH 003/156] PMM-7: 8.4 as default ps --- pmm_qa/pmm-framework.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index 5538e53b..a68476e5 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -37,7 +37,7 @@ "TARBALL": ""} }, "PS": { - "versions": ["5.7", "8.4", "8.0"], + "versions": ["8.4", "5.7", "8.0"], "configurations": {"QUERY_SOURCE": "perfschema", "SETUP_TYPE": "", "CLIENT_VERSION": "3-dev-latest", "TARBALL": "", "NODES_COUNT": 1} }, From 360639b8ad4765b1381fcc266800fdb0c9f6393f Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Fri, 23 May 2025 13:22:24 +0200 Subject: [PATCH 004/156] PMM-7: 8.4 as default ps --- pmm_qa/pmm-framework.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index a68476e5..cd4d1e53 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -37,7 +37,7 @@ "TARBALL": ""} }, "PS": { - "versions": ["8.4", "5.7", "8.0"], + "versions": ["5.7", "8.0", "8.4"], "configurations": {"QUERY_SOURCE": "perfschema", "SETUP_TYPE": "", "CLIENT_VERSION": "3-dev-latest", "TARBALL": "", "NODES_COUNT": 1} }, From aa6ecd27119f872711d8178df95ccd6b021e0c9e Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Fri, 23 May 2025 13:49:05 +0200 Subject: [PATCH 005/156] PMM-7: road on pdpgsql --- pmm_qa/pdpgsql_pgsm_setup.yml | 5 +++ pmm_qa/tasks/run_load_pgbench.yml | 53 +++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+) create mode 100644 pmm_qa/tasks/run_load_pgbench.yml diff --git a/pmm_qa/pdpgsql_pgsm_setup.yml b/pmm_qa/pdpgsql_pgsm_setup.yml index 09bb4dcb..fdc6d5e4 100644 --- a/pmm_qa/pdpgsql_pgsm_setup.yml +++ b/pmm_qa/pdpgsql_pgsm_setup.yml @@ -81,3 +81,8 @@ shell: "{{ item }}" with_items: - docker exec {{ pdpgsql_pgsm_container }} bash ./pgsm_run_queries.sh & + + - name: Setup Percona Server group replication + include_tasks: ./tasks/run_load_pgbench.yml + vars: + node_name: "{{ pdpgsql_pgsm_container }}" \ No newline at end of file diff --git a/pmm_qa/tasks/run_load_pgbench.yml b/pmm_qa/tasks/run_load_pgbench.yml new file mode 100644 index 00000000..e91cdfde --- /dev/null +++ b/pmm_qa/tasks/run_load_pgbench.yml @@ -0,0 +1,53 @@ +# This ansible file run load on docker container of pgsql with variable name node_name + +- name: Detect OS inside the container + community.docker.docker_container_exec: + container: "{{ node_name }}" + command: cat /etc/os-release + register: container_os_info + +- name: Set distro family (debian/rhel) + set_fact: + distro_family: >- + {{ + ( + 'debian' if 'debian' in container_os_info.stdout | lower else + 'rhel' if 'rhel' in container_os_info.stdout | lower or 'centos' in container_os_info.stdout | lower or 'fedora' in container_os_info.stdout | lower + else 'unknown' + ) | trim + }} +- name: Ensure pgbench is installed (Debian-based container) + community.docker.docker_container_exec: + container: "{{ node_name }}" + user: root + command: > + /bin/sh -c " + apt-get update && + apt-get install -y postgresql-contrib + " + when: distro_family == "debian" + +- name: Ensure pgbench is installed (RHEL-based container) + community.docker.docker_container_exec: + container: "{{ node_name }}" + user: root + command: microdnf install -y postgresql-contrib + when: distro_family == "rhel" + +- name: Initialize pgbench database + community.docker.docker_container_exec: + container: "{{ node_name }}" + user: postgres + command: > + pgbench -i -s {{ pgbench_scale }} pgbench +- name: Run pgbench benchmark + community.docker.docker_container_exec: + container: "{{ node_name }}" + user: postgres + command: > + pgbench -c {{ pgbench_clients }} -T {{ pgbench_time }} -j 4 pgbench + register: pgbench_result + +- name: Print pgbench results + debug: + var: pgbench_result.stdout_lines \ No newline at end of file From afe0e32961d7d2129b5e2ebaf707aaa0acfeb6f2 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Fri, 23 May 2025 14:05:08 +0200 Subject: [PATCH 006/156] PMM-7: road on pdpgsql --- pmm_qa/pdpgsql_pgsm_setup.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pmm_qa/pdpgsql_pgsm_setup.yml b/pmm_qa/pdpgsql_pgsm_setup.yml index fdc6d5e4..f47f0018 100644 --- a/pmm_qa/pdpgsql_pgsm_setup.yml +++ b/pmm_qa/pdpgsql_pgsm_setup.yml @@ -85,4 +85,7 @@ - name: Setup Percona Server group replication include_tasks: ./tasks/run_load_pgbench.yml vars: - node_name: "{{ pdpgsql_pgsm_container }}" \ No newline at end of file + node_name: "{{ pdpgsql_pgsm_container }}" + pgbench_clients: 10 + pgbench_time: 120 + pgbench_scale: 1000 \ No newline at end of file From bea45d5974e0ec30889e6d4ab2a7d0054d61c34b Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Fri, 23 May 2025 14:11:17 +0200 Subject: [PATCH 007/156] PMM-7: road on pdpgsql --- pmm_qa/tasks/run_load_pgbench.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/pmm_qa/tasks/run_load_pgbench.yml b/pmm_qa/tasks/run_load_pgbench.yml index e91cdfde..b654f528 100644 --- a/pmm_qa/tasks/run_load_pgbench.yml +++ b/pmm_qa/tasks/run_load_pgbench.yml @@ -1,4 +1,20 @@ # This ansible file run load on docker container of pgsql with variable name node_name +- name: Create custom database for pgbench + community.docker.docker_container_exec: + container: "{{ node_name }}" + user: postgres + command: > + bash -c " + echo \" + CREATE DATABASE pgbench; + \\c pgbench + GRANT CONNECT ON DATABASE pgbench TO pmm; + GRANT USAGE ON SCHEMA public TO pmm; + GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA public TO pmm; + ALTER DEFAULT PRIVILEGES IN SCHEMA public + GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO pmm; + \" | psql -U postgres -v ON_ERROR_STOP=1 + " - name: Detect OS inside the container community.docker.docker_container_exec: From 4e8e23c5cb4b07fb2b2173ef8946461d7445ffea Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Fri, 23 May 2025 14:29:19 +0200 Subject: [PATCH 008/156] PMM-7: load on pdpgsql --- pmm_qa/pdpgsql_pgsm_setup.yml | 32 ++++++++--- pmm_qa/scripts/pgsql_load.sql | 91 +++++++++++++++++++++++++++++++ pmm_qa/tasks/run_load_pgbench.yml | 69 ----------------------- 3 files changed, 116 insertions(+), 76 deletions(-) create mode 100644 pmm_qa/scripts/pgsql_load.sql delete mode 100644 pmm_qa/tasks/run_load_pgbench.yml diff --git a/pmm_qa/pdpgsql_pgsm_setup.yml b/pmm_qa/pdpgsql_pgsm_setup.yml index f47f0018..4d53aa31 100644 --- a/pmm_qa/pdpgsql_pgsm_setup.yml +++ b/pmm_qa/pdpgsql_pgsm_setup.yml @@ -82,10 +82,28 @@ with_items: - docker exec {{ pdpgsql_pgsm_container }} bash ./pgsm_run_queries.sh & - - name: Setup Percona Server group replication - include_tasks: ./tasks/run_load_pgbench.yml - vars: - node_name: "{{ pdpgsql_pgsm_container }}" - pgbench_clients: 10 - pgbench_time: 120 - pgbench_scale: 1000 \ No newline at end of file + - name: Create custom database for sample load + community.docker.docker_container_exec: + container: "{{ pdpgsql_pgsm_container}}" + user: postgres + command: > + bash -c " + echo \" + CREATE DATABASE school; + \\c pgbench + GRANT CONNECT ON DATABASE pgbench TO pmm; + GRANT USAGE ON SCHEMA public TO pmm; + GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA public TO pmm; + ALTER DEFAULT PRIVILEGES IN SCHEMA public + GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO pmm; + \" | psql -U postgres -v ON_ERROR_STOP=1 + " + + - name: Create sample load + community.docker.docker_container_exec: + container: "{{ pdpgsql_pgsm_container}}" + user: postgres + command: > + bash -c " + psql -U postgres -d school -f ./scripts/pgsql_load.sql + " diff --git a/pmm_qa/scripts/pgsql_load.sql b/pmm_qa/scripts/pgsql_load.sql new file mode 100644 index 00000000..e35dde50 --- /dev/null +++ b/pmm_qa/scripts/pgsql_load.sql @@ -0,0 +1,91 @@ +-- ======================================== +-- CREATE TABLES +-- ======================================== + +CREATE TABLE students ( + student_id SERIAL PRIMARY KEY, + first_name VARCHAR(50), + last_name VARCHAR(50), + birth_date DATE +); + +CREATE TABLE classes ( + class_id SERIAL PRIMARY KEY, + name VARCHAR(100), + teacher VARCHAR(100) +); + +CREATE TABLE enrollments ( + enrollment_id SERIAL PRIMARY KEY, + student_id INTEGER REFERENCES students(student_id), + class_id INTEGER REFERENCES classes(class_id), + enrollment_date DATE DEFAULT CURRENT_DATE +); + +-- ======================================== +-- INSERT MOCK DATA +-- ======================================== + +INSERT INTO students (first_name, last_name, birth_date) VALUES +('Alice', 'Smith', '2005-04-10'), +('Bob', 'Johnson', '2006-08-15'), +('Charlie', 'Brown', '2004-12-01'); + +INSERT INTO classes (name, teacher) VALUES +('Mathematics', 'Mrs. Taylor'), +('History', 'Mr. Anderson'), +('Science', 'Dr. Reynolds'); + +INSERT INTO enrollments (student_id, class_id) VALUES +(1, 1), +(1, 2), +(2, 2), +(3, 1), +(3, 3); + +-- ======================================== +-- SELECT QUERIES +-- ======================================== + +-- Get all students +SELECT * FROM students; + +-- Get all students enrolled in Mathematics +SELECT s.first_name, s.last_name +FROM students s +JOIN enrollments e ON s.student_id = e.student_id +JOIN classes c ON e.class_id = c.class_id +WHERE c.name = 'Mathematics'; + +-- Count students per class +SELECT c.name, COUNT(e.student_id) AS student_count +FROM classes c +LEFT JOIN enrollments e ON c.class_id = e.class_id +GROUP BY c.name; + +-- ======================================== +-- UPDATE QUERIES +-- ======================================== + +-- Change Bob's last name +UPDATE students +SET last_name = 'Williams' +WHERE first_name = 'Bob' AND last_name = 'Johnson'; + +-- Update the teacher for the History class +UPDATE classes +SET teacher = 'Ms. Carter' +WHERE name = 'History'; + +-- ======================================== +-- DELETE QUERIES +-- ======================================== + +-- Remove Charlie from Science class +DELETE FROM enrollments +WHERE student_id = (SELECT student_id FROM students WHERE first_name = 'Charlie') + AND class_id = (SELECT class_id FROM classes WHERE name = 'Science'); + +-- Delete a student completely +DELETE FROM students +WHERE first_name = 'Alice' AND last_name = 'Smith'; \ No newline at end of file diff --git a/pmm_qa/tasks/run_load_pgbench.yml b/pmm_qa/tasks/run_load_pgbench.yml deleted file mode 100644 index b654f528..00000000 --- a/pmm_qa/tasks/run_load_pgbench.yml +++ /dev/null @@ -1,69 +0,0 @@ -# This ansible file run load on docker container of pgsql with variable name node_name -- name: Create custom database for pgbench - community.docker.docker_container_exec: - container: "{{ node_name }}" - user: postgres - command: > - bash -c " - echo \" - CREATE DATABASE pgbench; - \\c pgbench - GRANT CONNECT ON DATABASE pgbench TO pmm; - GRANT USAGE ON SCHEMA public TO pmm; - GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA public TO pmm; - ALTER DEFAULT PRIVILEGES IN SCHEMA public - GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO pmm; - \" | psql -U postgres -v ON_ERROR_STOP=1 - " - -- name: Detect OS inside the container - community.docker.docker_container_exec: - container: "{{ node_name }}" - command: cat /etc/os-release - register: container_os_info - -- name: Set distro family (debian/rhel) - set_fact: - distro_family: >- - {{ - ( - 'debian' if 'debian' in container_os_info.stdout | lower else - 'rhel' if 'rhel' in container_os_info.stdout | lower or 'centos' in container_os_info.stdout | lower or 'fedora' in container_os_info.stdout | lower - else 'unknown' - ) | trim - }} -- name: Ensure pgbench is installed (Debian-based container) - community.docker.docker_container_exec: - container: "{{ node_name }}" - user: root - command: > - /bin/sh -c " - apt-get update && - apt-get install -y postgresql-contrib - " - when: distro_family == "debian" - -- name: Ensure pgbench is installed (RHEL-based container) - community.docker.docker_container_exec: - container: "{{ node_name }}" - user: root - command: microdnf install -y postgresql-contrib - when: distro_family == "rhel" - -- name: Initialize pgbench database - community.docker.docker_container_exec: - container: "{{ node_name }}" - user: postgres - command: > - pgbench -i -s {{ pgbench_scale }} pgbench -- name: Run pgbench benchmark - community.docker.docker_container_exec: - container: "{{ node_name }}" - user: postgres - command: > - pgbench -c {{ pgbench_clients }} -T {{ pgbench_time }} -j 4 pgbench - register: pgbench_result - -- name: Print pgbench results - debug: - var: pgbench_result.stdout_lines \ No newline at end of file From ee48ef30d14d622a736c663cbac0e831aed240a8 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Fri, 23 May 2025 14:37:54 +0200 Subject: [PATCH 009/156] PMM-7: load on pdpgsql --- pmm_qa/pdpgsql_pgsm_setup.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pmm_qa/pdpgsql_pgsm_setup.yml b/pmm_qa/pdpgsql_pgsm_setup.yml index 4d53aa31..2131335d 100644 --- a/pmm_qa/pdpgsql_pgsm_setup.yml +++ b/pmm_qa/pdpgsql_pgsm_setup.yml @@ -90,8 +90,8 @@ bash -c " echo \" CREATE DATABASE school; - \\c pgbench - GRANT CONNECT ON DATABASE pgbench TO pmm; + \\c school + GRANT CONNECT ON DATABASE school TO pmm; GRANT USAGE ON SCHEMA public TO pmm; GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA public TO pmm; ALTER DEFAULT PRIVILEGES IN SCHEMA public From 86f7a500b43e8a0619b7cd17606aa5fe3f4ece44 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Fri, 23 May 2025 14:43:24 +0200 Subject: [PATCH 010/156] PMM-7: load on pdpgsql --- pmm_qa/pdpgsql_pgsm_setup.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pmm_qa/pdpgsql_pgsm_setup.yml b/pmm_qa/pdpgsql_pgsm_setup.yml index 2131335d..8d6313ce 100644 --- a/pmm_qa/pdpgsql_pgsm_setup.yml +++ b/pmm_qa/pdpgsql_pgsm_setup.yml @@ -99,11 +99,17 @@ \" | psql -U postgres -v ON_ERROR_STOP=1 " + - name: Copy file into container + community.docker.docker_container_copy_to: + container: "{{ pdpgsql_pgsm_container}}" + src: ./scripts/pgsql_load.sql + dest: /pgsql_load.sql + - name: Create sample load community.docker.docker_container_exec: container: "{{ pdpgsql_pgsm_container}}" user: postgres command: > bash -c " - psql -U postgres -d school -f ./scripts/pgsql_load.sql + psql -U postgres -d school -f /pgsql_load.sql " From 7c008264a064491c71234aa61f8c37e31ddc2e7a Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Fri, 23 May 2025 14:46:22 +0200 Subject: [PATCH 011/156] PMM-7: load on pdpgsql --- pmm_qa/pdpgsql_pgsm_setup.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/pdpgsql_pgsm_setup.yml b/pmm_qa/pdpgsql_pgsm_setup.yml index 8d6313ce..d96e740e 100644 --- a/pmm_qa/pdpgsql_pgsm_setup.yml +++ b/pmm_qa/pdpgsql_pgsm_setup.yml @@ -100,7 +100,7 @@ " - name: Copy file into container - community.docker.docker_container_copy_to: + community.docker.docker_container_copy_into: container: "{{ pdpgsql_pgsm_container}}" src: ./scripts/pgsql_load.sql dest: /pgsql_load.sql From 4dfa4b568ad2d31cc095bda601ee86303907b6cc Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Fri, 23 May 2025 14:50:30 +0200 Subject: [PATCH 012/156] PMM-7: load on pdpgsql --- pmm_qa/pdpgsql_pgsm_setup.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pmm_qa/pdpgsql_pgsm_setup.yml b/pmm_qa/pdpgsql_pgsm_setup.yml index d96e740e..9309be1c 100644 --- a/pmm_qa/pdpgsql_pgsm_setup.yml +++ b/pmm_qa/pdpgsql_pgsm_setup.yml @@ -102,8 +102,8 @@ - name: Copy file into container community.docker.docker_container_copy_into: container: "{{ pdpgsql_pgsm_container}}" - src: ./scripts/pgsql_load.sql - dest: /pgsql_load.sql + path: ./scripts/pgsql_load.sql + container_path: /pgsql_load.sql - name: Create sample load community.docker.docker_container_exec: From 38233da4d12acb4b00783760e3c67b13f37a566a Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 26 May 2025 12:05:48 +0200 Subject: [PATCH 013/156] PMM-7: Correct name for client instalation file --- pmm_qa/pdpgsql_pgsm_setup.yml | 32 -------------------------------- 1 file changed, 32 deletions(-) diff --git a/pmm_qa/pdpgsql_pgsm_setup.yml b/pmm_qa/pdpgsql_pgsm_setup.yml index 9309be1c..09bb4dcb 100644 --- a/pmm_qa/pdpgsql_pgsm_setup.yml +++ b/pmm_qa/pdpgsql_pgsm_setup.yml @@ -81,35 +81,3 @@ shell: "{{ item }}" with_items: - docker exec {{ pdpgsql_pgsm_container }} bash ./pgsm_run_queries.sh & - - - name: Create custom database for sample load - community.docker.docker_container_exec: - container: "{{ pdpgsql_pgsm_container}}" - user: postgres - command: > - bash -c " - echo \" - CREATE DATABASE school; - \\c school - GRANT CONNECT ON DATABASE school TO pmm; - GRANT USAGE ON SCHEMA public TO pmm; - GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA public TO pmm; - ALTER DEFAULT PRIVILEGES IN SCHEMA public - GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO pmm; - \" | psql -U postgres -v ON_ERROR_STOP=1 - " - - - name: Copy file into container - community.docker.docker_container_copy_into: - container: "{{ pdpgsql_pgsm_container}}" - path: ./scripts/pgsql_load.sql - container_path: /pgsql_load.sql - - - name: Create sample load - community.docker.docker_container_exec: - container: "{{ pdpgsql_pgsm_container}}" - user: postgres - command: > - bash -c " - psql -U postgres -d school -f /pgsql_load.sql - " From 12886a7e3f929ba1f892fa2f8b46d4fe2f133ac8 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 26 May 2025 15:45:40 +0200 Subject: [PATCH 014/156] PMM-7: minio credentials --- pmm_psmdb-pbm_setup/docker-compose-rs.yaml | 4 ++-- pmm_psmdb-pbm_setup/docker-compose-sharded.yaml | 4 ++-- pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pmm_psmdb-pbm_setup/docker-compose-rs.yaml b/pmm_psmdb-pbm_setup/docker-compose-rs.yaml index b140258c..7d793853 100644 --- a/pmm_psmdb-pbm_setup/docker-compose-rs.yaml +++ b/pmm_psmdb-pbm_setup/docker-compose-rs.yaml @@ -198,8 +198,8 @@ services: volumes: - /tmp/minio/backups:/backups environment: - - "MINIO_ACCESS_KEY=minio1234" - - "MINIO_SECRET_KEY=minio1234" + - "MINIO_ROOT_USER=minio1234" + - "MINIO_ROOT_PASSWORD=minio1234" command: server /backups createbucket: diff --git a/pmm_psmdb-pbm_setup/docker-compose-sharded.yaml b/pmm_psmdb-pbm_setup/docker-compose-sharded.yaml index 0123ecbc..18f9c156 100644 --- a/pmm_psmdb-pbm_setup/docker-compose-sharded.yaml +++ b/pmm_psmdb-pbm_setup/docker-compose-sharded.yaml @@ -235,8 +235,8 @@ services: volumes: - backups:/backups environment: - - "MINIO_ACCESS_KEY=minio1234" - - "MINIO_SECRET_KEY=minio1234" + - MINIO_ROOT_USER=minio1234 + - MINIO_ROOT_PASSWORD=minio1234 command: server /backups createbucket: diff --git a/pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml b/pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml index 5ac5ff68..e3214860 100644 --- a/pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml +++ b/pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml @@ -104,8 +104,8 @@ services: volumes: - /tmp/minio/backups:/backups environment: - - "MINIO_ACCESS_KEY=minio1234" - - "MINIO_SECRET_KEY=minio1234" + - "MINIO_ROOT_USER=minio1234" + - "MINIO_ROOT_PASSWORD=minio1234" command: server /backups depends_on: psmdb-server: From 57c87e122eed237f41869e15da5b82b0f82e7839 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 26 May 2025 16:13:40 +0200 Subject: [PATCH 015/156] PMM-7: minio credentials --- pmm_psmdb-pbm_setup/docker-compose-rs.yaml | 2 +- pmm_psmdb-pbm_setup/docker-compose-sharded.yaml | 2 +- pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pmm_psmdb-pbm_setup/docker-compose-rs.yaml b/pmm_psmdb-pbm_setup/docker-compose-rs.yaml index 7d793853..77bcee00 100644 --- a/pmm_psmdb-pbm_setup/docker-compose-rs.yaml +++ b/pmm_psmdb-pbm_setup/docker-compose-rs.yaml @@ -182,7 +182,7 @@ services: hostname: rs203 minio: - image: minio/minio + image: minio/minio:RELEASE.2025-04-22T22-12-26Z profiles: ["classic", "extra"] container_name: minio ports: diff --git a/pmm_psmdb-pbm_setup/docker-compose-sharded.yaml b/pmm_psmdb-pbm_setup/docker-compose-sharded.yaml index 18f9c156..67207c58 100644 --- a/pmm_psmdb-pbm_setup/docker-compose-sharded.yaml +++ b/pmm_psmdb-pbm_setup/docker-compose-sharded.yaml @@ -226,7 +226,7 @@ services: - rscfg03 minio: - image: minio/minio + image: minio/minio:RELEASE.2025-04-22T22-12-26Z container_name: minio ports: - "9001:9000" diff --git a/pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml b/pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml index e3214860..dc067765 100644 --- a/pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml +++ b/pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml @@ -97,7 +97,7 @@ services: - 'openldap-data:/bitnami/openldap' minio: - image: minio/minio + image: minio/minio:RELEASE.2025-04-22T22-12-26Z container_name: minio ports: - "9001:9000" From b5e23f3625b9306e68776fb7e1a44815db6798ef Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 26 May 2025 16:24:31 +0200 Subject: [PATCH 016/156] PMM-7: minio credentials --- pmm_psmdb-pbm_setup/docker-compose-rs.yaml | 4 ++-- pmm_psmdb-pbm_setup/docker-compose-sharded.yaml | 2 +- pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pmm_psmdb-pbm_setup/docker-compose-rs.yaml b/pmm_psmdb-pbm_setup/docker-compose-rs.yaml index 77bcee00..613328f5 100644 --- a/pmm_psmdb-pbm_setup/docker-compose-rs.yaml +++ b/pmm_psmdb-pbm_setup/docker-compose-rs.yaml @@ -182,7 +182,7 @@ services: hostname: rs203 minio: - image: minio/minio:RELEASE.2025-04-22T22-12-26Z + image: minio/minio profiles: ["classic", "extra"] container_name: minio ports: @@ -215,7 +215,7 @@ services: depends_on: - minio entrypoint: > - /bin/sh -c " sleep 5; /usr/bin/mc config host add myminio http://minio:9000 minio1234 minio1234; /usr/bin/mc mb myminio/bcp; exit 0; " + /bin/sh -c " sleep 30; /usr/bin/mc config host add myminio http://minio:9000 minio1234 minio1234; /usr/bin/mc mb myminio/bcp; exit 0; " networks: qa-integration: diff --git a/pmm_psmdb-pbm_setup/docker-compose-sharded.yaml b/pmm_psmdb-pbm_setup/docker-compose-sharded.yaml index 67207c58..18f9c156 100644 --- a/pmm_psmdb-pbm_setup/docker-compose-sharded.yaml +++ b/pmm_psmdb-pbm_setup/docker-compose-sharded.yaml @@ -226,7 +226,7 @@ services: - rscfg03 minio: - image: minio/minio:RELEASE.2025-04-22T22-12-26Z + image: minio/minio container_name: minio ports: - "9001:9000" diff --git a/pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml b/pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml index dc067765..e3214860 100644 --- a/pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml +++ b/pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml @@ -97,7 +97,7 @@ services: - 'openldap-data:/bitnami/openldap' minio: - image: minio/minio:RELEASE.2025-04-22T22-12-26Z + image: minio/minio container_name: minio ports: - "9001:9000" From c7866ac40af41de92c83407da71096f125066e25 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 26 May 2025 17:13:21 +0200 Subject: [PATCH 017/156] PMM-7: minio credentials --- pmm_psmdb-pbm_setup/docker-compose-rs.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_psmdb-pbm_setup/docker-compose-rs.yaml b/pmm_psmdb-pbm_setup/docker-compose-rs.yaml index 613328f5..e2819a1c 100644 --- a/pmm_psmdb-pbm_setup/docker-compose-rs.yaml +++ b/pmm_psmdb-pbm_setup/docker-compose-rs.yaml @@ -215,7 +215,7 @@ services: depends_on: - minio entrypoint: > - /bin/sh -c " sleep 30; /usr/bin/mc config host add myminio http://minio:9000 minio1234 minio1234; /usr/bin/mc mb myminio/bcp; exit 0; " + /bin/sh -c " sleep 45; /usr/bin/mc config host add myminio http://minio:9000 minio1234 minio1234; /usr/bin/mc mb myminio/bcp; exit 0; " networks: qa-integration: From 1c494d8c69d2c7910874946f0683bac4e52ec258 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 26 May 2025 18:02:40 +0200 Subject: [PATCH 018/156] PMM-7: minio credentials --- pmm_psmdb-pbm_setup/docker-compose-rs.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_psmdb-pbm_setup/docker-compose-rs.yaml b/pmm_psmdb-pbm_setup/docker-compose-rs.yaml index e2819a1c..e5e1ee52 100644 --- a/pmm_psmdb-pbm_setup/docker-compose-rs.yaml +++ b/pmm_psmdb-pbm_setup/docker-compose-rs.yaml @@ -215,7 +215,7 @@ services: depends_on: - minio entrypoint: > - /bin/sh -c " sleep 45; /usr/bin/mc config host add myminio http://minio:9000 minio1234 minio1234; /usr/bin/mc mb myminio/bcp; exit 0; " + /bin/sh -c " sleep 60; /usr/bin/mc config host add myminio http://minio:9000 minio1234 minio1234; /usr/bin/mc mb myminio/bcp; exit 0; " networks: qa-integration: From aa5213c221855d0fb09cf4100c1e9dbd89e46e34 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 26 May 2025 19:33:47 +0200 Subject: [PATCH 019/156] PMM-7: minio credentials --- pmm_psmdb-pbm_setup/start-rs-only.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pmm_psmdb-pbm_setup/start-rs-only.sh b/pmm_psmdb-pbm_setup/start-rs-only.sh index d7f95dfc..84b3ba8c 100755 --- a/pmm_psmdb-pbm_setup/start-rs-only.sh +++ b/pmm_psmdb-pbm_setup/start-rs-only.sh @@ -35,3 +35,5 @@ if [ $profile = "extra" ]; then fi bash -x ./configure-extra-agents.sh fi + +docker exec minio /bin/sh -c " sleep 5; /usr/bin/mc config host add myminio http://minio:9000 minio1234 minio1234; /usr/bin/mc mb myminio/bcp; exit 0; " \ No newline at end of file From 06f0a613cb6946b8e6b270fa85c619118c91eca2 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 4 Jun 2025 23:55:15 +0200 Subject: [PATCH 020/156] PMM-7: New buckets --- pmm_psmdb-pbm_setup/start-rs-only.sh | 2 ++ pmm_psmdb-pbm_setup/start-sharded.sh | 2 ++ 2 files changed, 4 insertions(+) diff --git a/pmm_psmdb-pbm_setup/start-rs-only.sh b/pmm_psmdb-pbm_setup/start-rs-only.sh index d7f95dfc..c038bdba 100755 --- a/pmm_psmdb-pbm_setup/start-rs-only.sh +++ b/pmm_psmdb-pbm_setup/start-rs-only.sh @@ -35,3 +35,5 @@ if [ $profile = "extra" ]; then fi bash -x ./configure-extra-agents.sh fi + +docker exec minio /bin/sh -c " sleep 5; /usr/bin/mc alias set myminio http://minio:9000 minio1234 minio1234; /usr/bin/mc mb myminio/bcp-replicaset; exit 0; " diff --git a/pmm_psmdb-pbm_setup/start-sharded.sh b/pmm_psmdb-pbm_setup/start-sharded.sh index 4fe16b1b..0798fe48 100755 --- a/pmm_psmdb-pbm_setup/start-sharded.sh +++ b/pmm_psmdb-pbm_setup/start-sharded.sh @@ -259,3 +259,5 @@ if [ $cleanup != "no" ]; then else echo "skipping cleanup" fi + +docker exec minio /bin/sh -c " sleep 5; /usr/bin/mc alias set myminio http://minio:9000 minio1234 minio1234; /usr/bin/mc mb myminio/bcp-sharded; exit 0; " From 3238db92fcac0c25fe7cd51e8be8df2d3e3919c2 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Thu, 5 Jun 2025 22:32:23 +0200 Subject: [PATCH 021/156] PMM-7: Configure agents --- pmm_psmdb-pbm_setup/configure-agents.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pmm_psmdb-pbm_setup/configure-agents.sh b/pmm_psmdb-pbm_setup/configure-agents.sh index 93e9fa96..8505ce67 100644 --- a/pmm_psmdb-pbm_setup/configure-agents.sh +++ b/pmm_psmdb-pbm_setup/configure-agents.sh @@ -33,9 +33,9 @@ do echo "congiguring pmm agent on $node" docker compose -f docker-compose-rs.yaml exec -T -e PMM_AGENT_SETUP_NODE_NAME=${node}._${random_number} $node pmm-agent setup if [[ $mongo_setup_type == "psa" && $node == "rs103" ]]; then - docker compose -f docker-compose-rs.yaml exec -T $node pmm-admin add mongodb --enable-all-collectors --agent-password=mypass --cluster=replicaset --replication-set=rs ${node}_${random_number} 127.0.0.1:27017 + docker compose -f docker-compose-rs.yaml exec -T $node pmm-admin add mongodb --enable-all-collectors --agent-password=mypass --cluster=replicaset --environment=mongo-sharded-dev--replication-set=rs ${node}_${random_number} 127.0.0.1:27017 else - docker compose -f docker-compose-rs.yaml exec -T $node pmm-admin add mongodb --enable-all-collectors --agent-password=mypass --cluster=replicaset --replication-set=rs --username=${pmm_mongo_user} --password=${pmm_mongo_user_pass} ${node}_${random_number} 127.0.0.1:27017 + docker compose -f docker-compose-rs.yaml exec -T $node pmm-admin add mongodb --enable-all-collectors --agent-password=mypass --cluster=replicaset --environment=mongo-sharded-dev --replication-set=rs --username=${pmm_mongo_user} --password=${pmm_mongo_user_pass} ${node}_${random_number} 127.0.0.1:27017 fi done echo From aae13c60f16c2f040ebb6e4151b76f565f387f23 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Thu, 5 Jun 2025 22:52:27 +0200 Subject: [PATCH 022/156] PMM-7: Configure agents --- pmm_psmdb-pbm_setup/configure-agents.sh | 4 ++-- pmm_psmdb-pbm_setup/start-sharded.sh | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pmm_psmdb-pbm_setup/configure-agents.sh b/pmm_psmdb-pbm_setup/configure-agents.sh index 8505ce67..93e9fa96 100644 --- a/pmm_psmdb-pbm_setup/configure-agents.sh +++ b/pmm_psmdb-pbm_setup/configure-agents.sh @@ -33,9 +33,9 @@ do echo "congiguring pmm agent on $node" docker compose -f docker-compose-rs.yaml exec -T -e PMM_AGENT_SETUP_NODE_NAME=${node}._${random_number} $node pmm-agent setup if [[ $mongo_setup_type == "psa" && $node == "rs103" ]]; then - docker compose -f docker-compose-rs.yaml exec -T $node pmm-admin add mongodb --enable-all-collectors --agent-password=mypass --cluster=replicaset --environment=mongo-sharded-dev--replication-set=rs ${node}_${random_number} 127.0.0.1:27017 + docker compose -f docker-compose-rs.yaml exec -T $node pmm-admin add mongodb --enable-all-collectors --agent-password=mypass --cluster=replicaset --replication-set=rs ${node}_${random_number} 127.0.0.1:27017 else - docker compose -f docker-compose-rs.yaml exec -T $node pmm-admin add mongodb --enable-all-collectors --agent-password=mypass --cluster=replicaset --environment=mongo-sharded-dev --replication-set=rs --username=${pmm_mongo_user} --password=${pmm_mongo_user_pass} ${node}_${random_number} 127.0.0.1:27017 + docker compose -f docker-compose-rs.yaml exec -T $node pmm-admin add mongodb --enable-all-collectors --agent-password=mypass --cluster=replicaset --replication-set=rs --username=${pmm_mongo_user} --password=${pmm_mongo_user_pass} ${node}_${random_number} 127.0.0.1:27017 fi done echo diff --git a/pmm_psmdb-pbm_setup/start-sharded.sh b/pmm_psmdb-pbm_setup/start-sharded.sh index 0798fe48..78e72af6 100755 --- a/pmm_psmdb-pbm_setup/start-sharded.sh +++ b/pmm_psmdb-pbm_setup/start-sharded.sh @@ -237,10 +237,10 @@ do echo "congiguring pmm agent on $node" rs=$(echo $node | awk -F "0" '{print $1}') docker compose -f docker-compose-sharded.yaml exec -T -e PMM_AGENT_SETUP_NODE_NAME=${node}_${random_number} $node pmm-agent setup - docker compose -f docker-compose-sharded.yaml exec -T $node pmm-admin add mongodb --agent-password=mypass --cluster=sharded --username=${pmm_user} --password=${pmm_pass} ${node}_${random_number} 127.0.0.1:27017 + docker compose -f docker-compose-sharded.yaml exec -T $node pmm-admin add mongodb --agent-password=mypass --cluster=sharded --environment=mongo-sharded-dev --username=${pmm_user} --password=${pmm_pass} ${node}_${random_number} 127.0.0.1:27017 done echo "configuring pmm-agent on primary rscfg01 for mongos instance" -docker compose -f docker-compose-sharded.yaml exec -T rscfg01 pmm-admin add mongodb --agent-password=mypass --cluster=sharded --username=${pmm_user} --password=${pmm_pass} mongos_${random_number} mongos:27017 +docker compose -f docker-compose-sharded.yaml exec -T rscfg01 pmm-admin add mongodb --agent-password=mypass --cluster=sharded --environment=mongo-sharded-dev --username=${pmm_user} --password=${pmm_pass} mongos_${random_number} mongos:27017 echo "adding some data" docker compose -f docker-compose-sharded.yaml exec -T mongos mgodatagen -f /etc/datagen/sharded.json --uri=mongodb://root:root@127.0.0.1:27017 From ab67e16f3d4ccfbbb1e43e11fae009a44c22f9df Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Fri, 6 Jun 2025 10:41:32 +0200 Subject: [PATCH 023/156] PMM-7: Minio --- pmm_psmdb-pbm_setup/docker-compose-sharded.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pmm_psmdb-pbm_setup/docker-compose-sharded.yaml b/pmm_psmdb-pbm_setup/docker-compose-sharded.yaml index 4f4598a5..af16ad5d 100644 --- a/pmm_psmdb-pbm_setup/docker-compose-sharded.yaml +++ b/pmm_psmdb-pbm_setup/docker-compose-sharded.yaml @@ -229,7 +229,8 @@ services: image: minio/minio container_name: minio ports: - - "9001:9000" + - "9000:9000" + - "9001:9001" networks: - test-network volumes: From e3c4421f97061e464acecc6b764782e1059553f9 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Sun, 8 Jun 2025 00:35:30 +0200 Subject: [PATCH 024/156] PMM-7: Minio --- pmm_psmdb-pbm_setup/start-sharded.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/pmm_psmdb-pbm_setup/start-sharded.sh b/pmm_psmdb-pbm_setup/start-sharded.sh index 78e72af6..5ab2a50e 100755 --- a/pmm_psmdb-pbm_setup/start-sharded.sh +++ b/pmm_psmdb-pbm_setup/start-sharded.sh @@ -261,3 +261,4 @@ if [ $cleanup != "no" ]; then fi docker exec minio /bin/sh -c " sleep 5; /usr/bin/mc alias set myminio http://minio:9000 minio1234 minio1234; /usr/bin/mc mb myminio/bcp-sharded; exit 0; " +docker ps -a From 4498168accd9075e4d6889812910c723e91886dc Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Sun, 8 Jun 2025 00:39:03 +0200 Subject: [PATCH 025/156] PMM-7: Minio --- pmm_psmdb-pbm_setup/docker-compose-rs.yaml | 2 +- pmm_psmdb-pbm_setup/docker-compose-sharded.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pmm_psmdb-pbm_setup/docker-compose-rs.yaml b/pmm_psmdb-pbm_setup/docker-compose-rs.yaml index 9a328eaf..817092c3 100644 --- a/pmm_psmdb-pbm_setup/docker-compose-rs.yaml +++ b/pmm_psmdb-pbm_setup/docker-compose-rs.yaml @@ -200,7 +200,7 @@ services: environment: - "MINIO_ACCESS_KEY=minio1234" - "MINIO_SECRET_KEY=minio1234" - command: server /backups + command: server /backups --address 0.0.0.0:9000 --console-address 0.0.0.0:9001 createbucket: container_name: createbucket diff --git a/pmm_psmdb-pbm_setup/docker-compose-sharded.yaml b/pmm_psmdb-pbm_setup/docker-compose-sharded.yaml index af16ad5d..6f0b5ca4 100644 --- a/pmm_psmdb-pbm_setup/docker-compose-sharded.yaml +++ b/pmm_psmdb-pbm_setup/docker-compose-sharded.yaml @@ -238,7 +238,7 @@ services: environment: - "MINIO_ACCESS_KEY=minio1234" - "MINIO_SECRET_KEY=minio1234" - command: server /backups + command: server /backups --address 0.0.0.0:9000 --console-address 0.0.0.0:9001 createbucket: container_name: createbucket From da934d0386a0b82498f74991808fd7aa5313704f Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Sun, 8 Jun 2025 13:26:25 +0200 Subject: [PATCH 026/156] PMM-7: Minio --- pmm_qa/pmm-framework.py | 3 +++ pmm_qa/tasks/create_minio_container.yml | 30 +++++++++++++++++++++++++ 2 files changed, 33 insertions(+) create mode 100644 pmm_qa/tasks/create_minio_container.yml diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index 5538e53b..b3635890 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -894,12 +894,15 @@ def setup_database(db_type, db_version=None, db_config=None, args=None): parser.add_argument("--client-version", nargs='?', help='PMM Client version/tarball') parser.add_argument("--verbose", "--v", action='store_true', help='Display verbose information') parser.add_argument("--verbosity-level", nargs='?', help='Display verbose information level') + parser.add_argument("--bucket", nargs='?', help='Create MinIO S3 compatible bucket') args = parser.parse_args() if args.verbosity_level is not None and not args.verbosity_level.isnumeric(): print(f"Option {args.verbosity_level} is invalid verbosity level option, please provide number 1-5") exit(1) + print(args) + # Parse arguments try: for db in args.database: diff --git a/pmm_qa/tasks/create_minio_container.yml b/pmm_qa/tasks/create_minio_container.yml new file mode 100644 index 00000000..d9e234e1 --- /dev/null +++ b/pmm_qa/tasks/create_minio_container.yml @@ -0,0 +1,30 @@ +--- +- name: Deploy MinIO bucket. + hosts: localhost + connection: local + gather_facts: yes + vars: + minio_access_key: minio1234 + minio_secret_key: minio1234 + minio_volume_name: minio_backups + minio_ports: + - "9000:9000" + - "9001:9001" + + tasks: + - name: Create MinIO Docker volume + community.docker.docker_volume: + name: "{{ minio_volume_name }}" + + - name: Run MinIO container + community.docker.docker_container: + name: minio + image: minio/minio + restart_policy: unless-stopped + ports: "{{ minio_ports }}" + volumes: + - "{{ minio_volume_name }}:/backups" + env: + MINIO_ACCESS_KEY: "{{ minio_access_key }}" + MINIO_SECRET_KEY: "{{ minio_secret_key }}" + command: server /backups --address 0.0.0.0:9000 --console-address 0.0.0.0:9001 \ No newline at end of file From 4febefeda68e6a67f5ecf7eb726657ce720b66a1 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Sun, 8 Jun 2025 13:36:24 +0200 Subject: [PATCH 027/156] PMM-7: Minio --- pmm_qa/pmm-framework.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index b3635890..f63d6432 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -894,7 +894,7 @@ def setup_database(db_type, db_version=None, db_config=None, args=None): parser.add_argument("--client-version", nargs='?', help='PMM Client version/tarball') parser.add_argument("--verbose", "--v", action='store_true', help='Display verbose information') parser.add_argument("--verbosity-level", nargs='?', help='Display verbose information level') - parser.add_argument("--bucket", nargs='?', help='Create MinIO S3 compatible bucket') + parser.add_argument("--bucket", nargs='store_true', help='Create MinIO S3 compatible bucket') args = parser.parse_args() if args.verbosity_level is not None and not args.verbosity_level.isnumeric(): @@ -938,6 +938,7 @@ def setup_database(db_type, db_version=None, db_config=None, args=None): if args.verbose: print(f"Option {key} is not recognised, will be using default option") continue + # if args except KeyError as e: print(f"Option {key} is not recognised with error {e}, Please check and try again") parser.print_help() From 03c862563fc2279624094f61de3bf8a3df2c3025 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Sun, 8 Jun 2025 13:39:51 +0200 Subject: [PATCH 028/156] PMM-7: Minio --- pmm_qa/pmm-framework.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index f63d6432..a1557e34 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -894,7 +894,7 @@ def setup_database(db_type, db_version=None, db_config=None, args=None): parser.add_argument("--client-version", nargs='?', help='PMM Client version/tarball') parser.add_argument("--verbose", "--v", action='store_true', help='Display verbose information') parser.add_argument("--verbosity-level", nargs='?', help='Display verbose information level') - parser.add_argument("--bucket", nargs='store_true', help='Create MinIO S3 compatible bucket') + parser.add_argument("--bucket", nargs='+', help='Create MinIO S3 compatible bucket') args = parser.parse_args() if args.verbosity_level is not None and not args.verbosity_level.isnumeric(): From b77b0bfb44aa8d0147e18a363215e14c86c7d220 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Sun, 8 Jun 2025 13:41:17 +0200 Subject: [PATCH 029/156] PMM-7: Minio --- pmm_qa/pmm-framework.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index a1557e34..2d9ef44e 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -938,7 +938,8 @@ def setup_database(db_type, db_version=None, db_config=None, args=None): if args.verbose: print(f"Option {key} is not recognised, will be using default option") continue - # if args + if args.bucket: + print(f"Buckets are: {args.bucket}") except KeyError as e: print(f"Option {key} is not recognised with error {e}, Please check and try again") parser.print_help() From 8fd7b0e6e1ffbd6ff49e060c05199280e4529b79 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Sun, 8 Jun 2025 13:43:14 +0200 Subject: [PATCH 030/156] PMM-7: Minio --- pmm_qa/pmm-framework.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index 2d9ef44e..18237aa3 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -902,6 +902,7 @@ def setup_database(db_type, db_version=None, db_config=None, args=None): exit(1) print(args) + print(f"Buckets are {args.bucket}") # Parse arguments try: From 15c79302622b4d916727b51cb723b502bc4e67a3 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Sun, 8 Jun 2025 13:45:10 +0200 Subject: [PATCH 031/156] PMM-7: Minio --- pmm_qa/pmm-framework.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index 18237aa3..55871edf 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -902,10 +902,11 @@ def setup_database(db_type, db_version=None, db_config=None, args=None): exit(1) print(args) - print(f"Buckets are {args.bucket}") # Parse arguments try: + if args.bucket: + print(f"Buckets are: {args.bucket}") for db in args.database: db_parts = db[0].split(',') configs = db_parts[0:] if len(db_parts) > 1 else db[0:] @@ -939,8 +940,6 @@ def setup_database(db_type, db_version=None, db_config=None, args=None): if args.verbose: print(f"Option {key} is not recognised, will be using default option") continue - if args.bucket: - print(f"Buckets are: {args.bucket}") except KeyError as e: print(f"Option {key} is not recognised with error {e}, Please check and try again") parser.print_help() From 353b5f85a3e127954c90ad49055254020c19a727 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Sun, 8 Jun 2025 13:58:42 +0200 Subject: [PATCH 032/156] PMM-7: Minio --- pmm_qa/pmm-framework.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index 55871edf..73b22492 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -872,6 +872,9 @@ def setup_database(db_type, db_version=None, db_config=None, args=None): print(f"Database type {db_type} is not recognised, Exiting...") exit(1) +def setup_bucket(args=None): + print("Setting up bucket") + print(args) # Main if __name__ == "__main__": @@ -907,6 +910,7 @@ def setup_database(db_type, db_version=None, db_config=None, args=None): try: if args.bucket: print(f"Buckets are: {args.bucket}") + setup_bucket(args) for db in args.database: db_parts = db[0].split(',') configs = db_parts[0:] if len(db_parts) > 1 else db[0:] From 4a253839f1ec80b9c5e65b97db131a9304a94a37 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Sun, 8 Jun 2025 14:00:06 +0200 Subject: [PATCH 033/156] PMM-7: Minio --- pmm_qa/pmm-framework.py | 93 ++++++++++++++++++++++------------------- 1 file changed, 49 insertions(+), 44 deletions(-) diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index 73b22492..489b2f1d 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -92,8 +92,6 @@ def run_ansible_playbook(playbook_filename, env_vars, args): playbook_path = script_dir + "/" + playbook_filename verbosity_level = 1 - - if args.verbosity_level is not None: verbosity_level = int(args.verbosity_level) @@ -212,7 +210,8 @@ def setup_ps(db_type, db_version=None, db_config=None, args=None): 'PS_NODES': no_of_nodes, 'PS_VERSION': ps_version, 'PMM_SERVER_IP': args.pmm_server_ip or container_name or '127.0.0.1', - 'PS_CONTAINER': 'ps_pmm_' + str(ps_version) + ('_replica' if setup_type_value in ("replication", "replica") else ''), + 'PS_CONTAINER': 'ps_pmm_' + str(ps_version) + ( + '_replica' if setup_type_value in ("replication", "replica") else ''), 'PS_PORT': 3318 if setup_type_value in ("replication", "replica") else 3317, 'CLIENT_VERSION': get_value('CLIENT_VERSION', db_type, args, db_config), 'QUERY_SOURCE': get_value('QUERY_SOURCE', db_type, args, db_config), @@ -227,6 +226,7 @@ def setup_ps(db_type, db_version=None, db_config=None, args=None): # Call the function to run the Ansible playbook run_ansible_playbook(playbook_filename, env_vars, args) + def setup_mysql(db_type, db_version=None, db_config=None, args=None): # Check if PMM server is running container_name = get_running_container_name() @@ -470,6 +470,7 @@ def setup_mlaunch_psmdb(db_type, db_version=None, db_config=None, args=None): # Call the function to run the Ansible playbook run_ansible_playbook(playbook_filename, env_vars, args) + def setup_mlaunch_modb(db_type, db_version=None, db_config=None, args=None): # Check if PMM server is running container_name = get_running_container_name() @@ -479,7 +480,7 @@ def setup_mlaunch_modb(db_type, db_version=None, db_config=None, args=None): # Gather Version details modb_version = os.getenv('MODB_VERSION') or db_version or \ - database_configs[db_type]["versions"][-1] + database_configs[db_type]["versions"][-1] # Define environment variables for playbook env_vars = { @@ -599,7 +600,6 @@ def get_latest_psmdb_version(psmdb_version): # Extract the version number using regular expression version_number = [v.split('|')[0] for v in re.findall(r'value="([^"]*)"', response.text)] - if version_number: # Sort the version numbers and extract the latest one latest_version = sorted(version_number, key=lambda x: tuple(map(int, x.split('-')[-1].split('.'))))[-1] @@ -689,7 +689,9 @@ def mongo_ssl_setup(script_filename, args): subprocess.run( ['cp', f'{scripts_path}docker-compose-pmm-psmdb.yml', f'{compose_file_path}']) admin_password = os.getenv('ADMIN_PASSWORD') or args.pmm_server_password or 'admin' - subprocess.run(['sed', '-i', f's/PMM_AGENT_SERVER_PASSWORD=admin/PMM_AGENT_SERVER_PASSWORD={admin_password}/g', f'{compose_file_path}']) + subprocess.run( + ['sed', '-i', f's/PMM_AGENT_SERVER_PASSWORD=admin/PMM_AGENT_SERVER_PASSWORD={admin_password}/g', + f'{compose_file_path}']) subprocess.run(['sed', '-i', '/container_name/a\ networks:\\\n \\- pmm-qa', f'{compose_file_path}']) subprocess.run(['sed', '-i', '$a\\\nnetworks:\\\n pmm-qa:\\\n name: pmm-qa\\\n external: true', f'{compose_file_path}']) @@ -872,10 +874,12 @@ def setup_database(db_type, db_version=None, db_config=None, args=None): print(f"Database type {db_type} is not recognised, Exiting...") exit(1) + def setup_bucket(args=None): print("Setting up bucket") print(args) + # Main if __name__ == "__main__": parser = argparse.ArgumentParser(description='PMM Framework Script to setup Multiple Databases', @@ -911,45 +915,46 @@ def setup_bucket(args=None): if args.bucket: print(f"Buckets are: {args.bucket}") setup_bucket(args) - for db in args.database: - db_parts = db[0].split(',') - configs = db_parts[0:] if len(db_parts) > 1 else db[0:] - db_type = None - db_version = None - db_config = {} - - if configs: - for config in configs: - if "=" in config: - key, value = config.split('=') - else: - key, value = config, None - - # Convert all arguments/options only to uppercase - key = key.upper() - - try: - if key in database_configs: - db_type = key - if "versions" in database_configs[db_type]: - if value in database_configs[db_type]["versions"]: - db_version = value - else: - if args.verbose: - print( - f"Value {value} is not recognised for Option {key}, will be using Default value") - elif key in database_configs[db_type]["configurations"]: - db_config[key] = value + if args.database: + for db in args.database: + db_parts = db[0].split(',') + configs = db_parts[0:] if len(db_parts) > 1 else db[0:] + db_type = None + db_version = None + db_config = {} + + if configs: + for config in configs: + if "=" in config: + key, value = config.split('=') else: - if args.verbose: - print(f"Option {key} is not recognised, will be using default option") - continue - except KeyError as e: - print(f"Option {key} is not recognised with error {e}, Please check and try again") - parser.print_help() - exit(1) - # Set up the specified databases - setup_database(db_type, db_version, db_config, args) + key, value = config, None + + # Convert all arguments/options only to uppercase + key = key.upper() + + try: + if key in database_configs: + db_type = key + if "versions" in database_configs[db_type]: + if value in database_configs[db_type]["versions"]: + db_version = value + else: + if args.verbose: + print( + f"Value {value} is not recognised for Option {key}, will be using Default value") + elif key in database_configs[db_type]["configurations"]: + db_config[key] = value + else: + if args.verbose: + print(f"Option {key} is not recognised, will be using default option") + continue + except KeyError as e: + print(f"Option {key} is not recognised with error {e}, Please check and try again") + parser.print_help() + exit(1) + # Set up the specified databases + setup_database(db_type, db_version, db_config, args) except argparse.ArgumentError as e: print(f"Option is not recognised:", e) parser.print_help() From 9fe9408a207db00cd200eddd913d69fd83fc9306 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Sun, 8 Jun 2025 14:01:45 +0200 Subject: [PATCH 034/156] PMM-7: Minio --- pmm_qa/pmm-framework.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index 489b2f1d..2339a44e 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -877,8 +877,10 @@ def setup_database(db_type, db_version=None, db_config=None, args=None): def setup_bucket(args=None): print("Setting up bucket") - print(args) + print(args.bucket) + env_vars = {} + run_ansible_playbook('tasks/create_minio_container.yml', env_vars, args) # Main if __name__ == "__main__": @@ -913,7 +915,6 @@ def setup_bucket(args=None): # Parse arguments try: if args.bucket: - print(f"Buckets are: {args.bucket}") setup_bucket(args) if args.database: for db in args.database: From aacad1b6d12a27c513befe49fc29821895a5672e Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Sun, 8 Jun 2025 14:03:37 +0200 Subject: [PATCH 035/156] PMM-7: Minio --- pmm_qa/tasks/create_minio_container.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/pmm_qa/tasks/create_minio_container.yml b/pmm_qa/tasks/create_minio_container.yml index d9e234e1..e74583eb 100644 --- a/pmm_qa/tasks/create_minio_container.yml +++ b/pmm_qa/tasks/create_minio_container.yml @@ -12,6 +12,20 @@ - "9001:9001" tasks: + - name: Remove old MinIO docker container + community.docker.docker_container: + name: minio + image: minio/minio + restart_policy: always + state: absent + ignore_errors: yes + + - name: Remove MinIO Docker volume + community.docker.docker_volume: + name: "{{ minio_volume_name }}" + state: absent + ignore_errors: true + - name: Create MinIO Docker volume community.docker.docker_volume: name: "{{ minio_volume_name }}" From 1ab1024787cef988b3195d7e2771d4ce9050a690 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Sun, 8 Jun 2025 14:04:28 +0200 Subject: [PATCH 036/156] PMM-7: Minio --- pmm_qa/pmm-framework.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index 2339a44e..08c8d8d6 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -878,7 +878,9 @@ def setup_database(db_type, db_version=None, db_config=None, args=None): def setup_bucket(args=None): print("Setting up bucket") print(args.bucket) - env_vars = {} + env_vars = { + 'BUCKETS': args.bucket + } run_ansible_playbook('tasks/create_minio_container.yml', env_vars, args) From 875ce5f1c18ff7fc2ea76979b829096beb6aa908 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Sun, 8 Jun 2025 14:05:33 +0200 Subject: [PATCH 037/156] PMM-7: Minio --- pmm_qa/tasks/create_minio_container.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pmm_qa/tasks/create_minio_container.yml b/pmm_qa/tasks/create_minio_container.yml index e74583eb..667df81f 100644 --- a/pmm_qa/tasks/create_minio_container.yml +++ b/pmm_qa/tasks/create_minio_container.yml @@ -4,6 +4,7 @@ connection: local gather_facts: yes vars: + buckets: "{{ lookup('env', 'BUCKETS') | default('bcp', true) }}" minio_access_key: minio1234 minio_secret_key: minio1234 minio_volume_name: minio_backups @@ -41,4 +42,8 @@ env: MINIO_ACCESS_KEY: "{{ minio_access_key }}" MINIO_SECRET_KEY: "{{ minio_secret_key }}" - command: server /backups --address 0.0.0.0:9000 --console-address 0.0.0.0:9001 \ No newline at end of file + command: server /backups --address 0.0.0.0:9000 --console-address 0.0.0.0:9001 + + - name: Show the list of buckets + debug: + var: buckets \ No newline at end of file From 4c0d568d9fa21f9bd577f546456ecdc91607b657 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Sun, 8 Jun 2025 14:09:13 +0200 Subject: [PATCH 038/156] PMM-7: Minio --- pmm_qa/tasks/create_minio_container.yml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/pmm_qa/tasks/create_minio_container.yml b/pmm_qa/tasks/create_minio_container.yml index 667df81f..44a6aae1 100644 --- a/pmm_qa/tasks/create_minio_container.yml +++ b/pmm_qa/tasks/create_minio_container.yml @@ -46,4 +46,15 @@ - name: Show the list of buckets debug: - var: buckets \ No newline at end of file + var: buckets + + - name: Create MinIO buckets + community.docker.docker_container_exec: + container: "minio" + command: > + /bin/sh -c " + sleep 5; + /usr/bin/mc alias set myminio http://minio:9000 minio1234 minio1234; + /usr/bin/mc mb myminio/{{ item }}; + exit 0;" + loop: "{{ buckets }}" From 2085ac0ec222a3302b7ff0332bfe8329264d04ad Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Sun, 8 Jun 2025 14:11:56 +0200 Subject: [PATCH 039/156] PMM-7: Minio --- pmm_qa/tasks/create_minio_container.yml | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/pmm_qa/tasks/create_minio_container.yml b/pmm_qa/tasks/create_minio_container.yml index 44a6aae1..18dd05b9 100644 --- a/pmm_qa/tasks/create_minio_container.yml +++ b/pmm_qa/tasks/create_minio_container.yml @@ -48,13 +48,20 @@ debug: var: buckets - - name: Create MinIO buckets + - name: Set MinIO alias inside the container community.docker.docker_container_exec: container: "minio" command: > /bin/sh -c " sleep 5; /usr/bin/mc alias set myminio http://minio:9000 minio1234 minio1234; - /usr/bin/mc mb myminio/{{ item }}; exit 0;" - loop: "{{ buckets }}" + + - name: Create MinIO buckets + community.docker.docker_container_exec: + container: "minio" + command: > + /bin/sh -c " + /usr/bin/mc mb myminio/{{ item }} || echo 'Bucket {{ item }} already exists'; + exit 0;" + loop: "{{ buckets }}" From 7ac3513b9ee2ea844232e8c97a52835a433b6e07 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Sun, 8 Jun 2025 14:13:18 +0200 Subject: [PATCH 040/156] PMM-7: Minio --- pmm_qa/tasks/create_minio_container.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/tasks/create_minio_container.yml b/pmm_qa/tasks/create_minio_container.yml index 18dd05b9..ed8162c0 100644 --- a/pmm_qa/tasks/create_minio_container.yml +++ b/pmm_qa/tasks/create_minio_container.yml @@ -54,7 +54,7 @@ command: > /bin/sh -c " sleep 5; - /usr/bin/mc alias set myminio http://minio:9000 minio1234 minio1234; + /usr/bin/mc alias set myminio http://127.0.0.1:9000 minio1234 minio1234; exit 0;" - name: Create MinIO buckets From 6ba7e6e7510252789f86987eb3942becccfb060e Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Sun, 8 Jun 2025 15:49:35 +0200 Subject: [PATCH 041/156] PMM-7: Minio --- pmm_qa/tasks/create_minio_container.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/pmm_qa/tasks/create_minio_container.yml b/pmm_qa/tasks/create_minio_container.yml index ed8162c0..0e6cc1e5 100644 --- a/pmm_qa/tasks/create_minio_container.yml +++ b/pmm_qa/tasks/create_minio_container.yml @@ -4,6 +4,7 @@ connection: local gather_facts: yes vars: + network_name: "pmm-qa" buckets: "{{ lookup('env', 'BUCKETS') | default('bcp', true) }}" minio_access_key: minio1234 minio_secret_key: minio1234 @@ -13,6 +14,12 @@ - "9001:9001" tasks: + - name: Create Docker network + community.docker.docker_network: + name: "{{ network_name }}" + state: present + ignore_errors: yes + - name: Remove old MinIO docker container community.docker.docker_container: name: minio @@ -37,6 +44,8 @@ image: minio/minio restart_policy: unless-stopped ports: "{{ minio_ports }}" + networks: + - name: "{{ network_name }}" volumes: - "{{ minio_volume_name }}:/backups" env: From 9244f6f2fdf4ad224ae94a41afc979c08bafa105 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Sun, 8 Jun 2025 16:20:41 +0200 Subject: [PATCH 042/156] PMM-7: Minio --- pmm_qa/tasks/create_minio_container.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pmm_qa/tasks/create_minio_container.yml b/pmm_qa/tasks/create_minio_container.yml index 0e6cc1e5..cda41e7c 100644 --- a/pmm_qa/tasks/create_minio_container.yml +++ b/pmm_qa/tasks/create_minio_container.yml @@ -14,6 +14,10 @@ - "9001:9001" tasks: + - name: List running Docker containers + community.docker.docker_container_info: + gather_facts: false + - name: Create Docker network community.docker.docker_network: name: "{{ network_name }}" From 932aad098135174de0636a2c4a0d777de6a40c61 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Sun, 8 Jun 2025 16:31:49 +0200 Subject: [PATCH 043/156] PMM-7: Minio --- pmm_qa/tasks/create_minio_container.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pmm_qa/tasks/create_minio_container.yml b/pmm_qa/tasks/create_minio_container.yml index cda41e7c..60c4fa64 100644 --- a/pmm_qa/tasks/create_minio_container.yml +++ b/pmm_qa/tasks/create_minio_container.yml @@ -14,9 +14,9 @@ - "9001:9001" tasks: - - name: List running Docker containers - community.docker.docker_container_info: - gather_facts: false + - name: Run docker ps with port info + shell: docker ps -a + register: docker_ps_output - name: Create Docker network community.docker.docker_network: From 2183beb876839e50f059444b623206ae6c05120d Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Sun, 8 Jun 2025 16:46:22 +0200 Subject: [PATCH 044/156] PMM-7: Minio --- pmm_qa/tasks/create_minio_container.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pmm_qa/tasks/create_minio_container.yml b/pmm_qa/tasks/create_minio_container.yml index 60c4fa64..2e355b74 100644 --- a/pmm_qa/tasks/create_minio_container.yml +++ b/pmm_qa/tasks/create_minio_container.yml @@ -10,8 +10,8 @@ minio_secret_key: minio1234 minio_volume_name: minio_backups minio_ports: - - "9000:9000" - - "9001:9001" + - "90000:9000" + - "90001:9001" tasks: - name: Run docker ps with port info @@ -55,7 +55,7 @@ env: MINIO_ACCESS_KEY: "{{ minio_access_key }}" MINIO_SECRET_KEY: "{{ minio_secret_key }}" - command: server /backups --address 0.0.0.0:9000 --console-address 0.0.0.0:9001 + command: server /backups --address 0.0.0.0:90000 --console-address 0.0.0.0:90001 - name: Show the list of buckets debug: @@ -67,7 +67,7 @@ command: > /bin/sh -c " sleep 5; - /usr/bin/mc alias set myminio http://127.0.0.1:9000 minio1234 minio1234; + /usr/bin/mc alias set myminio http://127.0.0.1:90000 minio1234 minio1234; exit 0;" - name: Create MinIO buckets From 168b161f238e7a0e8bd73f8092f822c38375185e Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Sun, 8 Jun 2025 16:48:25 +0200 Subject: [PATCH 045/156] PMM-7: Minio --- pmm_qa/tasks/create_minio_container.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pmm_qa/tasks/create_minio_container.yml b/pmm_qa/tasks/create_minio_container.yml index 2e355b74..d5fc806d 100644 --- a/pmm_qa/tasks/create_minio_container.yml +++ b/pmm_qa/tasks/create_minio_container.yml @@ -55,7 +55,7 @@ env: MINIO_ACCESS_KEY: "{{ minio_access_key }}" MINIO_SECRET_KEY: "{{ minio_secret_key }}" - command: server /backups --address 0.0.0.0:90000 --console-address 0.0.0.0:90001 + command: server /backups --address 0.0.0.0:9000 --console-address 0.0.0.0:9001 - name: Show the list of buckets debug: @@ -67,7 +67,7 @@ command: > /bin/sh -c " sleep 5; - /usr/bin/mc alias set myminio http://127.0.0.1:90000 minio1234 minio1234; + /usr/bin/mc alias set myminio http://127.0.0.1:9000 minio1234 minio1234; exit 0;" - name: Create MinIO buckets From 1135804b606e71922d9d8c035d3bfe855f0eb456 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Sun, 8 Jun 2025 16:50:03 +0200 Subject: [PATCH 046/156] PMM-7: Minio --- pmm_qa/tasks/create_minio_container.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pmm_qa/tasks/create_minio_container.yml b/pmm_qa/tasks/create_minio_container.yml index d5fc806d..326043c6 100644 --- a/pmm_qa/tasks/create_minio_container.yml +++ b/pmm_qa/tasks/create_minio_container.yml @@ -10,8 +10,8 @@ minio_secret_key: minio1234 minio_volume_name: minio_backups minio_ports: - - "90000:9000" - - "90001:9001" + - "9010:9000" + - "9001:9001" tasks: - name: Run docker ps with port info From fe762fb26cbee0d464398be88a3c348a93d600cf Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 9 Jun 2025 07:58:37 +0200 Subject: [PATCH 047/156] PMM-7: Minio --- pmm_qa/scripts/pgsql_load.sql | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/pmm_qa/scripts/pgsql_load.sql b/pmm_qa/scripts/pgsql_load.sql index bd70b7f2..81a4fa90 100644 --- a/pmm_qa/scripts/pgsql_load.sql +++ b/pmm_qa/scripts/pgsql_load.sql @@ -43,6 +43,30 @@ INSERT INTO enrollments (student_id, class_id) VALUES (3, 1), (3, 3); +-- ======================================== +-- SIMULATE DEAD TUPLES +-- ======================================== + + +INSERT INTO students (first_name, last_name, birth_date) +SELECT 'John', 'Doe', CURRENT_DATE - (random() * 5000)::int +FROM generate_series(1, 100000); + +-- These updates and deletes will create dead tuples + +-- Update records (old versions become dead) +UPDATE students +SET last_name = last_name || '_updated' +WHERE student_id IN (1, 2); + +-- Delete records (deleted rows become dead) +DELETE FROM enrollments +WHERE enrollment_id IN (SELECT enrollment_id FROM enrollments LIMIT 2); + +-- Disable autovacuum temporarily (for demo) +ALTER TABLE students SET (autovacuum_enabled = false); +ALTER TABLE enrollments SET (autovacuum_enabled = false); + -- ======================================== -- SELECT QUERIES -- ======================================== From 7f9258fe654aa4714c45cb7beb9db205c1d67851 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 10 Jun 2025 09:02:26 +0200 Subject: [PATCH 048/156] PMM-7: Code cleanup --- pmm_psmdb-pbm_setup/docker-compose-rs.yaml | 2 +- pmm_psmdb-pbm_setup/docker-compose-sharded.yaml | 5 ++--- pmm_psmdb-pbm_setup/start-rs-only.sh | 2 -- pmm_psmdb-pbm_setup/start-sharded.sh | 3 --- pmm_qa/pmm-framework.py | 10 +++------- 5 files changed, 6 insertions(+), 16 deletions(-) diff --git a/pmm_psmdb-pbm_setup/docker-compose-rs.yaml b/pmm_psmdb-pbm_setup/docker-compose-rs.yaml index 817092c3..9a328eaf 100644 --- a/pmm_psmdb-pbm_setup/docker-compose-rs.yaml +++ b/pmm_psmdb-pbm_setup/docker-compose-rs.yaml @@ -200,7 +200,7 @@ services: environment: - "MINIO_ACCESS_KEY=minio1234" - "MINIO_SECRET_KEY=minio1234" - command: server /backups --address 0.0.0.0:9000 --console-address 0.0.0.0:9001 + command: server /backups createbucket: container_name: createbucket diff --git a/pmm_psmdb-pbm_setup/docker-compose-sharded.yaml b/pmm_psmdb-pbm_setup/docker-compose-sharded.yaml index 6f0b5ca4..4f4598a5 100644 --- a/pmm_psmdb-pbm_setup/docker-compose-sharded.yaml +++ b/pmm_psmdb-pbm_setup/docker-compose-sharded.yaml @@ -229,8 +229,7 @@ services: image: minio/minio container_name: minio ports: - - "9000:9000" - - "9001:9001" + - "9001:9000" networks: - test-network volumes: @@ -238,7 +237,7 @@ services: environment: - "MINIO_ACCESS_KEY=minio1234" - "MINIO_SECRET_KEY=minio1234" - command: server /backups --address 0.0.0.0:9000 --console-address 0.0.0.0:9001 + command: server /backups createbucket: container_name: createbucket diff --git a/pmm_psmdb-pbm_setup/start-rs-only.sh b/pmm_psmdb-pbm_setup/start-rs-only.sh index c038bdba..d7f95dfc 100755 --- a/pmm_psmdb-pbm_setup/start-rs-only.sh +++ b/pmm_psmdb-pbm_setup/start-rs-only.sh @@ -35,5 +35,3 @@ if [ $profile = "extra" ]; then fi bash -x ./configure-extra-agents.sh fi - -docker exec minio /bin/sh -c " sleep 5; /usr/bin/mc alias set myminio http://minio:9000 minio1234 minio1234; /usr/bin/mc mb myminio/bcp-replicaset; exit 0; " diff --git a/pmm_psmdb-pbm_setup/start-sharded.sh b/pmm_psmdb-pbm_setup/start-sharded.sh index 5ab2a50e..e63274eb 100755 --- a/pmm_psmdb-pbm_setup/start-sharded.sh +++ b/pmm_psmdb-pbm_setup/start-sharded.sh @@ -259,6 +259,3 @@ if [ $cleanup != "no" ]; then else echo "skipping cleanup" fi - -docker exec minio /bin/sh -c " sleep 5; /usr/bin/mc alias set myminio http://minio:9000 minio1234 minio1234; /usr/bin/mc mb myminio/bcp-sharded; exit 0; " -docker ps -a diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index 08c8d8d6..739dfdd1 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -470,7 +470,6 @@ def setup_mlaunch_psmdb(db_type, db_version=None, db_config=None, args=None): # Call the function to run the Ansible playbook run_ansible_playbook(playbook_filename, env_vars, args) - def setup_mlaunch_modb(db_type, db_version=None, db_config=None, args=None): # Check if PMM server is running container_name = get_running_container_name() @@ -600,6 +599,7 @@ def get_latest_psmdb_version(psmdb_version): # Extract the version number using regular expression version_number = [v.split('|')[0] for v in re.findall(r'value="([^"]*)"', response.text)] + if version_number: # Sort the version numbers and extract the latest one latest_version = sorted(version_number, key=lambda x: tuple(map(int, x.split('-')[-1].split('.'))))[-1] @@ -622,7 +622,7 @@ def setup_psmdb(db_type, db_version=None, db_config=None, args=None): # Gather Version details psmdb_version = os.getenv('PSMDB_VERSION') or get_latest_psmdb_version(db_version) or \ - database_configs[db_type]["versions"][-1] + database_configs[db_type]["versions"][-1] # Handle port address for external or internal address server_hostname = container_name @@ -689,9 +689,7 @@ def mongo_ssl_setup(script_filename, args): subprocess.run( ['cp', f'{scripts_path}docker-compose-pmm-psmdb.yml', f'{compose_file_path}']) admin_password = os.getenv('ADMIN_PASSWORD') or args.pmm_server_password or 'admin' - subprocess.run( - ['sed', '-i', f's/PMM_AGENT_SERVER_PASSWORD=admin/PMM_AGENT_SERVER_PASSWORD={admin_password}/g', - f'{compose_file_path}']) + subprocess.run(['sed', '-i', f's/PMM_AGENT_SERVER_PASSWORD=admin/PMM_AGENT_SERVER_PASSWORD={admin_password}/g', f'{compose_file_path}']) subprocess.run(['sed', '-i', '/container_name/a\ networks:\\\n \\- pmm-qa', f'{compose_file_path}']) subprocess.run(['sed', '-i', '$a\\\nnetworks:\\\n pmm-qa:\\\n name: pmm-qa\\\n external: true', f'{compose_file_path}']) @@ -912,8 +910,6 @@ def setup_bucket(args=None): print(f"Option {args.verbosity_level} is invalid verbosity level option, please provide number 1-5") exit(1) - print(args) - # Parse arguments try: if args.bucket: From db661f2c7fa2cca3a8e84cf8da3e0fd3284d4c3d Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 10 Jun 2025 09:04:03 +0200 Subject: [PATCH 049/156] PMM-7: Code cleanup --- pmm_qa/pmm-framework.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index 739dfdd1..7c225a86 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -92,6 +92,8 @@ def run_ansible_playbook(playbook_filename, env_vars, args): playbook_path = script_dir + "/" + playbook_filename verbosity_level = 1 + + if args.verbosity_level is not None: verbosity_level = int(args.verbosity_level) @@ -210,8 +212,7 @@ def setup_ps(db_type, db_version=None, db_config=None, args=None): 'PS_NODES': no_of_nodes, 'PS_VERSION': ps_version, 'PMM_SERVER_IP': args.pmm_server_ip or container_name or '127.0.0.1', - 'PS_CONTAINER': 'ps_pmm_' + str(ps_version) + ( - '_replica' if setup_type_value in ("replication", "replica") else ''), + 'PS_CONTAINER': 'ps_pmm_' + str(ps_version) + ('_replica' if setup_type_value in ("replication", "replica") else ''), 'PS_PORT': 3318 if setup_type_value in ("replication", "replica") else 3317, 'CLIENT_VERSION': get_value('CLIENT_VERSION', db_type, args, db_config), 'QUERY_SOURCE': get_value('QUERY_SOURCE', db_type, args, db_config), @@ -226,7 +227,6 @@ def setup_ps(db_type, db_version=None, db_config=None, args=None): # Call the function to run the Ansible playbook run_ansible_playbook(playbook_filename, env_vars, args) - def setup_mysql(db_type, db_version=None, db_config=None, args=None): # Check if PMM server is running container_name = get_running_container_name() @@ -479,7 +479,7 @@ def setup_mlaunch_modb(db_type, db_version=None, db_config=None, args=None): # Gather Version details modb_version = os.getenv('MODB_VERSION') or db_version or \ - database_configs[db_type]["versions"][-1] + database_configs[db_type]["versions"][-1] # Define environment variables for playbook env_vars = { @@ -622,7 +622,7 @@ def setup_psmdb(db_type, db_version=None, db_config=None, args=None): # Gather Version details psmdb_version = os.getenv('PSMDB_VERSION') or get_latest_psmdb_version(db_version) or \ - database_configs[db_type]["versions"][-1] + database_configs[db_type]["versions"][-1] # Handle port address for external or internal address server_hostname = container_name From dbea822495a2f15c34654cfaff6c6e14bf1c482c Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 10 Jun 2025 11:13:04 +0200 Subject: [PATCH 050/156] PMM-7: Default value for bucket --- pmm_qa/pmm-framework.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index 7c225a86..ac869822 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -82,6 +82,9 @@ "DOCKERCLIENTS": { "configurations": {} # Empty dictionary for consistency }, + "BUCKET": { + "configurations": { "BUCKET_NAME": 'bcp' } + } } From f689b3a0b6e063d98235e719a50af23385665be3 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 10 Jun 2025 11:14:38 +0200 Subject: [PATCH 051/156] PMM-7: Default value for bucket --- pmm_qa/pmm-framework.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index ac869822..667dc73a 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -83,7 +83,7 @@ "configurations": {} # Empty dictionary for consistency }, "BUCKET": { - "configurations": { "BUCKET_NAME": 'bcp' } + "configurations": { "BUCKET_NAMES": 'bcp' } } } From b7ff844875ce2ed660838c615fcc7f3838759b6f Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 10 Jun 2025 11:17:07 +0200 Subject: [PATCH 052/156] PMM-7: Default value for bucket --- pmm_qa/pmm-framework.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index 667dc73a..5da0b667 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -870,7 +870,8 @@ def setup_database(db_type, db_version=None, db_config=None, args=None): setup_mlaunch_modb(db_type, db_version, db_config, args) elif db_type == 'SSL_MLAUNCH': setup_ssl_mlaunch(db_type, db_version, db_config, args) - + elif db_type == 'BUCKET': + setup_bucket(args) else: print(f"Database type {db_type} is not recognised, Exiting...") exit(1) From 1f9ed10cb92f2fd8fed026a9dd3d82b38da14d7f Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 10 Jun 2025 11:19:55 +0200 Subject: [PATCH 053/156] PMM-7: Default value for bucket --- pmm_qa/pmm-framework.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index 5da0b667..860883a7 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -871,6 +871,7 @@ def setup_database(db_type, db_version=None, db_config=None, args=None): elif db_type == 'SSL_MLAUNCH': setup_ssl_mlaunch(db_type, db_version, db_config, args) elif db_type == 'BUCKET': + print(f'Args Are: {args}') setup_bucket(args) else: print(f"Database type {db_type} is not recognised, Exiting...") From 53769196709fd682c08f6d784ccc948f2f2f3d62 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 10 Jun 2025 11:22:08 +0200 Subject: [PATCH 054/156] PMM-7: Default value for bucket --- pmm_qa/pmm-framework.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index 860883a7..be372fab 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -880,7 +880,8 @@ def setup_database(db_type, db_version=None, db_config=None, args=None): def setup_bucket(args=None): print("Setting up bucket") - print(args.bucket) + bucket_names_value = get_value('BUCKET_NAMES', db_type, args, db_config).lower() + print(bucket_names_value) env_vars = { 'BUCKETS': args.bucket } From bbbd4a6fb0121670c0a7632e6bbdbd182874a6c3 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 10 Jun 2025 11:22:44 +0200 Subject: [PATCH 055/156] PMM-7: Default value for bucket --- pmm_qa/pmm-framework.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index be372fab..c7d660fc 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -880,7 +880,7 @@ def setup_database(db_type, db_version=None, db_config=None, args=None): def setup_bucket(args=None): print("Setting up bucket") - bucket_names_value = get_value('BUCKET_NAMES', db_type, args, db_config).lower() + bucket_names_value = get_value('BUCKET_NAMES', '', args, '').lower() print(bucket_names_value) env_vars = { 'BUCKETS': args.bucket From f91fe8ed2e9a3596cd29a50507302cab90851f51 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 10 Jun 2025 11:24:17 +0200 Subject: [PATCH 056/156] PMM-7: Default value for bucket --- pmm_qa/pmm-framework.py | 102 ++++++++++++++++++++-------------------- 1 file changed, 51 insertions(+), 51 deletions(-) diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index c7d660fc..3a7a1586 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -83,7 +83,7 @@ "configurations": {} # Empty dictionary for consistency }, "BUCKET": { - "configurations": { "BUCKET_NAMES": 'bcp' } + "configurations": {"BUCKET_NAMES": 'bcp'} } } @@ -95,8 +95,6 @@ def run_ansible_playbook(playbook_filename, env_vars, args): playbook_path = script_dir + "/" + playbook_filename verbosity_level = 1 - - if args.verbosity_level is not None: verbosity_level = int(args.verbosity_level) @@ -215,7 +213,8 @@ def setup_ps(db_type, db_version=None, db_config=None, args=None): 'PS_NODES': no_of_nodes, 'PS_VERSION': ps_version, 'PMM_SERVER_IP': args.pmm_server_ip or container_name or '127.0.0.1', - 'PS_CONTAINER': 'ps_pmm_' + str(ps_version) + ('_replica' if setup_type_value in ("replication", "replica") else ''), + 'PS_CONTAINER': 'ps_pmm_' + str(ps_version) + ( + '_replica' if setup_type_value in ("replication", "replica") else ''), 'PS_PORT': 3318 if setup_type_value in ("replication", "replica") else 3317, 'CLIENT_VERSION': get_value('CLIENT_VERSION', db_type, args, db_config), 'QUERY_SOURCE': get_value('QUERY_SOURCE', db_type, args, db_config), @@ -230,6 +229,7 @@ def setup_ps(db_type, db_version=None, db_config=None, args=None): # Call the function to run the Ansible playbook run_ansible_playbook(playbook_filename, env_vars, args) + def setup_mysql(db_type, db_version=None, db_config=None, args=None): # Check if PMM server is running container_name = get_running_container_name() @@ -473,6 +473,7 @@ def setup_mlaunch_psmdb(db_type, db_version=None, db_config=None, args=None): # Call the function to run the Ansible playbook run_ansible_playbook(playbook_filename, env_vars, args) + def setup_mlaunch_modb(db_type, db_version=None, db_config=None, args=None): # Check if PMM server is running container_name = get_running_container_name() @@ -482,7 +483,7 @@ def setup_mlaunch_modb(db_type, db_version=None, db_config=None, args=None): # Gather Version details modb_version = os.getenv('MODB_VERSION') or db_version or \ - database_configs[db_type]["versions"][-1] + database_configs[db_type]["versions"][-1] # Define environment variables for playbook env_vars = { @@ -602,7 +603,6 @@ def get_latest_psmdb_version(psmdb_version): # Extract the version number using regular expression version_number = [v.split('|')[0] for v in re.findall(r'value="([^"]*)"', response.text)] - if version_number: # Sort the version numbers and extract the latest one latest_version = sorted(version_number, key=lambda x: tuple(map(int, x.split('-')[-1].split('.'))))[-1] @@ -692,7 +692,9 @@ def mongo_ssl_setup(script_filename, args): subprocess.run( ['cp', f'{scripts_path}docker-compose-pmm-psmdb.yml', f'{compose_file_path}']) admin_password = os.getenv('ADMIN_PASSWORD') or args.pmm_server_password or 'admin' - subprocess.run(['sed', '-i', f's/PMM_AGENT_SERVER_PASSWORD=admin/PMM_AGENT_SERVER_PASSWORD={admin_password}/g', f'{compose_file_path}']) + subprocess.run( + ['sed', '-i', f's/PMM_AGENT_SERVER_PASSWORD=admin/PMM_AGENT_SERVER_PASSWORD={admin_password}/g', + f'{compose_file_path}']) subprocess.run(['sed', '-i', '/container_name/a\ networks:\\\n \\- pmm-qa', f'{compose_file_path}']) subprocess.run(['sed', '-i', '$a\\\nnetworks:\\\n pmm-qa:\\\n name: pmm-qa\\\n external: true', f'{compose_file_path}']) @@ -872,15 +874,15 @@ def setup_database(db_type, db_version=None, db_config=None, args=None): setup_ssl_mlaunch(db_type, db_version, db_config, args) elif db_type == 'BUCKET': print(f'Args Are: {args}') - setup_bucket(args) + setup_bucket(db_type, db_version, db_config, args) else: print(f"Database type {db_type} is not recognised, Exiting...") exit(1) -def setup_bucket(args=None): +def setup_bucket(db_type, db_version=None, db_config=None, args=None): print("Setting up bucket") - bucket_names_value = get_value('BUCKET_NAMES', '', args, '').lower() + bucket_names_value = get_value('BUCKET_NAMES', db_type, args, db_config).lower() print(bucket_names_value) env_vars = { 'BUCKETS': args.bucket @@ -888,6 +890,7 @@ def setup_bucket(args=None): run_ansible_playbook('tasks/create_minio_container.yml', env_vars, args) + # Main if __name__ == "__main__": parser = argparse.ArgumentParser(description='PMM Framework Script to setup Multiple Databases', @@ -918,48 +921,45 @@ def setup_bucket(args=None): # Parse arguments try: - if args.bucket: - setup_bucket(args) - if args.database: - for db in args.database: - db_parts = db[0].split(',') - configs = db_parts[0:] if len(db_parts) > 1 else db[0:] - db_type = None - db_version = None - db_config = {} - - if configs: - for config in configs: - if "=" in config: - key, value = config.split('=') + for db in args.database: + db_parts = db[0].split(',') + configs = db_parts[0:] if len(db_parts) > 1 else db[0:] + db_type = None + db_version = None + db_config = {} + + if configs: + for config in configs: + if "=" in config: + key, value = config.split('=') + else: + key, value = config, None + + # Convert all arguments/options only to uppercase + key = key.upper() + + try: + if key in database_configs: + db_type = key + if "versions" in database_configs[db_type]: + if value in database_configs[db_type]["versions"]: + db_version = value + else: + if args.verbose: + print( + f"Value {value} is not recognised for Option {key}, will be using Default value") + elif key in database_configs[db_type]["configurations"]: + db_config[key] = value else: - key, value = config, None - - # Convert all arguments/options only to uppercase - key = key.upper() - - try: - if key in database_configs: - db_type = key - if "versions" in database_configs[db_type]: - if value in database_configs[db_type]["versions"]: - db_version = value - else: - if args.verbose: - print( - f"Value {value} is not recognised for Option {key}, will be using Default value") - elif key in database_configs[db_type]["configurations"]: - db_config[key] = value - else: - if args.verbose: - print(f"Option {key} is not recognised, will be using default option") - continue - except KeyError as e: - print(f"Option {key} is not recognised with error {e}, Please check and try again") - parser.print_help() - exit(1) - # Set up the specified databases - setup_database(db_type, db_version, db_config, args) + if args.verbose: + print(f"Option {key} is not recognised, will be using default option") + continue + except KeyError as e: + print(f"Option {key} is not recognised with error {e}, Please check and try again") + parser.print_help() + exit(1) + # Set up the specified databases + setup_database(db_type, db_version, db_config, args) except argparse.ArgumentError as e: print(f"Option is not recognised:", e) parser.print_help() From 48e47d8dce0e5af012d26b02f054dd2f12edaefb Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 10 Jun 2025 11:25:42 +0200 Subject: [PATCH 057/156] PMM-7: Default value for bucket --- pmm_qa/pmm-framework.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index 3a7a1586..0c9fe70b 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -885,7 +885,7 @@ def setup_bucket(db_type, db_version=None, db_config=None, args=None): bucket_names_value = get_value('BUCKET_NAMES', db_type, args, db_config).lower() print(bucket_names_value) env_vars = { - 'BUCKETS': args.bucket + 'BUCKETS': bucket_names_value } run_ansible_playbook('tasks/create_minio_container.yml', env_vars, args) From f5c3e1efb93446ea0186a4c738801b6f114f7fab Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 10 Jun 2025 11:28:31 +0200 Subject: [PATCH 058/156] PMM-7: Default value for bucket --- pmm_qa/pmm-framework.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index 0c9fe70b..129c7fac 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -884,6 +884,8 @@ def setup_bucket(db_type, db_version=None, db_config=None, args=None): print("Setting up bucket") bucket_names_value = get_value('BUCKET_NAMES', db_type, args, db_config).lower() print(bucket_names_value) + bucket_names_value = bucket_names_value if isinstance(bucket_names_value, list) else [bucket_names_value] + print(bucket_names_value) env_vars = { 'BUCKETS': bucket_names_value } From 8aa72138af14b0c3ec8b46746863e42e330aace1 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 10 Jun 2025 11:42:15 +0200 Subject: [PATCH 059/156] PMM-7: Default value for bucket --- pmm_qa/pmm-framework.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index 129c7fac..d2b583b5 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -882,9 +882,7 @@ def setup_database(db_type, db_version=None, db_config=None, args=None): def setup_bucket(db_type, db_version=None, db_config=None, args=None): print("Setting up bucket") - bucket_names_value = get_value('BUCKET_NAMES', db_type, args, db_config).lower() - print(bucket_names_value) - bucket_names_value = bucket_names_value if isinstance(bucket_names_value, list) else [bucket_names_value] + bucket_names_value = get_value('BUCKET_NAMES', db_type, args, db_config).lower().split(';') print(bucket_names_value) env_vars = { 'BUCKETS': bucket_names_value From 799d8c415610e1c2aae40630d165c88e13d65e5a Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 10 Jun 2025 11:44:42 +0200 Subject: [PATCH 060/156] PMM-7: Default value for bucket --- pmm_qa/pmm-framework.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index d2b583b5..edca1182 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -603,6 +603,7 @@ def get_latest_psmdb_version(psmdb_version): # Extract the version number using regular expression version_number = [v.split('|')[0] for v in re.findall(r'value="([^"]*)"', response.text)] + if version_number: # Sort the version numbers and extract the latest one latest_version = sorted(version_number, key=lambda x: tuple(map(int, x.split('-')[-1].split('.'))))[-1] @@ -692,9 +693,7 @@ def mongo_ssl_setup(script_filename, args): subprocess.run( ['cp', f'{scripts_path}docker-compose-pmm-psmdb.yml', f'{compose_file_path}']) admin_password = os.getenv('ADMIN_PASSWORD') or args.pmm_server_password or 'admin' - subprocess.run( - ['sed', '-i', f's/PMM_AGENT_SERVER_PASSWORD=admin/PMM_AGENT_SERVER_PASSWORD={admin_password}/g', - f'{compose_file_path}']) + subprocess.run(['sed', '-i', f's/PMM_AGENT_SERVER_PASSWORD=admin/PMM_AGENT_SERVER_PASSWORD={admin_password}/g', f'{compose_file_path}']) subprocess.run(['sed', '-i', '/container_name/a\ networks:\\\n \\- pmm-qa', f'{compose_file_path}']) subprocess.run(['sed', '-i', '$a\\\nnetworks:\\\n pmm-qa:\\\n name: pmm-qa\\\n external: true', f'{compose_file_path}']) @@ -873,7 +872,6 @@ def setup_database(db_type, db_version=None, db_config=None, args=None): elif db_type == 'SSL_MLAUNCH': setup_ssl_mlaunch(db_type, db_version, db_config, args) elif db_type == 'BUCKET': - print(f'Args Are: {args}') setup_bucket(db_type, db_version, db_config, args) else: print(f"Database type {db_type} is not recognised, Exiting...") @@ -912,7 +910,6 @@ def setup_bucket(db_type, db_version=None, db_config=None, args=None): parser.add_argument("--client-version", nargs='?', help='PMM Client version/tarball') parser.add_argument("--verbose", "--v", action='store_true', help='Display verbose information') parser.add_argument("--verbosity-level", nargs='?', help='Display verbose information level') - parser.add_argument("--bucket", nargs='+', help='Create MinIO S3 compatible bucket') args = parser.parse_args() if args.verbosity_level is not None and not args.verbosity_level.isnumeric(): From 615de93705c3ec2e315e537f24b77c206082f298 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Fri, 13 Jun 2025 10:12:19 +0200 Subject: [PATCH 061/156] PMM-7: Add load to pgsql replication --- ...istribution-postgres-replication-setup.yml | 8 ++++++- pmm_qa/postgresql/tasks/load.sql | 24 +++++++++++++++++++ pmm_qa/postgresql/tasks/run_load_pgsql.yml | 6 +++++ 3 files changed, 37 insertions(+), 1 deletion(-) create mode 100644 pmm_qa/postgresql/tasks/load.sql diff --git a/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml b/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml index 242a984b..c218be61 100644 --- a/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml @@ -218,7 +218,7 @@ - name: Add service to pmm server community.docker.docker_container_exec: container: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" - command: pmm-admin add postgresql --username=pmm --password=pmm --query-source=pgstatmonitor pdpgsql_pmm_{{ pg_version }}_{{ item }}{{ random_service_name_value }} --debug 127.0.0.1:5432 + command: pmm-admin add postgresql --username=pmm --password=pmm --cluster=pdpgsq_replication_cluster--query-source=pgstatmonitor pdpgsql_pmm_{{ pg_version }}_{{ item }}{{ random_service_name_value }} --debug 127.0.0.1:5432 loop: "{{ range(1, nodes_count | int + 1) | list }}" - name: Run load on primary node. @@ -229,6 +229,12 @@ pgbench_time: 120 pgbench_scale: 1000 +- name: Create logical replication slots + community.docker.docker_container_exec: + container: "pdpgsql_pmm_{{ pg_version }}_1" + user: postgres + command: psql -U postgres -d postgres -c "SELECT * FROM pg_create_logical_replication_slot('test_slot', 'test_decoding');" + - name: Check replication status on primary community.docker.docker_container_exec: container: "pdpgsql_pmm_{{ pg_version }}_1" diff --git a/pmm_qa/postgresql/tasks/load.sql b/pmm_qa/postgresql/tasks/load.sql new file mode 100644 index 00000000..e5b9ca5f --- /dev/null +++ b/pmm_qa/postgresql/tasks/load.sql @@ -0,0 +1,24 @@ +-- Step 1: Show initial buffers_alloc value +SELECT 'Initial buffers_alloc' AS info, buffers_alloc FROM pg_stat_bgwriter; + +-- Step 2: Drop and create a large test table (~500MB+) +DROP TABLE IF EXISTS buffer_test; + +CREATE TABLE buffer_test AS +SELECT + generate_series(1, 10_000_000) AS id, + md5(random()::text) AS filler; + +ANALYZE buffer_test; + +-- Step 3: Perform repeated full-table scans to stress buffer allocation +DO $$ +BEGIN + FOR i IN 1..10 LOOP + RAISE NOTICE 'Running scan iteration %', i; + PERFORM COUNT(*) FROM buffer_test; + END LOOP; +END $$; + +-- Step 4: Show final buffers_alloc value +SELECT 'Final buffers_alloc' AS info, buffers_alloc FROM pg_stat_bgwriter; diff --git a/pmm_qa/postgresql/tasks/run_load_pgsql.yml b/pmm_qa/postgresql/tasks/run_load_pgsql.yml index 7e2cc3b0..f014288a 100644 --- a/pmm_qa/postgresql/tasks/run_load_pgsql.yml +++ b/pmm_qa/postgresql/tasks/run_load_pgsql.yml @@ -55,3 +55,9 @@ - name: Print pgbench results debug: var: pgbench_result.stdout_lines + +- name: Run sample load + community.docker.docker_container_exec: + container: "pdpgsql_pmm_{{ pg_version }}_1" + user: postgres + command: psql -U postgres -d postgres -f load.sql From 86f3d49bd90ecd22ce70cd31741b0ae1be20d707 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Fri, 13 Jun 2025 10:27:40 +0200 Subject: [PATCH 062/156] PMM-7: Add load to pgsql replication --- pmm_qa/{postgresql/tasks/load.sql => data/load_pgsql.sql} | 0 .../percona-distribution-postgres-replication-setup.yml | 6 ++++++ pmm_qa/postgresql/tasks/run_load_pgsql.yml | 6 +----- 3 files changed, 7 insertions(+), 5 deletions(-) rename pmm_qa/{postgresql/tasks/load.sql => data/load_pgsql.sql} (100%) diff --git a/pmm_qa/postgresql/tasks/load.sql b/pmm_qa/data/load_pgsql.sql similarity index 100% rename from pmm_qa/postgresql/tasks/load.sql rename to pmm_qa/data/load_pgsql.sql diff --git a/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml b/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml index c218be61..cdffc7fe 100644 --- a/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml @@ -229,6 +229,12 @@ pgbench_time: 120 pgbench_scale: 1000 +- name: Run sample load + community.docker.docker_container_exec: + container: "pdpgsql_pmm_{{ pg_version }}_1" + user: postgres + command: psql -U postgres -d postgres -f ../../data/load_pgsql.sql + - name: Create logical replication slots community.docker.docker_container_exec: container: "pdpgsql_pmm_{{ pg_version }}_1" diff --git a/pmm_qa/postgresql/tasks/run_load_pgsql.yml b/pmm_qa/postgresql/tasks/run_load_pgsql.yml index f014288a..172ad66c 100644 --- a/pmm_qa/postgresql/tasks/run_load_pgsql.yml +++ b/pmm_qa/postgresql/tasks/run_load_pgsql.yml @@ -56,8 +56,4 @@ debug: var: pgbench_result.stdout_lines -- name: Run sample load - community.docker.docker_container_exec: - container: "pdpgsql_pmm_{{ pg_version }}_1" - user: postgres - command: psql -U postgres -d postgres -f load.sql + From 864dcfc38c3f3a18f7549f8e7101ae2fe2a647a2 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Fri, 13 Jun 2025 10:37:03 +0200 Subject: [PATCH 063/156] PMM-7: Add load to pgsql replication --- .../tasks/percona-distribution-postgres-replication-setup.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml b/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml index cdffc7fe..47d25ce8 100644 --- a/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml @@ -233,7 +233,7 @@ community.docker.docker_container_exec: container: "pdpgsql_pmm_{{ pg_version }}_1" user: postgres - command: psql -U postgres -d postgres -f ../../data/load_pgsql.sql + command: psql -U postgres -d postgres -f ../data/load_pgsql.sql - name: Create logical replication slots community.docker.docker_container_exec: From 01987b2cdf08c9754a91f7d3ca8ab45175f2e1d8 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Fri, 13 Jun 2025 12:28:45 +0200 Subject: [PATCH 064/156] PMM-7: Add load to pgsql replication --- .../percona-distribution-postgres-replication-setup.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml b/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml index 47d25ce8..a80c4559 100644 --- a/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml @@ -229,11 +229,17 @@ pgbench_time: 120 pgbench_scale: 1000 +- name: Copy sample load into container + community.docker.docker_container_copy_into: + container: "{{ container_name }}" + path: ../data/load_pgsql.sql + container_path: /load_pgsql.sql + - name: Run sample load community.docker.docker_container_exec: container: "pdpgsql_pmm_{{ pg_version }}_1" user: postgres - command: psql -U postgres -d postgres -f ../data/load_pgsql.sql + command: psql -U postgres -d postgres -f /load_pgsql.sql - name: Create logical replication slots community.docker.docker_container_exec: From 8e0d2e1d5cbdaef0012530c4a99a58985012d541 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Fri, 13 Jun 2025 12:39:12 +0200 Subject: [PATCH 065/156] PMM-7: Add load to pgsql replication --- .../tasks/percona-distribution-postgres-replication-setup.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml b/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml index a80c4559..53186114 100644 --- a/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml @@ -231,7 +231,7 @@ - name: Copy sample load into container community.docker.docker_container_copy_into: - container: "{{ container_name }}" + container: "pdpgsql_pmm_{{ pg_version }}_1" path: ../data/load_pgsql.sql container_path: /load_pgsql.sql From 191683a04a7a877c31bcff28c09e4ebaacb04a31 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Fri, 13 Jun 2025 12:56:00 +0200 Subject: [PATCH 066/156] PMM-7: Add load to pgsql replication --- .../data/postgresql-primary.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/percona-distribution-postgresql/data/postgresql-primary.conf b/pmm_qa/percona-distribution-postgresql/data/postgresql-primary.conf index 1bff8edd..eb09610b 100644 --- a/pmm_qa/percona-distribution-postgresql/data/postgresql-primary.conf +++ b/pmm_qa/percona-distribution-postgresql/data/postgresql-primary.conf @@ -1,4 +1,4 @@ -wal_level = replica +wal_level = logical max_wal_senders = 10 wal_keep_size = 64MB hot_standby = on From 73f7c3cb643302567b6ddb3d1f64148644bf7415 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 16 Jun 2025 09:00:53 +0200 Subject: [PATCH 067/156] PMM-7: background load --- .../data/background_load.sql | 21 +++++++++++++++++++ ...istribution-postgres-replication-setup.yml | 12 +++++++++++ 2 files changed, 33 insertions(+) create mode 100644 pmm_qa/percona-distribution-postgresql/data/background_load.sql diff --git a/pmm_qa/percona-distribution-postgresql/data/background_load.sql b/pmm_qa/percona-distribution-postgresql/data/background_load.sql new file mode 100644 index 00000000..97cbfc3f --- /dev/null +++ b/pmm_qa/percona-distribution-postgresql/data/background_load.sql @@ -0,0 +1,21 @@ +-- Step 1: Create a test table +CREATE TABLE test_users ( + id INT PRIMARY KEY, + name VARCHAR(50), + email VARCHAR(100) +); + +-- Step 2: Insert test data +INSERT INTO test_users (id, name, email) VALUES +(1, 'Alice', 'alice@example.com'), +(2, 'Bob', 'bob@example.com'), +(3, 'Charlie', 'charlie@example.com'); + +-- Step 3: Query the data +SELECT * FROM test_users; + +-- Step 4: Delete the data +DELETE FROM test_users; + +-- Step 5: Drop the table +DROP TABLE test_users; diff --git a/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml b/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml index 53186114..6eae6ef3 100644 --- a/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml @@ -235,12 +235,24 @@ path: ../data/load_pgsql.sql container_path: /load_pgsql.sql +- name: Copy permanent load into container + community.docker.docker_container_copy_into: + container: "pdpgsql_pmm_{{ pg_version }}_1" + path: ./data/background_load.sql + container_path: /background_load.sql + - name: Run sample load community.docker.docker_container_exec: container: "pdpgsql_pmm_{{ pg_version }}_1" user: postgres command: psql -U postgres -d postgres -f /load_pgsql.sql +- name: Start permanent SQL load in background + community.docker.docker_container_exec: + container: "pdpgsql_pmm_{{ pg_version }}_1" + command: > + sh -c "nohup bash -c 'while true; do psql -U postgres -d postgres -f /background_load.sql; sleep 5; done' > /tmp/sql_loop.log 2>&1 &" + - name: Create logical replication slots community.docker.docker_container_exec: container: "pdpgsql_pmm_{{ pg_version }}_1" From 7aa3dd2cef0968c262ef90a08341de3c20ab7b6b Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 16 Jun 2025 09:17:53 +0200 Subject: [PATCH 068/156] PMM-7: background load --- .../data/background_load.sql | 19 +++++++- ...istribution-postgres-replication-setup.yml | 48 +++++++++---------- 2 files changed, 42 insertions(+), 25 deletions(-) diff --git a/pmm_qa/percona-distribution-postgresql/data/background_load.sql b/pmm_qa/percona-distribution-postgresql/data/background_load.sql index 97cbfc3f..122cd63d 100644 --- a/pmm_qa/percona-distribution-postgresql/data/background_load.sql +++ b/pmm_qa/percona-distribution-postgresql/data/background_load.sql @@ -9,7 +9,24 @@ CREATE TABLE test_users ( INSERT INTO test_users (id, name, email) VALUES (1, 'Alice', 'alice@example.com'), (2, 'Bob', 'bob@example.com'), -(3, 'Charlie', 'charlie@example.com'); +(3, 'Charlie', 'charlie@example.com'), +(4, 'David', 'david@example.com'), +(5, 'Eva', 'eva@example.com'), +(6, 'Frank', 'frank@example.com'), +(7, 'Grace', 'grace@example.com'), +(8, 'Hannah', 'hannah@example.com'), +(9, 'Ian', 'ian@example.com'), +(10, 'Julia', 'julia@example.com'), +(11, 'Kevin', 'kevin@example.com'), +(12, 'Laura', 'laura@example.com'), +(13, 'Mike', 'mike@example.com'), +(14, 'Nina', 'nina@example.com'), +(15, 'Oscar', 'oscar@example.com'), +(16, 'Paula', 'paula@example.com'), +(17, 'Quentin', 'quentin@example.com'), +(18, 'Rachel', 'rachel@example.com'), +(19, 'Steve', 'steve@example.com'), +(20, 'Tina', 'tina@example.com'); -- Step 3: Query the data SELECT * FROM test_users; diff --git a/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml b/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml index 6eae6ef3..e39c74f0 100644 --- a/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml @@ -121,22 +121,22 @@ " ' -- name: Create custom database for pgbench - community.docker.docker_container_exec: - container: "pdpgsql_pmm_{{ pg_version }}_1" - user: postgres - command: > - bash -c " - echo \" - CREATE DATABASE pgbench; - \\c pgbench - GRANT CONNECT ON DATABASE pgbench TO pmm; - GRANT USAGE ON SCHEMA public TO pmm; - GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA public TO pmm; - ALTER DEFAULT PRIVILEGES IN SCHEMA public - GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO pmm; - \" | psql -U postgres -v ON_ERROR_STOP=1 - " +#- name: Create custom database for pgbench +# community.docker.docker_container_exec: +# container: "pdpgsql_pmm_{{ pg_version }}_1" +# user: postgres +# command: > +# bash -c " +# echo \" +# CREATE DATABASE pgbench; +# \\c pgbench +# GRANT CONNECT ON DATABASE pgbench TO pmm; +# GRANT USAGE ON SCHEMA public TO pmm; +# GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA public TO pmm; +# ALTER DEFAULT PRIVILEGES IN SCHEMA public +# GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO pmm; +# \" | psql -U postgres -v ON_ERROR_STOP=1 +# " - name: Run pg_basebackup from primary to replica community.docker.docker_container_exec: @@ -221,13 +221,13 @@ command: pmm-admin add postgresql --username=pmm --password=pmm --cluster=pdpgsq_replication_cluster--query-source=pgstatmonitor pdpgsql_pmm_{{ pg_version }}_{{ item }}{{ random_service_name_value }} --debug 127.0.0.1:5432 loop: "{{ range(1, nodes_count | int + 1) | list }}" -- name: Run load on primary node. - include_tasks: ../postgresql/tasks/run_load_pgsql.yml - vars: - node_name: "pdpgsql_pmm_{{ pg_version }}_1" - pgbench_clients: 10 - pgbench_time: 120 - pgbench_scale: 1000 +#- name: Run load on primary node. +# include_tasks: ../postgresql/tasks/run_load_pgsql.yml +# vars: +# node_name: "pdpgsql_pmm_{{ pg_version }}_1" +# pgbench_clients: 10 +# pgbench_time: 120 +# pgbench_scale: 1000 - name: Copy sample load into container community.docker.docker_container_copy_into: @@ -251,7 +251,7 @@ community.docker.docker_container_exec: container: "pdpgsql_pmm_{{ pg_version }}_1" command: > - sh -c "nohup bash -c 'while true; do psql -U postgres -d postgres -f /background_load.sql; sleep 5; done' > /tmp/sql_loop.log 2>&1 &" + sh -c "nohup bash -c 'while true; do psql -U postgres -d postgres -f /background_load.sql; sleep 1; done' > /tmp/sql_loop.log 2>&1 &" - name: Create logical replication slots community.docker.docker_container_exec: From 544583c76a0c3cbc699c4371790d95b2654797be Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 16 Jun 2025 09:28:11 +0200 Subject: [PATCH 069/156] PMM-7: background load --- .../tasks/percona-distribution-postgres-replication-setup.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml b/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml index e39c74f0..4461f8bb 100644 --- a/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml @@ -251,7 +251,7 @@ community.docker.docker_container_exec: container: "pdpgsql_pmm_{{ pg_version }}_1" command: > - sh -c "nohup bash -c 'while true; do psql -U postgres -d postgres -f /background_load.sql; sleep 1; done' > /tmp/sql_loop.log 2>&1 &" + sh -c "nohup bash -c 'while true; do psql -U postgres -d postgres -f /background_load.sql; done' > /tmp/sql_loop.log 2>&1 &" - name: Create logical replication slots community.docker.docker_container_exec: From c3cb2d84abb6af9ac7b479639c53488318fe02d3 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 16 Jun 2025 09:34:33 +0200 Subject: [PATCH 070/156] PMM-7: background load --- .../data/background_load.sql | 39 +++++++------------ 1 file changed, 13 insertions(+), 26 deletions(-) diff --git a/pmm_qa/percona-distribution-postgresql/data/background_load.sql b/pmm_qa/percona-distribution-postgresql/data/background_load.sql index 122cd63d..d0dbb9a6 100644 --- a/pmm_qa/percona-distribution-postgresql/data/background_load.sql +++ b/pmm_qa/percona-distribution-postgresql/data/background_load.sql @@ -1,32 +1,19 @@ -- Step 1: Create a test table -CREATE TABLE test_users ( - id INT PRIMARY KEY, - name VARCHAR(50), - email VARCHAR(100) +CREATE TABLE IF NOT EXISTS test_users ( + id SERIAL PRIMARY KEY, + name VARCHAR(100), + email VARCHAR(150), + created_at TIMESTAMP DEFAULT NOW(), + bio TEXT ); --- Step 2: Insert test data -INSERT INTO test_users (id, name, email) VALUES -(1, 'Alice', 'alice@example.com'), -(2, 'Bob', 'bob@example.com'), -(3, 'Charlie', 'charlie@example.com'), -(4, 'David', 'david@example.com'), -(5, 'Eva', 'eva@example.com'), -(6, 'Frank', 'frank@example.com'), -(7, 'Grace', 'grace@example.com'), -(8, 'Hannah', 'hannah@example.com'), -(9, 'Ian', 'ian@example.com'), -(10, 'Julia', 'julia@example.com'), -(11, 'Kevin', 'kevin@example.com'), -(12, 'Laura', 'laura@example.com'), -(13, 'Mike', 'mike@example.com'), -(14, 'Nina', 'nina@example.com'), -(15, 'Oscar', 'oscar@example.com'), -(16, 'Paula', 'paula@example.com'), -(17, 'Quentin', 'quentin@example.com'), -(18, 'Rachel', 'rachel@example.com'), -(19, 'Steve', 'steve@example.com'), -(20, 'Tina', 'tina@example.com'); +-- Step 2: Insert 10,000 rows +INSERT INTO test_users (name, email, bio) +SELECT + 'User_' || gs::TEXT AS name, + 'user_' || gs::TEXT || '@example.com' AS email, + 'This is a generated bio for user #' || gs::TEXT +FROM generate_series(1, 10000) AS gs; -- Step 3: Query the data SELECT * FROM test_users; From bb843bc4217fb68fad63fdba276d5410e14bed44 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 16 Jun 2025 09:42:30 +0200 Subject: [PATCH 071/156] PMM-7: background load --- .../tasks/percona-distribution-postgres-replication-setup.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml b/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml index 4461f8bb..ececb77f 100644 --- a/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml @@ -251,7 +251,7 @@ community.docker.docker_container_exec: container: "pdpgsql_pmm_{{ pg_version }}_1" command: > - sh -c "nohup bash -c 'while true; do psql -U postgres -d postgres -f /background_load.sql; done' > /tmp/sql_loop.log 2>&1 &" + sh -c "nohup bash -c 'while true; do echo "Starting insert at $(date '+%Y-%m-%d %H:%M:%S')"; psql -U postgres -d postgres -f /background_load.sql; done' > /tmp/sql_loop.log 2>&1 &" - name: Create logical replication slots community.docker.docker_container_exec: From d9dc47a5d5151eb8405c6e0b7720e702693256bc Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 16 Jun 2025 09:49:02 +0200 Subject: [PATCH 072/156] PMM-7: background load --- .../tasks/percona-distribution-postgres-replication-setup.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml b/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml index ececb77f..5106b803 100644 --- a/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml @@ -251,7 +251,7 @@ community.docker.docker_container_exec: container: "pdpgsql_pmm_{{ pg_version }}_1" command: > - sh -c "nohup bash -c 'while true; do echo "Starting insert at $(date '+%Y-%m-%d %H:%M:%S')"; psql -U postgres -d postgres -f /background_load.sql; done' > /tmp/sql_loop.log 2>&1 &" + sh -c "nohup bash -c 'while true; do echo Starting insert at $(date +\"%Y-%m-%d %H:%M:%S\"); psql -U postgres -d postgres -f /background_load.sql; done' > /tmp/sql_loop.log 2>&1 &" - name: Create logical replication slots community.docker.docker_container_exec: From b74666f5ba0ee1e9ff4ff1238c7bdd0d4db56076 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 16 Jun 2025 09:57:05 +0200 Subject: [PATCH 073/156] PMM-7: background load --- ...istribution-postgres-replication-setup.yml | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml b/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml index 5106b803..d025b74c 100644 --- a/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml @@ -235,23 +235,23 @@ path: ../data/load_pgsql.sql container_path: /load_pgsql.sql -- name: Copy permanent load into container - community.docker.docker_container_copy_into: - container: "pdpgsql_pmm_{{ pg_version }}_1" - path: ./data/background_load.sql - container_path: /background_load.sql - -- name: Run sample load - community.docker.docker_container_exec: - container: "pdpgsql_pmm_{{ pg_version }}_1" - user: postgres - command: psql -U postgres -d postgres -f /load_pgsql.sql +#- name: Copy permanent load into container +# community.docker.docker_container_copy_into: +# container: "pdpgsql_pmm_{{ pg_version }}_1" +# path: ./data/background_load.sql +# container_path: /background_load.sql +# +#- name: Run sample load +# community.docker.docker_container_exec: +# container: "pdpgsql_pmm_{{ pg_version }}_1" +# user: postgres +# command: psql -U postgres -d postgres -f /load_pgsql.sql - name: Start permanent SQL load in background community.docker.docker_container_exec: container: "pdpgsql_pmm_{{ pg_version }}_1" command: > - sh -c "nohup bash -c 'while true; do echo Starting insert at $(date +\"%Y-%m-%d %H:%M:%S\"); psql -U postgres -d postgres -f /background_load.sql; done' > /tmp/sql_loop.log 2>&1 &" + sh -c "nohup bash -c 'while true; do echo Starting insert at $(date +\"%Y-%m-%d %H:%M:%S\"); psql -U postgres -d postgres -f /load_pgsql.sql; done' > /tmp/sql_loop.log 2>&1 &" - name: Create logical replication slots community.docker.docker_container_exec: From e20029cc87476e290be3031c36e474bb3508dc92 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 16 Jun 2025 10:31:59 +0200 Subject: [PATCH 074/156] PMM-7: cluster and env --- ...ona-distribution-postgres-replication-setup.yml | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml b/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml index d025b74c..38938eaa 100644 --- a/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml @@ -218,7 +218,7 @@ - name: Add service to pmm server community.docker.docker_container_exec: container: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" - command: pmm-admin add postgresql --username=pmm --password=pmm --cluster=pdpgsq_replication_cluster--query-source=pgstatmonitor pdpgsql_pmm_{{ pg_version }}_{{ item }}{{ random_service_name_value }} --debug 127.0.0.1:5432 + command: pmm-admin add postgresql --username=pmm --password=pmm --cluster=pdpgsql_replication_cluster --environment=pdpgsql_replication_environment --query-source=pgstatmonitor pdpgsql_pmm_{{ pg_version }}_{{ item }}{{ random_service_name_value }} --debug 127.0.0.1:5432 loop: "{{ range(1, nodes_count | int + 1) | list }}" #- name: Run load on primary node. @@ -235,18 +235,6 @@ path: ../data/load_pgsql.sql container_path: /load_pgsql.sql -#- name: Copy permanent load into container -# community.docker.docker_container_copy_into: -# container: "pdpgsql_pmm_{{ pg_version }}_1" -# path: ./data/background_load.sql -# container_path: /background_load.sql -# -#- name: Run sample load -# community.docker.docker_container_exec: -# container: "pdpgsql_pmm_{{ pg_version }}_1" -# user: postgres -# command: psql -U postgres -d postgres -f /load_pgsql.sql - - name: Start permanent SQL load in background community.docker.docker_container_exec: container: "pdpgsql_pmm_{{ pg_version }}_1" From d7ba180d4a5f346b7e9080757e2cf951acd09ced Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 16 Jun 2025 14:40:13 +0200 Subject: [PATCH 075/156] PMM-7: cluster and env --- pmm_qa/data/load_pgsql.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/pmm_qa/data/load_pgsql.sql b/pmm_qa/data/load_pgsql.sql index e5b9ca5f..ab208197 100644 --- a/pmm_qa/data/load_pgsql.sql +++ b/pmm_qa/data/load_pgsql.sql @@ -22,3 +22,4 @@ END $$; -- Step 4: Show final buffers_alloc value SELECT 'Final buffers_alloc' AS info, buffers_alloc FROM pg_stat_bgwriter; +VACUUM buffer_test; From 7a6c022a6253ddb8eb519c3c6bbaa09ab0404ed9 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 16 Jun 2025 15:29:24 +0200 Subject: [PATCH 076/156] PMM-7: cluster and env --- .../tasks/percona-distribution-postgres-replication-setup.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml b/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml index 38938eaa..58d2b667 100644 --- a/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml @@ -239,7 +239,7 @@ community.docker.docker_container_exec: container: "pdpgsql_pmm_{{ pg_version }}_1" command: > - sh -c "nohup bash -c 'while true; do echo Starting insert at $(date +\"%Y-%m-%d %H:%M:%S\"); psql -U postgres -d postgres -f /load_pgsql.sql; done' > /tmp/sql_loop.log 2>&1 &" + sh -c "nohup bash -c 'while true; do echo Starting insert at $(date +\"%Y-%m-%d %H:%M:%S\"); psql -U postgres -d postgres -f /load_pgsql.sql; sleep 5; done' > /tmp/sql_loop.log 2>&1 &" - name: Create logical replication slots community.docker.docker_container_exec: From ac47efbe6179328d94705ffeaa60a40bda0f0453 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 16 Jun 2025 15:58:53 +0200 Subject: [PATCH 077/156] PMM-7: cluster and env --- pmm_qa/data/load_pgsql.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/data/load_pgsql.sql b/pmm_qa/data/load_pgsql.sql index ab208197..486829ed 100644 --- a/pmm_qa/data/load_pgsql.sql +++ b/pmm_qa/data/load_pgsql.sql @@ -22,4 +22,4 @@ END $$; -- Step 4: Show final buffers_alloc value SELECT 'Final buffers_alloc' AS info, buffers_alloc FROM pg_stat_bgwriter; -VACUUM buffer_test; +VACUUM FULL; From 161d1c545d65bafcb3aa74a39662253fe4c092de Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 16 Jun 2025 16:42:19 +0200 Subject: [PATCH 078/156] PMM-7: cluster and env --- pmm_qa/data/load_pgsql.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pmm_qa/data/load_pgsql.sql b/pmm_qa/data/load_pgsql.sql index 486829ed..f9f484c4 100644 --- a/pmm_qa/data/load_pgsql.sql +++ b/pmm_qa/data/load_pgsql.sql @@ -22,4 +22,5 @@ END $$; -- Step 4: Show final buffers_alloc value SELECT 'Final buffers_alloc' AS info, buffers_alloc FROM pg_stat_bgwriter; -VACUUM FULL; +VACUUM; +SELECT pg_switch_wal(); From 8dd39ca34c7181ce7575218c312b6c42120fd183 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 17 Jun 2025 08:17:24 +0200 Subject: [PATCH 079/156] PMM-7: cluster and env --- .../tasks/percona-distribution-postgres-replication-setup.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml b/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml index 58d2b667..187d3846 100644 --- a/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml @@ -239,7 +239,7 @@ community.docker.docker_container_exec: container: "pdpgsql_pmm_{{ pg_version }}_1" command: > - sh -c "nohup bash -c 'while true; do echo Starting insert at $(date +\"%Y-%m-%d %H:%M:%S\"); psql -U postgres -d postgres -f /load_pgsql.sql; sleep 5; done' > /tmp/sql_loop.log 2>&1 &" + sh -c "nohup bash -c 'while true; do echo Starting insert at $(date +\"%Y-%m-%d %H:%M:%S\"); psql -U postgres -d postgres -f /load_pgsql.sql; sleep 15; done' > /tmp/sql_loop.log 2>&1 &" - name: Create logical replication slots community.docker.docker_container_exec: From 502e8ed1234b687a8b192f2fe9c05866f11e84e8 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 17 Jun 2025 08:36:53 +0200 Subject: [PATCH 080/156] PMM-7: cluster and env --- .../tasks/percona-distribution-postgres-replication-setup.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml b/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml index 187d3846..455979bd 100644 --- a/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml @@ -239,7 +239,7 @@ community.docker.docker_container_exec: container: "pdpgsql_pmm_{{ pg_version }}_1" command: > - sh -c "nohup bash -c 'while true; do echo Starting insert at $(date +\"%Y-%m-%d %H:%M:%S\"); psql -U postgres -d postgres -f /load_pgsql.sql; sleep 15; done' > /tmp/sql_loop.log 2>&1 &" + sh -c "nohup bash -c 'while true; do echo Starting insert at $(date +\"%Y-%m-%d %H:%M:%S\"); psql -U postgres -d postgres -f /load_pgsql.sql; sleep 30; done' > /tmp/sql_loop.log 2>&1 &" - name: Create logical replication slots community.docker.docker_container_exec: From 7ab0991ee85c21c98ea1f8aef17f172dde26cc1b Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 17 Jun 2025 08:49:50 +0200 Subject: [PATCH 081/156] PMM-7: cluster and env --- pmm_qa/data/load_pgsql.sql | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pmm_qa/data/load_pgsql.sql b/pmm_qa/data/load_pgsql.sql index f9f484c4..736559cc 100644 --- a/pmm_qa/data/load_pgsql.sql +++ b/pmm_qa/data/load_pgsql.sql @@ -24,3 +24,9 @@ END $$; SELECT 'Final buffers_alloc' AS info, buffers_alloc FROM pg_stat_bgwriter; VACUUM; SELECT pg_switch_wal(); + +-- Step 5: Query the data +SELECT * FROM buffer_test; + +-- Step 6: Delete the data +DELETE FROM buffer_test; From 431d945affb376140ccb25d48701f214a78ae857 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 17 Jun 2025 09:05:33 +0200 Subject: [PATCH 082/156] PMM-7: cluster and env --- pmm_qa/data/load_pgsql.sql | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/pmm_qa/data/load_pgsql.sql b/pmm_qa/data/load_pgsql.sql index 736559cc..06323196 100644 --- a/pmm_qa/data/load_pgsql.sql +++ b/pmm_qa/data/load_pgsql.sql @@ -11,17 +11,17 @@ SELECT ANALYZE buffer_test; --- Step 3: Perform repeated full-table scans to stress buffer allocation -DO $$ -BEGIN - FOR i IN 1..10 LOOP - RAISE NOTICE 'Running scan iteration %', i; - PERFORM COUNT(*) FROM buffer_test; - END LOOP; -END $$; - --- Step 4: Show final buffers_alloc value -SELECT 'Final buffers_alloc' AS info, buffers_alloc FROM pg_stat_bgwriter; +---- Step 3: Perform repeated full-table scans to stress buffer allocation +--DO $$ +--BEGIN +-- FOR i IN 1..10 LOOP +-- RAISE NOTICE 'Running scan iteration %', i; +-- PERFORM COUNT(*) FROM buffer_test; +-- END LOOP; +--END $$; +-- +---- Step 4: Show final buffers_alloc value +--SELECT 'Final buffers_alloc' AS info, buffers_alloc FROM pg_stat_bgwriter; VACUUM; SELECT pg_switch_wal(); From 19ce190609ad3746d96aeba5f3c1d66a0a665c8f Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 17 Jun 2025 09:28:32 +0200 Subject: [PATCH 083/156] PMM-7: cluster and env --- pmm_qa/data/load_pgsql.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/data/load_pgsql.sql b/pmm_qa/data/load_pgsql.sql index 06323196..f5759280 100644 --- a/pmm_qa/data/load_pgsql.sql +++ b/pmm_qa/data/load_pgsql.sql @@ -6,7 +6,7 @@ DROP TABLE IF EXISTS buffer_test; CREATE TABLE buffer_test AS SELECT - generate_series(1, 10_000_000) AS id, + generate_series(1, 1_000_000) AS id, md5(random()::text) AS filler; ANALYZE buffer_test; From e07d8929783110efff3e61a3d6d9e4e41ad4b349 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 17 Jun 2025 10:29:45 +0200 Subject: [PATCH 084/156] PMM-7: cluster and env --- ...istribution-postgres-replication-setup.yml | 44 +++++++++---------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml b/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml index 455979bd..fc5a549f 100644 --- a/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml @@ -36,7 +36,7 @@ - name: Remove old PostgreSQL primary container community.docker.docker_container: - name: "pdpgsql_pmm_{{ pg_version }}_1" + name: "pdpgsql_pmm_replication_{{ pg_version }}_1" image: "{{ docker_repo }}:{{ pg_version }}" restart_policy: always state: absent @@ -44,7 +44,7 @@ - name: Start PostgreSQL primary container community.docker.docker_container: - name: "pdpgsql_pmm_{{ pg_version }}_1" + name: "pdpgsql_pmm_replication_{{ pg_version }}_1" image: "{{ docker_repo }}:{{ pg_version }}" restart_policy: always state: started @@ -70,7 +70,7 @@ - name: Create replication user community.docker.docker_container_exec: - container: "pdpgsql_pmm_{{ pg_version }}_1" + container: "pdpgsql_pmm_replication_{{ pg_version }}_1" user: postgres command: > psql -c " @@ -79,14 +79,14 @@ - name: Stop and remove replica if exists community.docker.docker_container: - name: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" + name: "pdpgsql_pmm_replication_{{ pg_version }}_{{ item }}" state: absent loop: "{{ range(2, nodes_count | int + 1) | list }}" ignore_errors: yes - name: Start Percona Distribution PostgreSQL replica container community.docker.docker_container: - name: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" + name: "pdpgsql_pmm_replication_{{ pg_version }}_{{ item }}" image: "{{ docker_repo }}:{{ pg_version }}" restart_policy: "no" state: started @@ -104,14 +104,14 @@ - name: Wipe replica data directory before basebackup community.docker.docker_container_exec: - container: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" + container: "pdpgsql_pmm_replication_{{ pg_version }}_{{ item }}" user: root command: rm -rf /data/db/* loop: "{{ range(2, nodes_count | int + 1) | list }}" - name: Create PostgreSQL user 'pmm' with password community.docker.docker_container_exec: - container: "pdpgsql_pmm_{{ pg_version }}_1" + container: "pdpgsql_pmm_replication_{{ pg_version }}_1" user: postgres command: > bash -c ' @@ -123,7 +123,7 @@ #- name: Create custom database for pgbench # community.docker.docker_container_exec: -# container: "pdpgsql_pmm_{{ pg_version }}_1" +# container: "pdpgsql_pmm_replication_{{ pg_version }}_1" # user: postgres # command: > # bash -c " @@ -140,20 +140,20 @@ - name: Run pg_basebackup from primary to replica community.docker.docker_container_exec: - container: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" + container: "pdpgsql_pmm_replication_{{ pg_version }}_{{ item }}" user: root command: > bash -c " export PGPASSWORD='{{ replication_password }}' && \ timeout 120s \ pg_basebackup --pgdata=/data/db -R -v -Fp -Xs -P \ - --host=pdpgsql_pmm_{{ pg_version }}_1 --port=5432 -U {{ replication_user }} > /tmp/pg_basebackup.log 2>&1 + --host=pdpgsql_pmm_replication_{{ pg_version }}_1 --port=5432 -U {{ replication_user }} > /tmp/pg_basebackup.log 2>&1 " loop: "{{ range(2, nodes_count | int + 1) | list }}" - name: Remove temporary backup container docker_container: - name: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" + name: "pdpgsql_pmm_replication_{{ pg_version }}_{{ item }}" state: absent loop: "{{ range(2, nodes_count | int + 1) | list }}" @@ -168,7 +168,7 @@ - name: Restart Percona Distribution PostgreSQL container with custom command community.docker.docker_container: - name: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" + name: "pdpgsql_pmm_replication_{{ pg_version }}_{{ item }}" image: "{{ docker_repo }}:{{ pg_version }}" restart: true state: started @@ -188,17 +188,17 @@ - name: Install pg stat monitor. include_tasks: ./tasks/install_pg_stat-monitor.yml vars: - container_name: "pdpgsql_pmm_{{ pg_version }}" + container_name: "pdpgsql_pmm_replication_{{ pg_version }}" - name: Install and add pmm client. include_tasks: ../tasks/install_pmm_client.yml vars: - container_name: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" + container_name: "pdpgsql_pmm_replication_{{ pg_version }}_{{ item }}" loop: "{{ range(1, nodes_count | int + 1) | list }}" - name: Get already connected services to pmm server community.docker.docker_container_exec: - container: "pdpgsql_pmm_{{ pg_version }}_1" + container: "pdpgsql_pmm_replication_{{ pg_version }}_1" command: > sh -c 'curl --location --insecure -u"admin:{{ admin_password }}" -s --request GET "http://{{ pmm_server_ip }}:{{ '80' if pmm_server_ip is ansible.utils.ipv4 else '8080' }}/v1/management/services" | jq -r ".services[].service_name"' register: pmm_server_services @@ -217,38 +217,38 @@ - name: Add service to pmm server community.docker.docker_container_exec: - container: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" - command: pmm-admin add postgresql --username=pmm --password=pmm --cluster=pdpgsql_replication_cluster --environment=pdpgsql_replication_environment --query-source=pgstatmonitor pdpgsql_pmm_{{ pg_version }}_{{ item }}{{ random_service_name_value }} --debug 127.0.0.1:5432 + container: "pdpgsql_pmm_replication_{{ pg_version }}_{{ item }}" + command: pmm-admin add postgresql --username=pmm --password=pmm --cluster=pdpgsql_replication_cluster --environment=pdpgsql_replication_environment --query-source=pgstatmonitor pdpgsql_pmm_replication_{{ pg_version }}_{{ item }}{{ random_service_name_value }} --debug 127.0.0.1:5432 loop: "{{ range(1, nodes_count | int + 1) | list }}" #- name: Run load on primary node. # include_tasks: ../postgresql/tasks/run_load_pgsql.yml # vars: -# node_name: "pdpgsql_pmm_{{ pg_version }}_1" +# node_name: "pdpgsql_pmm_replication_{{ pg_version }}_1" # pgbench_clients: 10 # pgbench_time: 120 # pgbench_scale: 1000 - name: Copy sample load into container community.docker.docker_container_copy_into: - container: "pdpgsql_pmm_{{ pg_version }}_1" + container: "pdpgsql_pmm_replication_{{ pg_version }}_1" path: ../data/load_pgsql.sql container_path: /load_pgsql.sql - name: Start permanent SQL load in background community.docker.docker_container_exec: - container: "pdpgsql_pmm_{{ pg_version }}_1" + container: "pdpgsql_pmm_replication_{{ pg_version }}_1" command: > sh -c "nohup bash -c 'while true; do echo Starting insert at $(date +\"%Y-%m-%d %H:%M:%S\"); psql -U postgres -d postgres -f /load_pgsql.sql; sleep 30; done' > /tmp/sql_loop.log 2>&1 &" - name: Create logical replication slots community.docker.docker_container_exec: - container: "pdpgsql_pmm_{{ pg_version }}_1" + container: "pdpgsql_pmm_replication_{{ pg_version }}_1" user: postgres command: psql -U postgres -d postgres -c "SELECT * FROM pg_create_logical_replication_slot('test_slot', 'test_decoding');" - name: Check replication status on primary community.docker.docker_container_exec: - container: "pdpgsql_pmm_{{ pg_version }}_1" + container: "pdpgsql_pmm_replication_{{ pg_version }}_1" user: postgres command: psql -c "SELECT * FROM pg_stat_replication;" From d582fdfdebbf21c5e5e8239d0611c6e52e2a9498 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 17 Jun 2025 11:28:26 +0200 Subject: [PATCH 085/156] PMM-7: cluster and env --- ...istribution-postgres-replication-setup.yml | 42 +++++++++---------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml b/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml index fc5a549f..19003b9a 100644 --- a/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml @@ -36,7 +36,7 @@ - name: Remove old PostgreSQL primary container community.docker.docker_container: - name: "pdpgsql_pmm_replication_{{ pg_version }}_1" + name: "pdpgsql_pmm_replication_primary_{{ pg_version }}" image: "{{ docker_repo }}:{{ pg_version }}" restart_policy: always state: absent @@ -44,7 +44,7 @@ - name: Start PostgreSQL primary container community.docker.docker_container: - name: "pdpgsql_pmm_replication_{{ pg_version }}_1" + name: "pdpgsql_pmm_replication_primary_{{ pg_version }}" image: "{{ docker_repo }}:{{ pg_version }}" restart_policy: always state: started @@ -70,7 +70,7 @@ - name: Create replication user community.docker.docker_container_exec: - container: "pdpgsql_pmm_replication_{{ pg_version }}_1" + container: "pdpgsql_pmm_replication_primary_{{ pg_version }}" user: postgres command: > psql -c " @@ -79,14 +79,14 @@ - name: Stop and remove replica if exists community.docker.docker_container: - name: "pdpgsql_pmm_replication_{{ pg_version }}_{{ item }}" + name: "pdpgsql_pmm_replication_replica_{{ item }}_{{ pg_version }}" state: absent loop: "{{ range(2, nodes_count | int + 1) | list }}" ignore_errors: yes - name: Start Percona Distribution PostgreSQL replica container community.docker.docker_container: - name: "pdpgsql_pmm_replication_{{ pg_version }}_{{ item }}" + name: "pdpgsql_pmm_replication_replica_{{ item }}_{{ pg_version }}" image: "{{ docker_repo }}:{{ pg_version }}" restart_policy: "no" state: started @@ -104,14 +104,14 @@ - name: Wipe replica data directory before basebackup community.docker.docker_container_exec: - container: "pdpgsql_pmm_replication_{{ pg_version }}_{{ item }}" + container: "pdpgsql_pmm_replication_replica_{{ item }}_{{ pg_version }}" user: root command: rm -rf /data/db/* loop: "{{ range(2, nodes_count | int + 1) | list }}" - name: Create PostgreSQL user 'pmm' with password community.docker.docker_container_exec: - container: "pdpgsql_pmm_replication_{{ pg_version }}_1" + container: "pdpgsql_pmm_replication_primary_{{ pg_version }}" user: postgres command: > bash -c ' @@ -123,7 +123,7 @@ #- name: Create custom database for pgbench # community.docker.docker_container_exec: -# container: "pdpgsql_pmm_replication_{{ pg_version }}_1" +# container: "pdpgsql_pmm_replication_primary_{{ pg_version }}" # user: postgres # command: > # bash -c " @@ -140,20 +140,20 @@ - name: Run pg_basebackup from primary to replica community.docker.docker_container_exec: - container: "pdpgsql_pmm_replication_{{ pg_version }}_{{ item }}" + container: "pdpgsql_pmm_replication_replica_{{ item }}_{{ pg_version }}" user: root command: > bash -c " export PGPASSWORD='{{ replication_password }}' && \ timeout 120s \ pg_basebackup --pgdata=/data/db -R -v -Fp -Xs -P \ - --host=pdpgsql_pmm_replication_{{ pg_version }}_1 --port=5432 -U {{ replication_user }} > /tmp/pg_basebackup.log 2>&1 + --host=pdpgsql_pmm_replication_primary_{{ pg_version }} --port=5432 -U {{ replication_user }} > /tmp/pg_basebackup.log 2>&1 " loop: "{{ range(2, nodes_count | int + 1) | list }}" - name: Remove temporary backup container docker_container: - name: "pdpgsql_pmm_replication_{{ pg_version }}_{{ item }}" + name: "pdpgsql_pmm_replication_replica_{{ item }}_{{ pg_version }}" state: absent loop: "{{ range(2, nodes_count | int + 1) | list }}" @@ -168,7 +168,7 @@ - name: Restart Percona Distribution PostgreSQL container with custom command community.docker.docker_container: - name: "pdpgsql_pmm_replication_{{ pg_version }}_{{ item }}" + name: "pdpgsql_pmm_replication_replica_{{ item }}_{{ pg_version }}" image: "{{ docker_repo }}:{{ pg_version }}" restart: true state: started @@ -193,12 +193,12 @@ - name: Install and add pmm client. include_tasks: ../tasks/install_pmm_client.yml vars: - container_name: "pdpgsql_pmm_replication_{{ pg_version }}_{{ item }}" + container_name: "pdpgsql_pmm_replication_replica_{{ item }}_{{ pg_version }}" loop: "{{ range(1, nodes_count | int + 1) | list }}" - name: Get already connected services to pmm server community.docker.docker_container_exec: - container: "pdpgsql_pmm_replication_{{ pg_version }}_1" + container: "pdpgsql_pmm_replication_primary_{{ pg_version }}" command: > sh -c 'curl --location --insecure -u"admin:{{ admin_password }}" -s --request GET "http://{{ pmm_server_ip }}:{{ '80' if pmm_server_ip is ansible.utils.ipv4 else '8080' }}/v1/management/services" | jq -r ".services[].service_name"' register: pmm_server_services @@ -217,38 +217,38 @@ - name: Add service to pmm server community.docker.docker_container_exec: - container: "pdpgsql_pmm_replication_{{ pg_version }}_{{ item }}" - command: pmm-admin add postgresql --username=pmm --password=pmm --cluster=pdpgsql_replication_cluster --environment=pdpgsql_replication_environment --query-source=pgstatmonitor pdpgsql_pmm_replication_{{ pg_version }}_{{ item }}{{ random_service_name_value }} --debug 127.0.0.1:5432 + container: "pdpgsql_pmm_replication_replica_{{ item }}_{{ pg_version }}" + command: pmm-admin add postgresql --username=pmm --password=pmm --cluster=pdpgsql_replication_cluster --environment=pdpgsql_replication_environment --query-source=pgstatmonitor pdpgsql_pmm_replication_replica_{{ item }}_{{ pg_version }}{{ random_service_name_value }} --debug 127.0.0.1:5432 loop: "{{ range(1, nodes_count | int + 1) | list }}" #- name: Run load on primary node. # include_tasks: ../postgresql/tasks/run_load_pgsql.yml # vars: -# node_name: "pdpgsql_pmm_replication_{{ pg_version }}_1" +# node_name: "pdpgsql_pmm_replication_primary_{{ pg_version }}" # pgbench_clients: 10 # pgbench_time: 120 # pgbench_scale: 1000 - name: Copy sample load into container community.docker.docker_container_copy_into: - container: "pdpgsql_pmm_replication_{{ pg_version }}_1" + container: "pdpgsql_pmm_replication_primary_{{ pg_version }}" path: ../data/load_pgsql.sql container_path: /load_pgsql.sql - name: Start permanent SQL load in background community.docker.docker_container_exec: - container: "pdpgsql_pmm_replication_{{ pg_version }}_1" + container: "pdpgsql_pmm_replication_primary_{{ pg_version }}" command: > sh -c "nohup bash -c 'while true; do echo Starting insert at $(date +\"%Y-%m-%d %H:%M:%S\"); psql -U postgres -d postgres -f /load_pgsql.sql; sleep 30; done' > /tmp/sql_loop.log 2>&1 &" - name: Create logical replication slots community.docker.docker_container_exec: - container: "pdpgsql_pmm_replication_{{ pg_version }}_1" + container: "pdpgsql_pmm_replication_primary_{{ pg_version }}" user: postgres command: psql -U postgres -d postgres -c "SELECT * FROM pg_create_logical_replication_slot('test_slot', 'test_decoding');" - name: Check replication status on primary community.docker.docker_container_exec: - container: "pdpgsql_pmm_replication_{{ pg_version }}_1" + container: "pdpgsql_pmm_replication_primary_{{ pg_version }}" user: postgres command: psql -c "SELECT * FROM pg_stat_replication;" From 061cbf3fcc6d98d664c6d6a37e3dbc5bd87c85ca Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 17 Jun 2025 11:38:41 +0200 Subject: [PATCH 086/156] PMM-7: cluster and env --- ...istribution-postgres-replication-setup.yml | 42 +++++++++---------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml b/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml index 19003b9a..c9146f2d 100644 --- a/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml @@ -36,7 +36,7 @@ - name: Remove old PostgreSQL primary container community.docker.docker_container: - name: "pdpgsql_pmm_replication_primary_{{ pg_version }}" + name: "pdpgsql_pmm_replication_{{ pg_version }}_1" image: "{{ docker_repo }}:{{ pg_version }}" restart_policy: always state: absent @@ -44,7 +44,7 @@ - name: Start PostgreSQL primary container community.docker.docker_container: - name: "pdpgsql_pmm_replication_primary_{{ pg_version }}" + name: "pdpgsql_pmm_replication_{{ pg_version }}_1" image: "{{ docker_repo }}:{{ pg_version }}" restart_policy: always state: started @@ -70,7 +70,7 @@ - name: Create replication user community.docker.docker_container_exec: - container: "pdpgsql_pmm_replication_primary_{{ pg_version }}" + container: "pdpgsql_pmm_replication_{{ pg_version }}_1" user: postgres command: > psql -c " @@ -79,14 +79,14 @@ - name: Stop and remove replica if exists community.docker.docker_container: - name: "pdpgsql_pmm_replication_replica_{{ item }}_{{ pg_version }}" + name: "pdpgsql_pmm_replication_{{ item }}_{{ pg_version }}" state: absent loop: "{{ range(2, nodes_count | int + 1) | list }}" ignore_errors: yes - name: Start Percona Distribution PostgreSQL replica container community.docker.docker_container: - name: "pdpgsql_pmm_replication_replica_{{ item }}_{{ pg_version }}" + name: "pdpgsql_pmm_replication_{{ item }}_{{ pg_version }}" image: "{{ docker_repo }}:{{ pg_version }}" restart_policy: "no" state: started @@ -104,14 +104,14 @@ - name: Wipe replica data directory before basebackup community.docker.docker_container_exec: - container: "pdpgsql_pmm_replication_replica_{{ item }}_{{ pg_version }}" + container: "pdpgsql_pmm_replication_{{ item }}_{{ pg_version }}" user: root command: rm -rf /data/db/* loop: "{{ range(2, nodes_count | int + 1) | list }}" - name: Create PostgreSQL user 'pmm' with password community.docker.docker_container_exec: - container: "pdpgsql_pmm_replication_primary_{{ pg_version }}" + container: "pdpgsql_pmm_replication_{{ pg_version }}_1" user: postgres command: > bash -c ' @@ -123,7 +123,7 @@ #- name: Create custom database for pgbench # community.docker.docker_container_exec: -# container: "pdpgsql_pmm_replication_primary_{{ pg_version }}" +# container: "pdpgsql_pmm_replication_{{ pg_version }}_1" # user: postgres # command: > # bash -c " @@ -140,20 +140,20 @@ - name: Run pg_basebackup from primary to replica community.docker.docker_container_exec: - container: "pdpgsql_pmm_replication_replica_{{ item }}_{{ pg_version }}" + container: "pdpgsql_pmm_replication_{{ item }}_{{ pg_version }}" user: root command: > bash -c " export PGPASSWORD='{{ replication_password }}' && \ timeout 120s \ pg_basebackup --pgdata=/data/db -R -v -Fp -Xs -P \ - --host=pdpgsql_pmm_replication_primary_{{ pg_version }} --port=5432 -U {{ replication_user }} > /tmp/pg_basebackup.log 2>&1 + --host=pdpgsql_pmm_replication_{{ pg_version }}_1 --port=5432 -U {{ replication_user }} > /tmp/pg_basebackup.log 2>&1 " loop: "{{ range(2, nodes_count | int + 1) | list }}" - name: Remove temporary backup container docker_container: - name: "pdpgsql_pmm_replication_replica_{{ item }}_{{ pg_version }}" + name: "pdpgsql_pmm_replication_{{ item }}_{{ pg_version }}" state: absent loop: "{{ range(2, nodes_count | int + 1) | list }}" @@ -168,7 +168,7 @@ - name: Restart Percona Distribution PostgreSQL container with custom command community.docker.docker_container: - name: "pdpgsql_pmm_replication_replica_{{ item }}_{{ pg_version }}" + name: "pdpgsql_pmm_replication_{{ item }}_{{ pg_version }}" image: "{{ docker_repo }}:{{ pg_version }}" restart: true state: started @@ -193,12 +193,12 @@ - name: Install and add pmm client. include_tasks: ../tasks/install_pmm_client.yml vars: - container_name: "pdpgsql_pmm_replication_replica_{{ item }}_{{ pg_version }}" + container_name: "pdpgsql_pmm_replication_{{ item }}_{{ pg_version }}" loop: "{{ range(1, nodes_count | int + 1) | list }}" - name: Get already connected services to pmm server community.docker.docker_container_exec: - container: "pdpgsql_pmm_replication_primary_{{ pg_version }}" + container: "pdpgsql_pmm_replication_{{ pg_version }}_1" command: > sh -c 'curl --location --insecure -u"admin:{{ admin_password }}" -s --request GET "http://{{ pmm_server_ip }}:{{ '80' if pmm_server_ip is ansible.utils.ipv4 else '8080' }}/v1/management/services" | jq -r ".services[].service_name"' register: pmm_server_services @@ -217,38 +217,38 @@ - name: Add service to pmm server community.docker.docker_container_exec: - container: "pdpgsql_pmm_replication_replica_{{ item }}_{{ pg_version }}" - command: pmm-admin add postgresql --username=pmm --password=pmm --cluster=pdpgsql_replication_cluster --environment=pdpgsql_replication_environment --query-source=pgstatmonitor pdpgsql_pmm_replication_replica_{{ item }}_{{ pg_version }}{{ random_service_name_value }} --debug 127.0.0.1:5432 + container: "pdpgsql_pmm_replication_{{ item }}_{{ pg_version }}" + command: pmm-admin add postgresql --username=pmm --password=pmm --cluster=pdpgsql_replication_cluster --environment=pdpgsql_replication_environment --query-source=pgstatmonitor pdpgsql_pmm_replication_{{ item }}_{{ pg_version }}{{ random_service_name_value }} --debug 127.0.0.1:5432 loop: "{{ range(1, nodes_count | int + 1) | list }}" #- name: Run load on primary node. # include_tasks: ../postgresql/tasks/run_load_pgsql.yml # vars: -# node_name: "pdpgsql_pmm_replication_primary_{{ pg_version }}" +# node_name: "pdpgsql_pmm_replication_{{ pg_version }}_1" # pgbench_clients: 10 # pgbench_time: 120 # pgbench_scale: 1000 - name: Copy sample load into container community.docker.docker_container_copy_into: - container: "pdpgsql_pmm_replication_primary_{{ pg_version }}" + container: "pdpgsql_pmm_replication_{{ pg_version }}_1" path: ../data/load_pgsql.sql container_path: /load_pgsql.sql - name: Start permanent SQL load in background community.docker.docker_container_exec: - container: "pdpgsql_pmm_replication_primary_{{ pg_version }}" + container: "pdpgsql_pmm_replication_{{ pg_version }}_1" command: > sh -c "nohup bash -c 'while true; do echo Starting insert at $(date +\"%Y-%m-%d %H:%M:%S\"); psql -U postgres -d postgres -f /load_pgsql.sql; sleep 30; done' > /tmp/sql_loop.log 2>&1 &" - name: Create logical replication slots community.docker.docker_container_exec: - container: "pdpgsql_pmm_replication_primary_{{ pg_version }}" + container: "pdpgsql_pmm_replication_{{ pg_version }}_1" user: postgres command: psql -U postgres -d postgres -c "SELECT * FROM pg_create_logical_replication_slot('test_slot', 'test_decoding');" - name: Check replication status on primary community.docker.docker_container_exec: - container: "pdpgsql_pmm_replication_primary_{{ pg_version }}" + container: "pdpgsql_pmm_replication_{{ pg_version }}_1" user: postgres command: psql -c "SELECT * FROM pg_stat_replication;" From 97c5518036abe1b858b93f38458e12a0d29cb5d3 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 17 Jun 2025 11:48:22 +0200 Subject: [PATCH 087/156] PMM-7: cluster and env --- ...distribution-postgres-replication-setup.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml b/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml index c9146f2d..fc5a549f 100644 --- a/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml @@ -79,14 +79,14 @@ - name: Stop and remove replica if exists community.docker.docker_container: - name: "pdpgsql_pmm_replication_{{ item }}_{{ pg_version }}" + name: "pdpgsql_pmm_replication_{{ pg_version }}_{{ item }}" state: absent loop: "{{ range(2, nodes_count | int + 1) | list }}" ignore_errors: yes - name: Start Percona Distribution PostgreSQL replica container community.docker.docker_container: - name: "pdpgsql_pmm_replication_{{ item }}_{{ pg_version }}" + name: "pdpgsql_pmm_replication_{{ pg_version }}_{{ item }}" image: "{{ docker_repo }}:{{ pg_version }}" restart_policy: "no" state: started @@ -104,7 +104,7 @@ - name: Wipe replica data directory before basebackup community.docker.docker_container_exec: - container: "pdpgsql_pmm_replication_{{ item }}_{{ pg_version }}" + container: "pdpgsql_pmm_replication_{{ pg_version }}_{{ item }}" user: root command: rm -rf /data/db/* loop: "{{ range(2, nodes_count | int + 1) | list }}" @@ -140,7 +140,7 @@ - name: Run pg_basebackup from primary to replica community.docker.docker_container_exec: - container: "pdpgsql_pmm_replication_{{ item }}_{{ pg_version }}" + container: "pdpgsql_pmm_replication_{{ pg_version }}_{{ item }}" user: root command: > bash -c " @@ -153,7 +153,7 @@ - name: Remove temporary backup container docker_container: - name: "pdpgsql_pmm_replication_{{ item }}_{{ pg_version }}" + name: "pdpgsql_pmm_replication_{{ pg_version }}_{{ item }}" state: absent loop: "{{ range(2, nodes_count | int + 1) | list }}" @@ -168,7 +168,7 @@ - name: Restart Percona Distribution PostgreSQL container with custom command community.docker.docker_container: - name: "pdpgsql_pmm_replication_{{ item }}_{{ pg_version }}" + name: "pdpgsql_pmm_replication_{{ pg_version }}_{{ item }}" image: "{{ docker_repo }}:{{ pg_version }}" restart: true state: started @@ -193,7 +193,7 @@ - name: Install and add pmm client. include_tasks: ../tasks/install_pmm_client.yml vars: - container_name: "pdpgsql_pmm_replication_{{ item }}_{{ pg_version }}" + container_name: "pdpgsql_pmm_replication_{{ pg_version }}_{{ item }}" loop: "{{ range(1, nodes_count | int + 1) | list }}" - name: Get already connected services to pmm server @@ -217,8 +217,8 @@ - name: Add service to pmm server community.docker.docker_container_exec: - container: "pdpgsql_pmm_replication_{{ item }}_{{ pg_version }}" - command: pmm-admin add postgresql --username=pmm --password=pmm --cluster=pdpgsql_replication_cluster --environment=pdpgsql_replication_environment --query-source=pgstatmonitor pdpgsql_pmm_replication_{{ item }}_{{ pg_version }}{{ random_service_name_value }} --debug 127.0.0.1:5432 + container: "pdpgsql_pmm_replication_{{ pg_version }}_{{ item }}" + command: pmm-admin add postgresql --username=pmm --password=pmm --cluster=pdpgsql_replication_cluster --environment=pdpgsql_replication_environment --query-source=pgstatmonitor pdpgsql_pmm_replication_{{ pg_version }}_{{ item }}{{ random_service_name_value }} --debug 127.0.0.1:5432 loop: "{{ range(1, nodes_count | int + 1) | list }}" #- name: Run load on primary node. From 04cc2d65b210e57ed0c2d914a5a06fe3914157a0 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 17 Jun 2025 14:07:02 +0200 Subject: [PATCH 088/156] PMM-7: cluster and env --- ...istribution-postgres-replication-setup.yml | 32 +++++++++++++++---- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml b/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml index fc5a549f..bcf68dcc 100644 --- a/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml @@ -221,13 +221,31 @@ command: pmm-admin add postgresql --username=pmm --password=pmm --cluster=pdpgsql_replication_cluster --environment=pdpgsql_replication_environment --query-source=pgstatmonitor pdpgsql_pmm_replication_{{ pg_version }}_{{ item }}{{ random_service_name_value }} --debug 127.0.0.1:5432 loop: "{{ range(1, nodes_count | int + 1) | list }}" -#- name: Run load on primary node. -# include_tasks: ../postgresql/tasks/run_load_pgsql.yml -# vars: -# node_name: "pdpgsql_pmm_replication_{{ pg_version }}_1" -# pgbench_clients: 10 -# pgbench_time: 120 -# pgbench_scale: 1000 +- name: Create pg_custom_publication view using psql + community.docker.docker_container_exec: + container: "pdpgsql_pmm_replication_{{ pg_version }}_1" + user: postgres +- name: Create pg_custom_publication view using psql + community.docker.docker_container_exec: + container: "pdpgsql_pmm_replication_{{ pg_version }}_1" + user: postgres + command: > + psql -U postgres -d postgres -c " + CREATE OR REPLACE VIEW pg_custom_publication AS + SELECT + p.pubname AS publication, + pt.schemaname || '.' || pt.tablename AS fully_qualified_table, + p.puballtables, + p.pubinsert, + p.pubupdate, + p.pubdelete, + p.pubtruncate + FROM + pg_publication p + JOIN + pg_publication_tables pt ON p.pubname = pt.pubname; + " + - name: Copy sample load into container community.docker.docker_container_copy_into: From 046210721522d2344a7bc1d5d570c42a8fb6307b Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 17 Jun 2025 14:15:37 +0200 Subject: [PATCH 089/156] PMM-7: cluster and env --- .../tasks/percona-distribution-postgres-replication-setup.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml b/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml index bcf68dcc..d3ece338 100644 --- a/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml @@ -221,10 +221,6 @@ command: pmm-admin add postgresql --username=pmm --password=pmm --cluster=pdpgsql_replication_cluster --environment=pdpgsql_replication_environment --query-source=pgstatmonitor pdpgsql_pmm_replication_{{ pg_version }}_{{ item }}{{ random_service_name_value }} --debug 127.0.0.1:5432 loop: "{{ range(1, nodes_count | int + 1) | list }}" -- name: Create pg_custom_publication view using psql - community.docker.docker_container_exec: - container: "pdpgsql_pmm_replication_{{ pg_version }}_1" - user: postgres - name: Create pg_custom_publication view using psql community.docker.docker_container_exec: container: "pdpgsql_pmm_replication_{{ pg_version }}_1" From c56720dcc067a189243cd370b6b460fcd9f98ae0 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 17 Jun 2025 14:26:51 +0200 Subject: [PATCH 090/156] PMM-7: cluster and env --- ...istribution-postgres-replication-setup.yml | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml b/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml index d3ece338..f673bc30 100644 --- a/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml @@ -229,19 +229,14 @@ psql -U postgres -d postgres -c " CREATE OR REPLACE VIEW pg_custom_publication AS SELECT - p.pubname AS publication, - pt.schemaname || '.' || pt.tablename AS fully_qualified_table, - p.puballtables, - p.pubinsert, - p.pubupdate, - p.pubdelete, - p.pubtruncate + pubname, + puballtables, + pubinsert, + pubupdate, + pubdelete FROM - pg_publication p - JOIN - pg_publication_tables pt ON p.pubname = pt.pubname; - " - + pg_publication; + " - name: Copy sample load into container community.docker.docker_container_copy_into: From d85c3456bd0434cd42af82bba03f02fec55a93d5 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 17 Jun 2025 14:30:00 +0200 Subject: [PATCH 091/156] PMM-7: cluster and env --- ...istribution-postgres-replication-setup.yml | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml b/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml index f673bc30..80fdc746 100644 --- a/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml @@ -226,17 +226,17 @@ container: "pdpgsql_pmm_replication_{{ pg_version }}_1" user: postgres command: > - psql -U postgres -d postgres -c " - CREATE OR REPLACE VIEW pg_custom_publication AS - SELECT - pubname, - puballtables, - pubinsert, - pubupdate, - pubdelete - FROM - pg_publication; - " + bash -c 'psql -U postgres -d postgres < Date: Wed, 18 Jun 2025 09:16:30 +0200 Subject: [PATCH 092/156] PMM-7: cluster and env --- pmm_qa/data/load_pgsql.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/data/load_pgsql.sql b/pmm_qa/data/load_pgsql.sql index f5759280..b4c284b4 100644 --- a/pmm_qa/data/load_pgsql.sql +++ b/pmm_qa/data/load_pgsql.sql @@ -6,7 +6,7 @@ DROP TABLE IF EXISTS buffer_test; CREATE TABLE buffer_test AS SELECT - generate_series(1, 1_000_000) AS id, + generate_series(1, 500_000) AS id, md5(random()::text) AS filler; ANALYZE buffer_test; From b8581275a327f2b7edb307e7ea7ff5daf6710136 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 18 Jun 2025 09:27:54 +0200 Subject: [PATCH 093/156] PMM-7: cluster and env --- pmm_qa/data/load_pgsql.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/data/load_pgsql.sql b/pmm_qa/data/load_pgsql.sql index b4c284b4..f5759280 100644 --- a/pmm_qa/data/load_pgsql.sql +++ b/pmm_qa/data/load_pgsql.sql @@ -6,7 +6,7 @@ DROP TABLE IF EXISTS buffer_test; CREATE TABLE buffer_test AS SELECT - generate_series(1, 500_000) AS id, + generate_series(1, 1_000_000) AS id, md5(random()::text) AS filler; ANALYZE buffer_test; From 9632b2a054a57f0b79a79429f4d3a178f53702b1 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 18 Jun 2025 13:23:29 +0200 Subject: [PATCH 094/156] PMM-7: cluster and env --- ...istribution-postgres-replication-setup.yml | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml b/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml index 80fdc746..18e165b8 100644 --- a/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml @@ -121,22 +121,22 @@ " ' -#- name: Create custom database for pgbench -# community.docker.docker_container_exec: -# container: "pdpgsql_pmm_replication_{{ pg_version }}_1" -# user: postgres -# command: > -# bash -c " -# echo \" -# CREATE DATABASE pgbench; -# \\c pgbench -# GRANT CONNECT ON DATABASE pgbench TO pmm; -# GRANT USAGE ON SCHEMA public TO pmm; -# GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA public TO pmm; -# ALTER DEFAULT PRIVILEGES IN SCHEMA public -# GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO pmm; -# \" | psql -U postgres -v ON_ERROR_STOP=1 -# " +- name: Create custom database + community.docker.docker_container_exec: + container: "pdpgsql_pmm_replication_{{ pg_version }}_1" + user: postgres + command: > + bash -c " + echo \" + CREATE DATABASE test_database; + \\c test_database + GRANT CONNECT ON DATABASE test_database TO pmm; + GRANT USAGE ON SCHEMA public TO pmm; + GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA public TO pmm; + ALTER DEFAULT PRIVILEGES IN SCHEMA public + GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO pmm; + \" | psql -U postgres -v ON_ERROR_STOP=1 + " - name: Run pg_basebackup from primary to replica community.docker.docker_container_exec: @@ -248,7 +248,7 @@ community.docker.docker_container_exec: container: "pdpgsql_pmm_replication_{{ pg_version }}_1" command: > - sh -c "nohup bash -c 'while true; do echo Starting insert at $(date +\"%Y-%m-%d %H:%M:%S\"); psql -U postgres -d postgres -f /load_pgsql.sql; sleep 30; done' > /tmp/sql_loop.log 2>&1 &" + sh -c "nohup bash -c 'while true; do echo Starting insert at $(date +\"%Y-%m-%d %H:%M:%S\"); psql -U postgres -d test_database -f /load_pgsql.sql; sleep 30; done' > /tmp/sql_loop.log 2>&1 &" - name: Create logical replication slots community.docker.docker_container_exec: From 8bb6fc9002237660acda896751f3caff288e0611 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 18 Jun 2025 13:44:07 +0200 Subject: [PATCH 095/156] PMM-7: cluster and env --- ...percona-distribution-postgres-replication-setup.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml b/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml index 18e165b8..160e7a7c 100644 --- a/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml @@ -238,6 +238,16 @@ pg_publication; SQL' +- name: Create pg_stat_statements extension + community.docker.docker_container_exec: + container: "{{ container_name }}_1" + user: postgres + command: > + psql -U postgres -d test_database -c " + CREATE EXTENSION IF NOT EXISTS pg_stat_monitor; + SELECT pg_stat_monitor_version(); + " + - name: Copy sample load into container community.docker.docker_container_copy_into: container: "pdpgsql_pmm_replication_{{ pg_version }}_1" From 98978089d7959fe8b8583984832e75c39e254b66 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 18 Jun 2025 14:00:48 +0200 Subject: [PATCH 096/156] PMM-7: cluster and env --- .../tasks/percona-distribution-postgres-replication-setup.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml b/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml index 160e7a7c..98efcc50 100644 --- a/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml @@ -240,7 +240,7 @@ - name: Create pg_stat_statements extension community.docker.docker_container_exec: - container: "{{ container_name }}_1" + container: "pdpgsql_pmm_replication_{{ pg_version }}_1" user: postgres command: > psql -U postgres -d test_database -c " From be62009b3b083bbefca9e5e1b9caf4207585657a Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 18 Jun 2025 14:43:24 +0200 Subject: [PATCH 097/156] PMM-7: cluster and env --- .../tasks/percona-distribution-postgres-patroni-setup.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-patroni-setup.yml b/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-patroni-setup.yml index 4cab04b1..a2267627 100644 --- a/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-patroni-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-patroni-setup.yml @@ -173,13 +173,13 @@ - name: Add service to pmm server community.docker.docker_container_exec: container: "pdpgsql_pmm_patroni_{{ pg_version }}_{{ item }}" - command: pmm-admin add postgresql --username=postgres --password=postgres --query-source=pgstatmonitor pdpgsql_pmm_patroni_{{ pg_version }}_{{ item }}{{ random_service_name_value }} --debug 127.0.0.1:5432 + command: pmm-admin add postgresql --username=postgres --cluster=pdpgsql_patroni_cluster --environment=pdpgsql_patroni_environment --password=postgres --query-source=pgstatmonitor pdpgsql_pmm_patroni_{{ pg_version }}_{{ item }}{{ random_service_name_value }} --debug 127.0.0.1:5432 loop: "{{ range(1, nodes_count | int + 1) | list }}" - name: Add patroni service to pmm server community.docker.docker_container_exec: container: "pdpgsql_pmm_patroni_{{ pg_version }}_{{ item }}" - command: pmm-admin add external --listen-port=8008 --service-name=patroni_service_{{ item }}{{ random_service_name_value }} + command: pmm-admin add external --listen-port=8008 --cluster=pdpgsql_patroni_service_cluster --environment=pdpgsql_patroni_service_environment --service-name=patroni_service_{{ item }}{{ random_service_name_value }} loop: "{{ range(1, nodes_count | int + 1) | list }}" - name: Log Patroni cluster From 32950cdcfe99a97cdf1d92f07460d35e66bb2044 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 18 Jun 2025 15:01:43 +0200 Subject: [PATCH 098/156] PMM-7: cluster and env --- .../tasks/percona-distribution-postgres-patroni-setup.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-patroni-setup.yml b/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-patroni-setup.yml index a2267627..b83e1f5f 100644 --- a/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-patroni-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-patroni-setup.yml @@ -176,11 +176,16 @@ command: pmm-admin add postgresql --username=postgres --cluster=pdpgsql_patroni_cluster --environment=pdpgsql_patroni_environment --password=postgres --query-source=pgstatmonitor pdpgsql_pmm_patroni_{{ pg_version }}_{{ item }}{{ random_service_name_value }} --debug 127.0.0.1:5432 loop: "{{ range(1, nodes_count | int + 1) | list }}" +- name: Add patroni service to pmm server + community.docker.docker_container_exec: + container: "pdpgsql_pmm_patroni_{{ pg_version }}_1" + command: pmm-admin add external --listen-port=8008 --service-name=patroni_service_{{ item }}{{ random_service_name_value }} + - name: Add patroni service to pmm server community.docker.docker_container_exec: container: "pdpgsql_pmm_patroni_{{ pg_version }}_{{ item }}" command: pmm-admin add external --listen-port=8008 --cluster=pdpgsql_patroni_service_cluster --environment=pdpgsql_patroni_service_environment --service-name=patroni_service_{{ item }}{{ random_service_name_value }} - loop: "{{ range(1, nodes_count | int + 1) | list }}" + loop: "{{ range(2, nodes_count | int + 1) | list }}" - name: Log Patroni cluster community.docker.docker_container_exec: From f9dd5e9a55aee5c6a7f072ec880eeba25cb644fd Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 18 Jun 2025 15:08:17 +0200 Subject: [PATCH 099/156] PMM-7: cluster and env --- pmm_qa/pg_stat_monitor_setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/pg_stat_monitor_setup.sh b/pmm_qa/pg_stat_monitor_setup.sh index cc6fb54e..a7fd6abd 100644 --- a/pmm_qa/pg_stat_monitor_setup.sh +++ b/pmm_qa/pg_stat_monitor_setup.sh @@ -19,7 +19,7 @@ fi # If branch/tag is not provided then it will default to main branch if [ -z "$pgstat_monitor_branch" ] then - export pgstat_monitor_branch="2.1.0" + export pgstat_monitor_branch="2.1.1" fi # If repo is not provided then it will default to percona PGSM repository From d17bd14a32458c18529aebbe849ffdc658b3192b Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 18 Jun 2025 15:11:43 +0200 Subject: [PATCH 100/156] PMM-7: cluster and env --- .../tasks/percona-distribution-postgres-patroni-setup.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-patroni-setup.yml b/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-patroni-setup.yml index b83e1f5f..d5055843 100644 --- a/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-patroni-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-patroni-setup.yml @@ -179,7 +179,7 @@ - name: Add patroni service to pmm server community.docker.docker_container_exec: container: "pdpgsql_pmm_patroni_{{ pg_version }}_1" - command: pmm-admin add external --listen-port=8008 --service-name=patroni_service_{{ item }}{{ random_service_name_value }} + command: pmm-admin add external --listen-port=8008 --service-name=patroni_service_1{{ random_service_name_value }} - name: Add patroni service to pmm server community.docker.docker_container_exec: From c9ce13f4ca0461de0b6342373519aee8c96529fc Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 18 Jun 2025 17:36:17 +0200 Subject: [PATCH 101/156] PMM-7: cluster and env --- pmm_qa/pg_stat_monitor_setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/pg_stat_monitor_setup.sh b/pmm_qa/pg_stat_monitor_setup.sh index a7fd6abd..70cf8765 100644 --- a/pmm_qa/pg_stat_monitor_setup.sh +++ b/pmm_qa/pg_stat_monitor_setup.sh @@ -19,7 +19,7 @@ fi # If branch/tag is not provided then it will default to main branch if [ -z "$pgstat_monitor_branch" ] then - export pgstat_monitor_branch="2.1.1" + export pgstat_monitor_branch="main" fi # If repo is not provided then it will default to percona PGSM repository From 73b8f7863265b372a76a28a9cb8b6fd0f1d7aa8a Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Thu, 19 Jun 2025 07:22:09 +0200 Subject: [PATCH 102/156] PMM-7: install pgsm --- pmm_qa/pg_stat_monitor_setup.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/pmm_qa/pg_stat_monitor_setup.sh b/pmm_qa/pg_stat_monitor_setup.sh index 70cf8765..953cbb2a 100644 --- a/pmm_qa/pg_stat_monitor_setup.sh +++ b/pmm_qa/pg_stat_monitor_setup.sh @@ -54,13 +54,13 @@ then wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' apt update - apt -y install postgresql-${pgsql_version} postgresql-server-dev-${pgsql_version} + apt -y install postgresql-${pgsql_version} postgresql-server-dev-${pgsql_version} percona-pg-stat-monitor${pgsql_version} else wget https://repo.percona.com/apt/percona-release_latest.generic_all.deb dpkg -i percona-release_latest.generic_all.deb percona-release setup ppg-${pgsql_version} apt-get -y update - apt-get -y install percona-postgresql-${pgsql_version} percona-postgresql-contrib percona-postgresql-server-dev-all + apt-get -y install percona-postgresql-${pgsql_version} percona-postgresql-contrib percona-postgresql-server-dev-all percona-pg-stat-monitor${pgsql_version} fi sleep 10 @@ -84,16 +84,16 @@ echo $PATH cp /usr/lib/postgresql/${pgsql_version}/bin/pg_config /usr/bin # Clone PGSM repo and move to /home/postgres/pg_stat_monitor dir -cd /home/postgres -git clone --depth 1 --branch ${pgstat_monitor_branch} https://github.com/${pgstat_monitor_repo} -chown -R postgres:postgres pg_stat_monitor -cd pg_stat_monitor +#cd /home/postgres +#git clone --depth 1 --branch ${pgstat_monitor_branch} https://github.com/${pgstat_monitor_repo} +#chown -R postgres:postgres pg_stat_monitor +#cd pg_stat_monitor # Build PGSM -make USE_PGXS=1 +#make USE_PGXS=1 # Install built PGSM library into server -make USE_PGXS=1 install +#make USE_PGXS=1 install # Stop server and edit postgresql.conf to load PGSM library with required configurations service postgresql stop From 70a488cc37ec06842f10a98987de1a8e14157c13 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Thu, 19 Jun 2025 08:00:37 +0200 Subject: [PATCH 103/156] PMM-7: install pgsm --- .../tasks/percona-distribution-postgres-patroni-setup.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-patroni-setup.yml b/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-patroni-setup.yml index d5055843..fe609171 100644 --- a/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-patroni-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-patroni-setup.yml @@ -179,7 +179,7 @@ - name: Add patroni service to pmm server community.docker.docker_container_exec: container: "pdpgsql_pmm_patroni_{{ pg_version }}_1" - command: pmm-admin add external --listen-port=8008 --service-name=patroni_service_1{{ random_service_name_value }} + command: pmm-admin add external --listen-port=8008 --cluster=pdpgsql_patroni_service_cluster_test --environment=pdpgsql_patroni_service_environment_test --service-name=patroni_service_1{{ random_service_name_value }} - name: Add patroni service to pmm server community.docker.docker_container_exec: From 5c79b966104f0700baafbdfc0b013bfc3077098c Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Thu, 19 Jun 2025 08:11:09 +0200 Subject: [PATCH 104/156] PMM-7: install pgsm --- .../tasks/percona-distribution-postgres-patroni-setup.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-patroni-setup.yml b/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-patroni-setup.yml index fe609171..d5055843 100644 --- a/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-patroni-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-patroni-setup.yml @@ -179,7 +179,7 @@ - name: Add patroni service to pmm server community.docker.docker_container_exec: container: "pdpgsql_pmm_patroni_{{ pg_version }}_1" - command: pmm-admin add external --listen-port=8008 --cluster=pdpgsql_patroni_service_cluster_test --environment=pdpgsql_patroni_service_environment_test --service-name=patroni_service_1{{ random_service_name_value }} + command: pmm-admin add external --listen-port=8008 --service-name=patroni_service_1{{ random_service_name_value }} - name: Add patroni service to pmm server community.docker.docker_container_exec: From f3541aaed891b5e59c31348716d2333e187d17b1 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Thu, 19 Jun 2025 12:28:52 +0200 Subject: [PATCH 105/156] PMM-7: log --- pmm_qa/percona-distribution-postgresql/data/patroni.yml.j2 | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pmm_qa/percona-distribution-postgresql/data/patroni.yml.j2 b/pmm_qa/percona-distribution-postgresql/data/patroni.yml.j2 index 13ec02f5..ea680deb 100644 --- a/pmm_qa/percona-distribution-postgresql/data/patroni.yml.j2 +++ b/pmm_qa/percona-distribution-postgresql/data/patroni.yml.j2 @@ -30,6 +30,9 @@ bootstrap: archive_mode: "on" archive_timeout: 600s archive_command: "cp -f %p /home/postgres/archived/%f" + logging_collector: on + log_directory: 'log' + log_filename: 'postgresql.log' pg_hba: - host replication replicator 127.0.0.1/32 md5 - host replication replicator 172.18.0.0/16 md5 From 858bbea5207ac46db792bd6fb4a0b6d714a0f856 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Thu, 19 Jun 2025 15:11:10 +0200 Subject: [PATCH 106/156] PMM-7: log --- .../tasks/percona-distribution-postgres-patroni-setup.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-patroni-setup.yml b/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-patroni-setup.yml index d5055843..f828d78e 100644 --- a/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-patroni-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-patroni-setup.yml @@ -2,6 +2,9 @@ set_fact: nodes_count: 3 when: nodes_count | int < 3 +- name: Set external facing port in patroni setup + set_fact: + pdpgsql_port: 6432 - name: Remove old data folders shell: 'rm -fr {{ data_dir }}' From c3b62c2d88f05ae133050977dd2e2016de5fca5e Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 23 Jun 2025 08:34:21 +0200 Subject: [PATCH 107/156] PMM-7: log --- .../docker-compose-pmm-psmdb.yml | 19 ------------------- pmm_qa/data/load_pgsql.sql | 18 +++++++++--------- 2 files changed, 9 insertions(+), 28 deletions(-) diff --git a/pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml b/pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml index d5a49be6..54f04fd3 100644 --- a/pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml +++ b/pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml @@ -64,8 +64,6 @@ services: depends_on: pmm-server: condition: service_healthy - kerberos: - condition: service_healthy pmm-server: container_name: pmm-server @@ -100,23 +98,6 @@ services: volumes: - 'openldap-data:/bitnami/openldap' - kerberos: - image: kerberos/local - build: - dockerfile: ./Dockerfile-kerberos - context: . - container_name: kerberos - hostname: kerberos - environment: - - "KRB5_TRACE=/dev/stderr" - volumes: - - keytabs:/keytabs - healthcheck: - test: ["CMD", "kadmin.local", "-q", "listprincs"] - interval: 2s - timeout: 1s - retries: 5 - minio: image: minio/minio container_name: minio diff --git a/pmm_qa/data/load_pgsql.sql b/pmm_qa/data/load_pgsql.sql index f5759280..227e904c 100644 --- a/pmm_qa/data/load_pgsql.sql +++ b/pmm_qa/data/load_pgsql.sql @@ -11,15 +11,15 @@ SELECT ANALYZE buffer_test; ----- Step 3: Perform repeated full-table scans to stress buffer allocation ---DO $$ ---BEGIN --- FOR i IN 1..10 LOOP --- RAISE NOTICE 'Running scan iteration %', i; --- PERFORM COUNT(*) FROM buffer_test; --- END LOOP; ---END $$; --- +-- Step 3: Perform repeated full-table scans to stress buffer allocation +DO $$ +BEGIN + FOR i IN 1..10 LOOP + RAISE NOTICE 'Running scan iteration %', i; + PERFORM COUNT(*) FROM buffer_test; + END LOOP; +END $$; + ---- Step 4: Show final buffers_alloc value --SELECT 'Final buffers_alloc' AS info, buffers_alloc FROM pg_stat_bgwriter; VACUUM; From 0e2c900749e946a0e09ade123f4219b972ab5fe8 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 23 Jun 2025 08:40:48 +0200 Subject: [PATCH 108/156] PMM-7: log --- pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml b/pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml index 54f04fd3..6d6b7ca2 100644 --- a/pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml +++ b/pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml @@ -42,14 +42,12 @@ services: - -c - | cp /mongodb_certs/ca-certs.pem /etc/pki/tls/certs/ - chown -R mongod:mongod /keytabs exec /usr/sbin/init volumes: - ./conf:/etc/mongod - ./certs:/mongodb_certs - /sys/fs/cgroup:/sys/fs/cgroup:rw - 'psmdb-server-data:/data/db' - - keytabs:/keytabs - ../pmm_psmdb-pbm_setup/conf/datagen:/etc/datagen:ro privileged: true cgroup: host @@ -130,5 +128,3 @@ volumes: driver: local openldap-data: driver: local - keytabs: - driver: local From e488a8d994e1a0ce126901d70912974e68a0add2 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 23 Jun 2025 08:42:57 +0200 Subject: [PATCH 109/156] PMM-7: log --- pmm_psmdb_diffauth_setup/test-auth.sh | 134 +++++++++++++------------- 1 file changed, 67 insertions(+), 67 deletions(-) diff --git a/pmm_psmdb_diffauth_setup/test-auth.sh b/pmm_psmdb_diffauth_setup/test-auth.sh index ade6c3fa..2f3d160f 100755 --- a/pmm_psmdb_diffauth_setup/test-auth.sh +++ b/pmm_psmdb_diffauth_setup/test-auth.sh @@ -28,70 +28,70 @@ git clone https://github.com/OpenVPN/easy-rsa.git ./easy-rsa/easyrsa3/easyrsa --req-ou=server --subject-alt-name=DNS:psmdb-server --batch build-server-full psmdb-server nopass ./easy-rsa/easyrsa3/easyrsa --req-ou=client --batch build-client-full pmm-test nopass openssl dhparam -out certs/dhparam.pem 2048 - -cp pki/ca.crt certs/ca-certs.pem -cp pki/private/pmm-server.key certs/certificate.key -cp pki/issued/pmm-server.crt certs/certificate.crt -cat pki/private/psmdb-server.key pki/issued/psmdb-server.crt > certs/psmdb-server.pem -cat pki/private/pmm-test.key pki/issued/pmm-test.crt > certs/client.pem -find certs -type f -exec chmod 644 {} \; - -#Start setup -docker compose -f docker-compose-pmm-psmdb.yml down -v --remove-orphans -docker compose -f docker-compose-pmm-psmdb.yml build -docker compose -f docker-compose-pmm-psmdb.yml up -d - -#Add users -docker compose -f docker-compose-pmm-psmdb.yml exec -T psmdb-server mongo --quiet << EOF -db.getSiblingDB("admin").createUser({ user: "root", pwd: "root", roles: [ "root", "userAdminAnyDatabase", "clusterAdmin" ] }); -EOF -docker compose -f docker-compose-pmm-psmdb.yml exec -T psmdb-server mongo --quiet "mongodb://root:root@localhost/?replicaSet=rs0" < init/setup_psmdb.js - -#Configure PBM -docker compose -f docker-compose-pmm-psmdb.yml exec -T psmdb-server bash -c "echo \"PBM_MONGODB_URI=mongodb://pbm:pbmpass@127.0.0.1:27017\" > /etc/sysconfig/pbm-agent" -docker compose -f docker-compose-pmm-psmdb.yml exec -T psmdb-server systemctl restart pbm-agent - -#Configure PMM -set +e -i=1 -while [ $i -le 3 ]; do - output=$(docker compose -f docker-compose-pmm-psmdb.yml exec -T psmdb-server pmm-agent setup 2>&1) - exit_code=$? - - if [ $exit_code -ne 0 ] && [[ $output == *"500 Internal Server Error"* ]]; then - i=$((i + 1)) - else - break - fi - sleep 1 -done - -#Add Mongo Service -random_number=$RANDOM -docker compose -f docker-compose-pmm-psmdb.yml exec -T psmdb-server pmm-admin add mongodb psmdb-server_${random_number} --agent-password=mypass --username=pmm_mongodb --password="5M](Q%q/U+YQ<^m" --host psmdb-server --port 27017 --tls --tls-certificate-key-file=/mongodb_certs/client.pem --tls-ca-file=/mongodb_certs/ca-certs.pem --cluster=mycluster -#Add some data -docker compose -f docker-compose-pmm-psmdb.yml exec -T psmdb-server mgodatagen -f /etc/datagen/replicaset.json --username=pmm_mongodb --password="5M](Q%q/U+YQ<^m" --host psmdb-server --port 27017 --tlsCertificateKeyFile=/mongodb_certs/client.pem --tlsCAFile=/mongodb_certs/ca-certs.pem - -tests=${TESTS:-yes} -if [ $tests = "yes" ]; then - echo "running tests" - output=$(docker compose -f docker-compose-pmm-psmdb.yml run test pytest -s --verbose test.py) - else - echo "skipping tests" -fi - -cleanup=${CLEANUP:-yes} -if [ $cleanup = "yes" ]; then - echo "cleanup" - docker compose -f docker-compose-pmm-psmdb.yml down -v --remove-orphans - if [[ -n "$PSMDB_VERSION" ]] && [[ "$PSMDB_VERSION" == *"4.2"* ]]; then - sed -i 's/MONGODB-X509/MONGODB-X509,MONGODB-AWS/' conf/mongod.conf - fi - else - echo "skipping cleanup" -fi - -echo "$output" -if echo "$output" | grep -q "\bFAILED\b"; then - exit 1 -fi +# +#cp pki/ca.crt certs/ca-certs.pem +#cp pki/private/pmm-server.key certs/certificate.key +#cp pki/issued/pmm-server.crt certs/certificate.crt +#cat pki/private/psmdb-server.key pki/issued/psmdb-server.crt > certs/psmdb-server.pem +#cat pki/private/pmm-test.key pki/issued/pmm-test.crt > certs/client.pem +#find certs -type f -exec chmod 644 {} \; +# +##Start setup +#docker compose -f docker-compose-pmm-psmdb.yml down -v --remove-orphans +#docker compose -f docker-compose-pmm-psmdb.yml build +#docker compose -f docker-compose-pmm-psmdb.yml up -d +# +##Add users +#docker compose -f docker-compose-pmm-psmdb.yml exec -T psmdb-server mongo --quiet << EOF +#db.getSiblingDB("admin").createUser({ user: "root", pwd: "root", roles: [ "root", "userAdminAnyDatabase", "clusterAdmin" ] }); +#EOF +#docker compose -f docker-compose-pmm-psmdb.yml exec -T psmdb-server mongo --quiet "mongodb://root:root@localhost/?replicaSet=rs0" < init/setup_psmdb.js +# +##Configure PBM +#docker compose -f docker-compose-pmm-psmdb.yml exec -T psmdb-server bash -c "echo \"PBM_MONGODB_URI=mongodb://pbm:pbmpass@127.0.0.1:27017\" > /etc/sysconfig/pbm-agent" +#docker compose -f docker-compose-pmm-psmdb.yml exec -T psmdb-server systemctl restart pbm-agent +# +##Configure PMM +#set +e +#i=1 +#while [ $i -le 3 ]; do +# output=$(docker compose -f docker-compose-pmm-psmdb.yml exec -T psmdb-server pmm-agent setup 2>&1) +# exit_code=$? +# +# if [ $exit_code -ne 0 ] && [[ $output == *"500 Internal Server Error"* ]]; then +# i=$((i + 1)) +# else +# break +# fi +# sleep 1 +#done +# +##Add Mongo Service +#random_number=$RANDOM +#docker compose -f docker-compose-pmm-psmdb.yml exec -T psmdb-server pmm-admin add mongodb psmdb-server_${random_number} --agent-password=mypass --username=pmm_mongodb --password="5M](Q%q/U+YQ<^m" --host psmdb-server --port 27017 --tls --tls-certificate-key-file=/mongodb_certs/client.pem --tls-ca-file=/mongodb_certs/ca-certs.pem --cluster=mycluster +##Add some data +#docker compose -f docker-compose-pmm-psmdb.yml exec -T psmdb-server mgodatagen -f /etc/datagen/replicaset.json --username=pmm_mongodb --password="5M](Q%q/U+YQ<^m" --host psmdb-server --port 27017 --tlsCertificateKeyFile=/mongodb_certs/client.pem --tlsCAFile=/mongodb_certs/ca-certs.pem +# +#tests=${TESTS:-yes} +#if [ $tests = "yes" ]; then +# echo "running tests" +# output=$(docker compose -f docker-compose-pmm-psmdb.yml run test pytest -s --verbose test.py) +# else +# echo "skipping tests" +#fi +# +#cleanup=${CLEANUP:-yes} +#if [ $cleanup = "yes" ]; then +# echo "cleanup" +# docker compose -f docker-compose-pmm-psmdb.yml down -v --remove-orphans +# if [[ -n "$PSMDB_VERSION" ]] && [[ "$PSMDB_VERSION" == *"4.2"* ]]; then +# sed -i 's/MONGODB-X509/MONGODB-X509,MONGODB-AWS/' conf/mongod.conf +# fi +# else +# echo "skipping cleanup" +#fi +# +#echo "$output" +#if echo "$output" | grep -q "\bFAILED\b"; then +# exit 1 +#fi From 9d788aadd3b19a24f9c150b62e649cb5535e7995 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 23 Jun 2025 08:44:47 +0200 Subject: [PATCH 110/156] PMM-7: log --- pmm_psmdb_diffauth_setup/test-auth.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/pmm_psmdb_diffauth_setup/test-auth.sh b/pmm_psmdb_diffauth_setup/test-auth.sh index 2f3d160f..c5a66bce 100755 --- a/pmm_psmdb_diffauth_setup/test-auth.sh +++ b/pmm_psmdb_diffauth_setup/test-auth.sh @@ -28,14 +28,14 @@ git clone https://github.com/OpenVPN/easy-rsa.git ./easy-rsa/easyrsa3/easyrsa --req-ou=server --subject-alt-name=DNS:psmdb-server --batch build-server-full psmdb-server nopass ./easy-rsa/easyrsa3/easyrsa --req-ou=client --batch build-client-full pmm-test nopass openssl dhparam -out certs/dhparam.pem 2048 -# -#cp pki/ca.crt certs/ca-certs.pem -#cp pki/private/pmm-server.key certs/certificate.key -#cp pki/issued/pmm-server.crt certs/certificate.crt -#cat pki/private/psmdb-server.key pki/issued/psmdb-server.crt > certs/psmdb-server.pem -#cat pki/private/pmm-test.key pki/issued/pmm-test.crt > certs/client.pem -#find certs -type f -exec chmod 644 {} \; -# + +cp pki/ca.crt certs/ca-certs.pem +cp pki/private/pmm-server.key certs/certificate.key +cp pki/issued/pmm-server.crt certs/certificate.crt +cat pki/private/psmdb-server.key pki/issued/psmdb-server.crt > certs/psmdb-server.pem +cat pki/private/pmm-test.key pki/issued/pmm-test.crt > certs/client.pem +find certs -type f -exec chmod 644 {} \; + ##Start setup #docker compose -f docker-compose-pmm-psmdb.yml down -v --remove-orphans #docker compose -f docker-compose-pmm-psmdb.yml build From 746b8ef7712f6f88817929f4ae44c11d0e922165 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 23 Jun 2025 08:45:36 +0200 Subject: [PATCH 111/156] PMM-7: log --- pmm_psmdb_diffauth_setup/test-auth.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pmm_psmdb_diffauth_setup/test-auth.sh b/pmm_psmdb_diffauth_setup/test-auth.sh index c5a66bce..d79c1aab 100755 --- a/pmm_psmdb_diffauth_setup/test-auth.sh +++ b/pmm_psmdb_diffauth_setup/test-auth.sh @@ -36,11 +36,11 @@ cat pki/private/psmdb-server.key pki/issued/psmdb-server.crt > certs/psmdb-serve cat pki/private/pmm-test.key pki/issued/pmm-test.crt > certs/client.pem find certs -type f -exec chmod 644 {} \; -##Start setup -#docker compose -f docker-compose-pmm-psmdb.yml down -v --remove-orphans -#docker compose -f docker-compose-pmm-psmdb.yml build -#docker compose -f docker-compose-pmm-psmdb.yml up -d -# +#Start setup +docker compose -f docker-compose-pmm-psmdb.yml down -v --remove-orphans +docker compose -f docker-compose-pmm-psmdb.yml build +docker compose -f docker-compose-pmm-psmdb.yml up -d + ##Add users #docker compose -f docker-compose-pmm-psmdb.yml exec -T psmdb-server mongo --quiet << EOF #db.getSiblingDB("admin").createUser({ user: "root", pwd: "root", roles: [ "root", "userAdminAnyDatabase", "clusterAdmin" ] }); From 3fdb32066ee57042c93c79b0dee27430a22b81a5 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 23 Jun 2025 08:53:51 +0200 Subject: [PATCH 112/156] PMM-7: log --- pmm_psmdb_diffauth_setup/test-auth.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pmm_psmdb_diffauth_setup/test-auth.sh b/pmm_psmdb_diffauth_setup/test-auth.sh index d79c1aab..33d844ba 100755 --- a/pmm_psmdb_diffauth_setup/test-auth.sh +++ b/pmm_psmdb_diffauth_setup/test-auth.sh @@ -38,8 +38,8 @@ find certs -type f -exec chmod 644 {} \; #Start setup docker compose -f docker-compose-pmm-psmdb.yml down -v --remove-orphans -docker compose -f docker-compose-pmm-psmdb.yml build -docker compose -f docker-compose-pmm-psmdb.yml up -d +#docker compose -f docker-compose-pmm-psmdb.yml build +#docker compose -f docker-compose-pmm-psmdb.yml up -d ##Add users #docker compose -f docker-compose-pmm-psmdb.yml exec -T psmdb-server mongo --quiet << EOF From f37cab0de347b00a7d70c6e2a9e90d2208f9348b Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 23 Jun 2025 09:00:43 +0200 Subject: [PATCH 113/156] PMM-7: log --- pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml b/pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml index 6d6b7ca2..293b927c 100644 --- a/pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml +++ b/pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml @@ -6,11 +6,11 @@ services: dockerfile: ../pmm_psmdb-pbm_setup/Dockerfile context: ../pmm_psmdb-pbm_setup/ args: - - REPO=${REPO:-testing} - - PMM_REPO=${PMM_REPO:-experimental} - - PBM_VERSION=${PBM_VERSION:-latest} - - PSMDB_VERSION=${PSMDB_VERSION:-latest} - - PMM_CLIENT_VERSION=${PMM_CLIENT_VERSION:-3-dev-latest} + REPO: ${REPO:-testing} + PMM_REPO: ${PMM_REPO:-experimental} + PBM_VERSION: ${PBM_VERSION:-latest} + PSMDB_VERSION: ${PSMDB_VERSION:-latest} + PMM_CLIENT_VERSION: ${PMM_CLIENT_VERSION:-3-dev-latest} command: /bin/bash test: From d1a7bcfa91ebd72528c53dac6cc41c09f3369fcc Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 23 Jun 2025 09:04:43 +0200 Subject: [PATCH 114/156] PMM-7: log --- pmm_qa/pmm-framework.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index c5cba045..66e3cec9 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -606,7 +606,7 @@ def mongo_ssl_setup(script_filename, args): ['cp', f'{scripts_path}docker-compose-pmm-psmdb.yml', f'{compose_file_path}']) admin_password = os.getenv('ADMIN_PASSWORD') or args.pmm_server_password or 'admin' subprocess.run(['sed', '-i', f's/PMM_AGENT_SERVER_PASSWORD=admin/PMM_AGENT_SERVER_PASSWORD={admin_password}/g', f'{compose_file_path}']) - subprocess.run(['sed', '-i', '/container_name/a\\ networks:\\\\n \\\\- pmm-qa', f'{compose_file_path}']) + subprocess.run(['sed', '-i', r"/container_name/a\ networks:\n - pmm-qa", compose_file_path]) subprocess.run(['sed', '-i', '$a\\\nnetworks:\\\n pmm-qa:\\\n name: pmm-qa\\\n external: true', f'{compose_file_path}']) subprocess.run(['sed', '-i', From d65b0fa187a3ff4220c6a243d3b629e359a69ec0 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 23 Jun 2025 09:06:19 +0200 Subject: [PATCH 115/156] PMM-7: log --- pmm_qa/pmm-framework.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index 66e3cec9..1bfa6995 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -601,6 +601,10 @@ def mongo_ssl_setup(script_filename, args): # Add workaround (copy files) till sharding only support is ready. try: if no_server: + with open(compose_file_path, 'r') as f: + data = yaml.safe_load(f) + print("This is the parsed data: ") + print(data) # Search & Replace content in the temporary compose files subprocess.run( ['cp', f'{scripts_path}docker-compose-pmm-psmdb.yml', f'{compose_file_path}']) From 0dd637f4af98c67ee6a71a7f686539282a602bf6 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 23 Jun 2025 09:09:36 +0200 Subject: [PATCH 116/156] PMM-7: log --- pmm_qa/pmm-framework.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index 1bfa6995..91304b49 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -591,6 +591,7 @@ def mongo_ssl_setup(script_filename, args): # Temporary docker compose filename compose_filename = f'docker-compose-psmdb.yml' compose_file_path = scripts_path + compose_filename + file_path = os.path.dirname(current_dir) # Create pmm-qa n/w used in workaround result = subprocess.run(['docker', 'network', 'inspect', 'pmm-qa'], capture_output=True) @@ -601,6 +602,7 @@ def mongo_ssl_setup(script_filename, args): # Add workaround (copy files) till sharding only support is ready. try: if no_server: + print(f'File location is: {file_path}') with open(compose_file_path, 'r') as f: data = yaml.safe_load(f) print("This is the parsed data: ") From b09b3bfbb0bec5383c17fc95f93162b9886b30eb Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 23 Jun 2025 09:10:48 +0200 Subject: [PATCH 117/156] PMM-7: log --- pmm_qa/pmm-framework.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index 91304b49..b191cd07 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -591,7 +591,7 @@ def mongo_ssl_setup(script_filename, args): # Temporary docker compose filename compose_filename = f'docker-compose-psmdb.yml' compose_file_path = scripts_path + compose_filename - file_path = os.path.dirname(current_dir) + file_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) # Create pmm-qa n/w used in workaround result = subprocess.run(['docker', 'network', 'inspect', 'pmm-qa'], capture_output=True) From 56f2a23763b707ec01474cb1ee1ade4f48f11952 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 23 Jun 2025 09:11:58 +0200 Subject: [PATCH 118/156] PMM-7: log --- pmm_qa/pmm-framework.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index b191cd07..e09b400c 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -591,7 +591,7 @@ def mongo_ssl_setup(script_filename, args): # Temporary docker compose filename compose_filename = f'docker-compose-psmdb.yml' compose_file_path = scripts_path + compose_filename - file_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + file_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + '/pmm_psmdb_diffauth_setup/' + script_filename # Create pmm-qa n/w used in workaround result = subprocess.run(['docker', 'network', 'inspect', 'pmm-qa'], capture_output=True) @@ -603,7 +603,7 @@ def mongo_ssl_setup(script_filename, args): try: if no_server: print(f'File location is: {file_path}') - with open(compose_file_path, 'r') as f: + with open(file_path, 'r') as f: data = yaml.safe_load(f) print("This is the parsed data: ") print(data) From 530a75878de2a5c1f85fb29949563f72398d3d4b Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 23 Jun 2025 09:12:52 +0200 Subject: [PATCH 119/156] PMM-7: log --- pmm_qa/pmm-framework.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index e09b400c..e31b65e8 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -591,7 +591,7 @@ def mongo_ssl_setup(script_filename, args): # Temporary docker compose filename compose_filename = f'docker-compose-psmdb.yml' compose_file_path = scripts_path + compose_filename - file_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + '/pmm_psmdb_diffauth_setup/' + script_filename + file_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + '/pmm_psmdb_diffauth_setup/' + compose_filename # Create pmm-qa n/w used in workaround result = subprocess.run(['docker', 'network', 'inspect', 'pmm-qa'], capture_output=True) From 15b9cc62b45dfb389cfe60d70dbdd2df202f224c Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 23 Jun 2025 09:15:34 +0200 Subject: [PATCH 120/156] PMM-7: log --- pmm_qa/pmm-framework.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index e31b65e8..33bdb4ec 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -591,7 +591,7 @@ def mongo_ssl_setup(script_filename, args): # Temporary docker compose filename compose_filename = f'docker-compose-psmdb.yml' compose_file_path = scripts_path + compose_filename - file_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + '/pmm_psmdb_diffauth_setup/' + compose_filename + compose_file_folder = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + '/pmm_psmdb_diffauth_setup/' # Create pmm-qa n/w used in workaround result = subprocess.run(['docker', 'network', 'inspect', 'pmm-qa'], capture_output=True) @@ -602,6 +602,7 @@ def mongo_ssl_setup(script_filename, args): # Add workaround (copy files) till sharding only support is ready. try: if no_server: + shutil.copy(compose_file_folder + 'docker-compose-pmm-psmdb.yml', compose_file_folder + compose_filename) print(f'File location is: {file_path}') with open(file_path, 'r') as f: data = yaml.safe_load(f) From 27083f264ad08fe7056cdfd37a329560a89909c2 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 23 Jun 2025 09:16:08 +0200 Subject: [PATCH 121/156] PMM-7: log --- pmm_qa/pmm-framework.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index 33bdb4ec..7799d618 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -4,6 +4,7 @@ import sys import requests import re +import shutil from scripts.get_env_value import get_value from scripts.database_options import database_options as database_configs from scripts.run_ansible_playbook import run_ansible_playbook From ec429c36de0f3e27724aa55562d31ee6ef96ee0a Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 23 Jun 2025 09:17:43 +0200 Subject: [PATCH 122/156] PMM-7: log --- pmm_qa/pmm-framework.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index 7799d618..88f555f4 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -605,7 +605,7 @@ def mongo_ssl_setup(script_filename, args): if no_server: shutil.copy(compose_file_folder + 'docker-compose-pmm-psmdb.yml', compose_file_folder + compose_filename) print(f'File location is: {file_path}') - with open(file_path, 'r') as f: + with open(compose_file_path + compose_filename, 'r') as f: data = yaml.safe_load(f) print("This is the parsed data: ") print(data) From d57aa2c4ae298e1ac4a67707f531563a96f0bc1c Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 23 Jun 2025 09:18:37 +0200 Subject: [PATCH 123/156] PMM-7: log --- pmm_qa/pmm-framework.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index 88f555f4..e8a90aaa 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -604,7 +604,7 @@ def mongo_ssl_setup(script_filename, args): try: if no_server: shutil.copy(compose_file_folder + 'docker-compose-pmm-psmdb.yml', compose_file_folder + compose_filename) - print(f'File location is: {file_path}') + print(f'File location is: {compose_file_path + compose_filename}') with open(compose_file_path + compose_filename, 'r') as f: data = yaml.safe_load(f) print("This is the parsed data: ") From 1971a405dd47c6e171c1b4c626cf9cacee5d104d Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 23 Jun 2025 09:19:38 +0200 Subject: [PATCH 124/156] PMM-7: log --- pmm_qa/pmm-framework.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index e8a90aaa..20a8cdf7 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -604,8 +604,8 @@ def mongo_ssl_setup(script_filename, args): try: if no_server: shutil.copy(compose_file_folder + 'docker-compose-pmm-psmdb.yml', compose_file_folder + compose_filename) - print(f'File location is: {compose_file_path + compose_filename}') - with open(compose_file_path + compose_filename, 'r') as f: + print(f'File location is: {compose_file_folder + compose_filename}') + with open(compose_file_folder + compose_filename, 'r') as f: data = yaml.safe_load(f) print("This is the parsed data: ") print(data) From 8a274c1d16bdf1c3473d932b689a3f22943ec229 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 23 Jun 2025 09:22:04 +0200 Subject: [PATCH 125/156] PMM-7: log --- pmm_qa/pmm-framework.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index 20a8cdf7..c000f9ac 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -5,6 +5,7 @@ import requests import re import shutil +import yaml from scripts.get_env_value import get_value from scripts.database_options import database_options as database_configs from scripts.run_ansible_playbook import run_ansible_playbook From 77c8dd4eac09ab66dd9d9e0b7f94df3e08f442c9 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 23 Jun 2025 09:23:50 +0200 Subject: [PATCH 126/156] PMM-7: log --- pmm_qa/pmm-framework.py | 53 ++++++++++++++++++++++++++++------------- 1 file changed, 37 insertions(+), 16 deletions(-) diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index c000f9ac..21b9532a 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -610,24 +610,45 @@ def mongo_ssl_setup(script_filename, args): data = yaml.safe_load(f) print("This is the parsed data: ") print(data) - # Search & Replace content in the temporary compose files - subprocess.run( - ['cp', f'{scripts_path}docker-compose-pmm-psmdb.yml', f'{compose_file_path}']) - admin_password = os.getenv('ADMIN_PASSWORD') or args.pmm_server_password or 'admin' - subprocess.run(['sed', '-i', f's/PMM_AGENT_SERVER_PASSWORD=admin/PMM_AGENT_SERVER_PASSWORD={admin_password}/g', f'{compose_file_path}']) - subprocess.run(['sed', '-i', r"/container_name/a\ networks:\n - pmm-qa", compose_file_path]) - subprocess.run(['sed', '-i', '$a\\\nnetworks:\\\n pmm-qa:\\\n name: pmm-qa\\\n external: true', - f'{compose_file_path}']) - subprocess.run(['sed', '-i', - '/ depends_on:/{N;N;N;/ depends_on:\\\n pmm-server:\\\n condition: service_healthy/d;}', - f'{compose_file_path}']) - subprocess.run(['sed', '-i', '/^ pmm-server:/,/^$/{/^ ldap-server:/!d}', f'{compose_file_path}']) + for service in data.get('services', {}).values(): + networks = service.get('networks', []) + if isinstance(networks, list): + if 'pmm-qa' not in networks: + networks.append('pmm-qa') + service['networks'] = networks + elif isinstance(networks, dict): + networks['pmm-qa'] = {} + else: + service['networks'] = ['pmm-qa'] + + # Ensure the network is declared globally + if 'networks' not in data: + data['networks'] = {} + + data['networks']['pmm-qa'] = {'external': True, 'name': 'pmm-qa'} + + # Save it back + with open(compose_file_path, 'w') as f: + yaml.dump(data, f, sort_keys=False) + # Search & Replace content in the temporary compose files + # subprocess.run( + # ['cp', f'{scripts_path}docker-compose-pmm-psmdb.yml', f'{compose_file_path}']) + # admin_password = os.getenv('ADMIN_PASSWORD') or args.pmm_server_password or 'admin' + # subprocess.run(['sed', '-i', f's/PMM_AGENT_SERVER_PASSWORD=admin/PMM_AGENT_SERVER_PASSWORD={admin_password}/g', f'{compose_file_path}']) + # subprocess.run(['sed', '-i', r"/container_name/a\ networks:\n - pmm-qa", compose_file_path]) + # subprocess.run(['sed', '-i', '$a\\\nnetworks:\\\n pmm-qa:\\\n name: pmm-qa\\\n external: true', + # f'{compose_file_path}']) + # subprocess.run(['sed', '-i', + # '/ depends_on:/{N;N;N;/ depends_on:\\\n pmm-server:\\\n condition: service_healthy/d;}', + # f'{compose_file_path}']) + # subprocess.run(['sed', '-i', '/^ pmm-server:/,/^$/{/^ ldap-server:/!d}', f'{compose_file_path}']) + # # Search replace content in-line in shell file - subprocess.run(['sed', '-i', f's/pmm-agent setup 2/pmm-agent setup --server-insecure-tls 2/g', - f'{shellscript_file_path}']) - subprocess.run(['sed', '-i', f's/docker-compose-pmm-psmdb.yml/{compose_filename}/g', - f'{shellscript_file_path}']) + # subprocess.run(['sed', '-i', f's/pmm-agent setup 2/pmm-agent setup --server-insecure-tls 2/g', + # f'{shellscript_file_path}']) + # subprocess.run(['sed', '-i', f's/docker-compose-pmm-psmdb.yml/{compose_filename}/g', + # f'{shellscript_file_path}']) except subprocess.CalledProcessError as e: print(f"Error occurred: {e}") From a80e1fc2e1603952ea1b505c19f351ad51d8179e Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 23 Jun 2025 09:29:13 +0200 Subject: [PATCH 127/156] PMM-7: log --- pmm_qa/pmm-framework.py | 32 ++++++++++++++++++++++++++------ 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index 21b9532a..583bf6b8 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -606,10 +606,11 @@ def mongo_ssl_setup(script_filename, args): if no_server: shutil.copy(compose_file_folder + 'docker-compose-pmm-psmdb.yml', compose_file_folder + compose_filename) print(f'File location is: {compose_file_folder + compose_filename}') + # admin_password = os.getenv('ADMIN_PASSWORD') or args.pmm_server_password or 'admin' + admin_password = 'Test2' + print(f'Admin password is: {admin_password}') with open(compose_file_folder + compose_filename, 'r') as f: data = yaml.safe_load(f) - print("This is the parsed data: ") - print(data) for service in data.get('services', {}).values(): networks = service.get('networks', []) @@ -628,12 +629,31 @@ def mongo_ssl_setup(script_filename, args): data['networks']['pmm-qa'] = {'external': True, 'name': 'pmm-qa'} + psmdb_service = data.get('services', {}).get('psmdb-server') + if psmdb_service: + env = psmdb_service.get('environment', []) + + # If environment is a list (common in Docker Compose) + if isinstance(env, list): + for i, entry in enumerate(env): + if entry.startswith('PMM_AGENT_SERVER_PASSWORD='): + env[i] = f'PMM_AGENT_SERVER_PASSWORD={admin_password}' + break + else: + env.append(f'PMM_AGENT_SERVER_PASSWORD={admin_password}') + psmdb_service['environment'] = env + + # If environment is a dict (less common but valid) + elif isinstance(env, dict): + env['PMM_AGENT_SERVER_PASSWORD'] = admin_password + psmdb_service['environment'] = env + + print("This is the parsed data: ") + # print(data) + # Save it back with open(compose_file_path, 'w') as f: - yaml.dump(data, f, sort_keys=False) - # Search & Replace content in the temporary compose files - # subprocess.run( - # ['cp', f'{scripts_path}docker-compose-pmm-psmdb.yml', f'{compose_file_path}']) + print(yaml.dump(data, f, sort_keys=False)) # admin_password = os.getenv('ADMIN_PASSWORD') or args.pmm_server_password or 'admin' # subprocess.run(['sed', '-i', f's/PMM_AGENT_SERVER_PASSWORD=admin/PMM_AGENT_SERVER_PASSWORD={admin_password}/g', f'{compose_file_path}']) # subprocess.run(['sed', '-i', r"/container_name/a\ networks:\n - pmm-qa", compose_file_path]) From fd3eba518407e422cb18d369bb069218df5ef5a4 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 23 Jun 2025 09:31:13 +0200 Subject: [PATCH 128/156] PMM-7: log --- pmm_qa/pmm-framework.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index 583bf6b8..23a28df6 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -653,7 +653,7 @@ def mongo_ssl_setup(script_filename, args): # Save it back with open(compose_file_path, 'w') as f: - print(yaml.dump(data, f, sort_keys=False)) + print(yaml.dump(data, f, sort_keys=False, default_flow_style=False)) # admin_password = os.getenv('ADMIN_PASSWORD') or args.pmm_server_password or 'admin' # subprocess.run(['sed', '-i', f's/PMM_AGENT_SERVER_PASSWORD=admin/PMM_AGENT_SERVER_PASSWORD={admin_password}/g', f'{compose_file_path}']) # subprocess.run(['sed', '-i', r"/container_name/a\ networks:\n - pmm-qa", compose_file_path]) From b7062dfa5c7796c23659e2255212f107ce7ae3b1 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 23 Jun 2025 09:37:44 +0200 Subject: [PATCH 129/156] PMM-7: log --- pmm_qa/pmm-framework.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index 23a28df6..7cd3675a 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -648,20 +648,19 @@ def mongo_ssl_setup(script_filename, args): env['PMM_AGENT_SERVER_PASSWORD'] = admin_password psmdb_service['environment'] = env + depends_on = psmdb_service.get('depends_on') + if depends_on == {'pmm-server': {'condition': 'service_healthy'}}: + del psmdb_service['depends_on'] + print("This is the parsed data: ") - # print(data) + print(data) # Save it back with open(compose_file_path, 'w') as f: - print(yaml.dump(data, f, sort_keys=False, default_flow_style=False)) - # admin_password = os.getenv('ADMIN_PASSWORD') or args.pmm_server_password or 'admin' - # subprocess.run(['sed', '-i', f's/PMM_AGENT_SERVER_PASSWORD=admin/PMM_AGENT_SERVER_PASSWORD={admin_password}/g', f'{compose_file_path}']) - # subprocess.run(['sed', '-i', r"/container_name/a\ networks:\n - pmm-qa", compose_file_path]) - # subprocess.run(['sed', '-i', '$a\\\nnetworks:\\\n pmm-qa:\\\n name: pmm-qa\\\n external: true', - # f'{compose_file_path}']) - # subprocess.run(['sed', '-i', - # '/ depends_on:/{N;N;N;/ depends_on:\\\n pmm-server:\\\n condition: service_healthy/d;}', - # f'{compose_file_path}']) + yaml.dump(data, f, sort_keys=False, default_flow_style=False) + subprocess.run(['sed', '-i', + '/ depends_on:/{N;N;N;/ depends_on:\\\n pmm-server:\\\n condition: service_healthy/d;}', + f'{compose_file_path}']) # subprocess.run(['sed', '-i', '/^ pmm-server:/,/^$/{/^ ldap-server:/!d}', f'{compose_file_path}']) # # Search replace content in-line in shell file From 7b7495cb4373c8801fcd4c102ccff8e5c6072d9c Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 23 Jun 2025 09:39:56 +0200 Subject: [PATCH 130/156] PMM-7: log --- pmm_qa/pmm-framework.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index 7cd3675a..27cffd60 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -649,18 +649,16 @@ def mongo_ssl_setup(script_filename, args): psmdb_service['environment'] = env depends_on = psmdb_service.get('depends_on') + print(depends_on) if depends_on == {'pmm-server': {'condition': 'service_healthy'}}: del psmdb_service['depends_on'] - print("This is the parsed data: ") - print(data) - # Save it back with open(compose_file_path, 'w') as f: yaml.dump(data, f, sort_keys=False, default_flow_style=False) - subprocess.run(['sed', '-i', - '/ depends_on:/{N;N;N;/ depends_on:\\\n pmm-server:\\\n condition: service_healthy/d;}', - f'{compose_file_path}']) + # subprocess.run(['sed', '-i', + # '/ depends_on:/{N;N;N;/ depends_on:\\\n pmm-server:\\\n condition: service_healthy/d;}', + # f'{compose_file_path}']) # subprocess.run(['sed', '-i', '/^ pmm-server:/,/^$/{/^ ldap-server:/!d}', f'{compose_file_path}']) # # Search replace content in-line in shell file From 2cc3d2abf76c6d5a06a6e6b1694d580d75838ccc Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 23 Jun 2025 09:44:29 +0200 Subject: [PATCH 131/156] PMM-7: log --- pmm_qa/pmm-framework.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index 27cffd60..719293d8 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -606,12 +606,13 @@ def mongo_ssl_setup(script_filename, args): if no_server: shutil.copy(compose_file_folder + 'docker-compose-pmm-psmdb.yml', compose_file_folder + compose_filename) print(f'File location is: {compose_file_folder + compose_filename}') - # admin_password = os.getenv('ADMIN_PASSWORD') or args.pmm_server_password or 'admin' - admin_password = 'Test2' - print(f'Admin password is: {admin_password}') + admin_password = os.getenv('ADMIN_PASSWORD') or args.pmm_server_password or 'admin' with open(compose_file_folder + compose_filename, 'r') as f: data = yaml.safe_load(f) + if 'services' in data and 'pmm-server' in data['services']: + del data['services']['pmm-server'] + for service in data.get('services', {}).values(): networks = service.get('networks', []) if isinstance(networks, list): @@ -656,9 +657,6 @@ def mongo_ssl_setup(script_filename, args): # Save it back with open(compose_file_path, 'w') as f: yaml.dump(data, f, sort_keys=False, default_flow_style=False) - # subprocess.run(['sed', '-i', - # '/ depends_on:/{N;N;N;/ depends_on:\\\n pmm-server:\\\n condition: service_healthy/d;}', - # f'{compose_file_path}']) # subprocess.run(['sed', '-i', '/^ pmm-server:/,/^$/{/^ ldap-server:/!d}', f'{compose_file_path}']) # # Search replace content in-line in shell file From 52ba6db7e8ee7a366141e8f4b57bb0f5c05cad60 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 23 Jun 2025 09:47:22 +0200 Subject: [PATCH 132/156] PMM-7: log --- pmm_qa/pmm-framework.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index 719293d8..de993452 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -613,6 +613,8 @@ def mongo_ssl_setup(script_filename, args): if 'services' in data and 'pmm-server' in data['services']: del data['services']['pmm-server'] + content = data.replace('pmm-agent setup 2', 'pmm-agent setup --server-insecure-tls 2') + for service in data.get('services', {}).values(): networks = service.get('networks', []) if isinstance(networks, list): @@ -657,14 +659,7 @@ def mongo_ssl_setup(script_filename, args): # Save it back with open(compose_file_path, 'w') as f: yaml.dump(data, f, sort_keys=False, default_flow_style=False) - # subprocess.run(['sed', '-i', '/^ pmm-server:/,/^$/{/^ ldap-server:/!d}', f'{compose_file_path}']) - # - # Search replace content in-line in shell file - # subprocess.run(['sed', '-i', f's/pmm-agent setup 2/pmm-agent setup --server-insecure-tls 2/g', - # f'{shellscript_file_path}']) - # subprocess.run(['sed', '-i', f's/docker-compose-pmm-psmdb.yml/{compose_filename}/g', - # f'{shellscript_file_path}']) - except subprocess.CalledProcessError as e: + except yaml.YAMLError as e: print(f"Error occurred: {e}") From 8fc38320c5c9e20a11a457a4fabce22e8fdccc36 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 23 Jun 2025 09:50:56 +0200 Subject: [PATCH 133/156] PMM-7: log --- pmm_psmdb_diffauth_setup/test-auth.sh | 112 +++++++++++++------------- pmm_qa/pmm-framework.py | 3 +- 2 files changed, 58 insertions(+), 57 deletions(-) diff --git a/pmm_psmdb_diffauth_setup/test-auth.sh b/pmm_psmdb_diffauth_setup/test-auth.sh index 33d844ba..ade6c3fa 100755 --- a/pmm_psmdb_diffauth_setup/test-auth.sh +++ b/pmm_psmdb_diffauth_setup/test-auth.sh @@ -38,60 +38,60 @@ find certs -type f -exec chmod 644 {} \; #Start setup docker compose -f docker-compose-pmm-psmdb.yml down -v --remove-orphans -#docker compose -f docker-compose-pmm-psmdb.yml build -#docker compose -f docker-compose-pmm-psmdb.yml up -d +docker compose -f docker-compose-pmm-psmdb.yml build +docker compose -f docker-compose-pmm-psmdb.yml up -d -##Add users -#docker compose -f docker-compose-pmm-psmdb.yml exec -T psmdb-server mongo --quiet << EOF -#db.getSiblingDB("admin").createUser({ user: "root", pwd: "root", roles: [ "root", "userAdminAnyDatabase", "clusterAdmin" ] }); -#EOF -#docker compose -f docker-compose-pmm-psmdb.yml exec -T psmdb-server mongo --quiet "mongodb://root:root@localhost/?replicaSet=rs0" < init/setup_psmdb.js -# -##Configure PBM -#docker compose -f docker-compose-pmm-psmdb.yml exec -T psmdb-server bash -c "echo \"PBM_MONGODB_URI=mongodb://pbm:pbmpass@127.0.0.1:27017\" > /etc/sysconfig/pbm-agent" -#docker compose -f docker-compose-pmm-psmdb.yml exec -T psmdb-server systemctl restart pbm-agent -# -##Configure PMM -#set +e -#i=1 -#while [ $i -le 3 ]; do -# output=$(docker compose -f docker-compose-pmm-psmdb.yml exec -T psmdb-server pmm-agent setup 2>&1) -# exit_code=$? -# -# if [ $exit_code -ne 0 ] && [[ $output == *"500 Internal Server Error"* ]]; then -# i=$((i + 1)) -# else -# break -# fi -# sleep 1 -#done -# -##Add Mongo Service -#random_number=$RANDOM -#docker compose -f docker-compose-pmm-psmdb.yml exec -T psmdb-server pmm-admin add mongodb psmdb-server_${random_number} --agent-password=mypass --username=pmm_mongodb --password="5M](Q%q/U+YQ<^m" --host psmdb-server --port 27017 --tls --tls-certificate-key-file=/mongodb_certs/client.pem --tls-ca-file=/mongodb_certs/ca-certs.pem --cluster=mycluster -##Add some data -#docker compose -f docker-compose-pmm-psmdb.yml exec -T psmdb-server mgodatagen -f /etc/datagen/replicaset.json --username=pmm_mongodb --password="5M](Q%q/U+YQ<^m" --host psmdb-server --port 27017 --tlsCertificateKeyFile=/mongodb_certs/client.pem --tlsCAFile=/mongodb_certs/ca-certs.pem -# -#tests=${TESTS:-yes} -#if [ $tests = "yes" ]; then -# echo "running tests" -# output=$(docker compose -f docker-compose-pmm-psmdb.yml run test pytest -s --verbose test.py) -# else -# echo "skipping tests" -#fi -# -#cleanup=${CLEANUP:-yes} -#if [ $cleanup = "yes" ]; then -# echo "cleanup" -# docker compose -f docker-compose-pmm-psmdb.yml down -v --remove-orphans -# if [[ -n "$PSMDB_VERSION" ]] && [[ "$PSMDB_VERSION" == *"4.2"* ]]; then -# sed -i 's/MONGODB-X509/MONGODB-X509,MONGODB-AWS/' conf/mongod.conf -# fi -# else -# echo "skipping cleanup" -#fi -# -#echo "$output" -#if echo "$output" | grep -q "\bFAILED\b"; then -# exit 1 -#fi +#Add users +docker compose -f docker-compose-pmm-psmdb.yml exec -T psmdb-server mongo --quiet << EOF +db.getSiblingDB("admin").createUser({ user: "root", pwd: "root", roles: [ "root", "userAdminAnyDatabase", "clusterAdmin" ] }); +EOF +docker compose -f docker-compose-pmm-psmdb.yml exec -T psmdb-server mongo --quiet "mongodb://root:root@localhost/?replicaSet=rs0" < init/setup_psmdb.js + +#Configure PBM +docker compose -f docker-compose-pmm-psmdb.yml exec -T psmdb-server bash -c "echo \"PBM_MONGODB_URI=mongodb://pbm:pbmpass@127.0.0.1:27017\" > /etc/sysconfig/pbm-agent" +docker compose -f docker-compose-pmm-psmdb.yml exec -T psmdb-server systemctl restart pbm-agent + +#Configure PMM +set +e +i=1 +while [ $i -le 3 ]; do + output=$(docker compose -f docker-compose-pmm-psmdb.yml exec -T psmdb-server pmm-agent setup 2>&1) + exit_code=$? + + if [ $exit_code -ne 0 ] && [[ $output == *"500 Internal Server Error"* ]]; then + i=$((i + 1)) + else + break + fi + sleep 1 +done + +#Add Mongo Service +random_number=$RANDOM +docker compose -f docker-compose-pmm-psmdb.yml exec -T psmdb-server pmm-admin add mongodb psmdb-server_${random_number} --agent-password=mypass --username=pmm_mongodb --password="5M](Q%q/U+YQ<^m" --host psmdb-server --port 27017 --tls --tls-certificate-key-file=/mongodb_certs/client.pem --tls-ca-file=/mongodb_certs/ca-certs.pem --cluster=mycluster +#Add some data +docker compose -f docker-compose-pmm-psmdb.yml exec -T psmdb-server mgodatagen -f /etc/datagen/replicaset.json --username=pmm_mongodb --password="5M](Q%q/U+YQ<^m" --host psmdb-server --port 27017 --tlsCertificateKeyFile=/mongodb_certs/client.pem --tlsCAFile=/mongodb_certs/ca-certs.pem + +tests=${TESTS:-yes} +if [ $tests = "yes" ]; then + echo "running tests" + output=$(docker compose -f docker-compose-pmm-psmdb.yml run test pytest -s --verbose test.py) + else + echo "skipping tests" +fi + +cleanup=${CLEANUP:-yes} +if [ $cleanup = "yes" ]; then + echo "cleanup" + docker compose -f docker-compose-pmm-psmdb.yml down -v --remove-orphans + if [[ -n "$PSMDB_VERSION" ]] && [[ "$PSMDB_VERSION" == *"4.2"* ]]; then + sed -i 's/MONGODB-X509/MONGODB-X509,MONGODB-AWS/' conf/mongod.conf + fi + else + echo "skipping cleanup" +fi + +echo "$output" +if echo "$output" | grep -q "\bFAILED\b"; then + exit 1 +fi diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index de993452..455c4a72 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -613,7 +613,8 @@ def mongo_ssl_setup(script_filename, args): if 'services' in data and 'pmm-server' in data['services']: del data['services']['pmm-server'] - content = data.replace('pmm-agent setup 2', 'pmm-agent setup --server-insecure-tls 2') + if 'pmm-agent setup 2' in data: + data = data.replace('pmm-agent setup 2', 'pmm-agent setup --server-insecure-tls 2') for service in data.get('services', {}).values(): networks = service.get('networks', []) From 2005892e200b0f8d89547219502099250811990e Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 23 Jun 2025 09:58:41 +0200 Subject: [PATCH 134/156] PMM-7: log --- pmm_qa/pmm-framework.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index 455c4a72..997a0c4a 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -663,6 +663,10 @@ def mongo_ssl_setup(script_filename, args): except yaml.YAMLError as e: print(f"Error occurred: {e}") + try: + subprocess.run(['sed', '-i', f's/docker-compose-pmm-psmdb.yml/{compose_filename}/g', f'{shellscript_file_path}']) + except subprocess.CalledProcessError as e: + print(f"Error occurred: {e}") def setup_ssl_psmdb(db_type, db_version=None, db_config=None, args=None): # Check if PMM server is running From 4170f213b2fed95715909ed31a41f94d35ad1f95 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 23 Jun 2025 10:06:02 +0200 Subject: [PATCH 135/156] PMM-7: log --- pmm_psmdb_diffauth_setup/test-auth.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/pmm_psmdb_diffauth_setup/test-auth.sh b/pmm_psmdb_diffauth_setup/test-auth.sh index ade6c3fa..0bc3254e 100755 --- a/pmm_psmdb_diffauth_setup/test-auth.sh +++ b/pmm_psmdb_diffauth_setup/test-auth.sh @@ -32,6 +32,7 @@ openssl dhparam -out certs/dhparam.pem 2048 cp pki/ca.crt certs/ca-certs.pem cp pki/private/pmm-server.key certs/certificate.key cp pki/issued/pmm-server.crt certs/certificate.crt +cat pki/private/psmdb-server.key pki/issued/pmm-server.crt > certs/pmm-server.pem cat pki/private/psmdb-server.key pki/issued/psmdb-server.crt > certs/psmdb-server.pem cat pki/private/pmm-test.key pki/issued/pmm-test.crt > certs/client.pem find certs -type f -exec chmod 644 {} \; From f7afa2cd14a8d3bc92aba5cae12cde609c7d4547 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 23 Jun 2025 10:12:51 +0200 Subject: [PATCH 136/156] PMM-7: log --- pmm_psmdb_diffauth_setup/test-auth.sh | 82 +++++++++++++-------------- 1 file changed, 41 insertions(+), 41 deletions(-) diff --git a/pmm_psmdb_diffauth_setup/test-auth.sh b/pmm_psmdb_diffauth_setup/test-auth.sh index 0bc3254e..eded9cd3 100755 --- a/pmm_psmdb_diffauth_setup/test-auth.sh +++ b/pmm_psmdb_diffauth_setup/test-auth.sh @@ -55,44 +55,44 @@ docker compose -f docker-compose-pmm-psmdb.yml exec -T psmdb-server systemctl re #Configure PMM set +e i=1 -while [ $i -le 3 ]; do - output=$(docker compose -f docker-compose-pmm-psmdb.yml exec -T psmdb-server pmm-agent setup 2>&1) - exit_code=$? - - if [ $exit_code -ne 0 ] && [[ $output == *"500 Internal Server Error"* ]]; then - i=$((i + 1)) - else - break - fi - sleep 1 -done - -#Add Mongo Service -random_number=$RANDOM -docker compose -f docker-compose-pmm-psmdb.yml exec -T psmdb-server pmm-admin add mongodb psmdb-server_${random_number} --agent-password=mypass --username=pmm_mongodb --password="5M](Q%q/U+YQ<^m" --host psmdb-server --port 27017 --tls --tls-certificate-key-file=/mongodb_certs/client.pem --tls-ca-file=/mongodb_certs/ca-certs.pem --cluster=mycluster -#Add some data -docker compose -f docker-compose-pmm-psmdb.yml exec -T psmdb-server mgodatagen -f /etc/datagen/replicaset.json --username=pmm_mongodb --password="5M](Q%q/U+YQ<^m" --host psmdb-server --port 27017 --tlsCertificateKeyFile=/mongodb_certs/client.pem --tlsCAFile=/mongodb_certs/ca-certs.pem - -tests=${TESTS:-yes} -if [ $tests = "yes" ]; then - echo "running tests" - output=$(docker compose -f docker-compose-pmm-psmdb.yml run test pytest -s --verbose test.py) - else - echo "skipping tests" -fi - -cleanup=${CLEANUP:-yes} -if [ $cleanup = "yes" ]; then - echo "cleanup" - docker compose -f docker-compose-pmm-psmdb.yml down -v --remove-orphans - if [[ -n "$PSMDB_VERSION" ]] && [[ "$PSMDB_VERSION" == *"4.2"* ]]; then - sed -i 's/MONGODB-X509/MONGODB-X509,MONGODB-AWS/' conf/mongod.conf - fi - else - echo "skipping cleanup" -fi - -echo "$output" -if echo "$output" | grep -q "\bFAILED\b"; then - exit 1 -fi +#while [ $i -le 3 ]; do +# output=$(docker compose -f docker-compose-pmm-psmdb.yml exec -T psmdb-server pmm-agent setup 2>&1) +# exit_code=$? +# +# if [ $exit_code -ne 0 ] && [[ $output == *"500 Internal Server Error"* ]]; then +# i=$((i + 1)) +# else +# break +# fi +# sleep 1 +#done +# +Add Mongo Service +#random_number=$RANDOM +#docker compose -f docker-compose-pmm-psmdb.yml exec -T psmdb-server pmm-admin add mongodb psmdb-server_${random_number} --agent-password=mypass --username=pmm_mongodb --password="5M](Q%q/U+YQ<^m" --host psmdb-server --port 27017 --tls --tls-certificate-key-file=/mongodb_certs/client.pem --tls-ca-file=/mongodb_certs/ca-certs.pem --cluster=mycluster +Add some data +#docker compose -f docker-compose-pmm-psmdb.yml exec -T psmdb-server mgodatagen -f /etc/datagen/replicaset.json --username=pmm_mongodb --password="5M](Q%q/U+YQ<^m" --host psmdb-server --port 27017 --tlsCertificateKeyFile=/mongodb_certs/client.pem --tlsCAFile=/mongodb_certs/ca-certs.pem +# +#tests=${TESTS:-yes} +#if [ $tests = "yes" ]; then +# echo "running tests" +# output=$(docker compose -f docker-compose-pmm-psmdb.yml run test pytest -s --verbose test.py) +# else +# echo "skipping tests" +#fi +# +#cleanup=${CLEANUP:-yes} +#if [ $cleanup = "yes" ]; then +# echo "cleanup" +# docker compose -f docker-compose-pmm-psmdb.yml down -v --remove-orphans +# if [[ -n "$PSMDB_VERSION" ]] && [[ "$PSMDB_VERSION" == *"4.2"* ]]; then +# sed -i 's/MONGODB-X509/MONGODB-X509,MONGODB-AWS/' conf/mongod.conf +# fi +# else +# echo "skipping cleanup" +#fi +# +#echo "$output" +#if echo "$output" | grep -q "\bFAILED\b"; then +# exit 1 +#fi From 830a56e7b7c03aed1ae2df7c2acbe2ac497de30e Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 23 Jun 2025 10:26:58 +0200 Subject: [PATCH 137/156] PMM-7: log --- pmm_psmdb_diffauth_setup/test-auth.sh | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/pmm_psmdb_diffauth_setup/test-auth.sh b/pmm_psmdb_diffauth_setup/test-auth.sh index eded9cd3..e3b92f2a 100755 --- a/pmm_psmdb_diffauth_setup/test-auth.sh +++ b/pmm_psmdb_diffauth_setup/test-auth.sh @@ -55,18 +55,18 @@ docker compose -f docker-compose-pmm-psmdb.yml exec -T psmdb-server systemctl re #Configure PMM set +e i=1 -#while [ $i -le 3 ]; do -# output=$(docker compose -f docker-compose-pmm-psmdb.yml exec -T psmdb-server pmm-agent setup 2>&1) -# exit_code=$? -# -# if [ $exit_code -ne 0 ] && [[ $output == *"500 Internal Server Error"* ]]; then -# i=$((i + 1)) -# else -# break -# fi -# sleep 1 -#done -# +while [ $i -le 3 ]; do + output=$(docker compose -f docker-compose-pmm-psmdb.yml exec -T psmdb-server pmm-agent setup 2>&1) + exit_code=$? + + if [ $exit_code -ne 0 ] && [[ $output == *"500 Internal Server Error"* ]]; then + i=$((i + 1)) + else + break + fi + sleep 1 +done + Add Mongo Service #random_number=$RANDOM #docker compose -f docker-compose-pmm-psmdb.yml exec -T psmdb-server pmm-admin add mongodb psmdb-server_${random_number} --agent-password=mypass --username=pmm_mongodb --password="5M](Q%q/U+YQ<^m" --host psmdb-server --port 27017 --tls --tls-certificate-key-file=/mongodb_certs/client.pem --tls-ca-file=/mongodb_certs/ca-certs.pem --cluster=mycluster From 5c6fbada978e259b61ab8e48792512e54cc597f4 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 23 Jun 2025 10:39:40 +0200 Subject: [PATCH 138/156] PMM-7: log --- pmm_psmdb_diffauth_setup/test-auth.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pmm_psmdb_diffauth_setup/test-auth.sh b/pmm_psmdb_diffauth_setup/test-auth.sh index e3b92f2a..11bdeedf 100755 --- a/pmm_psmdb_diffauth_setup/test-auth.sh +++ b/pmm_psmdb_diffauth_setup/test-auth.sh @@ -67,10 +67,10 @@ while [ $i -le 3 ]; do sleep 1 done -Add Mongo Service -#random_number=$RANDOM -#docker compose -f docker-compose-pmm-psmdb.yml exec -T psmdb-server pmm-admin add mongodb psmdb-server_${random_number} --agent-password=mypass --username=pmm_mongodb --password="5M](Q%q/U+YQ<^m" --host psmdb-server --port 27017 --tls --tls-certificate-key-file=/mongodb_certs/client.pem --tls-ca-file=/mongodb_certs/ca-certs.pem --cluster=mycluster -Add some data +#Add Mongo Service +random_number=$RANDOM +docker compose -f docker-compose-pmm-psmdb.yml exec -T psmdb-server pmm-admin add mongodb psmdb-server_${random_number} --agent-password=mypass --username=pmm_mongodb --password="5M](Q%q/U+YQ<^m" --host psmdb-server --port 27017 --tls --tls-certificate-key-file=/mongodb_certs/client.pem --tls-ca-file=/mongodb_certs/ca-certs.pem --cluster=mycluster +#Add some data #docker compose -f docker-compose-pmm-psmdb.yml exec -T psmdb-server mgodatagen -f /etc/datagen/replicaset.json --username=pmm_mongodb --password="5M](Q%q/U+YQ<^m" --host psmdb-server --port 27017 --tlsCertificateKeyFile=/mongodb_certs/client.pem --tlsCAFile=/mongodb_certs/ca-certs.pem # #tests=${TESTS:-yes} From 9842d1a0d1d8d29876b74c5fc512b61a2773a314 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 23 Jun 2025 10:43:23 +0200 Subject: [PATCH 139/156] PMM-7: log --- pmm_psmdb_diffauth_setup/test-auth.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pmm_psmdb_diffauth_setup/test-auth.sh b/pmm_psmdb_diffauth_setup/test-auth.sh index 11bdeedf..f106a3b9 100755 --- a/pmm_psmdb_diffauth_setup/test-auth.sh +++ b/pmm_psmdb_diffauth_setup/test-auth.sh @@ -67,6 +67,8 @@ while [ $i -le 3 ]; do sleep 1 done +nohup pmm-agent --config-file=/usr/local/percona/pmm/config/pmm-agent.yaml > /var/log/pmm-agent.log 2>&1 & + #Add Mongo Service random_number=$RANDOM docker compose -f docker-compose-pmm-psmdb.yml exec -T psmdb-server pmm-admin add mongodb psmdb-server_${random_number} --agent-password=mypass --username=pmm_mongodb --password="5M](Q%q/U+YQ<^m" --host psmdb-server --port 27017 --tls --tls-certificate-key-file=/mongodb_certs/client.pem --tls-ca-file=/mongodb_certs/ca-certs.pem --cluster=mycluster From a6a6625c57aab92030b5e442cc020b79f27f5691 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 23 Jun 2025 10:49:40 +0200 Subject: [PATCH 140/156] PMM-7: log --- pmm_psmdb_diffauth_setup/test-auth.sh | 50 +++++++++++++-------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/pmm_psmdb_diffauth_setup/test-auth.sh b/pmm_psmdb_diffauth_setup/test-auth.sh index f106a3b9..afc106a9 100755 --- a/pmm_psmdb_diffauth_setup/test-auth.sh +++ b/pmm_psmdb_diffauth_setup/test-auth.sh @@ -73,28 +73,28 @@ nohup pmm-agent --config-file=/usr/local/percona/pmm/config/pmm-agent.yaml > /va random_number=$RANDOM docker compose -f docker-compose-pmm-psmdb.yml exec -T psmdb-server pmm-admin add mongodb psmdb-server_${random_number} --agent-password=mypass --username=pmm_mongodb --password="5M](Q%q/U+YQ<^m" --host psmdb-server --port 27017 --tls --tls-certificate-key-file=/mongodb_certs/client.pem --tls-ca-file=/mongodb_certs/ca-certs.pem --cluster=mycluster #Add some data -#docker compose -f docker-compose-pmm-psmdb.yml exec -T psmdb-server mgodatagen -f /etc/datagen/replicaset.json --username=pmm_mongodb --password="5M](Q%q/U+YQ<^m" --host psmdb-server --port 27017 --tlsCertificateKeyFile=/mongodb_certs/client.pem --tlsCAFile=/mongodb_certs/ca-certs.pem -# -#tests=${TESTS:-yes} -#if [ $tests = "yes" ]; then -# echo "running tests" -# output=$(docker compose -f docker-compose-pmm-psmdb.yml run test pytest -s --verbose test.py) -# else -# echo "skipping tests" -#fi -# -#cleanup=${CLEANUP:-yes} -#if [ $cleanup = "yes" ]; then -# echo "cleanup" -# docker compose -f docker-compose-pmm-psmdb.yml down -v --remove-orphans -# if [[ -n "$PSMDB_VERSION" ]] && [[ "$PSMDB_VERSION" == *"4.2"* ]]; then -# sed -i 's/MONGODB-X509/MONGODB-X509,MONGODB-AWS/' conf/mongod.conf -# fi -# else -# echo "skipping cleanup" -#fi -# -#echo "$output" -#if echo "$output" | grep -q "\bFAILED\b"; then -# exit 1 -#fi +docker compose -f docker-compose-pmm-psmdb.yml exec -T psmdb-server mgodatagen -f /etc/datagen/replicaset.json --username=pmm_mongodb --password="5M](Q%q/U+YQ<^m" --host psmdb-server --port 27017 --tlsCertificateKeyFile=/mongodb_certs/client.pem --tlsCAFile=/mongodb_certs/ca-certs.pem + +tests=${TESTS:-yes} +if [ $tests = "yes" ]; then + echo "running tests" + output=$(docker compose -f docker-compose-pmm-psmdb.yml run test pytest -s --verbose test.py) + else + echo "skipping tests" +fi + +cleanup=${CLEANUP:-yes} +if [ $cleanup = "yes" ]; then + echo "cleanup" + docker compose -f docker-compose-pmm-psmdb.yml down -v --remove-orphans + if [[ -n "$PSMDB_VERSION" ]] && [[ "$PSMDB_VERSION" == *"4.2"* ]]; then + sed -i 's/MONGODB-X509/MONGODB-X509,MONGODB-AWS/' conf/mongod.conf + fi + else + echo "skipping cleanup" +fi + +echo "$output" +if echo "$output" | grep -q "\bFAILED\b"; then + exit 1 +fi From 59dfad2aaae1b7c26220ae32ef2b4c3ab20a0fb5 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 23 Jun 2025 11:07:00 +0200 Subject: [PATCH 141/156] PMM-7: log --- pmm_psmdb_diffauth_setup/test-auth.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_psmdb_diffauth_setup/test-auth.sh b/pmm_psmdb_diffauth_setup/test-auth.sh index afc106a9..90c0daf6 100755 --- a/pmm_psmdb_diffauth_setup/test-auth.sh +++ b/pmm_psmdb_diffauth_setup/test-auth.sh @@ -67,7 +67,7 @@ while [ $i -le 3 ]; do sleep 1 done -nohup pmm-agent --config-file=/usr/local/percona/pmm/config/pmm-agent.yaml > /var/log/pmm-agent.log 2>&1 & +docker compose -f docker-compose-pmm-psmdb.yml exec -T pmm-agent --config-file=/usr/local/percona/pmm/config/pmm-agent.yaml #Add Mongo Service random_number=$RANDOM From 2d4d154ed8a65a273414e50fbe5ea0d543109ee8 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 23 Jun 2025 11:07:18 +0200 Subject: [PATCH 142/156] PMM-7: log --- pmm_psmdb_diffauth_setup/test-auth.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_psmdb_diffauth_setup/test-auth.sh b/pmm_psmdb_diffauth_setup/test-auth.sh index 90c0daf6..e7129b2c 100755 --- a/pmm_psmdb_diffauth_setup/test-auth.sh +++ b/pmm_psmdb_diffauth_setup/test-auth.sh @@ -67,7 +67,7 @@ while [ $i -le 3 ]; do sleep 1 done -docker compose -f docker-compose-pmm-psmdb.yml exec -T pmm-agent --config-file=/usr/local/percona/pmm/config/pmm-agent.yaml +docker compose -f docker-compose-pmm-psmdb.yml exec -T psmdb-server bash -c "pmm-agent --config-file=/usr/local/percona/pmm/config/pmm-agent.yaml" #Add Mongo Service random_number=$RANDOM From ad607afe70db8011d6b9e7098d64662112386543 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 23 Jun 2025 11:11:21 +0200 Subject: [PATCH 143/156] PMM-7: log --- pmm_psmdb_diffauth_setup/test-auth.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pmm_psmdb_diffauth_setup/test-auth.sh b/pmm_psmdb_diffauth_setup/test-auth.sh index e7129b2c..1af85bfc 100755 --- a/pmm_psmdb_diffauth_setup/test-auth.sh +++ b/pmm_psmdb_diffauth_setup/test-auth.sh @@ -58,6 +58,7 @@ i=1 while [ $i -le 3 ]; do output=$(docker compose -f docker-compose-pmm-psmdb.yml exec -T psmdb-server pmm-agent setup 2>&1) exit_code=$? + echo "Output is: $output" if [ $exit_code -ne 0 ] && [[ $output == *"500 Internal Server Error"* ]]; then i=$((i + 1)) @@ -67,8 +68,6 @@ while [ $i -le 3 ]; do sleep 1 done -docker compose -f docker-compose-pmm-psmdb.yml exec -T psmdb-server bash -c "pmm-agent --config-file=/usr/local/percona/pmm/config/pmm-agent.yaml" - #Add Mongo Service random_number=$RANDOM docker compose -f docker-compose-pmm-psmdb.yml exec -T psmdb-server pmm-admin add mongodb psmdb-server_${random_number} --agent-password=mypass --username=pmm_mongodb --password="5M](Q%q/U+YQ<^m" --host psmdb-server --port 27017 --tls --tls-certificate-key-file=/mongodb_certs/client.pem --tls-ca-file=/mongodb_certs/ca-certs.pem --cluster=mycluster From 51757c05a7ceaa900b783b67668b0349c99ef4dd Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 23 Jun 2025 11:34:44 +0200 Subject: [PATCH 144/156] PMM-7: log --- pmm_psmdb_diffauth_setup/test-auth.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_psmdb_diffauth_setup/test-auth.sh b/pmm_psmdb_diffauth_setup/test-auth.sh index 1af85bfc..224587d6 100755 --- a/pmm_psmdb_diffauth_setup/test-auth.sh +++ b/pmm_psmdb_diffauth_setup/test-auth.sh @@ -56,7 +56,7 @@ docker compose -f docker-compose-pmm-psmdb.yml exec -T psmdb-server systemctl re set +e i=1 while [ $i -le 3 ]; do - output=$(docker compose -f docker-compose-pmm-psmdb.yml exec -T psmdb-server pmm-agent setup 2>&1) + output=$(docker compose -f docker-compose-pmm-psmdb.yml exec -T psmdb-server pmm-agent setup --config-file=/usr/local/percona/pmm/config/pmm-agent.yaml --server-address=pmm-server:8443 --metrics-mode=auto --server-username=admin --server-password=admin --server-insecure-tls) exit_code=$? echo "Output is: $output" From ecab2efda8d92bb397ecb7ce9cd23eb4aedfa771 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 23 Jun 2025 11:38:03 +0200 Subject: [PATCH 145/156] PMM-7: cleanup --- pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml b/pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml index 293b927c..3395731f 100644 --- a/pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml +++ b/pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml @@ -104,8 +104,8 @@ services: volumes: - /tmp/minio/backups:/backups environment: - - "MINIO_ROOT_USER=minio1234" - - "MINIO_ROOT_PASSWORD=minio1234" + - "MINIO_ACCESS_KEY=minio1234" + - "MINIO_SECRET_KEY=minio1234" command: server /backups depends_on: psmdb-server: From bcb25532512a528918e9de184d04da2f1a59ac71 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 23 Jun 2025 11:38:51 +0200 Subject: [PATCH 146/156] PMM-7: cleanup --- pmm_psmdb-pbm_setup/docker-compose-rs.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pmm_psmdb-pbm_setup/docker-compose-rs.yaml b/pmm_psmdb-pbm_setup/docker-compose-rs.yaml index e5e1ee52..9a328eaf 100644 --- a/pmm_psmdb-pbm_setup/docker-compose-rs.yaml +++ b/pmm_psmdb-pbm_setup/docker-compose-rs.yaml @@ -198,8 +198,8 @@ services: volumes: - /tmp/minio/backups:/backups environment: - - "MINIO_ROOT_USER=minio1234" - - "MINIO_ROOT_PASSWORD=minio1234" + - "MINIO_ACCESS_KEY=minio1234" + - "MINIO_SECRET_KEY=minio1234" command: server /backups createbucket: @@ -215,7 +215,7 @@ services: depends_on: - minio entrypoint: > - /bin/sh -c " sleep 60; /usr/bin/mc config host add myminio http://minio:9000 minio1234 minio1234; /usr/bin/mc mb myminio/bcp; exit 0; " + /bin/sh -c " sleep 5; /usr/bin/mc alias set myminio http://minio:9000 minio1234 minio1234; /usr/bin/mc mb myminio/bcp; exit 0; " networks: qa-integration: From cc750c15e84f68515d5fb87603d0506629c0bb00 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 23 Jun 2025 11:39:23 +0200 Subject: [PATCH 147/156] PMM-7: cleanup --- pmm_psmdb-pbm_setup/docker-compose-sharded.yaml | 4 ++-- pmm_psmdb-pbm_setup/start-rs-only.sh | 2 -- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/pmm_psmdb-pbm_setup/docker-compose-sharded.yaml b/pmm_psmdb-pbm_setup/docker-compose-sharded.yaml index 7598b94e..4f4598a5 100644 --- a/pmm_psmdb-pbm_setup/docker-compose-sharded.yaml +++ b/pmm_psmdb-pbm_setup/docker-compose-sharded.yaml @@ -235,8 +235,8 @@ services: volumes: - backups:/backups environment: - - MINIO_ROOT_USER=minio1234 - - MINIO_ROOT_PASSWORD=minio1234 + - "MINIO_ACCESS_KEY=minio1234" + - "MINIO_SECRET_KEY=minio1234" command: server /backups createbucket: diff --git a/pmm_psmdb-pbm_setup/start-rs-only.sh b/pmm_psmdb-pbm_setup/start-rs-only.sh index 84b3ba8c..d7f95dfc 100755 --- a/pmm_psmdb-pbm_setup/start-rs-only.sh +++ b/pmm_psmdb-pbm_setup/start-rs-only.sh @@ -35,5 +35,3 @@ if [ $profile = "extra" ]; then fi bash -x ./configure-extra-agents.sh fi - -docker exec minio /bin/sh -c " sleep 5; /usr/bin/mc config host add myminio http://minio:9000 minio1234 minio1234; /usr/bin/mc mb myminio/bcp; exit 0; " \ No newline at end of file From 5393264b2f1f76ac5c8c1c2fbf54f099428468ba Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 23 Jun 2025 11:39:45 +0200 Subject: [PATCH 148/156] PMM-7: cleanup --- pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml b/pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml index 3395731f..0931fd26 100644 --- a/pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml +++ b/pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml @@ -6,11 +6,11 @@ services: dockerfile: ../pmm_psmdb-pbm_setup/Dockerfile context: ../pmm_psmdb-pbm_setup/ args: - REPO: ${REPO:-testing} - PMM_REPO: ${PMM_REPO:-experimental} - PBM_VERSION: ${PBM_VERSION:-latest} - PSMDB_VERSION: ${PSMDB_VERSION:-latest} - PMM_CLIENT_VERSION: ${PMM_CLIENT_VERSION:-3-dev-latest} + - REPO=${REPO:-testing} + - PMM_REPO=${PMM_REPO:-experimental} + - PBM_VERSION=${PBM_VERSION:-latest} + - PSMDB_VERSION=${PSMDB_VERSION:-latest} + - PMM_CLIENT_VERSION=${PMM_CLIENT_VERSION:-3-dev-latest} command: /bin/bash test: From 3665a472c70e64dbeee874286e54ad4d5507530c Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 23 Jun 2025 11:42:32 +0200 Subject: [PATCH 149/156] PMM-7: cleanup --- .../docker-compose-pmm-psmdb.yml | 17 +++++++++++++++++ pmm_qa/pmm-framework.py | 3 +++ 2 files changed, 20 insertions(+) diff --git a/pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml b/pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml index 0931fd26..883c0f15 100644 --- a/pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml +++ b/pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml @@ -96,6 +96,23 @@ services: volumes: - 'openldap-data:/bitnami/openldap' + kerberos: + image: kerberos/local + build: + dockerfile: ./Dockerfile-kerberos + context: . + container_name: kerberos + hostname: kerberos + environment: + - "KRB5_TRACE=/dev/stderr" + volumes: + - keytabs:/keytabs + healthcheck: + test: ["CMD", "kadmin.local", "-q", "listprincs"] + interval: 2s + timeout: 1s + retries: 5 + minio: image: minio/minio container_name: minio diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index 997a0c4a..50088c9d 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -613,6 +613,9 @@ def mongo_ssl_setup(script_filename, args): if 'services' in data and 'pmm-server' in data['services']: del data['services']['pmm-server'] + if 'services' in data and 'kerberos' in data['services']: + del data['services']['kerberos'] + if 'pmm-agent setup 2' in data: data = data.replace('pmm-agent setup 2', 'pmm-agent setup --server-insecure-tls 2') From 215b314b0a675a30c357d534c5d28d330b91dd17 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 23 Jun 2025 11:44:29 +0200 Subject: [PATCH 150/156] PMM-7: cleanup --- pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml | 2 ++ pmm_qa/pmm-framework.py | 6 ++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml b/pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml index 883c0f15..4d676053 100644 --- a/pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml +++ b/pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml @@ -62,6 +62,8 @@ services: depends_on: pmm-server: condition: service_healthy + kerberos: + condition: service_healthy pmm-server: container_name: pmm-server diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index 50088c9d..cc8bf5dc 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -656,9 +656,11 @@ def mongo_ssl_setup(script_filename, args): psmdb_service['environment'] = env depends_on = psmdb_service.get('depends_on') - print(depends_on) + print(f'Service depends on: {depends_on}') if depends_on == {'pmm-server': {'condition': 'service_healthy'}}: - del psmdb_service['depends_on'] + del psmdb_service['depends_on']['pmm-server'] + if depends_on == {'kerberos': {'condition': 'service_healthy'}}: + del psmdb_service['depends_on']['kerberos'] # Save it back with open(compose_file_path, 'w') as f: From 46555803a58f9ed0b1a1e5aa9594af6f3cfd5b99 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 23 Jun 2025 11:46:03 +0200 Subject: [PATCH 151/156] PMM-7: cleanup --- pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml b/pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml index 4d676053..3290fa83 100644 --- a/pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml +++ b/pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml @@ -42,12 +42,14 @@ services: - -c - | cp /mongodb_certs/ca-certs.pem /etc/pki/tls/certs/ + chown -R mongod:mongod /keytabs exec /usr/sbin/init volumes: - ./conf:/etc/mongod - ./certs:/mongodb_certs - /sys/fs/cgroup:/sys/fs/cgroup:rw - 'psmdb-server-data:/data/db' + - keytabs:/keytabs - ../pmm_psmdb-pbm_setup/conf/datagen:/etc/datagen:ro privileged: true cgroup: host @@ -147,3 +149,5 @@ volumes: driver: local openldap-data: driver: local + keytabs: + driver: local \ No newline at end of file From a51157053296cc00c0e8ba7e2a911e562f1fc8d5 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 23 Jun 2025 11:48:07 +0200 Subject: [PATCH 152/156] PMM-7: cleanup --- pmm_qa/pmm-framework.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index cc8bf5dc..d36d87f4 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -657,10 +657,8 @@ def mongo_ssl_setup(script_filename, args): depends_on = psmdb_service.get('depends_on') print(f'Service depends on: {depends_on}') - if depends_on == {'pmm-server': {'condition': 'service_healthy'}}: - del psmdb_service['depends_on']['pmm-server'] - if depends_on == {'kerberos': {'condition': 'service_healthy'}}: - del psmdb_service['depends_on']['kerberos'] + if depends_on == {'pmm-server': {'condition': 'service_healthy'}} or depends_on == {'kerberos': {'condition': 'service_healthy'}}: + del psmdb_service['depends_on'] # Save it back with open(compose_file_path, 'w') as f: From 71084f37aa781f36c015ae1ad8a14cb3ee52a9ca Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 23 Jun 2025 11:49:46 +0200 Subject: [PATCH 153/156] PMM-7: cleanup --- pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml | 2 +- pmm_psmdb_diffauth_setup/test-auth.sh | 2 -- pmm_qa/postgresql/tasks/run_load_pgsql.yml | 2 -- 3 files changed, 1 insertion(+), 5 deletions(-) diff --git a/pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml b/pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml index 3290fa83..fe955262 100644 --- a/pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml +++ b/pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml @@ -150,4 +150,4 @@ volumes: openldap-data: driver: local keytabs: - driver: local \ No newline at end of file + driver: local diff --git a/pmm_psmdb_diffauth_setup/test-auth.sh b/pmm_psmdb_diffauth_setup/test-auth.sh index 224587d6..7cca03a4 100755 --- a/pmm_psmdb_diffauth_setup/test-auth.sh +++ b/pmm_psmdb_diffauth_setup/test-auth.sh @@ -32,7 +32,6 @@ openssl dhparam -out certs/dhparam.pem 2048 cp pki/ca.crt certs/ca-certs.pem cp pki/private/pmm-server.key certs/certificate.key cp pki/issued/pmm-server.crt certs/certificate.crt -cat pki/private/psmdb-server.key pki/issued/pmm-server.crt > certs/pmm-server.pem cat pki/private/psmdb-server.key pki/issued/psmdb-server.crt > certs/psmdb-server.pem cat pki/private/pmm-test.key pki/issued/pmm-test.crt > certs/client.pem find certs -type f -exec chmod 644 {} \; @@ -58,7 +57,6 @@ i=1 while [ $i -le 3 ]; do output=$(docker compose -f docker-compose-pmm-psmdb.yml exec -T psmdb-server pmm-agent setup --config-file=/usr/local/percona/pmm/config/pmm-agent.yaml --server-address=pmm-server:8443 --metrics-mode=auto --server-username=admin --server-password=admin --server-insecure-tls) exit_code=$? - echo "Output is: $output" if [ $exit_code -ne 0 ] && [[ $output == *"500 Internal Server Error"* ]]; then i=$((i + 1)) diff --git a/pmm_qa/postgresql/tasks/run_load_pgsql.yml b/pmm_qa/postgresql/tasks/run_load_pgsql.yml index 172ad66c..7e2cc3b0 100644 --- a/pmm_qa/postgresql/tasks/run_load_pgsql.yml +++ b/pmm_qa/postgresql/tasks/run_load_pgsql.yml @@ -55,5 +55,3 @@ - name: Print pgbench results debug: var: pgbench_result.stdout_lines - - From 2aac8a0eee6ab40be9403ac64b4850a995709e08 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 23 Jun 2025 11:51:00 +0200 Subject: [PATCH 154/156] PMM-7: cleanup --- pmm_qa/pmm-framework.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index d36d87f4..498d80b7 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -657,7 +657,7 @@ def mongo_ssl_setup(script_filename, args): depends_on = psmdb_service.get('depends_on') print(f'Service depends on: {depends_on}') - if depends_on == {'pmm-server': {'condition': 'service_healthy'}} or depends_on == {'kerberos': {'condition': 'service_healthy'}}: + if 'pmm-server' in depends_on or 'kerberos' in depends_on: del psmdb_service['depends_on'] # Save it back From 98617fc51b26e42fe4d2dc7e6096e6160a1c2b64 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 23 Jun 2025 11:52:15 +0200 Subject: [PATCH 155/156] PMM-7: cleanup --- pmm_qa/pg_stat_monitor_setup.sh | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/pmm_qa/pg_stat_monitor_setup.sh b/pmm_qa/pg_stat_monitor_setup.sh index 953cbb2a..cc6fb54e 100644 --- a/pmm_qa/pg_stat_monitor_setup.sh +++ b/pmm_qa/pg_stat_monitor_setup.sh @@ -19,7 +19,7 @@ fi # If branch/tag is not provided then it will default to main branch if [ -z "$pgstat_monitor_branch" ] then - export pgstat_monitor_branch="main" + export pgstat_monitor_branch="2.1.0" fi # If repo is not provided then it will default to percona PGSM repository @@ -54,13 +54,13 @@ then wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' apt update - apt -y install postgresql-${pgsql_version} postgresql-server-dev-${pgsql_version} percona-pg-stat-monitor${pgsql_version} + apt -y install postgresql-${pgsql_version} postgresql-server-dev-${pgsql_version} else wget https://repo.percona.com/apt/percona-release_latest.generic_all.deb dpkg -i percona-release_latest.generic_all.deb percona-release setup ppg-${pgsql_version} apt-get -y update - apt-get -y install percona-postgresql-${pgsql_version} percona-postgresql-contrib percona-postgresql-server-dev-all percona-pg-stat-monitor${pgsql_version} + apt-get -y install percona-postgresql-${pgsql_version} percona-postgresql-contrib percona-postgresql-server-dev-all fi sleep 10 @@ -84,16 +84,16 @@ echo $PATH cp /usr/lib/postgresql/${pgsql_version}/bin/pg_config /usr/bin # Clone PGSM repo and move to /home/postgres/pg_stat_monitor dir -#cd /home/postgres -#git clone --depth 1 --branch ${pgstat_monitor_branch} https://github.com/${pgstat_monitor_repo} -#chown -R postgres:postgres pg_stat_monitor -#cd pg_stat_monitor +cd /home/postgres +git clone --depth 1 --branch ${pgstat_monitor_branch} https://github.com/${pgstat_monitor_repo} +chown -R postgres:postgres pg_stat_monitor +cd pg_stat_monitor # Build PGSM -#make USE_PGXS=1 +make USE_PGXS=1 # Install built PGSM library into server -#make USE_PGXS=1 install +make USE_PGXS=1 install # Stop server and edit postgresql.conf to load PGSM library with required configurations service postgresql stop From 0bd0aefa922fc65b242e9e50ad3d352c9729ec55 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 23 Jun 2025 11:53:14 +0200 Subject: [PATCH 156/156] PMM-7: cleanup --- pmm_qa/percona-distribution-postgresql/data/patroni.yml.j2 | 3 --- 1 file changed, 3 deletions(-) diff --git a/pmm_qa/percona-distribution-postgresql/data/patroni.yml.j2 b/pmm_qa/percona-distribution-postgresql/data/patroni.yml.j2 index ea680deb..13ec02f5 100644 --- a/pmm_qa/percona-distribution-postgresql/data/patroni.yml.j2 +++ b/pmm_qa/percona-distribution-postgresql/data/patroni.yml.j2 @@ -30,9 +30,6 @@ bootstrap: archive_mode: "on" archive_timeout: 600s archive_command: "cp -f %p /home/postgres/archived/%f" - logging_collector: on - log_directory: 'log' - log_filename: 'postgresql.log' pg_hba: - host replication replicator 127.0.0.1/32 md5 - host replication replicator 172.18.0.0/16 md5