diff --git a/.github/workflows/ci-enterprise.yaml b/.github/workflows/ci-enterprise.yaml index 10afd7a00e..4e9ac0e160 100644 --- a/.github/workflows/ci-enterprise.yaml +++ b/.github/workflows/ci-enterprise.yaml @@ -360,7 +360,7 @@ jobs: persist-credentials: false - uses: actions/setup-node@v4 with: - node-version: "16" + node-version: "22" - name: Semantic Release id: version uses: cycjimmy/semantic-release-action@v3 diff --git a/.github/workflows/ci-lite.yaml b/.github/workflows/ci-lite.yaml index e077642ee2..ad1381cfc7 100644 --- a/.github/workflows/ci-lite.yaml +++ b/.github/workflows/ci-lite.yaml @@ -55,7 +55,7 @@ jobs: persist-credentials: false - uses: actions/setup-node@v4 with: - node-version: "16" + node-version: "22" - name: Semantic Release id: version uses: cycjimmy/semantic-release-action@v3 @@ -423,7 +423,7 @@ jobs: persist-credentials: false - uses: actions/setup-node@v4 with: - node-version: "16" + node-version: "22" - name: Semantic Release id: version uses: cycjimmy/semantic-release-action@v3 diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index 42aca979ff..d71fd782c1 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -55,7 +55,7 @@ jobs: persist-credentials: false - uses: actions/setup-node@v4 with: - node-version: "20" + node-version: "22" - name: Semantic Release id: version uses: cycjimmy/semantic-release-action@v3 @@ -63,7 +63,6 @@ jobs: semantic_version: 18 extra_plugins: | @semantic-release/exec@v6.0.3 - @semantic-release/git semantic-release-helm @google/semantic-release-replace-plugin@1.2.0 @@ -450,7 +449,7 @@ jobs: persist-credentials: false - uses: actions/setup-node@v4 with: - node-version: "20" + node-version: "22" - name: Semantic Release id: version uses: cycjimmy/semantic-release-action@v3 diff --git a/.gitignore b/.gitignore index deccabd4a7..87b3169edf 100644 --- a/.gitignore +++ b/.gitignore @@ -304,7 +304,6 @@ tags # Persistent undo [._]*.un~ -<<<<<<< HEAD # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and WebStorm # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 diff --git a/ansible/Dockerfile b/ansible/Dockerfile index 8e03af55a2..b92f7662bb 100644 --- a/ansible/Dockerfile +++ b/ansible/Dockerfile @@ -1,16 +1,17 @@ # install requirements in venv FROM python:3.10 AS venv_builder -RUN pip install ansible~=6.1.0 --no-cache-dir -RUN pip install pywinrm>=0.4.2 --no-cache-dir -RUN pip install ansible-lint>=6.0.0 --no-cache-dir +RUN pip install ansible~=6.1.0 --no-cache-dir \ + && pip install pywinrm>=0.4.2 --no-cache-dir \ + && pip install ansible-lint>=6.0.0 --no-cache-dir WORKDIR /opt COPY ./ansible . COPY ./charts . -RUN apt-get update -y -RUN apt-get install parallel -y -RUN apt-get install sshpass +RUN apt update -y \ + && apt install --no-install-recommends -y build-essential parallel \ + && apt install --no-install-recommends -y sshpass \ + && apt clean ENV ANSIBLE_CONFIG /opt/ansible.cfg ENV ANSIBLE_HOST_KEY_CHECKING False diff --git a/ansible/app/docker-compose.yml b/ansible/app/docker-compose.yml index d27d227aa0..28485c9122 100644 --- a/ansible/app/docker-compose.yml +++ b/ansible/app/docker-compose.yml @@ -1,3 +1,4 @@ +--- version: "3.7" services: sc4s: @@ -7,18 +8,18 @@ services: condition: on-failure image: ghcr.io/splunk/splunk-connect-for-syslog/container3:latest ports: - - target: 514 - published: 514 - protocol: tcp - - target: 514 - published: 514 - protocol: udp - - target: 601 - published: 601 - protocol: tcp - - target: 6514 - published: 6514 - protocol: tcp + - target: 514 + published: 514 + protocol: tcp + - target: 514 + published: 514 + protocol: udp + - target: 601 + published: 601 + protocol: tcp + - target: 6514 + published: 6514 + protocol: tcp env_file: - /opt/sc4s/env_file volumes: @@ -30,4 +31,4 @@ services: # - /opt/sc4s/tls:/etc/syslog-ng/tls:z volumes: - splunk-sc4s-var: \ No newline at end of file + splunk-sc4s-var: diff --git a/ansible/docker-compose.yml b/ansible/docker-compose.yml index 1179535880..c803b52320 100644 --- a/ansible/docker-compose.yml +++ b/ansible/docker-compose.yml @@ -3,9 +3,9 @@ ## SPDX-License-Identifier: LicenseRef-Splunk-8-2021 ## ## +--- version: "3.7" services: - ansible_sc4s: build: context: ../ diff --git a/ansible/inventory/inventory.yaml b/ansible/inventory/inventory.yaml index ae69eaf588..a3307a2d99 100644 --- a/ansible/inventory/inventory.yaml +++ b/ansible/inventory/inventory.yaml @@ -1,7 +1,8 @@ +--- all: hosts: children: node: hosts: node_1: - ansible_host: \ No newline at end of file + ansible_host: diff --git a/ansible/inventory/inventory_microk8s.yaml b/ansible/inventory/inventory_microk8s.yaml index 0cac947154..a3307a2d99 100644 --- a/ansible/inventory/inventory_microk8s.yaml +++ b/ansible/inventory/inventory_microk8s.yaml @@ -1,3 +1,4 @@ +--- all: hosts: children: diff --git a/ansible/inventory/inventory_microk8s_ha.yaml b/ansible/inventory/inventory_microk8s_ha.yaml index 543d709600..78e32a3f0c 100644 --- a/ansible/inventory/inventory_microk8s_ha.yaml +++ b/ansible/inventory/inventory_microk8s_ha.yaml @@ -1,3 +1,4 @@ +--- all: hosts: children: diff --git a/ansible/inventory/inventory_swarm.yaml b/ansible/inventory/inventory_swarm.yaml index 8e1f2ac9d0..bb6486c1eb 100644 --- a/ansible/inventory/inventory_swarm.yaml +++ b/ansible/inventory/inventory_swarm.yaml @@ -1,3 +1,4 @@ +--- all: hosts: children: diff --git a/ansible/playbooks/docker.yml b/ansible/playbooks/docker.yml index 2c2a530322..655405449b 100644 --- a/ansible/playbooks/docker.yml +++ b/ansible/playbooks/docker.yml @@ -1,6 +1,6 @@ --- - hosts: node_1 - become: yes + become: true vars: iface: "{{ swarm_iface | default('eth0') }}" tasks: diff --git a/ansible/playbooks/docker_swarm.yml b/ansible/playbooks/docker_swarm.yml index 15276a69db..1c44a3eda8 100644 --- a/ansible/playbooks/docker_swarm.yml +++ b/ansible/playbooks/docker_swarm.yml @@ -1,6 +1,6 @@ --- - hosts: manager[0] - become: yes + become: true vars: iface: "{{ swarm_iface | default('eth0') }}" tasks: @@ -10,7 +10,7 @@ - include_tasks: ../tasks/docker_swarm/create_swarm.yml - hosts: manager, !manager[0] - become: yes + become: true vars: iface: "{{ swarm_iface | default('eth0') }}" tasks: @@ -20,7 +20,7 @@ - include_tasks: ../tasks/docker_swarm/join_managers.yml - hosts: worker - become: yes + become: true tasks: - name: Docker installation role include_role: @@ -28,6 +28,6 @@ - include_tasks: ../tasks/docker_swarm/join_workers.yml - hosts: manager[0] - become: yes + become: true tasks: - include_tasks: ../tasks/docker_swarm/deploy_stack.yml diff --git a/ansible/playbooks/microk8s.yml b/ansible/playbooks/microk8s.yml index 19a03ec940..27cb157070 100644 --- a/ansible/playbooks/microk8s.yml +++ b/ansible/playbooks/microk8s.yml @@ -1,9 +1,10 @@ --- - hosts: node_1 - become: yes + become: true vars: microk8s_plugins: - # Do not provide here metallb here as the installation process is different to standard plugins + # Do not provide here metallb here as the installation process + # is different to standard plugins helm3: true dns: true community: true diff --git a/ansible/playbooks/microk8s_ha.yml b/ansible/playbooks/microk8s_ha.yml index c81ec4b5a4..6e8055f656 100644 --- a/ansible/playbooks/microk8s_ha.yml +++ b/ansible/playbooks/microk8s_ha.yml @@ -1,9 +1,10 @@ --- - hosts: manager - become: yes + become: true vars: microk8s_plugins: - # Do not provide here metallb here as the installation process is different to standard plugins + # Do not provide here metallb here as the installation process + # is different to standard plugins helm3: true dns: true community: true @@ -19,10 +20,11 @@ - include_tasks: ../tasks/mk8s/update_etc_hosts.yml - hosts: workers - become: yes + become: true vars: microk8s_plugins: - # Do not provide here metallb here as the installation process is different to standard plugins + # Do not provide here metallb here as the installation process + # is different to standard plugins helm3: true dns: true community: true diff --git a/ansible/playbooks/podman.yml b/ansible/playbooks/podman.yml index 8242def719..cc5786d8fd 100644 --- a/ansible/playbooks/podman.yml +++ b/ansible/playbooks/podman.yml @@ -1,6 +1,6 @@ --- - hosts: node_1 - become: yes + become: true tasks: - name: Podman installation role include_role: diff --git a/ansible/roles/install_docker/tasks/install_docker_debian.yml b/ansible/roles/install_docker/tasks/install_docker_debian.yml index e183e43fa3..21a517907d 100644 --- a/ansible/roles/install_docker/tasks/install_docker_debian.yml +++ b/ansible/roles/install_docker/tasks/install_docker_debian.yml @@ -1,7 +1,7 @@ --- - name: Ensure dependencies are installed apt: - update-cache: yes + update-cache: true name: - ca-certificates - curl diff --git a/ansible/roles/install_docker/tasks/install_docker_rhel.yml b/ansible/roles/install_docker/tasks/install_docker_rhel.yml index b1257ef493..48a9f6e9bb 100644 --- a/ansible/roles/install_docker/tasks/install_docker_rhel.yml +++ b/ansible/roles/install_docker/tasks/install_docker_rhel.yml @@ -13,9 +13,10 @@ - name: Add Docker Repository (RedHat, centOS) shell: | dnf install -y yum-utils - yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo + yum-config-manager --add-repo + https://download.docker.com/linux/centos/docker-ce.repo args: - warn: no + warn: false - name: Install docker-ce (RedHat, CentOS) yum: diff --git a/ansible/tasks/docker/deploy_app.yml b/ansible/tasks/docker/deploy_app.yml index 98a4346e62..50462fac88 100644 --- a/ansible/tasks/docker/deploy_app.yml +++ b/ansible/tasks/docker/deploy_app.yml @@ -3,7 +3,7 @@ file: path: "{{ item }}" state: directory - mode: 0755 + mode: 0750 loop: - /opt/sc4s - /opt/sc4s/tls @@ -16,7 +16,7 @@ dest: "/lib/systemd/system/sc4s.service" owner: "{{ ansible_user }}" group: "{{ ansible_user }}" - mode: u=rw,g=rw,o=r + mode: u=rw,g=rw - name: Copying env_file file on the server copy: @@ -24,7 +24,7 @@ dest: "/opt/sc4s/env_file" owner: "{{ ansible_user }}" group: "{{ ansible_user }}" - mode: u=rw,g=rw,o=r + mode: u=rw,g=rw - name: Create a volume docker_volume: @@ -33,7 +33,6 @@ - name: Enable sc4s service ansible.builtin.systemd: name: sc4s - enabled: yes - daemon_reload: yes + enabled: true + daemon_reload: true state: started - diff --git a/ansible/tasks/docker_swarm/create_swarm.yml b/ansible/tasks/docker_swarm/create_swarm.yml index 9f06d2b73b..42b5afb77a 100644 --- a/ansible/tasks/docker_swarm/create_swarm.yml +++ b/ansible/tasks/docker_swarm/create_swarm.yml @@ -30,4 +30,4 @@ - name: "set fact: join_token_worker" set_fact: - join_token_worker: "{{ join_token_worker_command['stdout'] }}" \ No newline at end of file + join_token_worker: "{{ join_token_worker_command['stdout'] }}" diff --git a/ansible/tasks/docker_swarm/deploy_stack.yml b/ansible/tasks/docker_swarm/deploy_stack.yml index edcccbdcbe..ce7d70ff1d 100644 --- a/ansible/tasks/docker_swarm/deploy_stack.yml +++ b/ansible/tasks/docker_swarm/deploy_stack.yml @@ -5,13 +5,13 @@ dest: "/home/{{ ansible_user }}/docker-compose.yml" owner: "{{ ansible_user }}" group: "{{ ansible_user }}" - mode: u=rw,g=rw,o=r + mode: u=rw,g=rw - name: Create sc4s dependency directories file: path: "{{ item }}" state: directory - mode: 0755 + mode: 0750 loop: - /opt/sc4s - /opt/sc4s/tls @@ -24,7 +24,7 @@ dest: "/opt/sc4s/env_file" owner: "{{ ansible_user }}" group: "{{ ansible_user }}" - mode: u=rw,g=rw,o=r + mode: u=rw,g=rw - name: Create a volume docker_volume: @@ -36,4 +36,3 @@ name: SC4S compose: - "/home/{{ ansible_user }}/docker-compose.yml" - diff --git a/ansible/tasks/docker_swarm/join_managers.yml b/ansible/tasks/docker_swarm/join_managers.yml index b189748e2a..e088ce2bd5 100644 --- a/ansible/tasks/docker_swarm/join_managers.yml +++ b/ansible/tasks/docker_swarm/join_managers.yml @@ -7,7 +7,8 @@ - name: v2 add_host: - hostname: "{{ groups['manager'] | map('extract', hostvars, ['ansible_host']) | join(':2377,') }}:2377" + hostname: "{{ groups['manager'] | + map('extract', hostvars, ['ansible_host']) | join(':2377,') }}:2377" groups: main_nodes_ips_with_port with_items: "{{ ansible_play_hosts | default(play_hosts) }}" @@ -17,4 +18,4 @@ state: join timeout: 60 join_token: "{{ hostvars[first_swarm_manager_host].join_token_manager }}" - remote_addrs: "{{ groups['main_nodes_ips_with_port'][0] }}:2377" + remote_addrs: "{{ groups['main_nodes_ips_with_port'][0] }}:2377" diff --git a/ansible/tasks/docker_swarm/join_workers.yml b/ansible/tasks/docker_swarm/join_workers.yml index 28ce656ca0..d3d5c9da45 100644 --- a/ansible/tasks/docker_swarm/join_workers.yml +++ b/ansible/tasks/docker_swarm/join_workers.yml @@ -5,7 +5,8 @@ - name: v2 add_host: - hostname: "{{ groups['manager'] | map('extract', hostvars, ['ansible_host']) | join(':3000,') }}:3000" + hostname: "{{ groups['manager'] | map('extract', hostvars, + ['ansible_host']) | join(':3000,') }}:3000" groups: main_nodes_ips_with_port with_items: "{{ ansible_play_hosts | default(play_hosts) }}" @@ -14,4 +15,4 @@ state: join timeout: 60 join_token: "{{ hostvars[first_swarm_manager_host].join_token_worker }}" - remote_addrs: "{{ groups['main_nodes_ips_with_port'][0] }}:2377" + remote_addrs: "{{ groups['main_nodes_ips_with_port'][0] }}:2377" diff --git a/ansible/tasks/install_docker.yml b/ansible/tasks/install_docker.yml index b0afbea3dd..e63708a0c3 100644 --- a/ansible/tasks/install_docker.yml +++ b/ansible/tasks/install_docker.yml @@ -1,7 +1,7 @@ --- - name: Ensure dependencies are installed (Debian) apt: - update-cache: yes + update-cache: true name: - ca-certificates - curl @@ -42,9 +42,10 @@ - name: Add Docker Repository (RedHat, centOS) shell: | dnf install -y yum-utils - yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo + yum-config-manager --add-repo + https://download.docker.com/linux/centos/docker-ce.repo args: - warn: no + warn: false when: ansible_os_family == "RedHat" - name: install docker-ce (Debian, Ubuntu) diff --git a/ansible/tasks/mk8s/deploy_app.yml b/ansible/tasks/mk8s/deploy_app.yml index e71f754950..37770a0dbd 100644 --- a/ansible/tasks/mk8s/deploy_app.yml +++ b/ansible/tasks/mk8s/deploy_app.yml @@ -5,16 +5,20 @@ dest: "/home/{{ ansible_user }}/values.yaml" owner: "{{ ansible_user }}" group: "{{ ansible_user }}" - mode: u=rw,g=rw,o=r + mode: u=rw,g=rw - name: Deploy app or update it with new values if already deployed block: - name: Deploy sc4s app from templates with overwrites from values.yml - ansible.builtin.shell: microk8s helm3 install sc4s splunk-connect-for-syslog/splunk-connect-for-syslog -f values.yaml + ansible.builtin.shell: | + microk8s helm3 install sc4s + splunk-connect-for-syslog/splunk-connect-for-syslog -f values.yaml args: chdir: "/home/{{ ansible_user }}/" rescue: - - name: Update app with new values.yml - ansible.builtin.shell: microk8s helm3 upgrade sc4s splunk-connect-for-syslog/splunk-connect-for-syslog -f values.yaml - args: - chdir: "/home/{{ ansible_user }}/" \ No newline at end of file + - name: Update app with new values.yml + ansible.builtin.shell: | + microk8s helm3 upgrade sc4s + splunk-connect-for-syslog/splunk-connect-for-syslog -f values.yaml + args: + chdir: "/home/{{ ansible_user }}/" diff --git a/ansible/tasks/mk8s/deploy_secrets.yml b/ansible/tasks/mk8s/deploy_secrets.yml index 878d88d033..314da765da 100644 --- a/ansible/tasks/mk8s/deploy_secrets.yml +++ b/ansible/tasks/mk8s/deploy_secrets.yml @@ -4,8 +4,8 @@ file: "{{ item }}" with_first_found: - files: - - /opt/ansible/resources/k8s_secrets.yaml - - /opt/charts/splunk-connect-for-syslog/secrets.yaml + - /opt/ansible/resources/k8s_secrets.yaml + - /opt/charts/splunk-connect-for-syslog/secrets.yaml - name: Create mTLS secret ansible.builtin.shell: | diff --git a/ansible/tasks/mk8s/get_registration_token.yml b/ansible/tasks/mk8s/get_registration_token.yml index 30446326d6..3432676f0c 100644 --- a/ansible/tasks/mk8s/get_registration_token.yml +++ b/ansible/tasks/mk8s/get_registration_token.yml @@ -1,6 +1,6 @@ --- - name: Get registration token - become: yes + become: true command: cmd: microk8s add-node --token-ttl 3600 register: init_cluster diff --git a/ansible/tasks/mk8s/install_helm_repo.yml b/ansible/tasks/mk8s/install_helm_repo.yml index dd6ba6bb05..851d19db7b 100644 --- a/ansible/tasks/mk8s/install_helm_repo.yml +++ b/ansible/tasks/mk8s/install_helm_repo.yml @@ -1,6 +1,8 @@ --- - name: Add sc4s helm repository - ansible.builtin.shell: microk8s helm3 repo add splunk-connect-for-syslog https://splunk.github.io/splunk-connect-for-syslog + ansible.builtin.shell: | + microk8s helm3 repo add splunk-connect-for-syslog + https://splunk.github.io/splunk-connect-for-syslog - name: Separately update the repository cache ansible.builtin.shell: microk8s helm3 repo update diff --git a/ansible/tasks/mk8s/install_mk8s.yml b/ansible/tasks/mk8s/install_mk8s.yml index 200ecff132..7c03ac2652 100644 --- a/ansible/tasks/mk8s/install_mk8s.yml +++ b/ansible/tasks/mk8s/install_mk8s.yml @@ -2,17 +2,18 @@ - name: Install mk8s community.general.snap: name: microk8s - classic: yes + classic: true channel: 1.24 - name: Wait for microk8s to be ready - become: yes + become: true command: microk8s.status --wait-ready changed_when: false register: mk8sstatusout failed_when: - - "'This MicroK8s deployment is acting as a node in a cluster.' not in mk8sstatusout.stdout_lines" - - mk8sstatusout.rc > 0 + - "'This MicroK8s deployment is acting as a node in a cluster.' + not in mk8sstatusout.stdout_lines" + - mk8sstatusout.rc > 0 - name: Add current user to group of 'microk8s' ansible.builtin.shell: usermod -a -G microk8s $USER @@ -21,10 +22,10 @@ ansible.builtin.shell: chown -f -R $USER ~/.kube - name: Get current addons state - become: yes + become: true command: cmd: microk8s.status --format yaml - changed_when: no + changed_when: false register: microk8s_status - name: Set current state fact @@ -35,22 +36,26 @@ ansible.builtin.shell: microk8s enable community - name: Enable metallb - ansible.builtin.shell: echo "{{ ansible_ssh_host }}/32" | microk8s enable metallb + ansible.builtin.shell: | + echo "{{ ansible_ssh_host }}/32" | microk8s enable metallb - name: Enable addons - become: yes + become: true loop: "{{ microk8s_status.addons }}" loop_control: label: "{{ item.name }}" command: - cmd: microk8s.enable {{ item.name }}{% if microk8s_plugins[item.name] != True %}:{{ microk8s_plugins[item.name] }}{% endif %} + cmd: | + microk8s.enable {{ item.name }} + {% if microk8s_plugins[item.name] != True %}: + {{ microk8s_plugins[item.name] }}{% endif %} when: - item.status == 'disabled' - item.name in microk8s_plugins - microk8s_plugins[item.name] - name: Disable addons - become: yes + become: true loop: "{{ microk8s_status.addons }}" loop_control: label: "{{ item.name }}" diff --git a/ansible/tasks/mk8s/join_mk8s_cluster.yml b/ansible/tasks/mk8s/join_mk8s_cluster.yml index 85bc532108..831b2d5c07 100644 --- a/ansible/tasks/mk8s/join_mk8s_cluster.yml +++ b/ansible/tasks/mk8s/join_mk8s_cluster.yml @@ -7,6 +7,6 @@ var: hostvars - name: Join cluster - become: yes + become: true command: - cmd: "{{ hostvars[manager_host].join_token_worker }} " \ No newline at end of file + cmd: "{{ hostvars[manager_host].join_token_worker }} " diff --git a/ansible/tasks/mk8s/update_etc_hosts.yml b/ansible/tasks/mk8s/update_etc_hosts.yml index cd4a5b6d3e..c6ecff4a84 100644 --- a/ansible/tasks/mk8s/update_etc_hosts.yml +++ b/ansible/tasks/mk8s/update_etc_hosts.yml @@ -6,6 +6,8 @@ - name: Add IP address of all hosts to all hosts lineinfile: dest: /etc/hosts - line: "{{ hostvars[item].ansible_host }} {{ hostvars[item].inventory_hostname }} {{ hostvars[item].inventory_hostname_short }}" + line: "{{ hostvars[item].ansible_host }} + {{ hostvars[item].inventory_hostname }} + {{ hostvars[item].inventory_hostname_short }}" state: present - with_items: "{{ groups.all }}" \ No newline at end of file + with_items: "{{ groups.all }}" diff --git a/ansible/tasks/podman/deploy_app.yml b/ansible/tasks/podman/deploy_app.yml index 46c4f96aed..07632d8eaa 100644 --- a/ansible/tasks/podman/deploy_app.yml +++ b/ansible/tasks/podman/deploy_app.yml @@ -3,7 +3,7 @@ file: path: "{{ item }}" state: directory - mode: 0755 + mode: 0750 loop: - /opt/sc4s - /opt/sc4s/tls @@ -16,7 +16,7 @@ dest: "/lib/systemd/system/sc4s.service" owner: "{{ ansible_user }}" group: "{{ ansible_user }}" - mode: u=rw,g=rw,o=r + mode: u=rw,g=rw - name: Copying env_file file on the server copy: @@ -24,7 +24,7 @@ dest: "/opt/sc4s/env_file" owner: "{{ ansible_user }}" group: "{{ ansible_user }}" - mode: u=rw,g=rw,o=r + mode: u=rw,g=rw - name: Create a volume command: podman volume create splunk-sc4s-var @@ -33,7 +33,6 @@ - name: Enable sc4s service ansible.builtin.systemd: name: sc4s - enabled: yes - daemon_reload: yes + enabled: true + daemon_reload: true state: started - diff --git a/charts/splunk-connect-for-syslog/Chart.yaml b/charts/splunk-connect-for-syslog/Chart.yaml index 4dd0bef7c7..7df18203a3 100644 --- a/charts/splunk-connect-for-syslog/Chart.yaml +++ b/charts/splunk-connect-for-syslog/Chart.yaml @@ -1,3 +1,4 @@ +--- apiVersion: v2 name: splunk-connect-for-syslog description: Deploy Splunk Connect for Syslog diff --git a/charts/splunk-connect-for-syslog/values.yaml b/charts/splunk-connect-for-syslog/values.yaml index 33aa7b3f85..44cec2d82b 100644 --- a/charts/splunk-connect-for-syslog/values.yaml +++ b/charts/splunk-connect-for-syslog/values.yaml @@ -1,3 +1,4 @@ +--- # Default values for splunk-connect-for-syslog. # This is a YAML-formatted file. # Declare variables to be passed into your templates. @@ -10,7 +11,7 @@ splunk: # hec_token: "" # hec_verify_tls: "yes" -sc4s: +sc4s: {} # existingCert: example-com-tls # vendor_product: @@ -154,7 +155,8 @@ serviceAccount: # Annotations to add to the service account annotations: {} # The name of the service account to use. - # If not set and create is true, a name is generated using the fullname template + # If not set and create is true, + # a name is generated using the fullname template name: "" persistence: @@ -186,10 +188,11 @@ service: resources: {} - # We usually recommend not to specify default resources and to leave this as a conscious - # choice for the user. This also increases chances charts run on environments with little - # resources, such as Minikube. If you do want to specify resources, uncomment the following - # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # We usually recommend not to specify default resources and to leave this as + # a conscious choice for the user. This also increases chances charts run on + # environments with little resources, such as Minikube. If you do want to + # specify resources, uncomment the following lines, adjust them as necessary, + # and remove the curly braces after 'resources:'. # limits: # cpu: 100m # memory: 128Mi diff --git a/docs/architecture/lb/f5.md b/docs/architecture/lb/f5.md new file mode 100644 index 0000000000..d17739e4db --- /dev/null +++ b/docs/architecture/lb/f5.md @@ -0,0 +1,210 @@ +# F5 +If you're using F5 BIG-IP for syslog ingestion, consider the following: + +- **Uneven TCP traffic distribution**: Even with round-robin load balancing, TCP traffic may not be evenly distributed, which can lead to overloaded instances. This can cause growing queues, delays, data loss, and potential memory or disk issues. +- **UDP limitations**: UDP is a protocol prone to data loss, and load balancers can introduce another point of data loss. +- **High Availability (HA) mode**: Running the load balancer without HA makes it a single point of failure. + +**Note**: Splunk only supports SC4S. If you encounter issues related to the load balancer, please contact F5 support. + +## Set up your F5 BIG-IP +F5 BIG-IP is available in both hardware and virtual editions. This documentation assumes you already have a functioning F5 instance. If you need help with your F5 instance, see [external resource](https://clouddocs.f5.com/cloud/public/v1/aws/AWS_singleNIC.html). + +## Create a custom health check +F5 includes built-in monitors to track the health status of pool members. You can add a custom HTTP monitor that leverages SC4S’s health check endpoint. + +The health check endpoint is available in SC4S versions 3.34.0 and later. + +1. Navigate to **Local Traffic** > **Monitors** > **Create**. +2. In the General Properties section configure the following settings: + +- **Type**: `HTTP` +- **Parent Monitor**: `http` + +![Monitor - General Properties](./monitor_1.png) + +3. In the Configuration section, set **Interval** to 1 second and **Timeout** to 2 seconds, this minimizes UDP losses between health check queries. +4. Configure the following settings: + +- **Send String**: `GET /health HTTP/1.0\r\n\r\n` +- **Receive String**: `\"healthy\"` +- **Alias Service Port**: `8080` + +![Monitor - Configuration](./monitor_2.png) + + +5. Click **Finished** to save. + +## Define pools + +Create a pool for each port in use. The default SC4S TCP ports are 514 and 601, and the default SC4S UDP port is 514. Separate pools are not needed for different protocols, so a single pool named "sc4s_514" can handle both TCP and UDP traffic. + +1. Navigate to **Local Traffic** > **Pools** > **Create**. +2. Select the following health monitors: +- "tcp" +- "http_sc4s_health_check" ([Custom SC4S monitor](#create-a-custom-health-check)) + +3. Add members and click **Finished** to save the pool. + +![Pool configuration](./pool.png) + +4. Go to **Pools** > **sc4s_514** > **Members** and verify that all members have an **Active** status. +5. (Optional) Test availability by restarting an SC4S instance: +```bash +sudo systemctl restart sc4s +``` +Observe that the member temporarily becomes unavailable. +![Inactive pool member](./pool_2.png) + +## Preserve Source IP +In many cases, replacing the original source IP with the load balancer’s IP (typically using `automap`) is recommended. However, for syslog load balancing make sure you retain the original source IP. Otherwise, logs that do not specify a hostname in the message will appear with the load balancer's IP. + +This documentation covers two methods for preserving the source IP: +- PROXY protocol for TCP +- Stateless virtual server for UDP + +## TCP with PROXY + +TCP is a session-based protocol that requires acknowledgments. Load balancers typically replace the original source IP with their own to ensure that responses from the backend server are routed back to them. + +The simplest way to preserve the source IP is by using the PROXY protocol. BIG-IP does not natively support it, but it can be implemented using iRules. + +### Add the iRule +1. Navigate to **Local Traffic** > **iRules** > **Create**. +2. Add the following iRule and save it: +```tcl +when CLIENT_ACCEPTED { + set proxyheader "PROXY TCP[IP::version] [IP::remote_addr] [IP::local_addr] [TCP::remote_port] [TCP::local_port]\r\n" +} + +when SERVER_CONNECTED { + TCP::respond $proxyheader +} +``` +![iRule](./irule.png) + +### Enable PROXY in SC4S +For each SC4S server: + +1. Open the configuration file at `/opt/sc4s/env_file` and add the following line: +```conf +SC4S_SOURCE_PROXYCONNECT=yes +``` + +2. Restart the SC4S service: +```bash +sudo systemctl restart sc4s +``` + +**Important**: Once enabled, SC4S will expect TCP input to include PROXY headers. Sending TCP messages without PROXY headers will result in the error: "Unable to determine PROXY protocol version." + +### Create TCP Virtual Servers +1. Navigate to **Local Traffic** > **Virtual Servers** > **Create**. + +2. In the General Properties section, configure the following settings: +- **Type**: `Standard` +- **Destination Address/Mask**: `` +- **Service Port**: `` + +![TCP Virtual Server General Properties](./tcp_vs_1.png) + +3. In the Basic Configuration section, configure the following: +- **Protocol**: `TCP` +- **Protocol Profile (Client)**: `tcp` +- **Source Address Translation**: `Auto Map` +![TCP Virtual Server Configuration](./tcp_vs_2.png) + +4. In the Resources section, configure the following: +- **iRules**: `` +- **Default Pool**: `` + +![TCP Virtual Server Resources](./tcp_vs_3.png) + +5. Click **Finished** to save your changes and start the server. + +### Test your configuration +1. From an external location, send several TCP messages to your load balancer: +```bash +for i in {1..5}; do echo "$i" | nc ; done +``` + +2. Verify that all messages reached Splunk, the host is set to the source IP (not the load balancer’s IP), and traffic is properly distributed across the pool. +![TCP Virtual Server Resources](./tcp_test_1.png) + +### Run performance tests +Follow the [Check TCP Performance](../performance-tests.md#check-your-tcp-performance) section. + +**Test Configuration:** +- **Log Source**: `c5.2xlarge` +- **SC4S Instances**: `c5.4xlarge` +- **F5 Instance**: `r5.8xlarge, 32vCPU, 256GiB Memory, 10Gbps Network Performance` + +| Receiver | Performance | +|----------------------------|--------------------| +| Single SC4S Server | 69,192.11 msg/sec | +| Load Balancer + 2 Servers | 93,832.33 msg/sec | + +**Note:** Load balancer support and fine-tuning are beyond the scope of the SC4S team. For assistance in optimizing the TCP throughput of your load balancer instance, contact the F5 support team. + +## Stateless Virtual Server for UDP +UDP syslog traffic is unacknowledged and unidirectional by design. To preserve the source IP for UDP ports, configure [stateless virtual servers](https://my.f5.com/manage/s/article/K13675). + +### Turn off source/destination checks +If you host your load balancer instance in a cloud environment, your provider might enforce Source/Destination IP checks. This will block traffic because the original source IP is passed through. Disable this setting. + +![Change source/destination check](./udp_1.png) + +### Create a multiprocess UDP traffic profile +Configure a dedicated [UDP profile](https://my.f5.com/manage/s/article/K3605) to load balance UDP packets individually. + +1. Navigate to **Local Traffic** > **Profiles**. +2. Go to the **Protocol** tab, select **UDP**, and click **Create**. +3. Enable the **Custom** option, then check **Datagram LB** and set **Idle Timeout** to **Immediate**. + +![Add UDP Datagram LB Profile](./udp_2.png) + +4. Click **Finished** to create the new UDP profile. + +### Create a Virtual Server +1. Navigate to **Local Traffic** > **Virtual Servers** > **Create**. +2. In **General Properties**, configure the following: + +- **Type**: `Stateless` +- **Destination Address**: Load balancer's IP +- **Service Port**: Desired port + +![UDP Virtual Server General Properties](./udp_3.png) + +3. In the Configuration section, set: + +- **Protocol Profile**: `udp_datagram_lb` +- **Source Address Translation**: `None` + +![UDP Virtual Server Configuration](./udp_4.png) + +4. Select your pool and click **Finished**. + +### Test your configuration +1. From an external location, send several UDP messages to your load balancer: +``` +for i in {1..5}; do echo "$i" > /dev/udp//; done +``` + +2. Verify that all messages reached Splunk, the host is set to the source IP (not the load balancer’s IP), and traffic is properly distributed across the pool. + +### Run performance tests +Follow the [Check UDP Performance](../performance-tests.md#check-your-udp-performance) section. + +**Test Configuration:** +- **Log Source**: `c5.2xlarge` +- **SC4S Instances**: `c5.4xlarge` +- **F5 Instance**: `r5.8xlarge, 32vCPU, 256GiB Memory, 10Gbps Network Performance` + +| Receiver / Drops Rate for EPS (msgs/sec) | 4,500 | 9,000 | 27,000 | 50,000 | 150,000 | +|------------------------------------------|--------|--------|--------|--------|---------| +| Single SC4S Server | 0% | 0.34% | 58.89% | 78.30% | 92.19% | +| Load Balancer + 2 Servers | 0% | 0% | 15.66% | 54.44% | 84.16% | +| Single Finetuned SC4S Server | 0% | 0% | 0% | 0% | 48.62% | +| Load Balancer + 2 Finetuned Servers | 0% | 0% | 0.002% | 0.05% | 0.03% | + +**Note:** Load balancer support and fine-tuning are beyond the scope of the SC4S team. For assistance in minimizing UDP drops on the load balancer side, please contact the F5 support team. diff --git a/docs/architecture/lb/irule.png b/docs/architecture/lb/irule.png new file mode 100644 index 0000000000..a7428a0cac Binary files /dev/null and b/docs/architecture/lb/irule.png differ diff --git a/docs/architecture/lb/monitor_1.png b/docs/architecture/lb/monitor_1.png new file mode 100644 index 0000000000..b6c450beee Binary files /dev/null and b/docs/architecture/lb/monitor_1.png differ diff --git a/docs/architecture/lb/monitor_2.png b/docs/architecture/lb/monitor_2.png new file mode 100644 index 0000000000..01ba80b94f Binary files /dev/null and b/docs/architecture/lb/monitor_2.png differ diff --git a/docs/architecture/lb/pool.png b/docs/architecture/lb/pool.png new file mode 100644 index 0000000000..754ced43ab Binary files /dev/null and b/docs/architecture/lb/pool.png differ diff --git a/docs/architecture/lb/pool_2.png b/docs/architecture/lb/pool_2.png new file mode 100644 index 0000000000..e8f28e384b Binary files /dev/null and b/docs/architecture/lb/pool_2.png differ diff --git a/docs/architecture/lb/tcp_test_1.png b/docs/architecture/lb/tcp_test_1.png new file mode 100644 index 0000000000..ba9c1cdc06 Binary files /dev/null and b/docs/architecture/lb/tcp_test_1.png differ diff --git a/docs/architecture/lb/tcp_vs_1.png b/docs/architecture/lb/tcp_vs_1.png new file mode 100644 index 0000000000..a2ee0b1b17 Binary files /dev/null and b/docs/architecture/lb/tcp_vs_1.png differ diff --git a/docs/architecture/lb/tcp_vs_2.png b/docs/architecture/lb/tcp_vs_2.png new file mode 100644 index 0000000000..7900d36f41 Binary files /dev/null and b/docs/architecture/lb/tcp_vs_2.png differ diff --git a/docs/architecture/lb/tcp_vs_3.png b/docs/architecture/lb/tcp_vs_3.png new file mode 100644 index 0000000000..6293493dae Binary files /dev/null and b/docs/architecture/lb/tcp_vs_3.png differ diff --git a/docs/architecture/lb/udp_1.png b/docs/architecture/lb/udp_1.png new file mode 100644 index 0000000000..7a345ede82 Binary files /dev/null and b/docs/architecture/lb/udp_1.png differ diff --git a/docs/architecture/lb/udp_2.png b/docs/architecture/lb/udp_2.png new file mode 100644 index 0000000000..6e3871b06f Binary files /dev/null and b/docs/architecture/lb/udp_2.png differ diff --git a/docs/architecture/lb/udp_3.png b/docs/architecture/lb/udp_3.png new file mode 100644 index 0000000000..5339ced620 Binary files /dev/null and b/docs/architecture/lb/udp_3.png differ diff --git a/docs/architecture/lb/udp_4.png b/docs/architecture/lb/udp_4.png new file mode 100644 index 0000000000..eef1063680 Binary files /dev/null and b/docs/architecture/lb/udp_4.png differ diff --git a/docs/edge_processor.md b/docs/edge_processor.md index 13e8177476..f0e5c60a41 100644 --- a/docs/edge_processor.md +++ b/docs/edge_processor.md @@ -1,4 +1,4 @@ -# Edge Processor integration guide (Experimental) +# Edge Processor integration guide ## Intro diff --git a/docs/gettingstarted/byoe-rhel8.md b/docs/gettingstarted/byoe-rhel8.md index b327545561..a9a31cb5d4 100644 --- a/docs/gettingstarted/byoe-rhel8.md +++ b/docs/gettingstarted/byoe-rhel8.md @@ -12,7 +12,7 @@ system administration. This topic provides guidance for using the SC4S syslog-ng configuration files directly on the host OS running on a hardware server or virtual machine. You must provide: -* An appropriate host operating system, RHEL 8 is the example provided in this topic. +* An appropriate host operating system, RHEL 9 is the example provided in this topic. * An up-to-date syslog-ng installation built from source or installed from community-built RPMs. You must modify the base configuration for most @@ -29,7 +29,7 @@ The following installation instructions are summarized from a [blog](https://www.syslog-ng.com/community/b/blog/posts/introducing-the-syslog-ng-stable-rpm-repositories) maintained by the One Identity team. -1. Install CentOS or RHEL 8.0. See your OS documentation for instructions. +1. Install CentOS or RHEL 9.5. See your OS documentation for instructions. 2. Enable EPEL (Centos 8). diff --git a/docs/sources/vendor/PaloaltoNetworks/panos.md b/docs/sources/vendor/PaloaltoNetworks/panos.md index bcb3716cf9..d2d14fb471 100644 --- a/docs/sources/vendor/PaloaltoNetworks/panos.md +++ b/docs/sources/vendor/PaloaltoNetworks/panos.md @@ -10,7 +10,7 @@ | Ref | Link | |----------------|---------------------------------------------------------------------------------------------------------| -| Splunk Add-on | | +| Splunk Add-on | | | Product Manual | | ## Sourcetypes diff --git a/docs/sources/vendor/Vectra/cognito_json.md b/docs/sources/vendor/Vectra/cognito_json.md new file mode 100644 index 0000000000..a113f823e4 --- /dev/null +++ b/docs/sources/vendor/Vectra/cognito_json.md @@ -0,0 +1,42 @@ +# Cognito JSON + +## Key facts + +* MSG Format based filter +* Legacy BSD Format default port 514 + +## Links + +| Ref | Link | +|----------------|---------------------------------------------------------------------------------------------------------| +| Technology Add-On for Vectra Detect (JSON) | | + +## Sourcetypes + +| sourcetype | notes | +|----------------|---------------------------------------------------------------------------------------------------------| +|vectra:cognito:detect:json|| +|vectra:cognito:hostscoring:json|| +|vectra:cognito:hostdetect:json|| +|vectra:cognito:hostlockdown:json|| +|vectra:cognito:accountscoring:json|| +|vectra:cognito:accountdetect:json|| +|vectra:cognito:accountlockdown:json|| +|vectra:cognito:campaigns:json|| +|vectra:cognito:audit:json|| +|vectra:cognito:health:json|| + +### Index Configuration + +| key | sourcetype | index | notes | +|----------------|----------------|----------------|----------------| +|vectra_cognito detect_detect |vectra:cognito:detect:json |main| +|vectra_cognito detect_hostscoring |vectra:cognito:hostscoring:json |main| +|vectra_cognito detect_hostdetect |vectra:cognito:hostdetect:json |main| +|vectra_cognito detect_hostlockdown |vectra:cognito:hostlockdown:json |main| +|vectra_cognito detect_accountscoring |vectra:cognito:accountscoring:json |main| +|vectra_cognito detect_accountdetect |vectra:cognito:accountdetect:json |main| +|vectra_cognito detect_accountlockdown |vectra:cognito:accountlockdown:json |main| +|vectra_cognito detect_campaigns |vectra:cognito:campaigns:json |main| +|vectra_cognito detect_audit |vectra:cognito:audit:json |main| +|vectra_cognito detect_health |vectra:cognito:health:json |main| diff --git a/mkdocs.yml b/mkdocs.yml index 1209127bba..e7e58f5a71 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -43,6 +43,7 @@ nav: - Load Balancers: - Overview: "architecture/lb/index.md" - Nginx: "architecture/lb/nginx.md" + - F5: "architecture/lb/f5.md" - Getting Started: - Read First: "gettingstarted/index.md" - Quickstart Guide: "gettingstarted/quickstart_guide.md" @@ -70,7 +71,7 @@ nav: - SC4S Lite (Experimental): - Intro: "lite.md" - Pluggable modules: "pluggable_modules.md" - - Edge Processor (Experimental): "edge_processor.md" + - Edge Processor: "edge_processor.md" - Troubleshooting: - SC4S Startup and Validation: "troubleshooting/troubleshoot_SC4S_server.md" - SC4S Logging and Troubleshooting Resources: "troubleshooting/troubleshoot_resources.md" diff --git a/package/Dockerfile b/package/Dockerfile index 2069315339..3cf5bff04d 100644 --- a/package/Dockerfile +++ b/package/Dockerfile @@ -60,7 +60,7 @@ EXPOSE 6514/tcp #/dev/log a low priv user cannot read this and the container will fail in SC4S #and other uses the low user may be selected -HEALTHCHECK --interval=2m --timeout=5s --start-period=30s CMD /usr/sbin/syslog-ng-ctl healthcheck --timeout 5 +HEALTHCHECK --interval=2m --timeout=5s --start-period=30s CMD ["/usr/sbin/syslog-ng-ctl", "healthcheck", "--timeout", "5"] COPY pyproject.toml / COPY poetry.lock / @@ -83,6 +83,6 @@ COPY package/sbin/source_ports_validator.py / ENV SC4S_CONTAINER_OPTS=--no-caps ARG VERSION=unknown -RUN echo $VERSION>/etc/syslog-ng/VERSION +RUN echo "$VERSION">/etc/syslog-ng/VERSION ENTRYPOINT ["/entrypoint.sh"] diff --git a/package/Dockerfile.lite b/package/Dockerfile.lite index 5ef25b980e..d5223a9fb6 100644 --- a/package/Dockerfile.lite +++ b/package/Dockerfile.lite @@ -60,7 +60,7 @@ EXPOSE 6514/tcp #/dev/log a low priv user cannot read this and the container will fail in SC4S #and other uses the low user may be selected -HEALTHCHECK --interval=2m --timeout=5s --start-period=30s CMD /usr/sbin/syslog-ng-ctl healthcheck --timeout 5 +HEALTHCHECK --interval=2m --timeout=5s --start-period=30s CMD ["/usr/sbin/syslog-ng-ctl", "healthcheck", "--timeout", "5"] COPY pyproject.toml / COPY poetry.lock / @@ -109,6 +109,6 @@ RUN chmod -R 755 /etc/syslog-ng/ ENV SC4S_CONTAINER_OPTS=--no-caps ARG VERSION=unknown -RUN echo $VERSION>/etc/syslog-ng/VERSION +RUN echo "$VERSION">/etc/syslog-ng/VERSION ENTRYPOINT ["/entrypoint.sh"] diff --git a/package/etc/conf.d/conflib/netsource/app-netsource-netapp_ontap.conf b/package/etc/conf.d/conflib/netsource/app-netsource-netapp_ontap.conf index 08cd806164..60d72bcfd7 100644 --- a/package/etc/conf.d/conflib/netsource/app-netsource-netapp_ontap.conf +++ b/package/etc/conf.d/conflib/netsource/app-netsource-netapp_ontap.conf @@ -34,6 +34,26 @@ block parser app-netsource-netapp_ontap() { class('audit') ); }; + } elif { + parser { + regexp-parser( + prefix(".tmp.") + patterns('\[(?[^:]+):(?[^:]+):(?[^\]]+)\]: (?.*)') + template("${MESSAGE}") + ); + }; + rewrite { + set('${.tmp.message}' value('MESSAGE')); + set('${.tmp.host}' value('HOST')); + set('${.tmp.category}' value('fields.category')); + set('${.tmp.severity}' value('fields.severity')); + }; + rewrite { + r_set_splunk_dest_update_v2( + sourcetype('netapp:ontap:ems') + class('ems') + ); + }; } else { rewrite { r_set_splunk_dest_update_v2( @@ -46,10 +66,10 @@ block parser app-netsource-netapp_ontap() { }; application app-netsource-netapp_ontap[sc4s-network-source] { - filter { + filter { match("netapp", value('.netsource.sc4s_vendor'), type(string)) and match("ontap", value('.netsource.sc4s_product'), type(string)) and "`SC4S_NETAPP_ONTAP_NEW_FORMAT`" eq "yes" - }; + }; parser { app-netsource-netapp_ontap(); }; -}; +}; \ No newline at end of file diff --git a/package/etc/conf.d/conflib/syslog/app-syslog-vectra_json.conf b/package/etc/conf.d/conflib/syslog/app-syslog-vectra_json.conf new file mode 100644 index 0000000000..e618c550bd --- /dev/null +++ b/package/etc/conf.d/conflib/syslog/app-syslog-vectra_json.conf @@ -0,0 +1,104 @@ +block parser app-syslog-vectra-json() { + channel { + parser { + regexp-parser( + prefix(".tmp.") + patterns('\"vectra_timestamp\"\:\s\"(?[^\"]+)\"') + template("$MESSAGE") + ); + date-parser-nofilter( + format('%s') + template("${.tmp.timestamp}") + ); + }; + + rewrite { + subst('\-\:\s',"",value("MESSAGE")); + }; + + rewrite { + r_set_splunk_dest_default( + index("main") + sourcetype('vectra:cognito:detect:json') + vendor("vectra") + product("cognito detect") + class('detect') + template("t_msg_only") + ); + }; + + if (message('\"host_\w+\"\:')) { + rewrite { + r_set_splunk_dest_update_v2( + sourcetype('vectra:cognito:hostscoring:json') + class('hostscoring') + condition(message('\"HOST\sSCORING\"')) + ); + }; + rewrite { + r_set_splunk_dest_update_v2( + sourcetype('vectra:cognito:hostdetect:json') + class('hostdetect') + condition(message('\"detection_id\"\:')) + ); + }; + rewrite { + r_set_splunk_dest_update_v2( + sourcetype('vectra:cognito:hostlockdown:json') + class('hostlockdown') + condition(message('\"success\"\:')) + ); + }; + } elif (message('\"account_uid\"\:')) { + rewrite { + r_set_splunk_dest_update_v2( + sourcetype('vectra:cognito:accountscoring:json') + class('accountscoring') + condition(message('\"ACCOUNT\sSCORING\"')) + ); + }; + rewrite { + r_set_splunk_dest_update_v2( + sourcetype('vectra:cognito:accountdetect:json') + class('accountdetect') + condition(message('\"detection_id\"\:')) + ); + }; + rewrite { + r_set_splunk_dest_update_v2( + sourcetype('vectra:cognito:accountlockdown:json') + class('accountlockdown') + condition(message('\"success\"\:')) + ); + }; + } elif (message('\"campaign_id\"\:')) { + rewrite { + r_set_splunk_dest_update_v2( + sourcetype('vectra:cognito:campaigns:json') + class('campaigns') + ); + }; + } elif (message('\"role\"\:')) { + rewrite { + r_set_splunk_dest_update_v2( + sourcetype('vectra:cognito:audit:json') + class('audit') + ); + }; + } elif (message('\"type\"\:')) { + rewrite { + r_set_splunk_dest_update_v2( + sourcetype('vectra:cognito:health:json') + class('health') + ); + }; + } else {}; + }; +}; + +application app-syslog-vectra-json[sc4s-syslog-pgm] { + filter { + program('vectra_json' type(string) flags(prefix)); + }; + parser { app-syslog-vectra-json(); }; +}; \ No newline at end of file diff --git a/package/etc/conf.d/log_paths/2/lp_dest_alts_global/plugin.py b/package/etc/conf.d/log_paths/2/lp_dest_alts_global/plugin.py index 13bb14603d..76081f7f80 100755 --- a/package/etc/conf.d/log_paths/2/lp_dest_alts_global/plugin.py +++ b/package/etc/conf.d/log_paths/2/lp_dest_alts_global/plugin.py @@ -36,7 +36,7 @@ def normalize_env_variable_input(env_variable: str): modev = os.environ.get(f"SC4S_DEST_SPLUNK_HEC_{r}_MODE", "GLOBAL") if ( r == "DEFAULT" - and not os.environ.get("SC4S_DEST_SPLUNK_HEC_GLOBAL", "") == "" + and os.environ.get("SC4S_DEST_SPLUNK_HEC_GLOBAL", "") != "" ): if os.environ.get("SC4S_DEST_SPLUNK_HEC_GLOBAL", "yes").lower() in [ "true", @@ -68,9 +68,8 @@ def normalize_env_variable_input(env_variable: str): if r != "": modev = os.environ.get(f"SC4S_DEST_{t}_{r}_MODE", "GLOBAL") filter = os.environ.get(f"SC4S_DEST_{t}_{r}_FILTER", "") - if filter == "": - if t == "BSD": - filter = '"${MSG}" ne ""' + if filter == "" and t == "BSD": + filter = '"${MSG}" ne ""' if modev.upper() in ("GLOBAL", "SELECT"): global_dests[r] = { "destination": f"d_{t.lower()}_{r.lower()}", diff --git a/package/etc/conf.d/sources/source_syslog/plugin.py b/package/etc/conf.d/sources/source_syslog/plugin.py index fdf790b0df..1d48a8a6b1 100755 --- a/package/etc/conf.d/sources/source_syslog/plugin.py +++ b/package/etc/conf.d/sources/source_syslog/plugin.py @@ -53,8 +53,6 @@ def normalize_env_variable_input(env_variable: str): if len(port_parts) == 2 or len(port_parts) == 3: vendor = port_parts[0].lower() product = port_parts[1].lower() - else: - pass outputText = tm.render( vendor=vendor, @@ -131,8 +129,8 @@ def normalize_env_variable_input(env_variable: str): "HIGH:!aNULL:!eNULL:!kECDH:!aDH:!RC4:!3DES:!CAMELLIA:!MD5:!PSK:!SRP:!KRB5:@STRENGTH", ), ebpf_no_sockets=int(os.getenv("SC4S_EBPF_NO_SOCKETS", 4)), - enable_parallelize=normalize_env_variable_input(f"SC4S_ENABLE_PARALLELIZE"), - parallelize_no_partitions=int(os.getenv(f"SC4S_PARALLELIZE_NO_PARTITION", 4)), + enable_parallelize=normalize_env_variable_input("SC4S_ENABLE_PARALLELIZE"), + parallelize_no_partitions=int(os.getenv("SC4S_PARALLELIZE_NO_PARTITION", 4)), set_source_sc4s=normalize_env_variable_input("SC4S_SET_SOURCE_AS_SC4S"), ) print(outputText) diff --git a/package/etc/pylib/config_generator/template_generator.py b/package/etc/pylib/config_generator/template_generator.py index 7832f532d7..1d0964e696 100644 --- a/package/etc/pylib/config_generator/template_generator.py +++ b/package/etc/pylib/config_generator/template_generator.py @@ -6,7 +6,7 @@ def template_generator(template_path: Path, **kwargs) -> str: env = jinja2.Environment( loader=jinja2.FileSystemLoader(template_path.parent), - autoescape=False, + autoescape=True, ) template = env.get_template(template_path.name) return template.render(**kwargs) diff --git a/package/etc/pylib/parser_cef.py b/package/etc/pylib/parser_cef.py index 7d0673fd45..28fd68498e 100644 --- a/package/etc/pylib/parser_cef.py +++ b/package/etc/pylib/parser_cef.py @@ -21,14 +21,13 @@ def parse(self, log_message): try: data = log_message.get_as_str(".metadata.cef.ext", "") - rpairs = re.findall(r"([^=\s]+)=((?:[\\]=|[^=])+)(?:\s|$)", data) + rpairs = re.findall(r"([^=\s]+)=((?:\\=|[^=])+)(?:\s|$)", data) pairs = {} keys = [] for p in rpairs: pairs[p[0]] = p[1] keys.append(p[0]) - cleanpairs = {} for k in keys: if k.endswith("Label"): vk = k.rstrip("Label") @@ -51,4 +50,4 @@ def parse(self, log_message): self.logger.debug("".join("!! " + line for line in lines)) return False self.logger.debug("kvqf_parse.parse complete") - return True \ No newline at end of file + return True diff --git a/package/etc/pylib/parser_fix_dns.py b/package/etc/pylib/parser_fix_dns.py index de3bbd7d15..8e72c97e68 100644 --- a/package/etc/pylib/parser_fix_dns.py +++ b/package/etc/pylib/parser_fix_dns.py @@ -3,7 +3,6 @@ resolves IP to hostname value pair names are hard-coded """ -import re import socket try: @@ -25,7 +24,7 @@ def parse(self, log_message): try: ipaddr = log_message.get_as_str("SOURCEIP", "", repr="internal") - hostname, aliaslist, ipaddrlist = socket.gethostbyaddr(ipaddr) + hostname, _, _ = socket.gethostbyaddr(ipaddr) if hostname == ipaddr: return False @@ -51,7 +50,7 @@ def parse(self, log_message): try: ipaddr = log_message.get_as_str("SOURCEIP", "", repr="internal") - fqdn, aliaslist, ipaddrlist = socket.gethostbyaddr(ipaddr) + fqdn, _, _ = socket.gethostbyaddr(ipaddr) if fqdn == ipaddr: return False @@ -61,4 +60,4 @@ def parse(self, log_message): return False # return True, other way message is dropped - return True \ No newline at end of file + return True diff --git a/package/etc/pylib/parser_kvqf.py b/package/etc/pylib/parser_kvqf.py index c5596c55f0..99206af8d1 100644 --- a/package/etc/pylib/parser_kvqf.py +++ b/package/etc/pylib/parser_kvqf.py @@ -26,7 +26,7 @@ def parse(self, log_message): matches = re.finditer( regex, log_message.get_as_str(".tmp.pairs", ""), re.MULTILINE ) - for matchNum, match in enumerate(matches, start=1): + for match_num, match in enumerate(matches, start=1): k = match.groups()[0] v = match.groups()[1] log_message[f".values.{k}"] = v @@ -36,4 +36,4 @@ def parse(self, log_message): self.logger.debug("".join("!! " + line for line in lines)) return False self.logger.debug("kvqf_parse.parse complete") - return True \ No newline at end of file + return True diff --git a/package/etc/pylib/parser_leef.py b/package/etc/pylib/parser_leef.py index 1279701314..5c5010cb2f 100644 --- a/package/etc/pylib/parser_leef.py +++ b/package/etc/pylib/parser_leef.py @@ -1,7 +1,5 @@ import re import binascii -import sys -import traceback try: import syslogng @@ -17,45 +15,64 @@ def init(self, options): self.regex = r"( ?(?:[A-Z]{2,4}T|HAEC|IDLW|MSK|NT|UTC|THA))" self.logger = syslogng.Logger() return True + + def parse_message_from_pair(self, pair, log_message): + f, v = pair.split("=", 1) + if f == "devTime": + log_message[".leef." + f] = re.sub( + self.regex, "", v, 0, re.MULTILINE + ) + else: + log_message[".leef." + f] = v - def parse(self, log_message): + def parse_v1(self, log_message, event, structure, separator): + pairs = event.split(separator) + if len(pairs) < 4: + separator = "|" + pairs = structure[5:] + event = "\t".join(pairs) + log_message[".leef.event"] = event + return event, pairs, separator + + def parse_v2(self, event, structure, separator): + # V2 messages should always provide the sep but some fail do comply + # with the format spec if they don't assume tab + if len(structure) == 6 or not structure[5]: + pairs = event.split(separator) + else: + separator = structure[5] + if separator.startswith("0"): + separator = separator[1:] + pairs = event.split(separator) + return event, pairs, separator + def parse(self, log_message): try: msg = log_message.get_as_str("MESSAGE", "") # All LEEF message are | separated super structures structure = msg.split("|") - # Indexed fields for Splunk + # Indexed fields for Splunk log_message[".metadata.leef.version"] = structure[0][5:] log_message[".metadata.leef.vendor"] = structure[1] log_message[".metadata.leef.product"] = structure[2] log_message[".metadata.leef.product_version"] = structure[3] log_message[".metadata.leef.EventID"] = structure[4] + # We just want the event field event = structure[len(structure) - 1] log_message[".leef.event"] = event + + separator = "\t" + pairs = [] + # V1 will always use tab if structure[0][5:].startswith("1"): - separator = "\t" lv = "1" - pairs = event.split(separator) - if len(pairs) < 4: - separator = "|" - pairs = structure[5:] - event = "\t".join(pairs) - log_message[".leef.event"] = event + event, pairs, separator = self.parse_v1(log_message, event, structure, separator) else: lv = "2" - # V2 messages should always provide the sep but some fail do comply - # with the format spec if they don't assume tab - if len(structure) == 6 or not structure[5]: - separator = "\t" - pairs = event.split(separator) - else: - separator = structure[5] - if separator.startswith("0"): - separator = separator[1:] - pairs = event.split(separator) + event, pairs, separator = self.parse_v2(event, structure, separator) if separator.startswith("x"): hex_sep = f"0{separator.lower()}" @@ -70,15 +87,9 @@ def parse(self, log_message): log_message["fields.sc4s_product"] = structure[2] for p in pairs: - f, v = p.split("=", 1) - if f == "devTime": - log_message[".leef." + f] = re.sub( - self.regex, "", v, 0, re.MULTILINE - ) - else: - log_message[".leef." + f] = v + self.parse_message_from_pair(p, log_message) except Exception as e: log_message[".metadata.leef.exception"] = str(e) # return True, other way message is dropped - return True \ No newline at end of file + return True diff --git a/package/etc/pylib/parser_stealthbits.py b/package/etc/pylib/parser_stealthbits.py index bdf0369816..07a3c87f10 100644 --- a/package/etc/pylib/parser_stealthbits.py +++ b/package/etc/pylib/parser_stealthbits.py @@ -10,6 +10,7 @@ class LogParser: regex = r"^(.*[\.\!\?])?(.*:.*)" +alert_text_key = ".values.AlertText" class alerttext_kv(LogParser): @@ -17,13 +18,13 @@ def init(self, options): return True def parse(self, log_message): - match = re.search(regex, log_message.get_as_str(".values.AlertText", "")) + match = re.search(regex, log_message.get_as_str(alert_text_key, "")) if match: - log_message[".values.AlertText"] = match.groups()[0] + log_message[alert_text_key] = match.groups()[0] text = match.groups()[1] else: - text = log_message.get_as_str(".values.AlertText", "") - log_message[".values.AlertText"] = "" + text = log_message.get_as_str(alert_text_key, "") + log_message[alert_text_key] = "" pairs = text.split("; ") @@ -33,4 +34,4 @@ def parse(self, log_message): k, v = p.split(": ") cleank = k.replace(" ", "_").replace(".", "_") log_message[f".values.AlertTextValues.{cleank}"] = v.strip() - return True \ No newline at end of file + return True diff --git a/package/etc/pylib/parser_vps_cache.py b/package/etc/pylib/parser_vps_cache.py index 4c8cf21250..d545620158 100644 --- a/package/etc/pylib/parser_vps_cache.py +++ b/package/etc/pylib/parser_vps_cache.py @@ -95,7 +95,3 @@ def send(self, log_message): def flush(self): self.db.commit() return True - - -if __name__ == "__main__": - pass diff --git a/package/etc/test_parsers/app-vps-test-netapp_ontap.conf b/package/etc/test_parsers/app-vps-test-netapp_ontap.conf index 24b14274e0..1c2f06f5a5 100644 --- a/package/etc/test_parsers/app-vps-test-netapp_ontap.conf +++ b/package/etc/test_parsers/app-vps-test-netapp_ontap.conf @@ -1,6 +1,7 @@ application app-vps-test-netapp_ontap[sc4s-vps] { filter { host("netapp-ontap-" type(string) flags(prefix)) + or message("[netapp-ontap-" type(string) flags(prefix)) or ( message("netapp-ontap-" type(string) flags(prefix)) and program("netapp-ontap-" type(string) flags(prefix)) diff --git a/package/lite/etc/addons/arista/addon_metadata.yaml b/package/lite/etc/addons/arista/addon_metadata.yaml index 7967e0bd48..e2cc48e131 100644 --- a/package/lite/etc/addons/arista/addon_metadata.yaml +++ b/package/lite/etc/addons/arista/addon_metadata.yaml @@ -1,2 +1,2 @@ --- -name: "arista" \ No newline at end of file +name: "arista" diff --git a/package/lite/etc/addons/aruba/addon_metadata.yaml b/package/lite/etc/addons/aruba/addon_metadata.yaml index 9dd24354aa..a6c692ceae 100644 --- a/package/lite/etc/addons/aruba/addon_metadata.yaml +++ b/package/lite/etc/addons/aruba/addon_metadata.yaml @@ -1,2 +1,2 @@ --- -name: "aruba" \ No newline at end of file +name: "aruba" diff --git a/package/lite/etc/addons/avi/addon_metadata.yaml b/package/lite/etc/addons/avi/addon_metadata.yaml index 22c1b67bec..0f820c3eda 100644 --- a/package/lite/etc/addons/avi/addon_metadata.yaml +++ b/package/lite/etc/addons/avi/addon_metadata.yaml @@ -1,2 +1,2 @@ --- -name: "avi" \ No newline at end of file +name: "avi" diff --git a/package/lite/etc/addons/barracuda/addon_metadata.yaml b/package/lite/etc/addons/barracuda/addon_metadata.yaml index c6b6e6daa7..131ff1c470 100644 --- a/package/lite/etc/addons/barracuda/addon_metadata.yaml +++ b/package/lite/etc/addons/barracuda/addon_metadata.yaml @@ -1,2 +1,2 @@ --- -name: "barracuda" \ No newline at end of file +name: "barracuda" diff --git a/package/lite/etc/addons/beyondtrust/addon_metadata.yaml b/package/lite/etc/addons/beyondtrust/addon_metadata.yaml index 9f5c31940b..3101f0351e 100644 --- a/package/lite/etc/addons/beyondtrust/addon_metadata.yaml +++ b/package/lite/etc/addons/beyondtrust/addon_metadata.yaml @@ -1,2 +1,2 @@ --- -name: "beyondtrust" \ No newline at end of file +name: "beyondtrust" diff --git a/package/lite/etc/addons/broadcom/addon_metadata.yaml b/package/lite/etc/addons/broadcom/addon_metadata.yaml index 1c503fd3eb..c88bc49c81 100644 --- a/package/lite/etc/addons/broadcom/addon_metadata.yaml +++ b/package/lite/etc/addons/broadcom/addon_metadata.yaml @@ -1,2 +1,2 @@ --- -name: "broadcom" \ No newline at end of file +name: "broadcom" diff --git a/package/lite/etc/addons/checkpoint/addon_metadata.yaml b/package/lite/etc/addons/checkpoint/addon_metadata.yaml index 4f97ea1464..1bad87e3d8 100644 --- a/package/lite/etc/addons/checkpoint/addon_metadata.yaml +++ b/package/lite/etc/addons/checkpoint/addon_metadata.yaml @@ -1,2 +1,2 @@ --- -name: "checkpoint" \ No newline at end of file +name: "checkpoint" diff --git a/package/lite/etc/addons/cisco/addon_metadata.yaml b/package/lite/etc/addons/cisco/addon_metadata.yaml index 43ebf167c7..c4d6718b68 100644 --- a/package/lite/etc/addons/cisco/addon_metadata.yaml +++ b/package/lite/etc/addons/cisco/addon_metadata.yaml @@ -1,2 +1,2 @@ --- -name: "cisco" \ No newline at end of file +name: "cisco" diff --git a/package/lite/etc/addons/clearswift/addon_metadata.yaml b/package/lite/etc/addons/clearswift/addon_metadata.yaml index ef72b194b6..2a8cc0bf9f 100644 --- a/package/lite/etc/addons/clearswift/addon_metadata.yaml +++ b/package/lite/etc/addons/clearswift/addon_metadata.yaml @@ -1,2 +1,2 @@ --- -name: "clearswift" \ No newline at end of file +name: "clearswift" diff --git a/package/lite/etc/addons/cyberark/addon_metadata.yaml b/package/lite/etc/addons/cyberark/addon_metadata.yaml index 2c5e33f861..ec351d3cfb 100644 --- a/package/lite/etc/addons/cyberark/addon_metadata.yaml +++ b/package/lite/etc/addons/cyberark/addon_metadata.yaml @@ -1,2 +1,2 @@ --- -name: "cyberark" \ No newline at end of file +name: "cyberark" diff --git a/package/lite/etc/addons/cylance/addon_metadata.yaml b/package/lite/etc/addons/cylance/addon_metadata.yaml index 456a315497..abcb1dbc3e 100644 --- a/package/lite/etc/addons/cylance/addon_metadata.yaml +++ b/package/lite/etc/addons/cylance/addon_metadata.yaml @@ -1,2 +1,2 @@ --- -name: "cylance" \ No newline at end of file +name: "cylance" diff --git a/package/lite/etc/addons/darktrace/addon_metadata.yaml b/package/lite/etc/addons/darktrace/addon_metadata.yaml index 5d7b4f6e63..5ab2ef712c 100644 --- a/package/lite/etc/addons/darktrace/addon_metadata.yaml +++ b/package/lite/etc/addons/darktrace/addon_metadata.yaml @@ -1,2 +1,2 @@ --- -name: "darktrace" \ No newline at end of file +name: "darktrace" diff --git a/package/lite/etc/addons/dell/addon_metadata.yaml b/package/lite/etc/addons/dell/addon_metadata.yaml index e23728a52a..a2701ad6e4 100644 --- a/package/lite/etc/addons/dell/addon_metadata.yaml +++ b/package/lite/etc/addons/dell/addon_metadata.yaml @@ -1,2 +1,2 @@ --- -name: "dell" \ No newline at end of file +name: "dell" diff --git a/package/lite/etc/addons/epic/addon_metadata.yaml b/package/lite/etc/addons/epic/addon_metadata.yaml index b3a345bf78..f6fe71ecd7 100644 --- a/package/lite/etc/addons/epic/addon_metadata.yaml +++ b/package/lite/etc/addons/epic/addon_metadata.yaml @@ -1,2 +1,2 @@ --- -name: "epic" \ No newline at end of file +name: "epic" diff --git a/package/lite/etc/addons/example_addon/addon_metadata.yaml b/package/lite/etc/addons/example_addon/addon_metadata.yaml index ce7492d597..a439dd47d6 100644 --- a/package/lite/etc/addons/example_addon/addon_metadata.yaml +++ b/package/lite/etc/addons/example_addon/addon_metadata.yaml @@ -1,2 +1,2 @@ --- -name: "Example addon" \ No newline at end of file +name: "Example addon" diff --git a/package/lite/etc/addons/f5/addon_metadata.yaml b/package/lite/etc/addons/f5/addon_metadata.yaml index 92073dad32..50ffdc9835 100644 --- a/package/lite/etc/addons/f5/addon_metadata.yaml +++ b/package/lite/etc/addons/f5/addon_metadata.yaml @@ -1,2 +1,2 @@ --- -name: "f5" \ No newline at end of file +name: "f5" diff --git a/package/lite/etc/addons/fireeye/addon_metadata.yaml b/package/lite/etc/addons/fireeye/addon_metadata.yaml index 10968f7725..7fe003d527 100644 --- a/package/lite/etc/addons/fireeye/addon_metadata.yaml +++ b/package/lite/etc/addons/fireeye/addon_metadata.yaml @@ -1,2 +1,2 @@ --- -name: "fireeye" \ No newline at end of file +name: "fireeye" diff --git a/package/lite/etc/addons/forcepoint/addon_metadata.yaml b/package/lite/etc/addons/forcepoint/addon_metadata.yaml index 0f78bee803..a5ce331fe6 100644 --- a/package/lite/etc/addons/forcepoint/addon_metadata.yaml +++ b/package/lite/etc/addons/forcepoint/addon_metadata.yaml @@ -1,2 +1,2 @@ --- -name: "forcepoint" \ No newline at end of file +name: "forcepoint" diff --git a/package/lite/etc/addons/hp/addon_metadata.yaml b/package/lite/etc/addons/hp/addon_metadata.yaml index 7408122ca7..bfddb95a6e 100644 --- a/package/lite/etc/addons/hp/addon_metadata.yaml +++ b/package/lite/etc/addons/hp/addon_metadata.yaml @@ -1,2 +1,2 @@ --- -name: "hp" \ No newline at end of file +name: "hp" diff --git a/package/lite/etc/addons/ibm/addon_metadata.yaml b/package/lite/etc/addons/ibm/addon_metadata.yaml index 4b460ef77d..c008d59986 100644 --- a/package/lite/etc/addons/ibm/addon_metadata.yaml +++ b/package/lite/etc/addons/ibm/addon_metadata.yaml @@ -1,2 +1,2 @@ --- -name: "ibm" \ No newline at end of file +name: "ibm" diff --git a/package/lite/etc/addons/imperva/addon_metadata.yaml b/package/lite/etc/addons/imperva/addon_metadata.yaml index c94e72d497..027ffef0be 100644 --- a/package/lite/etc/addons/imperva/addon_metadata.yaml +++ b/package/lite/etc/addons/imperva/addon_metadata.yaml @@ -1,2 +1,2 @@ --- -name: "imperva" \ No newline at end of file +name: "imperva" diff --git a/package/lite/etc/addons/juniper/addon_metadata.yaml b/package/lite/etc/addons/juniper/addon_metadata.yaml index 0421487385..9362c5d57d 100644 --- a/package/lite/etc/addons/juniper/addon_metadata.yaml +++ b/package/lite/etc/addons/juniper/addon_metadata.yaml @@ -1,2 +1,2 @@ --- -name: "juniper" \ No newline at end of file +name: "juniper" diff --git a/package/lite/etc/addons/kaspersky/addon_metadata.yaml b/package/lite/etc/addons/kaspersky/addon_metadata.yaml index f04054ffbf..b481e19a6c 100644 --- a/package/lite/etc/addons/kaspersky/addon_metadata.yaml +++ b/package/lite/etc/addons/kaspersky/addon_metadata.yaml @@ -1,2 +1,2 @@ --- -name: "kaspersky" \ No newline at end of file +name: "kaspersky" diff --git a/package/lite/etc/addons/liveaction/addon_metadata.yaml b/package/lite/etc/addons/liveaction/addon_metadata.yaml index f4bc997119..03a8d8dde1 100644 --- a/package/lite/etc/addons/liveaction/addon_metadata.yaml +++ b/package/lite/etc/addons/liveaction/addon_metadata.yaml @@ -1,2 +1,2 @@ --- -name: "liveaction" \ No newline at end of file +name: "liveaction" diff --git a/package/lite/etc/addons/mcafee/addon_metadata.yaml b/package/lite/etc/addons/mcafee/addon_metadata.yaml index f20b6ea356..b9eaf43bcd 100644 --- a/package/lite/etc/addons/mcafee/addon_metadata.yaml +++ b/package/lite/etc/addons/mcafee/addon_metadata.yaml @@ -1,2 +1,2 @@ --- -name: "mcafee" \ No newline at end of file +name: "mcafee" diff --git a/package/lite/etc/addons/microfocus/addon_metadata.yaml b/package/lite/etc/addons/microfocus/addon_metadata.yaml index a4cf771328..6c4c777edc 100644 --- a/package/lite/etc/addons/microfocus/addon_metadata.yaml +++ b/package/lite/etc/addons/microfocus/addon_metadata.yaml @@ -1 +1,2 @@ +--- name: microfocus diff --git a/package/lite/etc/addons/netapp/addon_metadata.yaml b/package/lite/etc/addons/netapp/addon_metadata.yaml index 43b4089da1..4ad26ab248 100644 --- a/package/lite/etc/addons/netapp/addon_metadata.yaml +++ b/package/lite/etc/addons/netapp/addon_metadata.yaml @@ -1,2 +1,2 @@ --- -name: "netapp" \ No newline at end of file +name: "netapp" diff --git a/package/lite/etc/addons/netapp/app-netsource-netapp_ontap.conf b/package/lite/etc/addons/netapp/app-netsource-netapp_ontap.conf index 08cd806164..60d72bcfd7 100644 --- a/package/lite/etc/addons/netapp/app-netsource-netapp_ontap.conf +++ b/package/lite/etc/addons/netapp/app-netsource-netapp_ontap.conf @@ -34,6 +34,26 @@ block parser app-netsource-netapp_ontap() { class('audit') ); }; + } elif { + parser { + regexp-parser( + prefix(".tmp.") + patterns('\[(?[^:]+):(?[^:]+):(?[^\]]+)\]: (?.*)') + template("${MESSAGE}") + ); + }; + rewrite { + set('${.tmp.message}' value('MESSAGE')); + set('${.tmp.host}' value('HOST')); + set('${.tmp.category}' value('fields.category')); + set('${.tmp.severity}' value('fields.severity')); + }; + rewrite { + r_set_splunk_dest_update_v2( + sourcetype('netapp:ontap:ems') + class('ems') + ); + }; } else { rewrite { r_set_splunk_dest_update_v2( @@ -46,10 +66,10 @@ block parser app-netsource-netapp_ontap() { }; application app-netsource-netapp_ontap[sc4s-network-source] { - filter { + filter { match("netapp", value('.netsource.sc4s_vendor'), type(string)) and match("ontap", value('.netsource.sc4s_product'), type(string)) and "`SC4S_NETAPP_ONTAP_NEW_FORMAT`" eq "yes" - }; + }; parser { app-netsource-netapp_ontap(); }; -}; +}; \ No newline at end of file diff --git a/package/lite/etc/addons/palo-alto/addon_metadata.yaml b/package/lite/etc/addons/palo-alto/addon_metadata.yaml index b3c45ef289..768ef1e129 100644 --- a/package/lite/etc/addons/palo-alto/addon_metadata.yaml +++ b/package/lite/etc/addons/palo-alto/addon_metadata.yaml @@ -1,2 +1,2 @@ --- -name: "paloalto" \ No newline at end of file +name: "paloalto" diff --git a/package/lite/etc/addons/polycom/addon_metadata.yaml b/package/lite/etc/addons/polycom/addon_metadata.yaml index 7868d2bade..dc95feff59 100644 --- a/package/lite/etc/addons/polycom/addon_metadata.yaml +++ b/package/lite/etc/addons/polycom/addon_metadata.yaml @@ -1 +1,2 @@ +--- name: polycom diff --git a/package/lite/etc/addons/pulse/addon_metadata.yaml b/package/lite/etc/addons/pulse/addon_metadata.yaml index 8956b0f020..9342fd06c7 100644 --- a/package/lite/etc/addons/pulse/addon_metadata.yaml +++ b/package/lite/etc/addons/pulse/addon_metadata.yaml @@ -1 +1,2 @@ +--- name: pulse diff --git a/package/lite/etc/addons/qumulo/addon_metadata.yaml b/package/lite/etc/addons/qumulo/addon_metadata.yaml index 731f407fb7..244b465b66 100644 --- a/package/lite/etc/addons/qumulo/addon_metadata.yaml +++ b/package/lite/etc/addons/qumulo/addon_metadata.yaml @@ -1,2 +1,2 @@ --- -name: "qumulo" \ No newline at end of file +name: "qumulo" diff --git a/package/lite/etc/addons/tanium/addon_metadata.yaml b/package/lite/etc/addons/tanium/addon_metadata.yaml index ca24297f9d..7454768ef0 100644 --- a/package/lite/etc/addons/tanium/addon_metadata.yaml +++ b/package/lite/etc/addons/tanium/addon_metadata.yaml @@ -1,2 +1,2 @@ --- -name: "tanium" \ No newline at end of file +name: "tanium" diff --git a/package/lite/etc/addons/thales/addon_metadata.yaml b/package/lite/etc/addons/thales/addon_metadata.yaml index 9c958d18ee..22217ce83f 100644 --- a/package/lite/etc/addons/thales/addon_metadata.yaml +++ b/package/lite/etc/addons/thales/addon_metadata.yaml @@ -1,2 +1,2 @@ --- -name: "thales" \ No newline at end of file +name: "thales" diff --git a/package/lite/etc/addons/tintri/addon_metadata.yaml b/package/lite/etc/addons/tintri/addon_metadata.yaml index 18060cc0dd..a7f456825a 100644 --- a/package/lite/etc/addons/tintri/addon_metadata.yaml +++ b/package/lite/etc/addons/tintri/addon_metadata.yaml @@ -1 +1,2 @@ +--- name: tintri diff --git a/package/lite/etc/addons/trendmicro/addon_metadata.yaml b/package/lite/etc/addons/trendmicro/addon_metadata.yaml index c14a93487a..42733d35be 100644 --- a/package/lite/etc/addons/trendmicro/addon_metadata.yaml +++ b/package/lite/etc/addons/trendmicro/addon_metadata.yaml @@ -1,2 +1,2 @@ --- -name: "trendmicro" \ No newline at end of file +name: "trendmicro" diff --git a/package/lite/etc/addons/ubiquiti/addon_metadata.yaml b/package/lite/etc/addons/ubiquiti/addon_metadata.yaml index a6c51a579a..aa422590c9 100644 --- a/package/lite/etc/addons/ubiquiti/addon_metadata.yaml +++ b/package/lite/etc/addons/ubiquiti/addon_metadata.yaml @@ -1,2 +1,2 @@ --- -name: "ubiquiti" \ No newline at end of file +name: "ubiquiti" diff --git a/package/lite/etc/addons/vectra/app-syslog-vectra_json.conf b/package/lite/etc/addons/vectra/app-syslog-vectra_json.conf new file mode 100644 index 0000000000..e618c550bd --- /dev/null +++ b/package/lite/etc/addons/vectra/app-syslog-vectra_json.conf @@ -0,0 +1,104 @@ +block parser app-syslog-vectra-json() { + channel { + parser { + regexp-parser( + prefix(".tmp.") + patterns('\"vectra_timestamp\"\:\s\"(?[^\"]+)\"') + template("$MESSAGE") + ); + date-parser-nofilter( + format('%s') + template("${.tmp.timestamp}") + ); + }; + + rewrite { + subst('\-\:\s',"",value("MESSAGE")); + }; + + rewrite { + r_set_splunk_dest_default( + index("main") + sourcetype('vectra:cognito:detect:json') + vendor("vectra") + product("cognito detect") + class('detect') + template("t_msg_only") + ); + }; + + if (message('\"host_\w+\"\:')) { + rewrite { + r_set_splunk_dest_update_v2( + sourcetype('vectra:cognito:hostscoring:json') + class('hostscoring') + condition(message('\"HOST\sSCORING\"')) + ); + }; + rewrite { + r_set_splunk_dest_update_v2( + sourcetype('vectra:cognito:hostdetect:json') + class('hostdetect') + condition(message('\"detection_id\"\:')) + ); + }; + rewrite { + r_set_splunk_dest_update_v2( + sourcetype('vectra:cognito:hostlockdown:json') + class('hostlockdown') + condition(message('\"success\"\:')) + ); + }; + } elif (message('\"account_uid\"\:')) { + rewrite { + r_set_splunk_dest_update_v2( + sourcetype('vectra:cognito:accountscoring:json') + class('accountscoring') + condition(message('\"ACCOUNT\sSCORING\"')) + ); + }; + rewrite { + r_set_splunk_dest_update_v2( + sourcetype('vectra:cognito:accountdetect:json') + class('accountdetect') + condition(message('\"detection_id\"\:')) + ); + }; + rewrite { + r_set_splunk_dest_update_v2( + sourcetype('vectra:cognito:accountlockdown:json') + class('accountlockdown') + condition(message('\"success\"\:')) + ); + }; + } elif (message('\"campaign_id\"\:')) { + rewrite { + r_set_splunk_dest_update_v2( + sourcetype('vectra:cognito:campaigns:json') + class('campaigns') + ); + }; + } elif (message('\"role\"\:')) { + rewrite { + r_set_splunk_dest_update_v2( + sourcetype('vectra:cognito:audit:json') + class('audit') + ); + }; + } elif (message('\"type\"\:')) { + rewrite { + r_set_splunk_dest_update_v2( + sourcetype('vectra:cognito:health:json') + class('health') + ); + }; + } else {}; + }; +}; + +application app-syslog-vectra-json[sc4s-syslog-pgm] { + filter { + program('vectra_json' type(string) flags(prefix)); + }; + parser { app-syslog-vectra-json(); }; +}; \ No newline at end of file diff --git a/package/lite/etc/addons/vmware/addon_metadata.yaml b/package/lite/etc/addons/vmware/addon_metadata.yaml index 520ee96b87..1225c7ca1a 100644 --- a/package/lite/etc/addons/vmware/addon_metadata.yaml +++ b/package/lite/etc/addons/vmware/addon_metadata.yaml @@ -1,2 +1,2 @@ --- -name: "vmware" \ No newline at end of file +name: "vmware" diff --git a/package/lite/etc/addons/zscaler/addon_metadata.yaml b/package/lite/etc/addons/zscaler/addon_metadata.yaml index aebcdc7e8a..aebae9e5c5 100644 --- a/package/lite/etc/addons/zscaler/addon_metadata.yaml +++ b/package/lite/etc/addons/zscaler/addon_metadata.yaml @@ -1,2 +1,2 @@ --- -name: "zscaler" \ No newline at end of file +name: "zscaler" diff --git a/package/lite/etc/config.yaml b/package/lite/etc/config.yaml index bcd62f83b8..b2b4871b35 100644 --- a/package/lite/etc/config.yaml +++ b/package/lite/etc/config.yaml @@ -85,4 +85,4 @@ addons: - trellix - netwrix - aviatrix - - veeam \ No newline at end of file + - veeam diff --git a/package/sbin/entrypoint.sh b/package/sbin/entrypoint.sh index d5a36b0bac..daadbde39e 100755 --- a/package/sbin/entrypoint.sh +++ b/package/sbin/entrypoint.sh @@ -222,7 +222,7 @@ echo syslog-ng checking config export SC4S_VERSION=$(cat $SC4S_ETC/VERSION) echo sc4s version=$(cat $SC4S_ETC/VERSION) echo sc4s version=$(cat $SC4S_ETC/VERSION) >>$SC4S_VAR/log/syslog-ng.out -$SC4S_SBIN/syslog-ng --no-caps $SC4S_CONTAINER_OPTS -s >>$SC4S_VAR/log/syslog-ng.out 2>$SC4S_VAR/log/syslog-ng.err +"${SC4S_SBIN}"/syslog-ng --no-caps $SC4S_CONTAINER_OPTS -s >>$SC4S_VAR/log/syslog-ng.out 2>$SC4S_VAR/log/syslog-ng.err echo "Configuring the health check port to: $SC4S_LISTEN_STATUS_PORT" nohup gunicorn -b 0.0.0.0:$SC4S_LISTEN_STATUS_PORT healthcheck:app & @@ -249,7 +249,7 @@ fi while : do echo starting syslog-ng - $SC4S_SBIN/syslog-ng --no-caps $SC4S_CONTAINER_OPTS -F $@ & + "${SC4S_SBIN}"/syslog-ng --no-caps "${SC4S_CONTAINER_OPTS}" -F "${@}" & pid="$!" sleep 2 if [ "${SC4S_DEBUG_CONTAINER}" == "yes" ] diff --git a/package/sbin/healthcheck.py b/package/sbin/healthcheck.py index d6f515fc66..13cb443966 100644 --- a/package/sbin/healthcheck.py +++ b/package/sbin/healthcheck.py @@ -1,9 +1,14 @@ -from flask import Flask, jsonify import logging import os import subprocess +import re + +from flask_wtf.csrf import CSRFProtect +from flask import Flask, jsonify app = Flask(__name__) +csrf = CSRFProtect() +csrf.init_app(app) def str_to_bool(value): return str(value).strip().lower() in { @@ -14,14 +19,23 @@ def str_to_bool(value): 'yes' } +def get_list_of_destinations(): + found_destinations = [] + regex = r"^SC4S_DEST_SPLUNK_HEC_(.*)_URL$" + + for var_key, var_variable in os.environ.items(): + if re.search(regex, var_key): + found_destinations.append(var_variable) + return set(found_destinations) + class Config: - SC4S_DEST_SPLUNK_HEC_DEFAULT_URL = os.getenv('SC4S_DEST_SPLUNK_HEC_DEFAULT_URL') HEALTHCHECK_PORT = int(os.getenv('SC4S_LISTEN_STATUS_PORT', '8080')) CHECK_QUEUE_SIZE = str_to_bool(os.getenv('HEALTHCHECK_CHECK_QUEUE_SIZE', "false")) MAX_QUEUE_SIZE = int(os.getenv('HEALTHCHECK_MAX_QUEUE_SIZE', '10000')) + DESTINATIONS = get_list_of_destinations() logging.basicConfig( - format=f"%(asctime)s - healthcheck.py - %(levelname)s - %(message)s", + format="%(asctime)s - healthcheck.py - %(levelname)s - %(message)s", datefmt="%Y-%m-%d %H:%M:%S" ) logger = logging.getLogger(__name__) @@ -48,11 +62,11 @@ def check_syslog_ng_health() -> bool: return False def check_queue_size( - sc4s_dest_splunk_hec_default=Config.SC4S_DEST_SPLUNK_HEC_DEFAULT_URL, + sc4s_dest_splunk_hec_destinations=Config.DESTINATIONS, max_queue_size=Config.MAX_QUEUE_SIZE ) -> bool: """Check syslog-ng queue size and compare it against the configured maximum limit.""" - if not sc4s_dest_splunk_hec_default: + if not sc4s_dest_splunk_hec_destinations: logger.error( "SC4S_DEST_SPLUNK_HEC_DEFAULT_URL not configured. " "Ensure the default HEC destination is set, or disable HEALTHCHECK_CHECK_QUEUE_SIZE." @@ -71,15 +85,22 @@ def check_queue_size( return False stats = result.stdout.splitlines() - destination_stat = next( - (s for s in stats if ";queued;" in s and sc4s_dest_splunk_hec_default in s), - None - ) - if not destination_stat: - logger.error("No matching queue stats found for the destination URL.") - return False - queue_size = int(destination_stat.split(";")[-1]) + queue_sizes_all_destinations = [] + + for destination in sc4s_dest_splunk_hec_destinations: + destination_stat = next( + (s for s in stats if ";queued;" in s and destination in s), + None + ) + + if not destination_stat: + logger.error(f"No matching queue stats found for the destination URL {destination}.") + return False + + queue_sizes_all_destinations.append(int(destination_stat.split(";")[-1])) + + queue_size = max(queue_sizes_all_destinations) if queue_size > max_queue_size: logger.warning( f"Queue size {queue_size} exceeds the maximum limit of {max_queue_size}." diff --git a/poetry.lock b/poetry.lock index c7dfa6cc80..6f33c24119 100644 --- a/poetry.lock +++ b/poetry.lock @@ -376,6 +376,26 @@ Werkzeug = ">=3.1" async = ["asgiref (>=3.2)"] dotenv = ["python-dotenv"] +[[package]] +name = "flask-wtf" +version = "1.2.2" +description = "Form rendering, validation, and CSRF protection for Flask with WTForms." +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "flask_wtf-1.2.2-py3-none-any.whl", hash = "sha256:e93160c5c5b6b571cf99300b6e01b72f9a101027cab1579901f8b10c5daf0b70"}, + {file = "flask_wtf-1.2.2.tar.gz", hash = "sha256:79d2ee1e436cf570bccb7d916533fa18757a2f18c290accffab1b9a0b684666b"}, +] + +[package.dependencies] +flask = "*" +itsdangerous = "*" +wtforms = "*" + +[package.extras] +email = ["email-validator"] + [[package]] name = "ghp-import" version = "2.1.0" @@ -1578,6 +1598,24 @@ MarkupSafe = ">=2.1.1" [package.extras] watchdog = ["watchdog (>=2.3)"] +[[package]] +name = "wtforms" +version = "3.2.1" +description = "Form validation and rendering for Python web development." +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "wtforms-3.2.1-py3-none-any.whl", hash = "sha256:583bad77ba1dd7286463f21e11aa3043ca4869d03575921d1a1698d0715e0fd4"}, + {file = "wtforms-3.2.1.tar.gz", hash = "sha256:df3e6b70f3192e92623128123ec8dca3067df9cfadd43d59681e210cfb8d4682"}, +] + +[package.dependencies] +markupsafe = "*" + +[package.extras] +email = ["email-validator"] + [[package]] name = "zipp" version = "3.21.0" @@ -1602,4 +1640,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.1" python-versions = "^3.9" -content-hash = "fb25c962740281b32d148066dd58bd4dc0f3a5e1f82acfed313dc94348530d9b" +content-hash = "3591d4f12184582831eeef88cf47d886a56694639a9c097442d31c6c43fde536" diff --git a/pyproject.toml b/pyproject.toml index e7e3545a0f..b8a5e7095d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,6 +17,7 @@ restricted-sqlitedict = "^1.0.0" tornado = "^6.4.2" gunicorn = "^23.0.0" flask = "^3.1.0" +flask-wtf = "^1.2.2" [tool.poetry.group.dev.dependencies] pytest = "^8.0.0" diff --git a/pytest.ini b/pytest.ini index 3827f777e4..b2a59a5010 100644 --- a/pytest.ini +++ b/pytest.ini @@ -10,6 +10,11 @@ filterwarnings = ignore::DeprecationWarning junit_family=xunit1 +markers = + addons: mark a test as an addon test + lite: mark a test as a lite test + name_cache: mark a test as a name_cache test + ; rp_endpoint = https://rp.spl.guru/ ; rp_project = ; rp_launch = Regression Test Suite diff --git a/sonar-project.properties b/sonar-project.properties index dc0fcdc279..232fc682e1 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -1,3 +1,6 @@ sonar.projectKey=github-mirrors.splunk-connect-for-syslog -sonar.exclusions=docs/**, mkdocs.yml, mkdocs-rtd.yml -sonar.coverage.exclusions=tests/** \ No newline at end of file +sonar.exclusions=\ + /docs/**, mkdocs.yml, mkdocs-rtd.yml, /tests/**, \ + /charts/splunk-connect-for-syslog/templates/*.yaml, \ + /charts/splunk-connect-for-syslog/templates/tests/*.yaml, \ + /ansible/*.yaml, /ansible/*.yml, /ansible/*/*.yaml, /ansible/*/*.yml diff --git a/tests/Dockerfile.nc b/tests/Dockerfile.nc index 020d4da191..2c1a64f3aa 100644 --- a/tests/Dockerfile.nc +++ b/tests/Dockerfile.nc @@ -1,4 +1,4 @@ ARG SYSLOGNG_VERSION=4.9.0 FROM ghcr.io/axoflow/axosyslog:${SYSLOGNG_VERSION} -RUN apk add -U netcat-openbsd +RUN apk --no-cache add -U netcat-openbsd diff --git a/tests/conftest.py b/tests/conftest.py index c8cb68f8a7..7141695854 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -5,7 +5,6 @@ # https://opensource.org/licenses/BSD-2-Clause import os import uuid -import socket import shortuuid from time import sleep import random @@ -136,7 +135,7 @@ def splunk(request): request.fixturenames.append("splunk_docker") splunk = request.getfixturevalue("splunk_docker") else: - raise Exception + raise ValueError yield splunk @@ -150,7 +149,7 @@ def sc4s(request): request.fixturenames.append("sc4s_docker") sc4s = request.getfixturevalue("sc4s_docker") else: - raise Exception + raise ValueError yield sc4s diff --git a/tests/docker-compose.yml b/tests/docker-compose.yml index 23e6aae911..68b652f86a 100644 --- a/tests/docker-compose.yml +++ b/tests/docker-compose.yml @@ -1,11 +1,12 @@ -#Splunk Connect for Syslog (SC4S) by Splunk, Inc. +# Splunk Connect for Syslog (SC4S) by Splunk, Inc. # -#To the extent possible under law, the person who associated CC0 with -#Splunk Connect for Syslog (SC4S) has waived all copyright and related or neighboring rights -#to Splunk Connect for Syslog (SC4S). +# To the extent possible under law, the person who associated CC0 with +# Splunk Connect for Syslog (SC4S) has waived all copyright and related or +# neighboring rights to Splunk Connect for Syslog (SC4S). # -#You should have received a copy of the CC0 legalcode along with this -#work. If not, see . +# You should have received a copy of the CC0 legalcode along with this +# work. If not, see . +--- version: "3.7" services: @@ -37,7 +38,7 @@ services: - SC4S_DEST_SPLUNK_HEC_DEFAULT_URL=https://splunk:8088 - SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN=${SPLUNK_HEC_TOKEN} - SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_VERIFY=no - #- SC4S_DEST_SPLUNK_HEC_DEFAULT_HTTP_COMPRESSION=yes + # - SC4S_DEST_SPLUNK_HEC_DEFAULT_HTTP_COMPRESSION=yes - SC4S_DEST_SYSLOG_NCSYSLOG_HOST=ncsyslog - SC4S_DEST_SYSLOG_NCSYSLOG_PORT=2514 - SC4S_DEST_SYSLOG_NCSYSLOG_MODE=GLOBAL @@ -55,14 +56,14 @@ services: - SC4S_LISTEN_SIMPLE_TEST_ONE_TCP_PORT=5514 - SC4S_LISTEN_SIMPLE_TEST_ONE_UDP_PORT=5514 - SC4S_LISTEN_SIMPLE_TEST_TWO_TCP_PORT=5601 - #- SC4S_ARCHIVE_GLOBAL=yes + # - SC4S_ARCHIVE_GLOBAL=yes - SC4S_DEST_ALCATEL_SWITCH_ARCHIVE=yes - SC4S_DEST_ALCATEL_SWITCH_ALTERNATES=d_syslog_nc - SC4S_SOURCE_STORE_RAWMSG=yes - SC4S_LISTEN_CHECKPOINT_SPLUNK_NOISE_CONTROL=yes - SC4S_SOURCE_LISTEN_UDP_SOCKETS=2 - SC4S_LISTEN_DEFAULT_RFC6587_PORT=601,7001 - #- SC4S_DEST_GLOBAL_ALTERNATES=d_hec_debug + # - SC4S_DEST_GLOBAL_ALTERNATES=d_hec_debug - SC4S_DEST_SPECTRACOM_NTP_ALT_FILTER=f_is_rfc3164 - SC4S_DEST_SPECTRACOM_NTP_FILTERED_ALTERNATES=d_hec_debug,d_archive - SC4S_SOURCE_RICOH_SYSLOG_FIXHOST=yes @@ -71,21 +72,20 @@ services: - SC4S_SOURCE_TLS_ENABLE=yes - SC4S_SOURCE_TLS_SELFSIGNED=yes - SC4S_LISTEN_PROOFPOINT_PPS_TLS_PORT=7000 - #- SC4S_DEST_SPLUNK_HEC_GLOBAL=no - #- SC4S_DEST_CEF_SPLUNK_HEC=yes + # - SC4S_DEST_SPLUNK_HEC_GLOBAL=no + # - SC4S_DEST_CEF_SPLUNK_HEC=yes - SC4S_USE_NAME_CACHE=no - SC4S_CLEAR_NAME_CACHE=no - SC4S_SOURCE_VMWARE_VSPHERE_GROUPMSG=yes - SC4S_USE_VPS_CACHE=yes - - SC4S_SOURCE_UDP_SO_RCVBUFF=17039360 - #- SC4S_DEST_SPLUNK_HEC_GLOBAL=yes + # - SC4S_DEST_SPLUNK_HEC_GLOBAL=yes - SC4S_LISTEN_SIMPLE_ATA_SYSLOG_TCP_PORT=6500 - SC4S_LISTEN_SIMPLE_SURICATA_SYSLOG_TCP_PORT=6501 - SC4S_LISTEN_SIMPLE_BRO_SYSLOG_TCP_PORT=6502 - SC4S_LISTEN_DEFAULT_TCP_PORT=514,6504,6505,6506 - #SC4S_USE_REVERSE_DNS=YES - #SC4S_REVERSE_DNS_KEEP_FQDN=YES + # SC4S_USE_REVERSE_DNS=YES + # SC4S_REVERSE_DNS_KEEP_FQDN=YES splunk: image: docker.io/splunk/splunk:latest hostname: splunk diff --git a/tests/entrypoint.sh b/tests/entrypoint.sh index 157ddcc557..eb4c5c91c2 100755 --- a/tests/entrypoint.sh +++ b/tests/entrypoint.sh @@ -1,4 +1,4 @@ #!/bin/sh cd /work -pytest $@ +pytest "${@}" diff --git a/tests/mk8s_tests/values.yaml b/tests/mk8s_tests/values.yaml index 8d0f1491fb..e58ad21541 100644 --- a/tests/mk8s_tests/values.yaml +++ b/tests/mk8s_tests/values.yaml @@ -2,60 +2,59 @@ # This is a YAML-formatted file. # Declare variables to be passed into your templates. +--- replicaCount: 2 splunk: - - hec_url: "" - hec_token: "" - hec_verify_tls: "" - -sc4s: -# {} -# existingCert: example-com-tls - vendor_product: - - name: pfsense_firewall - ports: - tcp: [6000] - - name: cisco_esa - ports: - tcp: [9000] - - name: cisco_meraki - ports: - tcp: [9001] - context_files: - splunk_metadata.csv: |- - cisco_meraki,index,netops - host.csv: |- - <>,HOST,newhost - - - config_files: - app_parser_test_big.conf: |- - application app-dell-isilion-postfilter[sc4s-postfilter] { - filter { - host("test_host" type(glob)) - }; - parser { app-dell-isiolion-postfilter() }; - }; - block parser app-dell-isiolion-postfilter() { - channel { - rewrite { - r_set_splunk_dest_default( - index('netops') - source('dell:isilion') - sourcetype('dell:isilion') - vendor("dell") - product('isilion') - template('t_5424_hdr_sdata_msg') - ); - }; - }; - }; + hec_url: "" + hec_token: "" + hec_verify_tls: "" + +sc4s: + # existingCert: example-com-tls + vendor_product: + - name: pfsense_firewall + ports: + tcp: [6000] + - name: cisco_esa + ports: + tcp: [9000] + - name: cisco_meraki + ports: + tcp: [9001] + context_files: + splunk_metadata.csv: |- + cisco_meraki,index,netops + host.csv: |- + <>,HOST,newhost + + + config_files: + app_parser_test_big.conf: |- + application app-dell-isilion-postfilter[sc4s-postfilter] { + filter { + host("test_host" type(glob)) + }; + parser { app-dell-isiolion-postfilter() }; + }; + block parser app-dell-isiolion-postfilter() { + channel { + rewrite { + r_set_splunk_dest_default( + index('netops') + source('dell:isilion') + sourcetype('dell:isilion') + vendor("dell") + product('isilion') + template('t_5424_hdr_sdata_msg') + ); + }; + }; + }; image: - repository: ghcr.io/splunk/splunk-connect-for-syslog/container3 + repository: ghcr.io/splunk/splunk-connect-for-syslog/container3 pullPolicy: IfNotPresent # Overrides the image tag whose default is the chart appVersion. tag: "" @@ -70,7 +69,8 @@ serviceAccount: # Annotations to add to the service account annotations: {} # The name of the service account to use. - # If not set and create is true, a name is generated using the fullname template + # If not set and create is true, a name is generated using + # the fullname template name: "" persistence: @@ -102,10 +102,11 @@ service: resources: {} - # We usually recommend not to specify default resources and to leave this as a conscious - # choice for the user. This also increases chances charts run on environments with little - # resources, such as Minikube. If you do want to specify resources, uncomment the following - # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # We usually recommend not to specify default resources and to leave this + # as a conscious choice for the user. This also increases chances charts + # run on environments with little resources, such as Minikube. If you do + # want to specify resources, uncomment the following lines, adjust them as + # necessary, and remove the curly braces after 'resources:'. # limits: # cpu: 100m # memory: 128Mi diff --git a/tests/splunkutils.py b/tests/splunkutils.py index d0329f7fe9..dd8f92190f 100644 --- a/tests/splunkutils.py +++ b/tests/splunkutils.py @@ -15,7 +15,8 @@ def splunk_single(service, search, attempt_limit=10): # A normal search returns the job's SID right away, so we need to poll for completion while True: while not job.is_ready(): - pass + sleep(1) + stats = { "isDone": job["isDone"], "doneProgress": float(job["doneProgress"]) * 100, diff --git a/tests/test_aruba_clearpass.py b/tests/test_aruba_clearpass.py index 5e67bc5e51..725a1d6f6b 100644 --- a/tests/test_aruba_clearpass.py +++ b/tests/test_aruba_clearpass.py @@ -28,7 +28,7 @@ @pytest.mark.addons("aruba") @pytest.mark.parametrize("event", test_data_cppm) -def test_aruba_clearpass_CPPM( +def test_aruba_clearpass_cppm( record_property, setup_splunk, setup_sc4s, get_host_key, event ): host = "aruba-cp-" + get_host_key diff --git a/tests/test_avi_vantage.py b/tests/test_avi_vantage.py index ccd7b983de..0202305134 100644 --- a/tests/test_avi_vantage.py +++ b/tests/test_avi_vantage.py @@ -4,8 +4,6 @@ # license that can be found in the LICENSE-BSD2 file or at # https://opensource.org/licenses/BSD-2-Clause -import shortuuid - from jinja2 import Environment, select_autoescape from .sendmessage import sendsingle @@ -64,7 +62,7 @@ def test_avi_event_rfc( @pytest.mark.addons("avi") @pytest.mark.parametrize("event", test_data_JSON) -def test_avi_event_JSON( +def test_avi_event_json( record_property, setup_splunk, setup_sc4s, get_host_key, event ): host = get_host_key diff --git a/tests/test_checkpoint.py b/tests/test_checkpoint.py index 06d82f5683..4cc6514c21 100644 --- a/tests/test_checkpoint.py +++ b/tests/test_checkpoint.py @@ -387,7 +387,7 @@ def test_checkpoint_splunk_os_nested(record_property, setup_splunk, setup_sc4s): # Test endpoint source event # time=1586182935|hostname=abc|product=Endpoint Management|action=Drop|ifdir=inbound|loguid={0x60069850,0x0,0xe03ea00a,0x23654691}|origin=10.160.62.224|originsicname=cn\=cp_mgmt,o\=gw-8be69c..ba5xxz|sequencenum=2|version=5|audit_status=Success|endpointname=C7553927437.WORKGROUP|endpointuser=Administrator@C7553927437|operation=Access Key For Encryptor|subject=Endpoint Activity|uid=2E5FD596-BAEF-4453-BFB0-85598CD43DF6 @pytest.mark.addons("checkpoint") -def test_checkpoint_splunk_Endpoint_Management( +def test_checkpoint_splunk_endpoint_management( record_property, setup_splunk, setup_sc4s ): host = f"{shortuuid.ShortUUID().random(length=5).lower()}-{shortuuid.ShortUUID().random(length=5).lower()}" @@ -459,7 +459,7 @@ def test_checkpoint_splunk_ios_profile(record_property, setup_splunk, setup_sc4s # Test audit source event # time=1586182935|hostname=abc|product=SmartUpdate|action=Accept|ifdir=outbound|loguid={0x6023d54c,0x0,0x6563a00a,0x3431e7e4}|origin=10.160.99.101|originsicname=cn\=cp_mgmt,o\=gw-02bd87..4zrt7d|sequencenum=6|time={{ epoch }}|version=5|additional_info=Performed 'Attach License' on 10.160.99.101|administrator=admin|client_ip=10.160.99.102|machine=C1359997769|operation=Modify Object|operation_number=1|subject=Object Manipulation @pytest.mark.addons("checkpoint") -def test_checkpoint_splunk_SmartUpdate(record_property, setup_splunk, setup_sc4s): +def test_checkpoint_splunk_smartupdate(record_property, setup_splunk, setup_sc4s): host = f"{shortuuid.ShortUUID().random(length=5).lower()}-{shortuuid.ShortUUID().random(length=5).lower()}" dt = datetime.datetime.now() @@ -493,7 +493,7 @@ def test_checkpoint_splunk_SmartUpdate(record_property, setup_splunk, setup_sc4s # time=1611044939|hostname=gw-8be69c|severity=Low|product=Endpoint Compliance|ifdir=inbound|loguid={0x60069d03,0x0,0xe03ea00a,0x23654691}|origin=10.160.62.224|sequencenum=1|version=1|action_comment= |client_name=Check Point Endpoint Security Client|client_version=84.30.6614|description= |event_type=Policy Update|host_type=Desktop|installed_products=Media Encryption & Port Protection; Compliance; Anti-Malware; Url Filtering; Anti-Bot; Forensics; Threat Emulation|local_time=1611044939|machine_guid= |os_name=Windows Server 10.0 Standard Server Edition|os_version=10.0-14393-SP0.0-SMP|policy_date=1610103648|policy_guid={5E122911-49AE-40ED-A91B-0B56576E4549}|policy_name=default_compliance_policy|policy_type=60|policy_version=1|product_family=Endpoint|src=10.160.177.73|src_machine_name=C7553927437|src_user_name=Administrator|user_name= |user_sid=S-1-5-21-1704411108-3626445783-306313190-500 @pytest.mark.addons("checkpoint") -def test_checkpoint_splunk_Endpoint_Compliance( +def test_checkpoint_splunk_endpoint_compliance( record_property, setup_splunk, setup_sc4s ): host = f"{shortuuid.ShortUUID().random(length=5).lower()}-{shortuuid.ShortUUID().random(length=5).lower()}" @@ -529,7 +529,7 @@ def test_checkpoint_splunk_Endpoint_Compliance( # time=1613022553|hostname=gw-02bd87|product=Mobile Access|ifdir=outbound|loguid={0x6024c55a,0x2,0x6563a00a,0x346ce8b1}|origin=10.160.99.101|originsicname=cn\=cp_mgmt,o\=gw-02bd87..4zrt7d|sequencenum=2|time=1613022553|version=5|message=All gateways successfully notified about the revocation of certificate with serial no. '49681' @pytest.mark.addons("checkpoint") -def test_checkpoint_splunk_Mobile_Access(record_property, setup_splunk, setup_sc4s): +def test_checkpoint_splunk_mobile_access(record_property, setup_splunk, setup_sc4s): host = f"{shortuuid.ShortUUID().random(length=5).lower()}-{shortuuid.ShortUUID().random(length=5).lower()}" dt = datetime.datetime.now() @@ -563,7 +563,7 @@ def test_checkpoint_splunk_Mobile_Access(record_property, setup_splunk, setup_sc # time=1703062311|hostname=gw-313a04|product=Check Point GO Password Reset|action=Accept|ifdir=outbound|loguid={0x6582ab28,0x0,0xcc8ea00a,0x3fffbd01}|origin=10.160.142.204|originsicname=cn\=cp_mgmt,o\=gw-313a04..fhsx6t|sequencenum=1|time=1703062311|version=5|administrator=admin|client_ip=10.160.3.181|machine=C6828388989|operation=Log Out|operation_number=12|subject=Administrator Login @pytest.mark.addons("checkpoint") -def test_checkpoint_splunk_Check_Point_Go_Password_Reset( +def test_checkpoint_splunk_check_point_go_password_reset( record_property, setup_splunk, setup_sc4s ): host = f"{shortuuid.ShortUUID().random(length=5).lower()}-{shortuuid.ShortUUID().random(length=5).lower()}" @@ -599,7 +599,7 @@ def test_checkpoint_splunk_Check_Point_Go_Password_Reset( # time=1703070563|hostname=gw-313a04|product=Database Tool|action=Accept|ifdir=outbound|loguid={0x6582cb65,0x0,0xcc8ea00a,0x3fffbd01}|origin=10.160.142.204|originsicname=cn\=cp_mgmt,o\=gw-313a04..fhsx6t|sequencenum=1|time=1703070563|version=5|administrator=admin|client_ip=10.160.3.181|machine=C6828388989|operation=Log Out|operation_number=12|subject=Administrator Login @pytest.mark.addons("checkpoint") -def test_checkpoint_splunk_Database_Tool(record_property, setup_splunk, setup_sc4s): +def test_checkpoint_splunk_database_tool(record_property, setup_splunk, setup_sc4s): host = f"{shortuuid.ShortUUID().random(length=5).lower()}-{shortuuid.ShortUUID().random(length=5).lower()}" dt = datetime.datetime.now() @@ -633,7 +633,7 @@ def test_checkpoint_splunk_Database_Tool(record_property, setup_splunk, setup_sc # time=1703168085|hostname=gw-313a04|product=FG VPN-1 & FireWall-1|layer_name=Network|layer_name=URL_APP|layer_uuid=38271c2f-ab44-4e25-9aa4-e219cb6e12cf|layer_uuid=789a1bbd-8125-4f7a-a420-179ce276e60f|match_id=2|match_id=16777218|parent_rule=0|parent_rule=0|rule_action=Accept|rule_action=Accept|rule_name=Cleanup rule|rule_name=Cleanup rule|rule_uid=2b922948-da96-4c9d-a654-063e0183f9ae|rule_uid=b72f0dee-0224-4f76-b79e-563f1cf3d3ef|action=Accept|conn_direction=External|contextnum=1|ifdir=inbound|ifname=eth0|logid=6|loguid={0x76e2147d,0x528aea2e,0x75e2f4c,0x22ec9c69}|origin=10.160.142.204|originsicname=cn\=cp_mgmt,o\=gw-313a04..fhsx6t|sequencenum=1|time=1703168085|version=5|__nsons=0|__p_dport=0|__pos=7|bytes=152|client_inbound_bytes=76|client_inbound_interface=eth0|client_inbound_packets=1|client_outbound_bytes=76|client_outbound_packets=2|context_num=1|dst=40.119.6.228|elapsed=0|fg-1_client_in_rule_name=Default|fg-1_client_out_rule_name=Default|fg-1_server_in_rule_name=Default|fg-1_server_out_rule_name=Default|hll_key=6193773038144685443|inzone=External|lastupdatetime=1703168125|nat_addtnl_rulenum=0|nat_rule_uid=89feaaba-a367-4972-bc44-9da7878c59c1|nat_rulenum=1|outzone=External|packets=3|proto=17|s_port=123|segment_time=1703168085|server_inbound_bytes=76|server_inbound_packets=1|server_outbound_bytes=76|server_outbound_packets=2|service=123|service_id=ntp-udp|src=10.160.50.230|start_time=1703168085|xlatedport=0|xlatedst=0.0.0.0|xlatesport=43710|xlatesrc=10.160.142.204 @pytest.mark.addons("checkpoint") -def test_checkpoint_splunk_FG_VPN_and_Firewall( +def test_checkpoint_splunk_fg_vpn_and_firewall( record_property, setup_splunk, setup_sc4s ): host = f"{shortuuid.ShortUUID().random(length=5).lower()}-{shortuuid.ShortUUID().random(length=5).lower()}" @@ -669,7 +669,7 @@ def test_checkpoint_splunk_FG_VPN_and_Firewall( # time=1703168085|hostname=gw-313a04|product=QoS|action=Accept|ifdir=inbound|loguid={0xd5b78c6e,0x5faa26e,0xddf9bd6a,0x657f411d}|origin=10.160.142.204|originsicname=cn\=cp_mgmt,o\=gw-313a04..fhsx6t|sequencenum=3|time=1703168085|version=5|dst=40.119.6.228|fg-1_client_in_rule_name=Default|fg-1_client_out_rule_name=Default|fg-1_server_in_rule_name=Default|fg-1_server_out_rule_name=Default|lastupdatetime=1703168085|proto=17|s_port=123|service=123|src=10.160.50.230 @pytest.mark.addons("checkpoint") -def test_checkpoint_splunk_QoS(record_property, setup_splunk, setup_sc4s): +def test_checkpoint_splunk_qos(record_property, setup_splunk, setup_sc4s): host = f"{shortuuid.ShortUUID().random(length=5).lower()}-{shortuuid.ShortUUID().random(length=5).lower()}" dt = datetime.datetime.now() @@ -773,7 +773,7 @@ def test_checkpoint_splunk_query_database(record_property, setup_splunk, setup_s # time=1692614010|hostname=nevis2-backup-mlm|severity=Low|confidence_level=High|product=Anti Phishing|action=Detect|ifdir=inbound|ifname=MTA|loguid={0x8745b56b,0x40b00845,0x924c2a3c,0x150a9782}|origin=194.29.38.29|originsicname=CN\=il-dmz-tls-05.dummydomain.com,O\=natasha..8ye75g|sequencenum=5|time=1692614010|version=5|dst=194.29.38.29|email_subject= The Morning: Flight risk|from=chrisr+caf_\=chrisri\=dummydomain.com@avanan.com|log_id=0|original_queue_id=4RTpjJ6qfGz6mkC|protection_type=SPAM|proto=6|s_port=0|service=25|src=194.29.47.47|to=chrisri@dummydomain.com|triggered_by=MTA @pytest.mark.addons("checkpoint") -def test_checkpoint_splunk_Anti_Phishing(record_property, setup_splunk, setup_sc4s): +def test_checkpoint_splunk_anti_phishing(record_property, setup_splunk, setup_sc4s): host = f"{shortuuid.ShortUUID().random(length=5).lower()}-{shortuuid.ShortUUID().random(length=5).lower()}" dt = datetime.datetime.now() @@ -807,7 +807,7 @@ def test_checkpoint_splunk_Anti_Phishing(record_property, setup_splunk, setup_sc # time=1693202076|hostname=nevis2-backup-mlm|product=Anti-Spam and Email Security|action=Accept|ifdir=inbound|ifname=MTA|loguid={0x64ec369d,0xc4,0xe07fa8c0,0x205ec071}|origin=194.29.38.29|sequencenum=1|time=1693202076|version=5|dst=194.29.38.29|email_control=Content Anti Spam|email_id=1|email_recipients_num=1|email_session_id={64EC3698-0-1D261DC2-2A3D}|email_spam_category=Non Spam|email_subject= Harmony Mobile - Warning Severity Audit Alert|from=bounces+2173712-9999-adonati\=dummydomain.com@sendgrid.net|proto=6|s_port=0|scan_direction=to/from this gateway|service=25|src=194.29.47.47|src_country=ISR|to=adonati@dummydomain.com @pytest.mark.addons("checkpoint") -def test_checkpoint_splunk_Anti_Spam_and_Email_Security( +def test_checkpoint_splunk_anti_spam_and_email_security( record_property, setup_splunk, setup_sc4s ): host = f"{shortuuid.ShortUUID().random(length=5).lower()}-{shortuuid.ShortUUID().random(length=5).lower()}" diff --git a/tests/test_checkpoint_syslog_rfc5424.py b/tests/test_checkpoint_syslog_rfc5424.py index 29bcbe79a9..7108b3b627 100644 --- a/tests/test_checkpoint_syslog_rfc5424.py +++ b/tests/test_checkpoint_syslog_rfc5424.py @@ -232,7 +232,7 @@ def test_checkpoint_syslog_cli( # Test iOS Profiles # <134>1 2021-02-08T10:19:34Z gw-02bd87 CheckPoint 26203 - [sc4s@2620 flags="131072" ifdir="inbound" loguid="{0x60215107,0x169a,0xd10617ac,0x4468886}" origin="10.1.46.86" sequencenum="4138" time="1612795822" version="5" calc_geo_location="calc_geo_location0" client_name="SandBlast Mobile Protect" client_version="2.72.8.3943" dashboard_orig="dashboard_orig0" device_identification="4624" email_address="email_address44" hardware_model="iPhone / iPhone 8" host_type="Mobile" incident_time="2018-06-03T17:33:09Z" jailbreak_message="False" mdm_id="E726405B-4BCF-46C6-8D1B-6F1A71E67D5D" os_name="IPhone" os_version="11.3.1" phone_number="phone_number24" product="iOS Profiles" protection_type="Global proxy" severity="0" src_user_name="Mike Johnson1" status="Removed"] @pytest.mark.addons("checkpoint") -def test_checkpoint_syslog_iOS_profiles( +def test_checkpoint_syslog_ios_profiles( record_property, setup_splunk, setup_sc4s ): host = f"{shortuuid.ShortUUID().random(length=5).lower()}-{shortuuid.ShortUUID().random(length=5).lower()}" @@ -268,7 +268,7 @@ def test_checkpoint_syslog_iOS_profiles( # Test Endpoint Compliance # <134>1 2021-02-08T10:19:34Z gw-02bd87 CheckPoint 26203 - [sc4s@2620 flags="131072" ifdir="inbound" loguid="{0x60215107,0x169a,0xd10617ac,0x4468886}" origin="10.1.46.86" sequencenum="4138" time="1612795822" version="5" calc_geo_location="calc_geo_location0" client_name="SandBlast Mobile Protect" client_version="2.72.8.3943" dashboard_orig="dashboard_orig0" device_identification="4624" email_address="email_address44" hardware_model="iPhone / iPhone 8" host_type="Mobile" incident_time="2018-06-03T17:33:09Z" jailbreak_message="False" mdm_id="E726405B-4BCF-46C6-8D1B-6F1A71E67D5D" os_name="IPhone" os_version="11.3.1" phone_number="phone_number24" product="Endpoint Compliance" protection_type="Global proxy" severity="0" src_user_name="Mike Johnson1" status="Removed"] @pytest.mark.addons("checkpoint") -def test_checkpoint_syslog_Endpoint_Compliance( +def test_checkpoint_syslog_endpoint_compliance( record_property, setup_splunk, setup_sc4s ): host = f"{shortuuid.ShortUUID().random(length=5).lower()}-{shortuuid.ShortUUID().random(length=5).lower()}" @@ -302,7 +302,7 @@ def test_checkpoint_syslog_Endpoint_Compliance( @pytest.mark.addons("checkpoint") -def test_checkpoint_syslog_Endpoint( +def test_checkpoint_syslog_endpoint( record_property, setup_splunk, setup_sc4s ): host = f"{shortuuid.ShortUUID().random(length=5).lower()}-{shortuuid.ShortUUID().random(length=5).lower()}" @@ -336,7 +336,7 @@ def test_checkpoint_syslog_Endpoint( # Test Identity Awareness @pytest.mark.addons("checkpoint") -def test_checkpoint_syslog_Identity_Awareness( +def test_checkpoint_syslog_identity_awareness( record_property, setup_splunk, setup_sc4s ): host = f"{shortuuid.ShortUUID().random(length=5).lower()}-{shortuuid.ShortUUID().random(length=5).lower()}" @@ -371,7 +371,7 @@ def test_checkpoint_syslog_Identity_Awareness( # Test Mobile Access # <134>1 2021-02-08T14:50:06Z r81-t279-leui-main-take-2 CheckPoint 2182 - [sc4s@2620 flags="131072" ifdir="inbound" loguid="{0x60215106,0xb,0xd10617ac,0x4468886}" origin="10.2.46.86" sequencenum="12" time="1612795806" version="5" app_repackaged="False" app_sig_id="3343cf41cb8736ad452453276b4f7c806ab83143eca0b3ad1e1bc6045e37f6a9" app_version="3.1.15" appi_name="iPGMail" calc_geo_location="calc_geo_location0" client_name="SandBlast Mobile Protect" client_version="2.73.0.3968" dashboard_orig="dashboard_orig0" device_identification="4768" email_address="email_address0" hardware_model="iPhone / iPhone 5S" host_type="Mobile" incident_time="2018-06-04T00:03:41Z" jailbreak_message="False" mdm_id="F2FCB053-5C28-4917-9FED-4821349B86A5" os_name="IPhone" os_version="11.4" phone_number="phone_number0" product="Mobile Access" protection_type="Backup Tool" severity="0" src_user_name="Allen Newsom" status="Installed" @pytest.mark.addons("checkpoint") -def test_checkpoint_syslog_Mobile_Access( +def test_checkpoint_syslog_mobile_access( record_property, setup_splunk, setup_sc4s ): host = f"{shortuuid.ShortUUID().random(length=5).lower()}-{shortuuid.ShortUUID().random(length=5).lower()}" diff --git a/tests/test_citrix_netscaler.py b/tests/test_citrix_netscaler.py index a5ac7b014a..b7e6413298 100644 --- a/tests/test_citrix_netscaler.py +++ b/tests/test_citrix_netscaler.py @@ -94,7 +94,7 @@ def test_citrix_netscaler_sdx( # [289]: AAA Message : In receive_ldap_user_search_event: ldap_first_entry returned null, user ssgconfig not found @pytest.mark.addons("citrix") -def test_citrix_netscaler_sdx_AAA( +def test_citrix_netscaler_sdx_aaa( record_property, setup_splunk, setup_sc4s, get_pid ): host = f"test-ctitrixns-host-{shortuuid.ShortUUID().random(length=5).lower()}-{shortuuid.ShortUUID().random(length=5).lower()}" diff --git a/tests/test_common.py b/tests/test_common.py index 5b45f11ec2..3673b1b96d 100644 --- a/tests/test_common.py +++ b/tests/test_common.py @@ -281,18 +281,17 @@ def test_check_config_version_multiple( assert result_count == 0 -# This test fails on circle; Cisco ACS single test seems to trigger a utf8 error. -# def test_check_utf8(record_property, setup_splunk, setup_sc4s): -# st = env.from_string( -# 'search earliest=-50m@m latest=+1m@m index=main sourcetype="sc4s:events" "Input is valid utf8"' -# ) -# search = st.render() +def test_check_utf8(record_property, setup_splunk, setup_sc4s): + st = env.from_string( + 'search earliest=-50m@m latest=+1m@m index=main sourcetype="sc4s:events" "Input is valid utf8"' + ) + search = st.render() -# result_count, _ = splunk_single(setup_splunk, search) + result_count, _ = splunk_single(setup_splunk, search) -# record_property("resultCount", result_count) + record_property("resultCount", result_count) -# assert result_count == 0 + assert result_count == 0 def test_check_sc4s_version(record_property, setup_splunk, setup_sc4s): diff --git a/tests/test_f5_bigip.py b/tests/test_f5_bigip.py index d97d671ce4..f8bf00a986 100644 --- a/tests/test_f5_bigip.py +++ b/tests/test_f5_bigip.py @@ -588,7 +588,6 @@ def test_f5_bigip_irule_lb_failed( def test_f5_bigip_asm_syslog( record_property, get_host_key, setup_splunk, setup_sc4s ): - host = get_host_key host = "bigip-2.test_domain.com" dt = datetime.datetime.now() diff --git a/tests/test_fortinet_web.py b/tests/test_fortinet_web.py index c143fd109d..e24da32a99 100644 --- a/tests/test_fortinet_web.py +++ b/tests/test_fortinet_web.py @@ -10,7 +10,7 @@ from .sendmessage import sendsingle from .splunkutils import splunk_single -from .timeutils import time_operations, insert_char, removeZero +from .timeutils import time_operations, insert_char, remove_zero import datetime env = Environment(autoescape=select_autoescape(default_for_string=False)) @@ -25,7 +25,7 @@ def test_fortinet_fwb_event(record_property, setup_splunk, setup_sc4s): # Tune time functions for Fortiweb time = time[:-7] - tzoffset = removeZero(insert_char(tzoffset, ":", 3)) + tzoffset = remove_zero(insert_char(tzoffset, ":", 3)) epoch = epoch[:-7] mt = env.from_string( @@ -61,7 +61,7 @@ def test_fortinet_fwb_traffic( # Tune time functions for Fortiweb time = time[:-7] - tzoffset = removeZero(insert_char(tzoffset, ":", 3)) + tzoffset = remove_zero(insert_char(tzoffset, ":", 3)) epoch = epoch[:-7] mt = env.from_string( @@ -95,7 +95,7 @@ def test_fortinet_fwb_attack(record_property, setup_splunk, setup_sc4s): # Tune time functions for Fortiweb time = time[:-7] - tzoffset = removeZero(insert_char(tzoffset, ":", 3)) + tzoffset = remove_zero(insert_char(tzoffset, ":", 3)) epoch = epoch[:-7] mt = env.from_string( @@ -128,7 +128,7 @@ def test_fortinet_fortimail(record_property, setup_splunk, setup_sc4s): # Tune time functions for Fortiweb time = time[:-7] - tzoffset = removeZero(insert_char(tzoffset, ":", 3)) + tzoffset = remove_zero(insert_char(tzoffset, ":", 3)) epoch = epoch[:-7] mt = env.from_string( diff --git a/tests/test_healthcheck_unit_tests.py b/tests/test_healthcheck_unit_tests.py index 53f98aeccc..5c523cca2a 100644 --- a/tests/test_healthcheck_unit_tests.py +++ b/tests/test_healthcheck_unit_tests.py @@ -8,6 +8,7 @@ check_syslog_ng_health, subprocess, check_queue_size, + get_list_of_destinations, ) # str_to_bool @@ -52,9 +53,9 @@ def test_check_syslog_ng_health_exception(mock_run): # check_queue_size def test_check_queue_size_no_url(): """ - If sc4s_dest_splunk_hec_default is not set, check_queue_size should fail. + If sc4s_dest_splunk_hec_destinations is not set, check_queue_size should fail. """ - assert check_queue_size(sc4s_dest_splunk_hec_default=None, max_queue_size=1000) is False + assert check_queue_size(sc4s_dest_splunk_hec_destinations=None, max_queue_size=1000) is False @patch("subprocess.run") def test_check_queue_size_stats_fail(mock_run): @@ -63,7 +64,7 @@ def test_check_queue_size_stats_fail(mock_run): """ mock_run.return_value.returncode = 1 mock_run.return_value.stderr = "stats error" - assert check_queue_size(sc4s_dest_splunk_hec_default="http://example.com:8088", max_queue_size=1000) is False + assert check_queue_size(sc4s_dest_splunk_hec_destinations={"http://example.com:8088"}, max_queue_size=1000) is False @patch("subprocess.run") def test_check_queue_size_no_matching_stats(mock_run): @@ -72,7 +73,7 @@ def test_check_queue_size_no_matching_stats(mock_run): """ mock_run.return_value.returncode = 0 mock_run.return_value.stdout = "some;other;stat;line\nanother;stat" - assert check_queue_size(sc4s_dest_splunk_hec_default="http://example.com:8088", max_queue_size=1000) is False + assert check_queue_size(sc4s_dest_splunk_hec_destinations={"http://example.com:8088"}, max_queue_size=1000) is False @patch("subprocess.run") def test_check_queue_size_exceeds_limit(mock_run): @@ -84,7 +85,7 @@ def test_check_queue_size_exceeds_limit(mock_run): "destination;queued;http://example.com:8088;2000\n" "another;queued;http://other-url.com;1234" ) - assert check_queue_size(sc4s_dest_splunk_hec_default="http://example.com:8088", max_queue_size=1000) is False + assert check_queue_size(sc4s_dest_splunk_hec_destinations={"http://example.com:8088"}, max_queue_size=1000) is False @patch("subprocess.run") def test_check_queue_size_under_limit(mock_run): @@ -96,7 +97,7 @@ def test_check_queue_size_under_limit(mock_run): "destination;queued;http://example.com:8088;500\n" "another;queued;http://other-url.com;1234" ) - assert check_queue_size(sc4s_dest_splunk_hec_default="http://example.com:8088", max_queue_size=1000) is True + assert check_queue_size(sc4s_dest_splunk_hec_destinations={"http://example.com:8088"}, max_queue_size=1000) is True @patch("subprocess.run") def test_check_queue_size_equals_limit(mock_run): @@ -108,7 +109,62 @@ def test_check_queue_size_equals_limit(mock_run): "destination;queued;http://example.com:8088;1000\n" "another;queued;http://other-url.com;1234" ) - assert check_queue_size(sc4s_dest_splunk_hec_default="http://example.com:8088", max_queue_size=1000) is True + assert check_queue_size(sc4s_dest_splunk_hec_destinations={"http://example.com:8088"}, max_queue_size=1000) is True + +@patch("subprocess.run") +def test_check_queue_size_multiple_destinations(mock_run): + """ + If queue size for all destinations is <= HEALTHCHECK_MAX_QUEUE_SIZE, check_queue_size should pass. + """ + mock_run.return_value.returncode = 0 + mock_run.return_value.stdout = ( + "destination;queued;http://example.com:8088;300\n" + "destination;queued;http://another.com:8088;500\n" + "another;queued;http://other-url.com;1234" + ) + assert check_queue_size(sc4s_dest_splunk_hec_destinations={"http://example.com:8088", "http://another.com:8088"}, + max_queue_size=1000) is True + +@patch("subprocess.run") +def test_check_queue_size_multiple_destinations_over_limit(mock_run): + """ + If queue size for at least one destination is > HEALTHCHECK_MAX_QUEUE_SIZE, check_queue_size should fail. + """ + mock_run.return_value.returncode = 0 + mock_run.return_value.stdout = ( + "destination;queued;http://example.com:8088;1300\n" + "destination;queued;http://another.com:8088;500\n" + "another;queued;http://other-url.com;1234" + ) + assert check_queue_size(sc4s_dest_splunk_hec_destinations={"http://example.com:8088", "http://another.com:8088"}, + max_queue_size=1000) is False + +@patch("subprocess.run") +def test_check_queue_size_multiple_destinations_all_over_limit(mock_run): + """ + If queue size for all destinations is > HEALTHCHECK_MAX_QUEUE_SIZE, check_queue_size should fail. + """ + mock_run.return_value.returncode = 0 + mock_run.return_value.stdout = ( + "destination;queued;http://example.com:8088;1300\n" + "destination;queued;http://another.com:8088;1500\n" + "another;queued;http://other-url.com;1234" + ) + assert check_queue_size(sc4s_dest_splunk_hec_destinations={"http://example.com:8088", "http://another.com:8088"}, + max_queue_size=1000) is False + +@patch("subprocess.run") +def test_check_queue_size_multiple_incomplete_info(mock_run): + """ + If stats run successfully but do not contain stats for one of the desired destinations, it should fail. + """ + mock_run.return_value.returncode = 0 + mock_run.return_value.stdout = ( + "destination;queued;http://example.com:8088;300\n" + "another;queued;http://other-url.com;1234" + ) + assert check_queue_size(sc4s_dest_splunk_hec_destinations={"http://example.com:8088", "http://another.com:8088"}, + max_queue_size=1000) is False @patch("subprocess.run", side_effect=Exception("some exception")) def test_check_queue_size_exception(mock_run): @@ -139,4 +195,26 @@ def test_health_endpoint_no_queue_check(mock_run, client): response = client.get("/health") assert response.status_code == 200 - assert response.json["status"] == "healthy" \ No newline at end of file + assert response.json["status"] == "healthy" + +@patch.dict( + os.environ, + { + "SC4S_DEST_SPLUNK_HEC_DEFAULT_URL": "http://my_test_url:1234", + "SC4S_DEST_SPLUNK_HEC_OTHER_URL": "http://my_hec:1234", + "SOME_OTHER_URL": "http://my_url/test_url", + "SOME_OTHER_ENV_VARIABLE": "my_variable", + "SC4S_LISTEN_STATUS_PORT": "1234", + }, + clear=True +) +def test_get_destinations(): + """ + Check if get_list_of_destinations method parses and returns the expected + destinations from environment variables. + """ + destinations = get_list_of_destinations() + + assert len(destinations) == 2 + assert "http://my_test_url:1234" in destinations + assert "http://my_hec:1234" in destinations diff --git a/tests/test_name_cache.py b/tests/test_name_cache.py index fb3a55865f..bc68b395fb 100644 --- a/tests/test_name_cache.py +++ b/tests/test_name_cache.py @@ -84,7 +84,7 @@ def test_ipv6_utils(): @pytest.mark.name_cache -def test_RestrictedSqliteDict_stores_and_retrieves_string(): +def test_restricted_sqlitedict_stores_and_retrieves_string(): with tempfile.NamedTemporaryFile(delete=True) as temp_db_file: cache = SqliteDict(f"{temp_db_file.name}.db") cache["key"] = "value" @@ -97,7 +97,7 @@ def test_RestrictedSqliteDict_stores_and_retrieves_string(): @pytest.mark.name_cache -def test_RestrictedSqliteDict_prevents_code_injection(): +def test_restricted_sqlitedict_prevents_code_injection(): class InjectionTestClass: def __reduce__(self): import os diff --git a/tests/test_netapp.py b/tests/test_netapp.py index 6451deac06..a05aead4c2 100644 --- a/tests/test_netapp.py +++ b/tests/test_netapp.py @@ -87,4 +87,36 @@ def test_netapp_ontap_ems_rfc5424( record_property("resultCount", result_count) record_property("message", message) + assert result_count == 1 + + +# Netapp Ontap EMS event +# <13>Feb 10 11:36:10 [cluster-01:secd.conn.auth.failure:notice]: Vserver (datavserver) could not make a connection over the network to server (ip 2.3.3.3, port 389). Error: Operation timed out (Service: LDAP (Active Directory), Operation: SiteDiscovery). +@pytest.mark.addons("netapp") +def test_netapp_ontap_ems( + record_property, get_host_key, setup_splunk, setup_sc4s +): + host = "netapp-ontap-" + get_host_key + + dt = datetime.datetime.now(datetime.timezone.utc) + _, bsd, _, _, _, _, epoch = time_operations(dt) + + # Tune time functions + epoch = epoch[:-7] + mt = env.from_string( + "{{ mark }}{{ bsd }} [{{ host }}:{{ category }}:{{ severity }}]: Vserver (datavserver) could not make a connection over the network to server (ip 2.3.3.3, port 389). Error: Operation timed out (Service: LDAP (Active Directory), Operation: SiteDiscovery)") + message = mt.render(mark="<13>", bsd=bsd, host=host, category="secd.conn.auth.failure", severity="notice") + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + st = env.from_string( + 'search index=infraops _time={{ epoch }} sourcetype="netapp:ontap:ems" host="{{ host }}"' + ) + search = st.render(epoch=epoch, host=host) + + result_count, _ = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", result_count) + record_property("message", message) + assert result_count == 1 \ No newline at end of file diff --git a/tests/test_symantec_proxy.py b/tests/test_symantec_proxy.py index 7c60462b97..755a2d1b62 100644 --- a/tests/test_symantec_proxy.py +++ b/tests/test_symantec_proxy.py @@ -17,7 +17,7 @@ # <134>1 2019-08-21T17:42:08.000z "sample_logs bluecoat[0]:SPLV5.1 c-ip=192.0.0.6 cs-bytes=6269 cs-categories="unavailable" cs-host=gg.hhh.iii.com cs-ip=192.0.0.6 cs-method=GET cs-uri-path=/Sample/abc-xyz-01.pqr_sample_Internal.crt/MFAwTqADAgEAMEcwRTBDMAkGBSsOAwIaBQAEFOoaVMtyzC9gObESY9g1eXf1VM8VBBTl1mBq2WFf4cYqBI6c08kr4S302gIKUCIZdgAAAAAnQA%3D%3D cs-uri-port=8000 cs-uri-scheme=http cs-User-Agent="ocspd/1.0.3" cs-username=user4 clientduration=0 rs-status=0 s-action=TCP_HIT s-ip=10.0.0.6 serveripservice.name="Explicit HTTP" service.group="Standard" s-supplier-ip=10.0.0.6 s-supplier-name=gg.hhh.iii.com sc-bytes=9469 sc-filter-result=OBSERVED sc-status=200 time-taken=20 x-bluecoat-appliance-name="10.0.0.6-sample_logs" x-bluecoat-appliance-primary-address=10.0.0.6 x-bluecoat-proxy-primary-address=10.0.0.6 x-bluecoat-transaction-uuid=35d24c931c0erecta-0003000012161a77e70-00042100041002145cc859ed c-url="http://randomserver:8000/en-US/app/examples/" @pytest.mark.addons("broadcom") -def test_bluecoatproxySG_kv(record_property, setup_splunk, setup_sc4s): +def test_bluecoatproxysg_kv(record_property, setup_splunk, setup_sc4s): host = f"{shortuuid.ShortUUID().random(length=5).lower()}-{shortuuid.ShortUUID().random(length=5).lower()}" dt = datetime.datetime.now(datetime.timezone.utc) @@ -51,7 +51,7 @@ def test_bluecoatproxySG_kv(record_property, setup_splunk, setup_sc4s): # <111>1 $(date)T$(x-bluecoat-hour-utc):$(x-bluecoat-minute-utc):$(x-bluecoat-second-utc).000z $(x-bluecoat-appliance-name) bluecoat - splunk_format - c-ip=$(c-ip) Content-Type=$(quot)$(rs(Content-Type))$(quot) cs-auth-group=$(cs-auth-group) cs-bytes=$(cs-bytes) cs-categories=$(quot)$(cs-categories)$(quot) cs-host=$(cs-host) cs-ip=$(cs-ip) cs-method=$(cs-method) cs-uri-extension=$(cs-uri-extension) cs-uri-path=$(cs-uri-path) cs-uri-port=$(cs-uri-port) cs-uri-query=$(quot)$(cs-uri-query)$(quot) cs-uri-scheme=$(cs-uri-scheme) cs-User-Agent=$(quot)$(cs(User-Agent))$(quot) cs-username=$(cs-username) dnslookup-time=$(dnslookup-time) duration=$(duration) rs-status=$(rs-status) rs-version=$(rs-version) rs_Content_Type=$(rs-Content-Type) s-action=$(s-action) s-ip=$(s-ip) service.name=$(service.name) service.group=$(service.group) s-supplier-ip=$(s-supplier-ip) s-supplier-name=$(s-supplier-name) sc-bytes=$(sc-bytes) sc-filter-result=$(sc-filter-result) sc-filter-result=$(sc-filter-result) sc-status=$(sc-status) time-taken=$(time-taken) x-bluecoat-appliance-name=$(x-bluecoat-appliance-name) x-bluecoat-appliance-primary-address=$(x-bluecoat-appliance-primary-address) x-bluecoat-application-name=$(x-bluecoat-application-name) x-bluecoat-application-operation=$(x-bluecoat-application-operation) x-bluecoat-proxy-primary-address=$(x-bluecoat-proxy-primary-address) x-bluecoat-transaction-uuid=$(x-bluecoat-transaction-uuid) x-exception-id=$(x-exception-id) x-virus-id=$(x-virus-id) c-url=$(quot)$(url)$(quot) cs-Referer=$(quot)$(cs(Referer))$(quot) c-cpu=$(c-cpu) c-uri-pathquery=$(c-uri-pathquery) connect-time=$(connect-time) cs-auth-groups=$(cs-auth-groups) cs-headerlength=$(cs-headerlength) cs-threat-risk=$(cs-threat-risk) r-ip=$(r-ip) r-supplier-ip=$(r-supplier-ip) rs-time-taken=$(rs-time-taken) rs-server=$(rs(server)) s-connect-type=$(s-connect-type) s-icap-status=$(s-icap-status) s-sitename=$(s-sitename) s-source-port=$(s-source-port) s-supplier-country=$(s-supplier-country) sc-Content-Encoding=$(sc(Content-Encoding)) sr-Accept-Encoding=$(sr(Accept-Encoding)) x-auth-credential-type=$(x-auth-credential-type) x-cookie-date=$(x-cookie-date) x-cs-certificate-subject=$(x-cs-certificate-subject) x-cs-connection-negotiated-cipher=$(x-cs-connection-negotiated-cipher) x-cs-connection-negotiated-cipher-size=$(x-cs-connection-negotiated-cipher-size) x-cs-connection-negotiated-ssl-version=$(x-cs-connection-negotiated-ssl-version) x-cs-ocsp-error=$(x-cs-ocsp-error) x-cs-Referer-uri=$(x-cs(Referer)-uri) x-cs-Referer-uri-address=$(x-cs(Referer)-uri-address) x-cs-Referer-uri-extension=$(x-cs(Referer)-uri-extension) x-cs-Referer-uri-host=$(x-cs(Referer)-uri-host) x-cs-Referer-uri-hostname=$(x-cs(Referer)-uri-hostname) x-cs-Referer-uri-path=$(x-cs(Referer)-uri-path) x-cs-Referer-uri-pathquery=$(x-cs(Referer)-uri-pathquery) x-cs-Referer-uri-port=$(x-cs(Referer)-uri-port) x-cs-Referer-uri-query=$(x-cs(Referer)-uri-query) x-cs-Referer-uri-scheme=$(x-cs(Referer)-uri-scheme) x-cs-Referer-uri-stem=$(x-cs(Referer)-uri-stem) x-exception-category=$(x-exception-category) x-exception-category-review-message=$(x-exception-category-review-message) x-exception-company-name=$(x-exception-company-name) x-exception-contact=$(x-exception-contact) x-exception-details=$(x-exception-details) x-exception-header=$(x-exception-header) x-exception-help=$(x-exception-help) x-exception-last-error=$(x-exception-last-error) x-exception-reason=$(x-exception-reason) x-exception-sourcefile=$(x-exception-sourcefile) x-exception-sourceline=$(x-exception-sourceline) x-exception-summary=$(x-exception-summary) x-icap-error-code=$(x-icap-error-code) x-rs-certificate-hostname=$(x-rs-certificate-hostname) x-rs-certificate-hostname-category=$(x-rs-certificate-hostname-category) x-rs-certificate-observed-errors=$(x-rs-certificate-observed-errors) x-rs-certificate-subject=$(x-rs-certificate-subject) x-rs-certificate-validate-status=$(x-rs-certificate-validate-status) x-rs-connection-negotiated-cipher=$(x-rs-connection-negotiated-cipher) x-rs-connection-negotiated-cipher-size=$(x-rs-connection-negotiated-cipher-size) x-rs-connection-negotiated-ssl-version=$(x-rs-connection-negotiated-ssl-version) x-rs-ocsp-error=$(x-rs-ocsp-error) # @pytest.mark.addons("broadcom") -def test_bluecoatproxySG_kv_5424( +def test_bluecoatproxysg_kv_5424( record_property, setup_splunk, setup_sc4s ): host = f"{shortuuid.ShortUUID().random(length=5).lower()}-{shortuuid.ShortUUID().random(length=5).lower()}" @@ -86,7 +86,7 @@ def test_bluecoatproxySG_kv_5424( # <111>1 $(date)T$(x-bluecoat-hour-utc):$(x-bluecoat-minute-utc):$(x-bluecoat-second-utc).000z $(x-bluecoat-appliance-name) bluecoat - splunk_format - c-ip=$(c-ip) Content-Type=$(quot)$(rs(Content-Type))$(quot) cs-auth-group=$(cs-auth-group) cs-bytes=$(cs-bytes) cs-categories=$(quot)$(cs-categories)$(quot) cs-host=$(cs-host) cs-ip=$(cs-ip) cs-method=$(cs-method) cs-uri-extension=$(cs-uri-extension) cs-uri-path=$(cs-uri-path) cs-uri-port=$(cs-uri-port) cs-uri-query=$(quot)$(cs-uri-query)$(quot) cs-uri-scheme=$(cs-uri-scheme) cs-User-Agent=$(quot)$(cs(User-Agent))$(quot) cs-username=$(cs-username) dnslookup-time=$(dnslookup-time) duration=$(duration) rs-status=$(rs-status) rs-version=$(rs-version) rs_Content_Type=$(rs-Content-Type) s-action=$(s-action) s-ip=$(s-ip) service.name=$(service.name) service.group=$(service.group) s-supplier-ip=$(s-supplier-ip) s-supplier-name=$(s-supplier-name) sc-bytes=$(sc-bytes) sc-filter-result=$(sc-filter-result) sc-filter-result=$(sc-filter-result) sc-status=$(sc-status) time-taken=$(time-taken) x-bluecoat-appliance-name=$(x-bluecoat-appliance-name) x-bluecoat-appliance-primary-address=$(x-bluecoat-appliance-primary-address) x-bluecoat-application-name=$(x-bluecoat-application-name) x-bluecoat-application-operation=$(x-bluecoat-application-operation) x-bluecoat-proxy-primary-address=$(x-bluecoat-proxy-primary-address) x-bluecoat-transaction-uuid=$(x-bluecoat-transaction-uuid) x-exception-id=$(x-exception-id) x-virus-id=$(x-virus-id) c-url=$(quot)$(url)$(quot) cs-Referer=$(quot)$(cs(Referer))$(quot) c-cpu=$(c-cpu) c-uri-pathquery=$(c-uri-pathquery) connect-time=$(connect-time) cs-auth-groups=$(cs-auth-groups) cs-headerlength=$(cs-headerlength) cs-threat-risk=$(cs-threat-risk) r-ip=$(r-ip) r-supplier-ip=$(r-supplier-ip) rs-time-taken=$(rs-time-taken) rs-server=$(rs(server)) s-connect-type=$(s-connect-type) s-icap-status=$(s-icap-status) s-sitename=$(s-sitename) s-source-port=$(s-source-port) s-supplier-country=$(s-supplier-country) sc-Content-Encoding=$(sc(Content-Encoding)) sr-Accept-Encoding=$(sr(Accept-Encoding)) x-auth-credential-type=$(x-auth-credential-type) x-cookie-date=$(x-cookie-date) x-cs-certificate-subject=$(x-cs-certificate-subject) x-cs-connection-negotiated-cipher=$(x-cs-connection-negotiated-cipher) x-cs-connection-negotiated-cipher-size=$(x-cs-connection-negotiated-cipher-size) x-cs-connection-negotiated-ssl-version=$(x-cs-connection-negotiated-ssl-version) x-cs-ocsp-error=$(x-cs-ocsp-error) x-cs-Referer-uri=$(x-cs(Referer)-uri) x-cs-Referer-uri-address=$(x-cs(Referer)-uri-address) x-cs-Referer-uri-extension=$(x-cs(Referer)-uri-extension) x-cs-Referer-uri-host=$(x-cs(Referer)-uri-host) x-cs-Referer-uri-hostname=$(x-cs(Referer)-uri-hostname) x-cs-Referer-uri-path=$(x-cs(Referer)-uri-path) x-cs-Referer-uri-pathquery=$(x-cs(Referer)-uri-pathquery) x-cs-Referer-uri-port=$(x-cs(Referer)-uri-port) x-cs-Referer-uri-query=$(x-cs(Referer)-uri-query) x-cs-Referer-uri-scheme=$(x-cs(Referer)-uri-scheme) x-cs-Referer-uri-stem=$(x-cs(Referer)-uri-stem) x-exception-category=$(x-exception-category) x-exception-category-review-message=$(x-exception-category-review-message) x-exception-company-name=$(x-exception-company-name) x-exception-contact=$(x-exception-contact) x-exception-details=$(x-exception-details) x-exception-header=$(x-exception-header) x-exception-help=$(x-exception-help) x-exception-last-error=$(x-exception-last-error) x-exception-reason=$(x-exception-reason) x-exception-sourcefile=$(x-exception-sourcefile) x-exception-sourceline=$(x-exception-sourceline) x-exception-summary=$(x-exception-summary) x-icap-error-code=$(x-icap-error-code) x-rs-certificate-hostname=$(x-rs-certificate-hostname) x-rs-certificate-hostname-category=$(x-rs-certificate-hostname-category) x-rs-certificate-observed-errors=$(x-rs-certificate-observed-errors) x-rs-certificate-subject=$(x-rs-certificate-subject) x-rs-certificate-validate-status=$(x-rs-certificate-validate-status) x-rs-connection-negotiated-cipher=$(x-rs-connection-negotiated-cipher) x-rs-connection-negotiated-cipher-size=$(x-rs-connection-negotiated-cipher-size) x-rs-connection-negotiated-ssl-version=$(x-rs-connection-negotiated-ssl-version) x-rs-ocsp-error=$(x-rs-ocsp-error) # @pytest.mark.addons("broadcom") -def test_bluecoatproxySG_syslog( +def test_bluecoatproxysg_syslog( record_property, setup_splunk, setup_sc4s ): host = f"{shortuuid.ShortUUID().random(length=5).lower()}-{shortuuid.ShortUUID().random(length=5).lower()}" diff --git a/tests/test_vectra_json.py b/tests/test_vectra_json.py new file mode 100644 index 0000000000..a01247ebf1 --- /dev/null +++ b/tests/test_vectra_json.py @@ -0,0 +1,303 @@ +import shortuuid +import pytest + +from jinja2 import Environment, select_autoescape + +from .sendmessage import sendsingle +from .splunkutils import splunk_single +from .timeutils import time_operations +import datetime + +env = Environment(autoescape=select_autoescape(default_for_string=False)) + +# <13>Jan 16 11:51:35 xxxxxxx vectra_json_v2 -: {"version": "$version", "dvchost": "$dvchost", "host_ip": "$host_ip", "href": "$href", "src_key_asset": $src_key_asset, "host_id": $host_id, "headend_addr": "$headend_addr", "category": "HOST SCORING", "dst_key_asset": $dst_key_asset, "privilege": $privilege, "certainty": $certainty, "score_decreases": $score_decreases, "vectra_timestamp": "$timestamp", "host_name": "$host_name", "threat": $threat} + +@pytest.mark.addons("vectra") +def test_vectra_ai_hostscoring_json(record_property, setup_splunk, setup_sc4s): + host = f"{shortuuid.ShortUUID().random(length=5).lower()}-{shortuuid.ShortUUID().random(length=5).lower()}" + + dt = datetime.datetime.now() + _, bsd, _, _, _, _, epoch = time_operations(dt) + + # Tune time functions + epoch = epoch[:-7] + + mt = env.from_string( + '{{ mark }}{{ bsd }} {{ host }} vectra_json_v2 -: {"version": "$version", "dvchost": "$dvchost", "host_ip": "$host_ip", "href": "$href", "src_key_asset": $src_key_asset, "host_id": $host_id, "headend_addr": "$headend_addr", "category": "HOST SCORING", "dst_key_asset": $dst_key_asset, "privilege": $privilege, "certainty": $certainty, "score_decreases": $score_decreases, "vectra_timestamp": "$timestamp", "host_name": "$host_name", "threat": $threat}' + ) + message = mt.render(mark="<13>", bsd=bsd, host=host) + + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + st = env.from_string( + 'search _time={{ epoch }} index=main host="{{ host }}" sourcetype="vectra:cognito:hostscoring:json"' + ) + search = st.render(epoch=epoch, host=host) + + result_count, _ = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", result_count) + record_property("message", message) + + assert result_count == 1 + + +# <13>Jan 16 11:51:35 xxxxxxx vectra_json_account_v2 -: {"category": "ACCOUNT SCORING", "account_id": $account_id, "href": "$href", "certainty": $certainty, "privilege": $privilege, "score_decreases": $score_decreases, "version": "$version", "vectra_timestamp": "$timestamp", "headend_addr": "$headend_addr", "threat": $threat, "account_uid": "$account_uid"} +@pytest.mark.addons("vectra") +def test_vectra_ai_accountscoring_json(record_property, setup_splunk, setup_sc4s): + host = f"{shortuuid.ShortUUID().random(length=5).lower()}-{shortuuid.ShortUUID().random(length=5).lower()}" + + dt = datetime.datetime.now() + _, bsd, _, _, _, _, epoch = time_operations(dt) + + # Tune time functions + epoch = epoch[:-7] + + mt = env.from_string( + '{{ mark }}{{ bsd }} {{ host }} vectra_json_account_v2 -: {"category": "ACCOUNT SCORING", "account_id": $account_id, "href": "$href", "certainty": $certainty, "privilege": $privilege, "score_decreases": $score_decreases, "version": "$version", "vectra_timestamp": "$timestamp", "headend_addr": "$headend_addr", "threat": $threat, "account_uid": "$account_uid"}' + ) + message = mt.render(mark="<13>", bsd=bsd, host=host) + + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + st = env.from_string( + 'search _time={{ epoch }} index=main host="{{ host }}" sourcetype="vectra:cognito:accountscoring:json"' + ) + search = st.render(epoch=epoch, host=host) + + result_count, _ = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", result_count) + record_property("message", message) + + assert result_count == 1 + + +# <13>Jan 16 11:51:35 xxxxxxx vectra_json_v2 -: {"d_type_vname": "$d_type_vname", "dvchost": "$dvchost", "host_ip": "$host_ip", "href": "$href", "detection_id": $detection_id, "dd_bytes_sent": $dd_bytes_sent, "headend_addr": "$headend_addr", "dd_dst_port": $dd_dst_port, "category": "$category", "dd_bytes_rcvd": $dd_bytes_rcvd, "dd_dst_dns": "$dd_dst_dns", "severity": $severity, "certainty": $certainty, "triaged": $triaged, "vectra_timestamp": "$timestamp", "version": "$version", "host_name": "$host_name", "threat": $threat, "dd_dst_ip": "$dd_dst_ip", "dd_proto": "$dd_proto", "d_type": "$d_type"} +@pytest.mark.addons("vectra") +def test_vectra_ai_hostdetect_json( + record_property, setup_splunk, setup_sc4s +): + host = f"{shortuuid.ShortUUID().random(length=5).lower()}-{shortuuid.ShortUUID().random(length=5).lower()}" + + dt = datetime.datetime.now() + _, bsd, _, _, _, _, epoch = time_operations(dt) + + # Tune time functions + epoch = epoch[:-7] + + mt = env.from_string( + '{{ mark }}{{ bsd }} {{ host }} vectra_json_v2 -: {"d_type_vname": "$d_type_vname", "dvchost": "$dvchost", "host_ip": "$host_ip", "href": "$href", "detection_id": $detection_id, "dd_bytes_sent": $dd_bytes_sent, "headend_addr": "$headend_addr", "dd_dst_port": $dd_dst_port, "category": "$category", "dd_bytes_rcvd": $dd_bytes_rcvd, "dd_dst_dns": "$dd_dst_dns", "severity": $severity, "certainty": $certainty, "triaged": $triaged, "vectra_timestamp": "$timestamp", "version": "$version", "host_name": "$host_name", "threat": $threat, "dd_dst_ip": "$dd_dst_ip", "dd_proto": "$dd_proto", "d_type": "$d_type"}' + ) + message = mt.render(mark="<13>", bsd=bsd, host=host) + + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + st = env.from_string( + 'search _time={{ epoch }} index=main host="{{ host }}" sourcetype="vectra:cognito:hostdetect:json"' + ) + search = st.render(epoch=epoch, host=host) + + result_count, _ = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", result_count) + record_property("message", message) + + assert result_count == 1 + + +# <13>Jan 16 11:51:35 xxxxxxx vectra_json_account_v2 -: {"d_type_vname": "$d_type_vname", "dvchost": "$dvchost", "href": "$href", "detection_id": $detect ion_id, "dd_bytes_sent": $dd_bytes_sent, "headend_addr": "$headend_addr", "dd_dst_port": $dd_dst_ port, "category": "$category", "dd_bytes_rcvd": $dd_bytes_rcvd, "dd_dst_dns": "$dd_dst_dns", "sev erity": $severity, "certainty": $certainty, "triaged": $triaged, "vectra_timestamp": "$timestamp", "account_uid": "$account_uid", "version": "$version", "threat": $threat, "dd_dst_ip": "$dd_dst_ip", "d_type": "$d_type"} +@pytest.mark.addons("vectra") +def test_vectra_ai_accountdetect_json( + record_property, setup_splunk, setup_sc4s +): + host = f"{shortuuid.ShortUUID().random(length=5).lower()}-{shortuuid.ShortUUID().random(length=5).lower()}" + + dt = datetime.datetime.now() + _, bsd, _, _, _, _, epoch = time_operations(dt) + + # Tune time functions + epoch = epoch[:-7] + + mt = env.from_string( + '{{ mark }}{{ bsd }} {{ host }} vectra_json_account_v2 -: {"d_type_vname": "$d_type_vname", "dvchost": "$dvchost", "href": "$href", "detection_id": $detect ion_id, "dd_bytes_sent": $dd_bytes_sent, "headend_addr": "$headend_addr", "dd_dst_port": $dd_dst_ port, "category": "$category", "dd_bytes_rcvd": $dd_bytes_rcvd, "dd_dst_dns": "$dd_dst_dns", "sev erity": $severity, "certainty": $certainty, "triaged": $triaged, "vectra_timestamp": "$timestamp", "account_uid": "$account_uid", "version": "$version", "threat": $threat, "dd_dst_ip": "$dd_dst_ip", "d_type": "$d_type"}' + ) + message = mt.render(mark="<13>", bsd=bsd, host=host) + + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + st = env.from_string( + 'search _time={{ epoch }} index=main host="{{ host }}" sourcetype="vectra:cognito:accountdetect:json"' + ) + search = st.render(epoch=epoch, host=host) + + result_count, _ = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", result_count) + record_property("message", message) + + assert result_count == 1 + + +# <13>Jan 16 11:51:35 xxxxxxx vectra_json_v2 -: {"category": "$category ", "version": "$version", "success": "$success", "vectra_timestamp": "$UTCTime", "will_retry": "$retry", "href": "$href", "host_name": "$host_name", "action": "$action", "host_id": "$host_id", "headend_addr": "$headend_addr", "user": "$user"} +@pytest.mark.addons("vectra") +def test_vectra_ai_hostlockdown_json(record_property, setup_splunk, setup_sc4s): + host = f"{shortuuid.ShortUUID().random(length=5).lower()}-{shortuuid.ShortUUID().random(length=5).lower()}" + + dt = datetime.datetime.now() + _, bsd, _, _, _, _, epoch = time_operations(dt) + + # Tune time functions + epoch = epoch[:-7] + + mt = env.from_string( + '{{ mark }}{{ bsd }} {{ host }} vectra_json_v2 -: {"category": "$category ", "version": "$version", "success": "$success", "vectra_timestamp": "$UTCTime", "will_retry": "$retry", "href": "$href", "host_name": "$host_name", "action": "$action", "host_id": "$host_id", "headend_addr": "$headend_addr", "user": "$user"}' + ) + message = mt.render(mark="<13>", bsd=bsd, host=host) + + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + st = env.from_string( + 'search _time={{ epoch }} index=main host="{{ host }}" sourcetype="vectra:cognito:hostlockdown:json"' + ) + search = st.render(epoch=epoch, host=host) + + result_count, _ = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", result_count) + record_property("message", message) + + assert result_count == 1 + + +# <13>Jan 16 11:51:35 xxxxxxx vectra_json_account_v2 -: {"category": "$category", "account_id": $account_id, "success": $success, "href": "$href", "vectra_timestamp": "$UTCTime", "headend_addr": "$headend_addr", "user": "$user", "version": "$version", "action": "$action", "account_uid": "$account_name"} +@pytest.mark.addons("vectra") +def test_vectra_ai_accountlockdown_json(record_property, setup_splunk, setup_sc4s): + host = f"{shortuuid.ShortUUID().random(length=5).lower()}-{shortuuid.ShortUUID().random(length=5).lower()}" + + dt = datetime.datetime.now() + _, bsd, _, _, _, _, epoch = time_operations(dt) + + # Tune time functions + epoch = epoch[:-7] + + mt = env.from_string( + '{{ mark }}{{ bsd }} {{ host }} vectra_json_account_v2 -: {"category": "$category", "account_id": $account_id, "success": $success, "href": "$href", "vectra_timestamp": "$UTCTime", "headend_addr": "$headend_addr", "user": "$user", "version": "$version", "action": "$action", "account_uid": "$account_name"}' + ) + message = mt.render(mark="<13>", bsd=bsd, host=host) + + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + st = env.from_string( + 'search _time={{ epoch }} index=main host="{{ host }}" sourcetype="vectra:cognito:accountlockdown:json"' + ) + search = st.render(epoch=epoch, host=host) + + result_count, _ = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", result_count) + record_property("message", message) + + assert result_count == 1 + + +# <13>Jan 16 11:51:35 xxxxxxx vectra_json_v2 -: {"src_hid": "$src_hid", "timestamp": "$syslog_timestamp", "dvchost": "$dvchost", "campaign_id": "$campaign_id", "reason": "$reason", "src_name": "$src_name", "campaign_name": "$campaign_name", "campaign_link": "$campaign_link", "headend_addr": "$headend_addr", "dest_name": "$dest_name", "dest_id": "$dest_id", "vectra_timestamp": "$vectra_timestamp", "src_ip": "$src_ip", "version": "$version", "action": "$action", "dest_ip": "$dest_ip", "det_id": "$det_id"} +@pytest.mark.addons("vectra") +def test_vectra_ai_campaign_json(record_property, setup_splunk, setup_sc4s): + host = f"{shortuuid.ShortUUID().random(length=5).lower()}-{shortuuid.ShortUUID().random(length=5).lower()}" + + dt = datetime.datetime.now() + _, bsd, _, _, _, _, epoch = time_operations(dt) + + # Tune time functions + epoch = epoch[:-7] + + mt = env.from_string( + '{{ mark }}{{ bsd }} {{ host }} vectra_json_v2 -: {"src_hid": "$src_hid", "timestamp": "$syslog_timestamp", "dvchost": "$dvchost", "campaign_id": "$campaign_id", "reason": "$reason", "src_name": "$src_name", "campaign_name": "$campaign_name", "campaign_link": "$campaign_link", "headend_addr": "$headend_addr", "dest_name": "$dest_name", "dest_id": "$dest_id", "vectra_timestamp": "$vectra_timestamp", "src_ip": "$src_ip", "version": "$version", "action": "$action", "dest_ip": "$dest_ip", "det_id": "$det_id"}' + ) + message = mt.render(mark="<13>", bsd=bsd, host=host) + + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + st = env.from_string( + 'search _time={{ epoch }} index=main host="{{ host }}" sourcetype="vectra:cognito:campaigns:json"' + ) + search = st.render(epoch=epoch, host=host) + + result_count, _ = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", result_count) + record_property("message", message) + + assert result_count == 1 + + +# <13>Jan 16 11:51:35 xxxxxxx vectra_json_v2 -: {"source_ip": "$source_ip", "dvchost": "$dvchost", "version": "$version", "role": "$role", "user": "$user", "message": "$message", "vectra_timestamp": "$vectra_timestamp", "headend_addr": "$headend_addr", "result": "$result"} +@pytest.mark.addons("vectra") +def test_vectra_ai_audit_json(record_property, setup_splunk, setup_sc4s): + host = f"{shortuuid.ShortUUID().random(length=5).lower()}-{shortuuid.ShortUUID().random(length=5).lower()}" + + dt = datetime.datetime.now() + _, bsd, _, _, _, _, epoch = time_operations(dt) + + # Tune time functions + epoch = epoch[:-7] + + mt = env.from_string( + '{{ mark }}{{ bsd }} {{ host }} vectra_json_v2 -: {"source_ip": "$source_ip", "dvchost": "$dvchost", "version": "$version", "role": "$role", "user": "$user", "message": "$message", "vectra_timestamp": "$vectra_timestamp", "headend_addr": "$headend_addr", "result": "$result"}' + ) + message = mt.render(mark="<13>", bsd=bsd, host=host) + + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + st = env.from_string( + 'search _time={{ epoch }} index=main host="{{ host }}" sourcetype="vectra:cognito:audit:json"' + ) + search = st.render(epoch=epoch, host=host) + + result_count, _ = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", result_count) + record_property("message", message) + + assert result_count == 1 + + +# <13>Jan 16 11:51:35 xxxxxxx vectra_json_v2 -: {"vectra_timestamp": "$vectra_timestamp", "version": "$version", "result": "$result", "type": "$type", "source_ip": "$source_ip", "message": "$message", "dvchost": "$dvchost", "headend_addr": "$headend_addr"} +@pytest.mark.addons("vectra") +def test_vectra_ai_health(record_property, setup_splunk, setup_sc4s): + host = f"{shortuuid.ShortUUID().random(length=5).lower()}-{shortuuid.ShortUUID().random(length=5).lower()}" + + dt = datetime.datetime.now() + _, bsd, _, _, _, _, epoch = time_operations(dt) + + # Tune time functions + epoch = epoch[:-7] + + mt = env.from_string( + '{{ mark }}{{ bsd }} {{ host }} vectra_json_v2 -: {"vectra_timestamp": "$vectra_timestamp", "version": "$version", "result": "$result", "type": "$type", "source_ip": "$source_ip", "message": "$message", "dvchost": "$dvchost", "headend_addr": "$headend_addr"}' + ) + message = mt.render(mark="<13>", bsd=bsd, host=host) + + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + st = env.from_string( + 'search _time={{ epoch }} index=main host="{{ host }}" sourcetype="vectra:cognito:health:json"' + ) + search = st.render(epoch=epoch, host=host) + + result_count, _ = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", result_count) + record_property("message", message) + + assert result_count == 1 diff --git a/tests/timeutils.py b/tests/timeutils.py index b4341fc4c4..b00f6202ac 100644 --- a/tests/timeutils.py +++ b/tests/timeutils.py @@ -7,7 +7,7 @@ def insert_char(string, char, integer): return string[0:integer] + char + string[integer:] -def removeZero(tz): +def remove_zero(tz): return re.sub(r"\b0+(\d)(?=:)", r"\1", tz)