feat: #31 add bind mount ignore patterns and option to disable Overlay2 scan (#44)

Author: amerkurev

README.md

@@ -92,7 +92,9 @@ Doku can be configured using environment variables. You can set these either dir
 | SCAN_INTERVAL | How often to collect basic Docker usage data (in seconds) | 60 |
 | SCAN_LOGFILE_INTERVAL | How frequently to check container log sizes (in seconds) | 300 |
 | SCAN_BINDMOUNTS_INTERVAL | Time between bind mount scanning operations (in seconds) | 3600 |
+| BINDMOUNT_IGNORE_PATTERNS | Paths matching these patterns will be excluded from bind mount scanning (semicolon-separated) (e.g., `/home/*;/tmp/*;*/.git/*`) | "" |
 | SCAN_OVERLAY2_INTERVAL | How often to analyze Overlay2 storage (in seconds) | 86400 |
+| DISABLE_OVERLAY2_SCAN | Disable Overlay2 storage scanning | false |
 | SCAN_INTENSITY | Performance impact level: "aggressive" (highest CPU usage), "normal" (balanced), or "light" (lowest impact) | normal |
 | SCAN_USE_DU | Use the faster system `du` command for disk calculations instead of slower built-in methods | true |
 | UVICORN_WORKERS | Number of web server worker processes | 1 |

app/scan/du.py

@@ -25,9 +25,12 @@ def main():
     schedule.every(settings.SCAN_BINDMOUNTS_INTERVAL).seconds.do(scanner.scan)
 
     ### Docker Overlay2 Scanner ###
-    scanner = Overlay2Scanner(is_stop=signal_.is_stop)
-    scanner.scan()  # run once immediately
-    schedule.every(settings.SCAN_OVERLAY2_INTERVAL).seconds.do(scanner.scan)
+    if settings.DISABLE_OVERLAY2_SCAN:
+        logger.warning('Overlay2 scanner disabled.')
+    else:
+        scanner = Overlay2Scanner(is_stop=signal_.is_stop)
+        scanner.scan()  # run once immediately
+        schedule.every(settings.SCAN_OVERLAY2_INTERVAL).seconds.do(scanner.scan)
 
     # main loop
     while not signal_.is_stop():

app/scan/scanner.py

@@ -1,4 +1,5 @@
 import time
+import fnmatch
 from collections.abc import Callable
 from pathlib import Path
 
@@ -229,6 +230,13 @@ def database_name(self):
     def table_name(self):
         return settings.TABLE_BINDMOUNTS
 
+    def should_ignore_path(self, path: str) -> bool:
+        """Check if the path matches any ignore pattern."""
+        for pattern in settings.BINDMOUNT_IGNORE_PATTERNS:
+            if fnmatch.fnmatch(path, pattern):
+                return True
+        return False
+
     def scan(self):
         if not self.doku_mounts:
             return
@@ -281,6 +289,11 @@ def scan(self):
                     if mnt.dst == '/var/run/docker.sock' or mnt.dst.startswith('/run/secrets/'):
                         continue
 
+                    # Skip paths matching ignore patterns
+                    if self.should_ignore_path(mnt.src):
+                        self.logger.debug(f'Skipping bind mount {mnt.src} as it matches an ignore pattern')
+                        continue
+
                     if mnt.src in already_scanned:
                         # skip already scanned bind mounts, but update the list of containers
                         obj = already_scanned[mnt.src]

app/settings.py

@@ -64,11 +64,22 @@ class Settings(BaseSettings):
         default=60 * 60,
         description='Time between bind mount scanning operations (in seconds)',
     )
+    bindmount_ignore_patterns: str = Field(
+        alias='BINDMOUNT_IGNORE_PATTERNS',
+        default='',
+        examples=['/home/*;/tmp/*;*/.git/*'],
+        description='Paths matching these patterns will be excluded from bind mount scanning (semicolon-separated)',
+    )
     scan_overlay2_interval: PositiveInt = Field(
         alias='SCAN_OVERLAY2_INTERVAL',
         default=60 * 60 * 24,
         description='How often to analyze Overlay2 storage (in seconds)',
     )
+    disable_overlay2_scan: bool = Field(
+        alias='DISABLE_OVERLAY2_SCAN',
+        default=False,
+        description='Disable Overlay2 storage scanning',
+    )
     scan_intensity: ScanIntensity = Field(
         alias='SCAN_INTENSITY',
         default=ScanIntensity.NORMAL,
@@ -136,6 +147,20 @@ def log_level_num(self) -> int:
         }
         return level_map[self.log_level]
 
+    @cached_property
+    def bindmount_ignore_patterns_list(self) -> list[str]:
+        patterns = self.bindmount_ignore_patterns
+
+        # Remove surrounding quotes from the entire string if present
+        if (patterns.startswith('"') and patterns.endswith('"')) or (
+            patterns.startswith("'") and patterns.endswith("'")
+        ):
+            patterns = patterns[1:-1]
+
+        # Split and filter empty values
+        result = list(filter(None, map(str.strip, patterns.split(';'))))
+        return result
+
 
 try:
     _settings = Settings()
@@ -168,7 +193,9 @@ def log_level_num(self) -> int:
 SCAN_INTERVAL = _settings.scan_interval
 SCAN_LOGFILE_INTERVAL = _settings.scan_logfile_interval
 SCAN_BINDMOUNTS_INTERVAL = _settings.scan_bindmounts_interval
+BINDMOUNT_IGNORE_PATTERNS = _settings.bindmount_ignore_patterns_list
 SCAN_OVERLAY2_INTERVAL = _settings.scan_overlay2_interval
+DISABLE_OVERLAY2_SCAN = _settings.disable_overlay2_scan
 SCAN_INTENSITY = _settings.scan_intensity
 SCAN_SLEEP_DURATION = {
     ScanIntensity.AGGRESSIVE: 0,  # no sleep, but CPU throttling
@@ -242,7 +269,9 @@ def to_string() -> str:
             'scan_interval',
             'scan_logfile_interval',
             'scan_bindmounts_interval',
+            'bindmount_ignore_patterns',
             'scan_overlay2_interval',
+            'disable_overlay2_scan',
             'scan_intensity',
             'scan_use_du',
         ],

app/test_main.py

@@ -1,6 +1,6 @@
 from unittest.mock import patch
 
-from settings import to_string as settings_to_string
+from settings import Settings, to_string as settings_to_string
 from main import main
 
 
@@ -76,6 +76,8 @@ def test_settings_to_string():
             'ssl_keyfile_password': 'secret',
             'ssl_certfile': '/.ssl/cert.pem',
             'scan_interval': 60,
+            'bindmount_ignore_patterns': '/home/*;/tmp/*;*/.git/*',
+            'disable_overlay2_scan': False,
             'workers': 4,
             'docker_host': 'unix:///var/run/docker.sock',
             'git_tag': 'v1.2.3',
@@ -105,6 +107,11 @@ def test_settings_to_string():
         assert 'log_level: info' in result
         assert 'root_path: /doku' in result
 
+        # Scan settings
+        assert 'scan_interval: 60' in result
+        assert 'bindmount_ignore_patterns: /home/*;/tmp/*;*/.git/*' in result
+        assert 'disable_overlay2_scan: False' in result
+
         # Verify password is masked
         assert 'ssl_keyfile_password: ********' in result
         assert 'ssl_keyfile_password: secret' not in result
@@ -119,3 +126,47 @@ def test_settings_to_string():
         # Verify version info
         assert 'git_tag: v1.2.3' in result
         assert 'git_sha: abcdef1234567890' in result
+
+
+def patterns_assert(patterns: list[str]):
+    assert len(patterns) == 3
+    assert '/home/*' in patterns
+    assert '/tmp/*' in patterns
+    assert '*/.git/*' in patterns
+
+
+def test_bindmount_ignore_patterns_list():
+    with patch('os.environ', {'BINDMOUNT_IGNORE_PATTERNS': '/home/*;/tmp/*;*/.git/*'}):
+        s = Settings()
+        patterns_assert(s.bindmount_ignore_patterns_list)
+
+    # Test with entire string in double quotes
+    with patch('os.environ', {'BINDMOUNT_IGNORE_PATTERNS': '"/home/*;/tmp/*;*/.git/*"'}):
+        s = Settings()
+        patterns_assert(s.bindmount_ignore_patterns_list)
+
+    # Test with entire string in single quotes
+    with patch('os.environ', {'BINDMOUNT_IGNORE_PATTERNS': "';/home/*;/tmp/*;*/.git/*;'"}):
+        s = Settings()
+        patterns_assert(s.bindmount_ignore_patterns_list)
+
+    # Test with a single pattern
+    with patch('os.environ', {'BINDMOUNT_IGNORE_PATTERNS': '/var/log/*;'}):
+        s = Settings()
+        patterns = s.bindmount_ignore_patterns_list
+        assert len(patterns) == 1
+        assert '/var/log/*' in patterns
+
+    # Test with empty string
+    with patch('os.environ', {'BINDMOUNT_IGNORE_PATTERNS': ';;'}):
+        s = Settings()
+        patterns = s.bindmount_ignore_patterns_list
+        assert len(patterns) == 0
+
+    # Test with whitespace
+    with patch('os.environ', {'BINDMOUNT_IGNORE_PATTERNS': '  /path1/*;  /path2/*  '}):
+        s = Settings()
+        patterns = s.bindmount_ignore_patterns_list
+        assert len(patterns) == 2
+        assert '/path1/*' in patterns
+        assert '/path2/*' in patterns