Merge pull request #171 from adamrtalbot/do_not_validate_az_and_gcp_paths

Does not validate Azure or Google blob storage

Author: adamrtalbot

CHANGELOG.md

@@ -2,6 +2,10 @@
 
 # Version 1.1.4
 
+## Improvements
+
+- No longer does false validation on Azure and GCP cloud storage paths ([#171](https://github.com/nextflow-io/nf-validation/pull/171))
+
 ## Vulnerability fix
 
 - Updated the org.json package to version `20240303` ([#172](https://github.com/nextflow-io/nf-validation/pull/172))

plugins/nf-validation/src/main/nextflow/validation/FormatValidators/DirectoryPathValidator.groovy

@@ -11,8 +11,8 @@ public class DirectoryPathValidator implements FormatValidator {
 
     @Override
     public Optional<String> validate(final String subject) {
-        if (subject.startsWith('s3://')) {
-            log.debug("S3 paths are not supported by 'DirectoryPathValidator': '${subject}'")
+        if (subject.matches("(s3://|az://|gs://).*")) {
+            log.debug("Cloud storage paths are not supported by 'DirectoryPathValidator': '${subject}'")
             return Optional.empty()
         }
         Path file = Nextflow.file(subject) as Path

plugins/nf-validation/src/main/nextflow/validation/FormatValidators/FilePathPatternValidator.groovy

@@ -11,8 +11,8 @@ public class FilePathPatternValidator implements FormatValidator {
 
     @Override
     public Optional<String> validate(final String subject) {
-        if (subject.startsWith('s3://')) {
-            log.debug("S3 paths are not supported by 'FilePathPatternValidator': '${subject}'")
+        if (subject.matches("(s3://|az://|gs://).*")) {
+            log.debug("Cloud storage paths are not supported by 'FilePathPatternValidator': '${subject}'")
             return Optional.empty()
         }
         ArrayList files = Nextflow.files(subject)

plugins/nf-validation/src/main/nextflow/validation/FormatValidators/FilePathValidator.groovy

@@ -11,8 +11,8 @@ public class FilePathValidator implements FormatValidator {
 
     @Override
     public Optional<String> validate(final String subject) {
-        if (subject.startsWith('s3://')) {
-            log.debug("S3 paths are not supported by 'FilePathValidator': '${subject}'")
+        if (subject.matches("(s3://|az://|gs://).*")) {
+            log.debug("Cloud storage paths are not supported by 'FilePathValidator': '${subject}'")
             return Optional.empty()
         }
         Path file = Nextflow.file(subject) as Path  

plugins/nf-validation/src/main/nextflow/validation/FormatValidators/PathValidator.groovy

@@ -11,8 +11,8 @@ public class PathValidator implements FormatValidator {
 
     @Override
     public Optional<String> validate(final String subject) {
-        if (subject.startsWith('s3://')) {
-            log.debug("S3 paths are not supported by 'PathValidator': '${subject}'")
+        if (subject.matches("(s3://|az://|gs://).*")) {
+            log.debug("Cloud storage paths are not supported by 'PathValidator': '${subject}'")
             return Optional.empty()
         }
         Path file = Nextflow.file(subject) as Path  

plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy

@@ -615,8 +615,8 @@ class SchemaValidator extends PluginExtensionPoint {
     // Function to check if a file or directory exists
     //
     List pathExists(String path, String paramName, Boolean s3PathCheck) {
-        if (path.startsWith('s3://') && !s3PathCheck) {
-            log.debug "Ignoring validation of S3 URL path '${path}'".toString()
+        if (path.matches("(s3://|az://|gs://).*") && !s3PathCheck) {
+            log.debug "Ignoring validation of cloud storage path '${path}'".toString()
         } else {
             def Path file = Nextflow.file(path) as Path
             if (!file.exists()) {

plugins/nf-validation/src/resources/META-INF/MANIFEST.MF

@@ -1,6 +1,6 @@
 Manifest-Version: 1.0
 Plugin-Id: nf-validation
-Plugin-Version: 1.1.3
+Plugin-Version: 1.1.4
 Plugin-Class: nextflow.validation.ValidationPlugin
 Plugin-Provider: nextflow
 Plugin-Requires: >=22.10.0

plugins/nf-validation/src/test/nextflow/validation/PluginExtensionMethodsTest.groovy

@@ -445,6 +445,72 @@ class PluginExtensionMethodsTest extends Dsl2Spec{
         !stdout
     }
 
+    def 'should ignore s3 path' () {
+        given:
+        def schema = Path.of('src/testResources/nextflow_schema_file_cloud_path.json').toAbsolutePath().toString()
+        def  SCRIPT_TEXT = """
+            params.input = 's3://fake/path'
+            include { validateParameters } from 'plugin/nf-validation'
+            
+            validateParameters(parameters_schema: '$schema')
+        """
+
+        when:
+        dsl_eval(SCRIPT_TEXT)
+        def stdout = capture
+                .toString()
+                .readLines()
+                .findResults {it.contains('WARN nextflow.validation.SchemaValidator') || it.startsWith('* --') ? it : null }
+
+        then:
+        noExceptionThrown()
+        !stdout
+    }
+
+    def 'should ignore az path' () {
+        given:
+        def schema = Path.of('src/testResources/nextflow_schema_file_cloud_path.json').toAbsolutePath().toString()
+        def  SCRIPT_TEXT = """
+            params.input = 'az://fake/path'
+            include { validateParameters } from 'plugin/nf-validation'
+            
+            validateParameters(parameters_schema: '$schema')
+        """
+
+        when:
+        dsl_eval(SCRIPT_TEXT)
+        def stdout = capture
+                .toString()
+                .readLines()
+                .findResults {it.contains('WARN nextflow.validation.SchemaValidator') || it.startsWith('* --') ? it : null }
+
+        then:
+        noExceptionThrown()
+        !stdout
+    }
+
+    def 'should ignore gs path' () {
+        given:
+        def schema = Path.of('src/testResources/nextflow_schema_file_cloud_path.json').toAbsolutePath().toString()
+        def  SCRIPT_TEXT = """
+            params.input = 'gs://fake/path'
+            include { validateParameters } from 'plugin/nf-validation'
+            
+            validateParameters(parameters_schema: '$schema')
+        """
+
+        when:
+        dsl_eval(SCRIPT_TEXT)
+        def stdout = capture
+                .toString()
+                .readLines()
+                .findResults {it.contains('WARN nextflow.validation.SchemaValidator') || it.startsWith('* --') ? it : null }
+
+        then:
+        noExceptionThrown()
+        !stdout
+    }
+
     def 'correct validation of numbers with lenient mode' () {
         given:
         def schema = Path.of('src/testResources/nextflow_schema.json').toAbsolutePath().toString()

plugins/nf-validation/src/testResources/nextflow_schema_file_cloud_path.json

@@ -0,0 +1,26 @@
+{
+    "$schema": "http://json-schema.org/draft-07/schema",
+    "$id": "https://raw.githubusercontent.com/nf-core/testpipeline/master/nextflow_schema.json",
+    "title": "nf-core/testpipeline pipeline parameters",
+    "description": "this is a test",
+    "type": "object",
+    "definitions": {
+        "file_patterns": {
+            "title": "Input/output options",
+            "type": "object",
+            "fa_icon": "fas fa-terminal",
+            "properties": {
+                "input": {
+                    "type": "string",
+                    "format": "file-path",
+                    "exists": true
+                }
+            }
+        }
+    },
+    "allOf": [
+        {
+            "$ref": "#/definitions/file_patterns"
+        }
+    ]
+}