test components change

.github/workflows/components-pr.yaml

@@ -0,0 +1,247 @@
+# TODO combine this with publish-components.yml
+name: Components Checks
+
+on:
+  pull_request:
+    branches:
+      - master
+    paths:
+      - 'components/**'
+
+jobs:
+  check_version:
+    name: Ensure component commits modify component versions
+    runs-on: ubuntu-latest
+
+    permissions: 
+      contents: write
+      pull-requests: write
+
+    steps:
+      - uses: actions/[email protected]
+        name: Checkout repo
+        with:
+          # See https://github.com/actions/checkout#checkout-v2
+          # This will be slow. The intent is to fetch all commits
+          # since the merge-base (the commit where we branched off)
+          # so we can check the git diff against all changed files.
+          # By default, the checkout action only returns the last commit,
+          # There's no native way to do this in the checkout action, so
+          # we have to fetch the entire history. See
+          # https://github.com/actions/checkout/issues/266#issuecomment-638346893
+          fetch-depth: 0
+      - uses: jitterbit/get-changed-files@v1
+        id: changed_files
+        name: Get changed files
+        with:
+          format: json
+      - name: Check git diff for version changes
+        uses: ./.github/actions/git-diff-on-components
+        with:
+          all_files: ${{ steps.changed_files.outputs.all }}
+          base_commit: ${{ github.event.pull_request.base.sha }}
+          head_commit: ${{ github.event.pull_request.head.sha }}
+
+  check_components:
+    name: Check component keys and app props
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout Code
+        uses: actions/[email protected]
+        with:
+          # Full git history is needed to get a proper list of changed files
+          # within `super-linter`
+          fetch-depth: 0
+      - uses: pnpm/[email protected]
+        with:
+          version: 7.33.6
+      - name: Get pnpm store directory
+        id: pnpm-cache
+        run: |
+          echo "::set-output name=pnpm_cache_dir::$(pnpm store path)"
+      - uses: actions/cache@v4
+        name: Setup pnpm cache
+        with:
+          path: ${{ steps.pnpm-cache.outputs.pnpm_cache_dir }}
+          key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }}
+          restore-keys: |
+            ${{ runner.os }}-pnpm-store-
+      - name: Install dependencies
+        run: pnpm install -r
+      - name: Setup Node Env
+        uses: actions/[email protected]
+        with:
+          node-version: 18
+          registry-url: https://registry.npmjs.org/
+          cache: 'pnpm'
+      - name: Compile TypeScript
+        run: npm run build
+      - name: Get Changed Files
+        id: changed_files
+        uses: jitterbit/get-changed-files@v1
+        with:
+          format: 'csv'
+      - name: Check component keys
+        run: node scripts/findBadKeys.js ${{ steps.changed_files.outputs.added_modified }} ${{ steps.changed_files.outputs.renamed }}
+      - name: Check component app prop
+        run: node scripts/checkComponentAppProp.js ${{ steps.changed_files.outputs.added_modified }} ${{ steps.changed_files.outputs.renamed }}
+      - name: Check for duplicate component keys
+        run: node scripts/findDuplicateKeys.js
+
+  verify-typescript-components:
+    name: Verify TypeScript components
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/[email protected]
+      - uses: pnpm/[email protected]
+        with:
+          version: 7.33.6
+      - name: Get pnpm store directory
+        id: pnpm-cache
+        run: |
+          echo "::set-output name=pnpm_cache_dir::$(pnpm store path)"
+      - uses: actions/cache@v4
+        name: Setup pnpm cache
+        with:
+          path: ${{ steps.pnpm-cache.outputs.pnpm_cache_dir }}
+          key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }}
+          restore-keys: |
+            ${{ runner.os }}-pnpm-store-
+      - name: Setup Node Env
+        uses: actions/[email protected]
+        with:
+          node-version: 18
+          registry-url: https://registry.npmjs.org/
+          cache: 'pnpm'
+      - name: Install Dependencies
+        run: pnpm install -r
+      - name: Compile TypeScript
+        id: compile
+        run: npm run build > files.txt
+      - name: Get Changed Files
+        id: files
+        uses: jitterbit/get-changed-files@v1
+        with:
+          format: 'csv'
+      - name: Check For Compiled TypeScript Files
+        run: |
+          IFS=$'\n'
+          # Remove initial tsc output
+          output_files=$(cat files.txt | sed 1,3d)
+          declare -a ERRORS
+          declare -a SKIPPED
+          mapfile -d ',' -t added_modified_renamed_files < <(printf '%s,%s' '${{ steps.files.outputs.added_modified }}' '${{ steps.files.outputs.renamed }}')
+          for added_modified_file in "${added_modified_renamed_files[@]}";
+          do
+            echo "Checking if $added_modified_file is a publishable ts file"
+            if [[ $added_modified_file == components/* ]] && [[ $added_modified_file == *.ts ]] && [[ $added_modified_file != *.app.ts ]] \
+                && [[ $added_modified_file != */common*.ts ]] && [[ $added_modified_file != */common/* ]]
+            then
+              # XXX This is a hacky way to publish only TS components with changes. If a changed
+              # file "path-a/path-b/c.ts" has a corresponding compiled output file
+              # "path-a/dist/path-b/c.mjs", attempt to publish the output `.mjs` file.
+              changed_output_file=""
+              for f in $output_files; # check each output file for a match
+              do
+                # Replaces /dist/path/filename.mjs with /path/filename.ts
+                maybe_source_file=$(echo "$f" | sed 's/\/dist\//\//;s/.mjs/\.ts/')
+                if [[ ${maybe_source_file} == **/${added_modified_file} ]]
+                then
+                  changed_output_file=${f}
+                  break
+                fi
+              done
+              if [[ $changed_output_file == "" ]]
+              then
+                ERROR_MESSAGE="cannot find an output .mjs file with ${added_modified_file}"
+                echo $ERROR_MESSAGE
+                ERRORS+=("*${ERROR_MESSAGE}")
+              fi
+            else
+              echo "$added_modified_file will not be added to the registry"
+              SKIPPED+=("*$added_modified_file")
+            fi
+          done
+          if [[ ${#SKIPPED[@]} -ne 0 ]]; then
+            echo "the following files were skipped:"
+            printf '%s\n' "${SKIPPED[@]}"
+          fi
+          if [[ ${#ERRORS[@]} -ne 0 ]]; then
+            echo "the following files generated errors:"
+            printf '%s\n' "${ERRORS[@]}"
+            echo "Please check if the components above were successfully compiled"
+            echo "More information here: https://pipedream.com/docs/components/typescript/#developing-typescript-components-in-the-pipedreamhq-pipedream-registry"
+            exit 1
+          fi
+          unset IFS
+
+  publish-typescript-components-dry-run:
+    name: Publish TypeScript components
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/[email protected]
+      - uses: pnpm/[email protected]
+        with:
+          version: 7.33.6
+      - name: Get pnpm store directory
+        id: pnpm-cache
+        run: |
+          echo "::set-output name=pnpm_cache_dir::$(pnpm store path)"
+      - uses: actions/cache@v4
+        name: Setup pnpm cache
+        with:
+          path: ${{ steps.pnpm-cache.outputs.pnpm_cache_dir }}
+          key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }}
+          restore-keys: |
+            ${{ runner.os }}-pnpm-store-
+      - name: Setup Node Env
+        uses: actions/[email protected]
+        with:
+          node-version: 18
+          registry-url: https://registry.npmjs.org/
+          cache: 'pnpm'
+      - name: Install Dependencies
+        run: pnpm install -r
+      - name: Compile TypeScript
+        id: compile
+        run: npm run build > files.txt
+      - name: Get Changed Files
+        id: files
+        uses: jitterbit/get-changed-files@v1
+        with:
+          format: 'csv'
+      - name: Publish TypeScript components (dry run)
+        shell: bash {0} # don't fast fail
+        run: |
+          IFS=$'\n'
+          mapfile -d ',' -t added_modified_renamed_files < <(printf '%s,%s' '${{ steps.files.outputs.added_modified }}' '${{ steps.files.outputs.renamed }}')
+          # Remove initial tsc output
+          output_files=$(cat files.txt | sed 1,3d)
+          echo "The following files will be published on merge:"
+          for added_modified_file in "${added_modified_renamed_files[@]}";
+          do
+            # starts with components, ends with .ts and not app.ts, doesn't end with /common*.ts,
+            # and doesn't follow */common/
+            if [[ $added_modified_file == components/* ]] && [[ $added_modified_file == *.ts ]] && [[ $added_modified_file != *.app.ts ]] \
+                && [[ $added_modified_file != */common*.ts ]] && [[ $added_modified_file != */common/* ]]
+            then
+              # XXX This is a hacky way to publish only TS components with changes. If a changed
+              # file "path-a/path-b/c.ts" has a corresponding compiled output file
+              # "path-a/dist/path-b/c.mjs", attempt to publish the output `.mjs` file.
+              for f in $output_files;
+              do
+                # Replaces /dist/path/filename.mjs with /path/filename.ts
+                maybe_source_file=$(echo "$f" | sed 's/\/dist\//\//;s/.mjs/\.ts/')
+                if [[ ${maybe_source_file} == **/${added_modified_file} ]]
+                then
+                  echo "$f"
+                fi
+              done
+            fi
+          done
+          unset IFS

.github/workflows/publish-components.yaml

@@ -2,6 +2,8 @@ on:
   push:
     branches:
       - master
+    paths:
+      - 'components/**'
 
 jobs:
   publish-components:

.github/workflows/publish-marketplace-content.yaml

@@ -2,6 +2,8 @@ on:
   push:
     branches:
       - master
+    paths:
+      - 'components/**'
 
 jobs:
   publish-components:

.github/workflows/publish-packages.yaml

@@ -2,19 +2,42 @@ on:
   push:
     branches:
       - master
+    paths-ignore:
+      - 'docs/**'
+      - 'docs-v2/**'
+  pull_request:
+    branches:
+      - master
+    paths-ignore:
+      - 'docs/**'
+      - 'docs-v2/**'
 
 jobs:
   # See https://pnpm.io/continuous-integration#github-actions
   publish:
+    name: pnpm publish
     runs-on: ubuntu-latest
     steps:
       - uses: actions/[email protected]
       - uses: pnpm/[email protected]
         with:
           version: 7.33.6
+      - name: Get pnpm store directory
+        if: github.ref != 'refs/heads/master' # Cache is used only for dry runs
+        id: pnpm-cache
+        run: |
+          echo "::set-output name=pnpm_cache_dir::$(pnpm store path)"
+      - uses: actions/cache@v4
+        if: github.ref != 'refs/heads/master' # Cache is used only for dry runs
+        name: Setup pnpm cache
+        with:
+          path: ${{ steps.pnpm-cache.outputs.pnpm_cache_dir }}
+          key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }}
+          restore-keys: |
+            ${{ runner.os }}-pnpm-store-
       - uses: actions/[email protected]
         with:
-          node-version: 14
+          node-version: 18
           registry-url: https://registry.npmjs.org/
           cache: 'pnpm'
       - name: pnpm install
@@ -24,5 +47,9 @@ jobs:
       # See https://pnpm.io/using-changesets
       - name: Setup npmrc for pnpm publish
         run: echo "//registry.npmjs.org/:_authToken=${{ secrets.NPM_TOKEN }}" > .npmrc
+      - name: pnpm publish (dry run)
+        if: github.ref != 'refs/heads/master'
+        run: pnpm publish -r --no-git-checks --dry-run
       - name: pnpm publish
+        if: github.ref == 'refs/heads/master'
         run: pnpm publish -r --no-git-checks