From 27f28ee20537441fcaf056c532d90371c1dcd110 Mon Sep 17 00:00:00 2001 From: Nataly Carbonell Date: Thu, 7 Aug 2025 16:49:12 -0400 Subject: [PATCH 1/7] CLOUDP-333846 Schema Analysis Redux Integration for Collection Plugin --- .../src/modules/collection-tab.ts | 219 +++++++++++++++++- .../src/stores/collection-tab.spec.ts | 66 +++++- .../src/stores/collection-tab.ts | 20 ++ 3 files changed, 292 insertions(+), 13 deletions(-) diff --git a/packages/compass-collection/src/modules/collection-tab.ts b/packages/compass-collection/src/modules/collection-tab.ts index 12655bc0dce..aef4ac3e2b4 100644 --- a/packages/compass-collection/src/modules/collection-tab.ts +++ b/packages/compass-collection/src/modules/collection-tab.ts @@ -1,4 +1,10 @@ import type { Reducer, AnyAction, Action } from 'redux'; +import { + analyzeDocuments, + SchemaParseOptions, + type Schema, +} from 'mongodb-schema'; + import type { CollectionMetadata } from 'mongodb-collection-model'; import type { ThunkAction } from 'redux-thunk'; import type AppRegistry from '@mongodb-js/compass-app-registry'; @@ -6,6 +12,14 @@ import type { workspacesServiceLocator } from '@mongodb-js/compass-workspaces/pr import type { CollectionSubtab } from '@mongodb-js/compass-workspaces'; import type { DataService } from '@mongodb-js/compass-connections/provider'; import type { experimentationServiceLocator } from '@mongodb-js/compass-telemetry/provider'; +import { calculateSchemaMetadata } from '@mongodb-js/compass-schema'; +import type { Logger } from '@mongodb-js/compass-logging/provider'; +import { type PreferencesAccess } from 'compass-preferences-model/provider'; +import { isInternalFieldPath } from 'hadron-document'; +import { mongoLogId } from '@mongodb-js/compass-logging'; +import toNS from 'mongodb-ns'; + +const DEFAULT_SAMPLE_SIZE = 100; function isAction( action: AnyAction, @@ -22,19 +36,44 @@ type CollectionThunkAction = ThunkAction< dataService: DataService; workspaces: ReturnType; experimentationServices: ReturnType; + logger: Logger; + preferences: PreferencesAccess; + analysisAbortControllerRef: { current?: AbortController }; }, A >; +export enum SchemaAnalysisStatus { + INITIAL = 'initial', + ANALYZING = 'analyzing', + COMPLETED = 'completed', + ERROR = 'error', +} + +type SchemaAnalysis = { + status: SchemaAnalysisStatus; + schema: Schema | null; + sampleDocument: Document | null; + schemaMetadata: { + maxNestingDepth: number; + validationRules: Document; + } | null; + error: string | null; +}; + export type CollectionState = { workspaceTabId: string; namespace: string; metadata: CollectionMetadata | null; editViewName?: string; + schemaAnalysis: SchemaAnalysis; }; -enum CollectionActions { +export enum CollectionActions { CollectionMetadataFetched = 'compass-collection/CollectionMetadataFetched', + SchemaAnalysisStarted = 'compass-collection/SchemaAnalysisStarted', + SchemaAnalysisFinished = 'compass-collection/SchemaAnalysisFinished', + SchemaAnalysisFailed = 'compass-collection/SchemaAnalysisFailed', } interface CollectionMetadataFetchedAction { @@ -42,12 +81,34 @@ interface CollectionMetadataFetchedAction { metadata: CollectionMetadata; } +interface SchemaAnalysisStartedAction { + type: CollectionActions.SchemaAnalysisStarted; + analysisStartTime: number; +} + +interface SchemaAnalysisFinishedAction { + type: CollectionActions.SchemaAnalysisFinished; + schemaAnalysis: SchemaAnalysis; +} + +interface SchemaAnalysisFailedAction { + type: CollectionActions.SchemaAnalysisFailed; + error: Error; +} + const reducer: Reducer = ( state = { // TODO(COMPASS-7782): use hook to get the workspace tab id instead workspaceTabId: '', namespace: '', metadata: null, + schemaAnalysis: { + status: SchemaAnalysisStatus.INITIAL, + schema: null, + sampleDocument: null, + schemaMetadata: null, + error: null, + }, }, action ) => { @@ -62,6 +123,53 @@ const reducer: Reducer = ( metadata: action.metadata, }; } + + if ( + isAction( + action, + CollectionActions.SchemaAnalysisStarted + ) + ) { + return { + ...state, + schemaAnalysis: { + status: SchemaAnalysisStatus.ANALYZING, + schema: null, + sampleDocument: null, + schemaMetadata: null, + error: null, + }, + }; + } + + if ( + isAction( + action, + CollectionActions.SchemaAnalysisFinished + ) + ) { + return { + ...state, + schemaAnalysis: action.schemaAnalysis, + }; + } + + if ( + isAction( + action, + CollectionActions.SchemaAnalysisFailed + ) + ) { + return { + ...state, + schemaAnalysis: { + ...state.schemaAnalysis, + status: SchemaAnalysisStatus.ERROR, + error: action.error.message, + }, + }; + } + return state; }; @@ -82,6 +190,115 @@ export const selectTab = ( }; }; +export const analyzeCollectionSchema = (): CollectionThunkAction => { + return async ( + dispatch, + getState, + { analysisAbortControllerRef, dataService, preferences, logger } + ) => { + const { schemaAnalysis, namespace } = getState(); + const analysisStatus = schemaAnalysis.status; + if (analysisStatus === SchemaAnalysisStatus.ANALYZING) { + logger.debug( + 'Schema analysis is already in progress, skipping new analysis.' + ); + return; + } + + analysisAbortControllerRef.current = new AbortController(); + const abortSignal = analysisAbortControllerRef.current.signal; + + const analysisStartTime = Date.now(); + + try { + logger.debug('Schema analysis started.'); + + dispatch({ + type: CollectionActions.SchemaAnalysisStarted, + analysisStartTime, + }); + + // Sample documents + const samplingOptions = { size: DEFAULT_SAMPLE_SIZE }; + const driverOptions = { + maxTimeMS: preferences.getPreferences().maxTimeMS, + signal: abortSignal, + }; + const sampleCursor = dataService.sampleCursor( + namespace, + samplingOptions, + driverOptions, + { + fallbackReadPreference: 'secondaryPreferred', + } + ); + const sampleDocuments = await sampleCursor.toArray(); + + // Analyze sampled documents + const schemaParseOptions: SchemaParseOptions = { + signal: abortSignal, + }; + const schemaAccessor = await analyzeDocuments( + sampleDocuments, + schemaParseOptions + ); + if (abortSignal?.aborted) { + throw new Error(abortSignal?.reason || new Error('Operation aborted')); + } + + let schema: Schema | null = null; + if (schemaAccessor) { + schema = await schemaAccessor.getInternalSchema(); + // Filter out internal fields from the schema + schema.fields = schema.fields.filter( + ({ path }) => !isInternalFieldPath(path[0]) + ); + // TODO: Transform schema to structure that will be used by the LLM. + } + + let schemaMetadata = null; + if (schema !== null) { + const { schema_depth } = await calculateSchemaMetadata(schema); + const { database, collection } = toNS(namespace); + const collInfo = await dataService.collectionInfo(database, collection); + schemaMetadata = { + maxNestingDepth: schema_depth, + validationRules: collInfo?.validation?.validator || null, + }; + } + dispatch({ + type: CollectionActions.SchemaAnalysisFinished, + schemaAnalysis: { + status: SchemaAnalysisStatus.COMPLETED, + schema, + sampleDocument: sampleDocuments[0] ?? null, + schemaMetadata, + }, + }); + } catch (err: any) { + logger.log.error( + mongoLogId(1_001_000_363), + 'Collection', + 'Schema analysis failed', + { + namespace, + error: err.message, + aborted: abortSignal.aborted, + ...(abortSignal.aborted + ? { abortReason: abortSignal.reason?.message ?? abortSignal.reason } + : {}), + } + ); + dispatch({ + type: CollectionActions.SchemaAnalysisFailed, + error: err as Error, + }); + } finally { + analysisAbortControllerRef.current = undefined; + } + }; +}; + export type CollectionTabPluginMetadata = CollectionMetadata & { /** * Initial query for the query bar diff --git a/packages/compass-collection/src/stores/collection-tab.spec.ts b/packages/compass-collection/src/stores/collection-tab.spec.ts index 3d49b319c35..84d9f4b41fd 100644 --- a/packages/compass-collection/src/stores/collection-tab.spec.ts +++ b/packages/compass-collection/src/stores/collection-tab.spec.ts @@ -1,6 +1,7 @@ import type { CollectionTabOptions } from './collection-tab'; import { activatePlugin } from './collection-tab'; import { selectTab } from '../modules/collection-tab'; +import * as collectionTabModule from '../modules/collection-tab'; import { waitFor } from '@mongodb-js/testing-library-compass'; import Sinon from 'sinon'; import AppRegistry from '@mongodb-js/compass-app-registry'; @@ -11,6 +12,7 @@ import type { connectionInfoRefLocator } from '@mongodb-js/compass-connections/p import { createNoopLogger } from '@mongodb-js/compass-logging/provider'; import { ReadOnlyPreferenceAccess } from 'compass-preferences-model/provider'; import { ExperimentTestName } from '@mongodb-js/compass-telemetry/provider'; +import { CollectionMetadata } from 'mongodb-collection-model'; const defaultMetadata = { namespace: 'test.foo', @@ -27,16 +29,6 @@ const defaultTabOptions = { namespace: defaultMetadata.namespace, }; -const mockCollection = { - _id: defaultMetadata.namespace, - fetchMetadata() { - return Promise.resolve(defaultMetadata); - }, - toJSON() { - return this; - }, -}; - const mockAtlasConnectionInfo = { current: { id: 'test-connection', @@ -67,6 +59,9 @@ describe('Collection Tab Content store', function () { const sandbox = Sinon.createSandbox(); const localAppRegistry = sandbox.spy(new AppRegistry()); + const analyzeCollectionSchemaStub = sandbox + .stub(collectionTabModule, 'analyzeCollectionSchema') + .returns(() => async () => {}); const dataService = {} as any; let store: ReturnType['store']; let deactivate: ReturnType['deactivate']; @@ -85,8 +80,19 @@ describe('Collection Tab Content store', function () { enableGenAIFeatures: true, enableGenAIFeaturesAtlasOrg: true, cloudFeatureRolloutAccess: { GEN_AI_COMPASS: true }, - }) + }), + collectionMetadata: Partial = defaultMetadata, + analysisAbortControllerRef: { current?: AbortController } = {} ) => { + const mockCollection = { + _id: collectionMetadata.namespace, + fetchMetadata() { + return Promise.resolve(collectionMetadata); + }, + toJSON() { + return this; + }, + }; ({ store, deactivate } = activatePlugin( { ...defaultTabOptions, @@ -107,7 +113,7 @@ describe('Collection Tab Content store', function () { await waitFor(() => { expect(store.getState()) .to.have.property('metadata') - .deep.eq(defaultMetadata); + .deep.eq(collectionMetadata); }); return store; }; @@ -231,4 +237,40 @@ describe('Collection Tab Content store', function () { }); }); }); + + describe('schema analysis on collection load', function () { + it('should start schema analysis if collection is not read-only and not time-series', async function () { + await configureStore(); + + expect(analyzeCollectionSchemaStub).to.have.been.calledOnce; + }); + + it('should not start schema analysis if collection is read-only', async function () { + await configureStore( + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + { ...defaultMetadata, isReadonly: true } + ); + + expect(analyzeCollectionSchemaStub).to.not.have.been.called; + }); + + it('should not start schema analysis if collection is time-series', async function () { + await configureStore( + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + { ...defaultMetadata, isTimeSeries: true } + ); + + expect(analyzeCollectionSchemaStub).to.not.have.been.called; + }); + }); }); diff --git a/packages/compass-collection/src/stores/collection-tab.ts b/packages/compass-collection/src/stores/collection-tab.ts index 925572bf7e3..2a62864d03a 100644 --- a/packages/compass-collection/src/stores/collection-tab.ts +++ b/packages/compass-collection/src/stores/collection-tab.ts @@ -1,10 +1,13 @@ import type AppRegistry from '@mongodb-js/compass-app-registry'; import type { DataService } from '@mongodb-js/compass-connections/provider'; import { createStore, applyMiddleware } from 'redux'; + import thunk from 'redux-thunk'; import reducer, { selectTab, collectionMetadataFetched, + analyzeCollectionSchema, + SchemaAnalysisStatus, } from '../modules/collection-tab'; import type { Collection } from '@mongodb-js/compass-app-stores/provider'; import type { ActivateHelpers } from '@mongodb-js/compass-app-registry'; @@ -77,6 +80,13 @@ export function activatePlugin( namespace, metadata: null, editViewName, + schemaAnalysis: { + status: SchemaAnalysisStatus.INITIAL, + schema: null, + sampleDocument: null, + schemaMetadata: null, + error: null, + }, }, applyMiddleware( thunk.withExtraArgument({ @@ -84,6 +94,11 @@ export function activatePlugin( workspaces, localAppRegistry, experimentationServices, + logger, + preferences, + analysisAbortControllerRef: { + current: undefined, + }, }) ) ); @@ -125,6 +140,11 @@ export function activatePlugin( }); }); } + + if (!metadata.isReadonly && !metadata.isTimeSeries) { + // TODO: Consider checking experiment variant + store.dispatch(analyzeCollectionSchema()); + } }); return { From fa541f54eb1729a097721270488dfcfe211f35c2 Mon Sep 17 00:00:00 2001 From: Nataly Carbonell Date: Thu, 7 Aug 2025 17:50:10 -0400 Subject: [PATCH 2/7] cleanup --- package-lock.json | 8 ++++++-- packages/compass-collection/package.json | 5 ++++- .../compass-collection/src/modules/collection-tab.ts | 9 +++++---- .../compass-collection/src/stores/collection-tab.spec.ts | 7 +++---- packages/compass-collection/src/stores/collection-tab.ts | 2 +- 5 files changed, 19 insertions(+), 12 deletions(-) diff --git a/package-lock.json b/package-lock.json index f25675ffda8..cd842747bc0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -44117,13 +44117,16 @@ "@mongodb-js/compass-components": "^1.47.0", "@mongodb-js/compass-connections": "^1.69.0", "@mongodb-js/compass-logging": "^1.7.10", + "@mongodb-js/compass-schema": "^6.70.0", "@mongodb-js/compass-telemetry": "^1.12.0", "@mongodb-js/compass-workspaces": "^0.50.0", "@mongodb-js/connection-info": "^0.17.0", "@mongodb-js/mongodb-constants": "^0.12.2", "compass-preferences-model": "^2.49.0", + "hadron-document": "^8.9.4", "mongodb-collection-model": "^5.31.0", "mongodb-ns": "^2.4.2", + "mongodb-schema": "^12.6.2", "react": "^17.0.2", "react-redux": "^8.1.3", "redux": "^4.2.1", @@ -47207,7 +47210,6 @@ "license": "SSPL", "dependencies": { "@mongodb-js/compass-app-registry": "^9.4.18", - "@mongodb-js/compass-collection": "^4.68.0", "@mongodb-js/compass-components": "^1.47.0", "@mongodb-js/compass-connections": "^1.69.0", "@mongodb-js/compass-editor": "^0.49.0", @@ -56755,6 +56757,7 @@ "@mongodb-js/compass-components": "^1.47.0", "@mongodb-js/compass-connections": "^1.69.0", "@mongodb-js/compass-logging": "^1.7.10", + "@mongodb-js/compass-schema": "^6.70.0", "@mongodb-js/compass-telemetry": "^1.12.0", "@mongodb-js/compass-workspaces": "^0.50.0", "@mongodb-js/connection-info": "^0.17.0", @@ -56774,9 +56777,11 @@ "compass-preferences-model": "^2.49.0", "depcheck": "^1.4.1", "electron-mocha": "^12.2.0", + "hadron-document": "^8.9.4", "mocha": "^10.2.0", "mongodb-collection-model": "^5.31.0", "mongodb-ns": "^2.4.2", + "mongodb-schema": "^12.6.2", "nyc": "^15.1.0", "react": "^17.0.2", "react-dom": "^17.0.2", @@ -58948,7 +58953,6 @@ "version": "file:packages/compass-schema", "requires": { "@mongodb-js/compass-app-registry": "^9.4.18", - "@mongodb-js/compass-collection": "^4.68.0", "@mongodb-js/compass-components": "^1.47.0", "@mongodb-js/compass-connections": "^1.69.0", "@mongodb-js/compass-editor": "^0.49.0", diff --git a/packages/compass-collection/package.json b/packages/compass-collection/package.json index 5fefe25c6c4..47fbcdb9973 100644 --- a/packages/compass-collection/package.json +++ b/packages/compass-collection/package.json @@ -48,18 +48,21 @@ "reformat": "npm run eslint . -- --fix && npm run prettier -- --write ." }, "dependencies": { + "@mongodb-js/compass-app-registry": "^9.4.18", "@mongodb-js/compass-app-stores": "^7.55.0", "@mongodb-js/compass-components": "^1.47.0", "@mongodb-js/compass-connections": "^1.69.0", "@mongodb-js/compass-logging": "^1.7.10", + "@mongodb-js/compass-schema": "^6.70.0", "@mongodb-js/compass-telemetry": "^1.12.0", "@mongodb-js/compass-workspaces": "^0.50.0", "@mongodb-js/connection-info": "^0.17.0", "@mongodb-js/mongodb-constants": "^0.12.2", "compass-preferences-model": "^2.49.0", - "@mongodb-js/compass-app-registry": "^9.4.18", + "hadron-document": "^8.9.4", "mongodb-collection-model": "^5.31.0", "mongodb-ns": "^2.4.2", + "mongodb-schema": "^12.6.2", "react": "^17.0.2", "react-redux": "^8.1.3", "redux": "^4.2.1", diff --git a/packages/compass-collection/src/modules/collection-tab.ts b/packages/compass-collection/src/modules/collection-tab.ts index aef4ac3e2b4..c05c260e10c 100644 --- a/packages/compass-collection/src/modules/collection-tab.ts +++ b/packages/compass-collection/src/modules/collection-tab.ts @@ -1,7 +1,7 @@ import type { Reducer, AnyAction, Action } from 'redux'; import { analyzeDocuments, - SchemaParseOptions, + type SchemaParseOptions, type Schema, } from 'mongodb-schema'; @@ -13,10 +13,9 @@ import type { CollectionSubtab } from '@mongodb-js/compass-workspaces'; import type { DataService } from '@mongodb-js/compass-connections/provider'; import type { experimentationServiceLocator } from '@mongodb-js/compass-telemetry/provider'; import { calculateSchemaMetadata } from '@mongodb-js/compass-schema'; -import type { Logger } from '@mongodb-js/compass-logging/provider'; +import { type Logger, mongoLogId } from '@mongodb-js/compass-logging/provider'; import { type PreferencesAccess } from 'compass-preferences-model/provider'; import { isInternalFieldPath } from 'hadron-document'; -import { mongoLogId } from '@mongodb-js/compass-logging'; import toNS from 'mongodb-ns'; const DEFAULT_SAMPLE_SIZE = 100; @@ -190,7 +189,9 @@ export const selectTab = ( }; }; -export const analyzeCollectionSchema = (): CollectionThunkAction => { +export const analyzeCollectionSchema = (): CollectionThunkAction< + Promise +> => { return async ( dispatch, getState, diff --git a/packages/compass-collection/src/stores/collection-tab.spec.ts b/packages/compass-collection/src/stores/collection-tab.spec.ts index 84d9f4b41fd..d200d24b0d9 100644 --- a/packages/compass-collection/src/stores/collection-tab.spec.ts +++ b/packages/compass-collection/src/stores/collection-tab.spec.ts @@ -12,7 +12,7 @@ import type { connectionInfoRefLocator } from '@mongodb-js/compass-connections/p import { createNoopLogger } from '@mongodb-js/compass-logging/provider'; import { ReadOnlyPreferenceAccess } from 'compass-preferences-model/provider'; import { ExperimentTestName } from '@mongodb-js/compass-telemetry/provider'; -import { CollectionMetadata } from 'mongodb-collection-model'; +import { type CollectionMetadata } from 'mongodb-collection-model'; const defaultMetadata = { namespace: 'test.foo', @@ -61,7 +61,7 @@ describe('Collection Tab Content store', function () { const localAppRegistry = sandbox.spy(new AppRegistry()); const analyzeCollectionSchemaStub = sandbox .stub(collectionTabModule, 'analyzeCollectionSchema') - .returns(() => async () => {}); + .returns(async () => {}); const dataService = {} as any; let store: ReturnType['store']; let deactivate: ReturnType['deactivate']; @@ -81,8 +81,7 @@ describe('Collection Tab Content store', function () { enableGenAIFeaturesAtlasOrg: true, cloudFeatureRolloutAccess: { GEN_AI_COMPASS: true }, }), - collectionMetadata: Partial = defaultMetadata, - analysisAbortControllerRef: { current?: AbortController } = {} + collectionMetadata: Partial = defaultMetadata ) => { const mockCollection = { _id: collectionMetadata.namespace, diff --git a/packages/compass-collection/src/stores/collection-tab.ts b/packages/compass-collection/src/stores/collection-tab.ts index 2a62864d03a..580e7b15ddd 100644 --- a/packages/compass-collection/src/stores/collection-tab.ts +++ b/packages/compass-collection/src/stores/collection-tab.ts @@ -143,7 +143,7 @@ export function activatePlugin( if (!metadata.isReadonly && !metadata.isTimeSeries) { // TODO: Consider checking experiment variant - store.dispatch(analyzeCollectionSchema()); + void store.dispatch(analyzeCollectionSchema()); } }); From e8f2d114fb30b4d9f88ab7247535122419dce7fb Mon Sep 17 00:00:00 2001 From: Nataly Carbonell Date: Mon, 11 Aug 2025 00:09:00 -0400 Subject: [PATCH 3/7] Address comments and remove abort controller code --- .../src/modules/collection-tab.ts | 68 +++++++------------ .../src/stores/collection-tab.ts | 3 - 2 files changed, 23 insertions(+), 48 deletions(-) diff --git a/packages/compass-collection/src/modules/collection-tab.ts b/packages/compass-collection/src/modules/collection-tab.ts index c05c260e10c..32b655c703d 100644 --- a/packages/compass-collection/src/modules/collection-tab.ts +++ b/packages/compass-collection/src/modules/collection-tab.ts @@ -1,9 +1,5 @@ import type { Reducer, AnyAction, Action } from 'redux'; -import { - analyzeDocuments, - type SchemaParseOptions, - type Schema, -} from 'mongodb-schema'; +import { analyzeDocuments, type Schema } from 'mongodb-schema'; import type { CollectionMetadata } from 'mongodb-collection-model'; import type { ThunkAction } from 'redux-thunk'; @@ -37,7 +33,6 @@ type CollectionThunkAction = ThunkAction< experimentationServices: ReturnType; logger: Logger; preferences: PreferencesAccess; - analysisAbortControllerRef: { current?: AbortController }; }, A >; @@ -82,12 +77,16 @@ interface CollectionMetadataFetchedAction { interface SchemaAnalysisStartedAction { type: CollectionActions.SchemaAnalysisStarted; - analysisStartTime: number; } interface SchemaAnalysisFinishedAction { type: CollectionActions.SchemaAnalysisFinished; - schemaAnalysis: SchemaAnalysis; + schema: Schema | null; + sampleDocument: Document | null; + schemaMetadata: { + maxNestingDepth: number; + validationRules: Document; + } | null; } interface SchemaAnalysisFailedAction { @@ -149,7 +148,13 @@ const reducer: Reducer = ( ) { return { ...state, - schemaAnalysis: action.schemaAnalysis, + schemaAnalysis: { + status: SchemaAnalysisStatus.COMPLETED, + schema: action.schema, + sampleDocument: action.sampleDocument, + schemaMetadata: action.schemaMetadata, + error: null, + }, }; } @@ -162,7 +167,9 @@ const reducer: Reducer = ( return { ...state, schemaAnalysis: { - ...state.schemaAnalysis, + schema: null, + sampleDocument: null, + schemaMetadata: null, status: SchemaAnalysisStatus.ERROR, error: action.error.message, }, @@ -192,11 +199,7 @@ export const selectTab = ( export const analyzeCollectionSchema = (): CollectionThunkAction< Promise > => { - return async ( - dispatch, - getState, - { analysisAbortControllerRef, dataService, preferences, logger } - ) => { + return async (dispatch, getState, { dataService, preferences, logger }) => { const { schemaAnalysis, namespace } = getState(); const analysisStatus = schemaAnalysis.status; if (analysisStatus === SchemaAnalysisStatus.ANALYZING) { @@ -206,24 +209,17 @@ export const analyzeCollectionSchema = (): CollectionThunkAction< return; } - analysisAbortControllerRef.current = new AbortController(); - const abortSignal = analysisAbortControllerRef.current.signal; - - const analysisStartTime = Date.now(); - try { logger.debug('Schema analysis started.'); dispatch({ type: CollectionActions.SchemaAnalysisStarted, - analysisStartTime, }); // Sample documents const samplingOptions = { size: DEFAULT_SAMPLE_SIZE }; const driverOptions = { maxTimeMS: preferences.getPreferences().maxTimeMS, - signal: abortSignal, }; const sampleCursor = dataService.sampleCursor( namespace, @@ -236,16 +232,7 @@ export const analyzeCollectionSchema = (): CollectionThunkAction< const sampleDocuments = await sampleCursor.toArray(); // Analyze sampled documents - const schemaParseOptions: SchemaParseOptions = { - signal: abortSignal, - }; - const schemaAccessor = await analyzeDocuments( - sampleDocuments, - schemaParseOptions - ); - if (abortSignal?.aborted) { - throw new Error(abortSignal?.reason || new Error('Operation aborted')); - } + const schemaAccessor = await analyzeDocuments(sampleDocuments); let schema: Schema | null = null; if (schemaAccessor) { @@ -264,17 +251,14 @@ export const analyzeCollectionSchema = (): CollectionThunkAction< const collInfo = await dataService.collectionInfo(database, collection); schemaMetadata = { maxNestingDepth: schema_depth, - validationRules: collInfo?.validation?.validator || null, + validationRules: collInfo?.validation?.validator ?? null, }; } dispatch({ type: CollectionActions.SchemaAnalysisFinished, - schemaAnalysis: { - status: SchemaAnalysisStatus.COMPLETED, - schema, - sampleDocument: sampleDocuments[0] ?? null, - schemaMetadata, - }, + schema, + sampleDocument: sampleDocuments[0] ?? null, + schemaMetadata, }); } catch (err: any) { logger.log.error( @@ -284,18 +268,12 @@ export const analyzeCollectionSchema = (): CollectionThunkAction< { namespace, error: err.message, - aborted: abortSignal.aborted, - ...(abortSignal.aborted - ? { abortReason: abortSignal.reason?.message ?? abortSignal.reason } - : {}), } ); dispatch({ type: CollectionActions.SchemaAnalysisFailed, error: err as Error, }); - } finally { - analysisAbortControllerRef.current = undefined; } }; }; diff --git a/packages/compass-collection/src/stores/collection-tab.ts b/packages/compass-collection/src/stores/collection-tab.ts index 580e7b15ddd..527d99d7a58 100644 --- a/packages/compass-collection/src/stores/collection-tab.ts +++ b/packages/compass-collection/src/stores/collection-tab.ts @@ -96,9 +96,6 @@ export function activatePlugin( experimentationServices, logger, preferences, - analysisAbortControllerRef: { - current: undefined, - }, }) ) ); From 81f92f2900c4d580d5f255d06703c8ece7b58b40 Mon Sep 17 00:00:00 2001 From: Nataly Carbonell Date: Tue, 12 Aug 2025 15:50:52 -0400 Subject: [PATCH 4/7] address comments --- package-lock.json | 38 ++-- packages/compass-collection/package.json | 1 - .../src/calculate-schema-depth.spec.ts | 165 ++++++++++++++++++ .../src/calculate-schema-depth.ts | 57 ++++++ .../src/modules/collection-tab.ts | 104 +++++------ .../src/schema-analysis-types.ts | 41 +++++ .../src/stores/collection-tab.ts | 8 +- 7 files changed, 329 insertions(+), 85 deletions(-) create mode 100644 packages/compass-collection/src/calculate-schema-depth.spec.ts create mode 100644 packages/compass-collection/src/calculate-schema-depth.ts create mode 100644 packages/compass-collection/src/schema-analysis-types.ts diff --git a/package-lock.json b/package-lock.json index cd842747bc0..e1c8f05e10c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -44117,7 +44117,6 @@ "@mongodb-js/compass-components": "^1.47.0", "@mongodb-js/compass-connections": "^1.69.0", "@mongodb-js/compass-logging": "^1.7.10", - "@mongodb-js/compass-schema": "^6.70.0", "@mongodb-js/compass-telemetry": "^1.12.0", "@mongodb-js/compass-workspaces": "^0.50.0", "@mongodb-js/connection-info": "^0.17.0", @@ -47210,6 +47209,7 @@ "license": "SSPL", "dependencies": { "@mongodb-js/compass-app-registry": "^9.4.18", + "@mongodb-js/compass-collection": "^4.68.0", "@mongodb-js/compass-components": "^1.47.0", "@mongodb-js/compass-connections": "^1.69.0", "@mongodb-js/compass-editor": "^0.49.0", @@ -56757,7 +56757,6 @@ "@mongodb-js/compass-components": "^1.47.0", "@mongodb-js/compass-connections": "^1.69.0", "@mongodb-js/compass-logging": "^1.7.10", - "@mongodb-js/compass-schema": "^6.70.0", "@mongodb-js/compass-telemetry": "^1.12.0", "@mongodb-js/compass-workspaces": "^0.50.0", "@mongodb-js/connection-info": "^0.17.0", @@ -58953,6 +58952,7 @@ "version": "file:packages/compass-schema", "requires": { "@mongodb-js/compass-app-registry": "^9.4.18", + "@mongodb-js/compass-collection": "^4.68.0", "@mongodb-js/compass-components": "^1.47.0", "@mongodb-js/compass-connections": "^1.69.0", "@mongodb-js/compass-editor": "^0.49.0", @@ -60698,14 +60698,14 @@ "@babel/eslint-parser": "^7.22.7", "@babel/preset-env": "^7.22.7", "@babel/preset-react": "^7.22.5", - "@typescript-eslint/eslint-plugin": "^8.39.0", - "@typescript-eslint/parser": "^8.39.0", + "@typescript-eslint/eslint-plugin": "^5.59.0", + "@typescript-eslint/parser": "^5.59.0", "eslint-config-prettier": "^8.3.0", "eslint-plugin-filename-rules": "^1.2.0", - "eslint-plugin-jsx-a11y": "^6.10.2", + "eslint-plugin-jsx-a11y": "^6.4.1", "eslint-plugin-mocha": "^8.0.0", - "eslint-plugin-react": "^7.37.5", - "eslint-plugin-react-hooks": "^5.2.0" + "eslint-plugin-react": "^7.24.0", + "eslint-plugin-react-hooks": "^4.2.0" } }, "@mongodb-js/eslint-plugin-compass": { @@ -65159,7 +65159,7 @@ "requires": { "@babel/runtime": "^7.12.5", "@testing-library/dom": "^8.0.0", - "@types/react-dom": "^17.0.25" + "@types/react-dom": "<18.0.0" } }, "@testing-library/react-hooks": { @@ -65168,8 +65168,8 @@ "integrity": "sha512-dYxpz8u9m4q1TuzfcUApqi8iFfR6R0FaMbr2hjZJy1uC8z+bO/K4v8Gs9eogGKYQop7QsrBTFkv/BCF7MzD2Cg==", "requires": { "@babel/runtime": "^7.12.5", - "@types/react": "^17.0.83", - "@types/react-dom": "^17.0.25", + "@types/react": ">=16.9.0", + "@types/react-dom": ">=16.9.0", "@types/react-test-renderer": ">=16.9.0", "react-error-boundary": "^3.1.0" } @@ -65451,7 +65451,7 @@ "dev": true, "requires": { "@types/cheerio": "*", - "@types/react": "^17.0.83" + "@types/react": "^16" } }, "@types/estree": { @@ -65532,7 +65532,7 @@ "resolved": "https://registry.npmjs.org/@types/hoist-non-react-statics/-/hoist-non-react-statics-3.3.1.tgz", "integrity": "sha512-iMIqiko6ooLrTh1joXodJK5X9xeEALT1kM5G3ZLhD3hszxBdIEd5C75U834D9mLcINgD4OyZf5uQXjkuYydWvA==", "requires": { - "@types/react": "^17.0.83", + "@types/react": "*", "hoist-non-react-statics": "^3.3.0" } }, @@ -65734,7 +65734,7 @@ "resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-17.0.25.tgz", "integrity": "sha512-urx7A7UxkZQmThYA4So0NelOVjx3V4rNFVJwp0WZlbIK5eM4rNJDiN3R/E9ix0MBh6kAEojk/9YL+Te6D9zHNA==", "requires": { - "@types/react": "^17.0.83" + "@types/react": "^17" } }, "@types/react-is": { @@ -65742,7 +65742,7 @@ "resolved": "https://registry.npmjs.org/@types/react-is/-/react-is-18.3.1.tgz", "integrity": "sha512-zts4lhQn5ia0cF/y2+3V6Riu0MAfez9/LJYavdM8TvcVl+S91A/7VWxyBT8hbRuWspmuCaiGI0F41OJYGrKhRA==", "requires": { - "@types/react": "^17.0.83" + "@types/react": "^18" } }, "@types/react-test-renderer": { @@ -65750,7 +65750,7 @@ "resolved": "https://registry.npmjs.org/@types/react-test-renderer/-/react-test-renderer-17.0.1.tgz", "integrity": "sha512-3Fi2O6Zzq/f3QR9dRnlnHso9bMl7weKCviFmfF6B4LS1Uat6Hkm15k0ZAQuDz+UBq6B3+g+NM6IT2nr5QgPzCw==", "requires": { - "@types/react": "^17.0.83" + "@types/react": "*" } }, "@types/react-transition-group": { @@ -65765,7 +65765,7 @@ "integrity": "sha512-GH8sAnBEM5GV9LTeiz56r4ZhMOUSrP43tAQNSRVxNexDjcNKLCEtnxusAItg1owFUFE6k0NslV26gqVClVvong==", "dev": true, "requires": { - "@types/react": "^17.0.83" + "@types/react": "*" } }, "@types/react-window": { @@ -65774,7 +65774,7 @@ "integrity": "sha512-V9q3CvhC9Jk9bWBOysPGaWy/Z0lxYcTXLtLipkt2cnRj1JOSFNF7wqGpkScSXMgBwC+fnVRg/7shwgddBG5ICw==", "dev": true, "requires": { - "@types/react": "^17.0.83" + "@types/react": "*" } }, "@types/reflux": { @@ -65783,7 +65783,7 @@ "integrity": "sha512-nRTTsQmy0prliP0I0GvpAbE27k7+I+MqD15gs4YuQGkuZjRHK65QHPLkykgHnPTdjZYNaY0sOvMQ7OtbcoDkKA==", "dev": true, "requires": { - "@types/react": "^17.0.83" + "@types/react": "*" } }, "@types/relateurl": { @@ -72179,7 +72179,7 @@ "integrity": "sha512-Dw8/Gs4vRjxY6/6i9wU0V+utmQO9kvh9XLnz3LIudviOnVYDEe2ec+0k+NQoMamn1VrjKgCUOWj5jG/5M5M0Qw==", "requires": { "array.prototype.flat": "^1.2.3", - "cheerio": "1.0.0-rc.10", + "cheerio": "^1.0.0-rc.3", "enzyme-shallow-equal": "^1.0.1", "function.prototype.name": "^1.1.2", "has": "^1.0.3", diff --git a/packages/compass-collection/package.json b/packages/compass-collection/package.json index 47fbcdb9973..68b7186b639 100644 --- a/packages/compass-collection/package.json +++ b/packages/compass-collection/package.json @@ -53,7 +53,6 @@ "@mongodb-js/compass-components": "^1.47.0", "@mongodb-js/compass-connections": "^1.69.0", "@mongodb-js/compass-logging": "^1.7.10", - "@mongodb-js/compass-schema": "^6.70.0", "@mongodb-js/compass-telemetry": "^1.12.0", "@mongodb-js/compass-workspaces": "^0.50.0", "@mongodb-js/connection-info": "^0.17.0", diff --git a/packages/compass-collection/src/calculate-schema-depth.spec.ts b/packages/compass-collection/src/calculate-schema-depth.spec.ts new file mode 100644 index 00000000000..743b833da67 --- /dev/null +++ b/packages/compass-collection/src/calculate-schema-depth.spec.ts @@ -0,0 +1,165 @@ +import { expect } from 'chai'; +import { calculateSchemaDepth } from './calculate-schema-depth'; +import type { + Schema, + SchemaField, + DocumentSchemaType, + ArraySchemaType, +} from 'mongodb-schema'; + +describe('calculateSchemaDepth', function () { + it('returns 1 for flat schema', async function () { + const schema: Schema = { + fields: [ + { name: 'a', types: [{ bsonType: 'String' }] } as SchemaField, + { name: 'b', types: [{ bsonType: 'Number' }] } as SchemaField, + ], + count: 2, + }; + const depth = await calculateSchemaDepth(schema); + expect(depth).to.equal(1); + }); + + it('returns correct depth for nested document', async function () { + const schema: Schema = { + fields: [ + { + name: 'a', + types: [ + { + bsonType: 'Document', + fields: [ + { + name: 'b', + types: [ + { + bsonType: 'Document', + fields: [ + { + name: 'c', + types: [{ bsonType: 'String' }], + } as SchemaField, + ], + } as DocumentSchemaType, + ], + } as SchemaField, + ], + } as DocumentSchemaType, + ], + } as SchemaField, + ], + count: 1, + }; + const depth = await calculateSchemaDepth(schema); + expect(depth).to.equal(3); + }); + + it('returns correct depth for nested arrays', async function () { + const schema: Schema = { + fields: [ + { + name: 'arr', + types: [ + { + bsonType: 'Array', + types: [ + { + bsonType: 'Array', + types: [ + { + bsonType: 'Document', + fields: [ + { + name: 'x', + types: [{ bsonType: 'String' }], + } as SchemaField, + ], + } as DocumentSchemaType, + ], + } as ArraySchemaType, + ], + } as ArraySchemaType, + ], + } as SchemaField, + ], + count: 1, + }; + const depth = await calculateSchemaDepth(schema); + expect(depth).to.equal(4); + }); + + it('returns 0 for empty schema', async function () { + const schema: Schema = { fields: [], count: 0 }; + const depth = await calculateSchemaDepth(schema); + expect(depth).to.equal(0); + }); + + it('handles mixed types at root', async function () { + const schema: Schema = { + fields: [ + { + name: 'a', + types: [ + { bsonType: 'String' }, + { + bsonType: 'Document', + fields: [ + { + name: 'b', + types: [{ bsonType: 'Number' }], + } as SchemaField, + ], + } as DocumentSchemaType, + ], + } as SchemaField, + ], + count: 1, + }; + const depth = await calculateSchemaDepth(schema); + expect(depth).to.equal(2); + }); + + it('handles deeply nested mixed arrays and documents', async function () { + const schema: Schema = { + fields: [ + { + name: 'root', + types: [ + { + bsonType: 'Array', + types: [ + { + bsonType: 'Document', + fields: [ + { + name: 'nestedArr', + types: [ + { + bsonType: 'Array', + types: [ + { + bsonType: 'Document', + fields: [ + { + name: 'leaf', + types: [{ bsonType: 'String' }], + } as SchemaField, + ], + } as DocumentSchemaType, + ], + } as ArraySchemaType, + ], + } as SchemaField, + ], + } as DocumentSchemaType, + ], + } as ArraySchemaType, + ], + } as SchemaField, + ], + count: 1, + }; + const depth = await calculateSchemaDepth(schema); + expect(depth).to.equal(5); + }); +}); diff --git a/packages/compass-collection/src/calculate-schema-depth.ts b/packages/compass-collection/src/calculate-schema-depth.ts new file mode 100644 index 00000000000..82a0ed02cb9 --- /dev/null +++ b/packages/compass-collection/src/calculate-schema-depth.ts @@ -0,0 +1,57 @@ +import type { + ArraySchemaType, + DocumentSchemaType, + Schema, + SchemaField, + SchemaType, +} from 'mongodb-schema'; + +// Every 1000 iterations, unblock the thread. +const UNBLOCK_INTERVAL_COUNT = 1000; +const unblockThread = async () => + new Promise((resolve) => setTimeout(resolve)); + +export async function calculateSchemaDepth(schema: Schema): Promise { + let unblockThreadCounter = 0; + let deepestPath = 0; + + async function traverseSchemaTree( + fieldsOrTypes: SchemaField[] | SchemaType[], + depth: number + ): Promise { + unblockThreadCounter++; + if (unblockThreadCounter === UNBLOCK_INTERVAL_COUNT) { + unblockThreadCounter = 0; + await unblockThread(); + } + + if (!fieldsOrTypes || fieldsOrTypes.length === 0) { + return; + } + + deepestPath = Math.max(depth, deepestPath); + + for (const fieldOrType of fieldsOrTypes) { + if ((fieldOrType as DocumentSchemaType).bsonType === 'Document') { + await traverseSchemaTree( + (fieldOrType as DocumentSchemaType).fields, + depth + 1 // Increment by one when we go a level deeper. + ); + } else if ( + (fieldOrType as ArraySchemaType).bsonType === 'Array' || + (fieldOrType as SchemaField).types + ) { + const increment = + (fieldOrType as ArraySchemaType).bsonType === 'Array' ? 1 : 0; + await traverseSchemaTree( + (fieldOrType as ArraySchemaType | SchemaField).types, + depth + increment // Increment by one when we go a level deeper. + ); + } + } + } + + await traverseSchemaTree(schema.fields, 1); + + return deepestPath; +} diff --git a/packages/compass-collection/src/modules/collection-tab.ts b/packages/compass-collection/src/modules/collection-tab.ts index 32b655c703d..4ad96ac8ee5 100644 --- a/packages/compass-collection/src/modules/collection-tab.ts +++ b/packages/compass-collection/src/modules/collection-tab.ts @@ -8,14 +8,23 @@ import type { workspacesServiceLocator } from '@mongodb-js/compass-workspaces/pr import type { CollectionSubtab } from '@mongodb-js/compass-workspaces'; import type { DataService } from '@mongodb-js/compass-connections/provider'; import type { experimentationServiceLocator } from '@mongodb-js/compass-telemetry/provider'; -import { calculateSchemaMetadata } from '@mongodb-js/compass-schema'; import { type Logger, mongoLogId } from '@mongodb-js/compass-logging/provider'; import { type PreferencesAccess } from 'compass-preferences-model/provider'; import { isInternalFieldPath } from 'hadron-document'; import toNS from 'mongodb-ns'; +import { + SCHEMA_ANALYSIS_STATE_ANALYZING, + SCHEMA_ANALYSIS_STATE_COMPLETE, + SCHEMA_ANALYSIS_STATE_ERROR, + SCHEMA_ANALYSIS_STATE_INITIAL, + type SchemaAnalysis, +} from '../schema-analysis-types'; +import { calculateSchemaDepth } from '../calculate-schema-depth'; const DEFAULT_SAMPLE_SIZE = 100; +const NO_DOCUMENTS_ERROR = 'No documents found in the collection to analyze.'; + function isAction( action: AnyAction, type: A['type'] @@ -37,24 +46,6 @@ type CollectionThunkAction = ThunkAction< A >; -export enum SchemaAnalysisStatus { - INITIAL = 'initial', - ANALYZING = 'analyzing', - COMPLETED = 'completed', - ERROR = 'error', -} - -type SchemaAnalysis = { - status: SchemaAnalysisStatus; - schema: Schema | null; - sampleDocument: Document | null; - schemaMetadata: { - maxNestingDepth: number; - validationRules: Document; - } | null; - error: string | null; -}; - export type CollectionState = { workspaceTabId: string; namespace: string; @@ -81,12 +72,12 @@ interface SchemaAnalysisStartedAction { interface SchemaAnalysisFinishedAction { type: CollectionActions.SchemaAnalysisFinished; - schema: Schema | null; - sampleDocument: Document | null; + schema: Schema; + sampleDocument: Document; schemaMetadata: { maxNestingDepth: number; - validationRules: Document; - } | null; + validationRules: Document | null; + }; } interface SchemaAnalysisFailedAction { @@ -101,11 +92,7 @@ const reducer: Reducer = ( namespace: '', metadata: null, schemaAnalysis: { - status: SchemaAnalysisStatus.INITIAL, - schema: null, - sampleDocument: null, - schemaMetadata: null, - error: null, + status: SCHEMA_ANALYSIS_STATE_INITIAL, }, }, action @@ -131,11 +118,11 @@ const reducer: Reducer = ( return { ...state, schemaAnalysis: { - status: SchemaAnalysisStatus.ANALYZING, + status: SCHEMA_ANALYSIS_STATE_ANALYZING, + error: null, schema: null, sampleDocument: null, schemaMetadata: null, - error: null, }, }; } @@ -149,11 +136,10 @@ const reducer: Reducer = ( return { ...state, schemaAnalysis: { - status: SchemaAnalysisStatus.COMPLETED, + status: SCHEMA_ANALYSIS_STATE_COMPLETE, schema: action.schema, sampleDocument: action.sampleDocument, schemaMetadata: action.schemaMetadata, - error: null, }, }; } @@ -167,11 +153,8 @@ const reducer: Reducer = ( return { ...state, schemaAnalysis: { - schema: null, - sampleDocument: null, - schemaMetadata: null, - status: SchemaAnalysisStatus.ERROR, - error: action.error.message, + status: SCHEMA_ANALYSIS_STATE_ERROR, + error: action.error, }, }; } @@ -202,7 +185,7 @@ export const analyzeCollectionSchema = (): CollectionThunkAction< return async (dispatch, getState, { dataService, preferences, logger }) => { const { schemaAnalysis, namespace } = getState(); const analysisStatus = schemaAnalysis.status; - if (analysisStatus === SchemaAnalysisStatus.ANALYZING) { + if (analysisStatus === SCHEMA_ANALYSIS_STATE_ANALYZING) { logger.debug( 'Schema analysis is already in progress, skipping new analysis.' ); @@ -229,35 +212,38 @@ export const analyzeCollectionSchema = (): CollectionThunkAction< fallbackReadPreference: 'secondaryPreferred', } ); - const sampleDocuments = await sampleCursor.toArray(); + const sampleDocuments: Array = await sampleCursor.toArray(); + if (sampleDocuments.length === 0) { + logger.debug(NO_DOCUMENTS_ERROR); + dispatch({ + type: CollectionActions.SchemaAnalysisFailed, + error: new Error(NO_DOCUMENTS_ERROR), + }); + return; + } // Analyze sampled documents const schemaAccessor = await analyzeDocuments(sampleDocuments); + const schema = await schemaAccessor.getInternalSchema(); - let schema: Schema | null = null; - if (schemaAccessor) { - schema = await schemaAccessor.getInternalSchema(); - // Filter out internal fields from the schema - schema.fields = schema.fields.filter( - ({ path }) => !isInternalFieldPath(path[0]) - ); - // TODO: Transform schema to structure that will be used by the LLM. - } + // Filter out internal fields from the schema + schema.fields = schema.fields.filter( + ({ path }) => !isInternalFieldPath(path[0]) + ); + // TODO: Transform schema to structure that will be used by the LLM. - let schemaMetadata = null; - if (schema !== null) { - const { schema_depth } = await calculateSchemaMetadata(schema); - const { database, collection } = toNS(namespace); - const collInfo = await dataService.collectionInfo(database, collection); - schemaMetadata = { - maxNestingDepth: schema_depth, - validationRules: collInfo?.validation?.validator ?? null, - }; - } + const maxNestingDepth = await calculateSchemaDepth(schema); + const { database, collection } = toNS(namespace); + const collInfo = await dataService.collectionInfo(database, collection); + const validationRules = collInfo?.validation?.validator ?? null; + const schemaMetadata = { + maxNestingDepth, + validationRules, + }; dispatch({ type: CollectionActions.SchemaAnalysisFinished, schema, - sampleDocument: sampleDocuments[0] ?? null, + sampleDocument: sampleDocuments[0], schemaMetadata, }); } catch (err: any) { diff --git a/packages/compass-collection/src/schema-analysis-types.ts b/packages/compass-collection/src/schema-analysis-types.ts new file mode 100644 index 00000000000..d8a1f723a0c --- /dev/null +++ b/packages/compass-collection/src/schema-analysis-types.ts @@ -0,0 +1,41 @@ +import { type Schema } from 'mongodb-schema'; + +export const SCHEMA_ANALYSIS_STATE_INITIAL = 'initial'; +export const SCHEMA_ANALYSIS_STATE_ANALYZING = 'analyzing'; +export const SCHEMA_ANALYSIS_STATE_COMPLETE = 'complete'; +export const SCHEMA_ANALYSIS_STATE_ERROR = 'error'; + +export type SchemaAnalysisStatus = + | typeof SCHEMA_ANALYSIS_STATE_INITIAL + | typeof SCHEMA_ANALYSIS_STATE_ANALYZING + | typeof SCHEMA_ANALYSIS_STATE_COMPLETE + | typeof SCHEMA_ANALYSIS_STATE_ERROR; + +export type SchemaAnalysisInitial = { + status: typeof SCHEMA_ANALYSIS_STATE_INITIAL; +}; + +export type SchemaAnalysisStarted = { + status: typeof SCHEMA_ANALYSIS_STATE_ANALYZING; +}; + +export type SchemaAnalysisError = { + status: typeof SCHEMA_ANALYSIS_STATE_ERROR; + error: Error; +}; + +export type SchemaAnalysisCompleted = { + status: typeof SCHEMA_ANALYSIS_STATE_COMPLETE; + schema: Schema; + sampleDocument: Document; + schemaMetadata: { + maxNestingDepth: number; + validationRules: Document | null; + }; +}; + +export type SchemaAnalysis = + | SchemaAnalysisError + | SchemaAnalysisInitial + | SchemaAnalysisStarted + | SchemaAnalysisCompleted; diff --git a/packages/compass-collection/src/stores/collection-tab.ts b/packages/compass-collection/src/stores/collection-tab.ts index 527d99d7a58..f5b4170f9d2 100644 --- a/packages/compass-collection/src/stores/collection-tab.ts +++ b/packages/compass-collection/src/stores/collection-tab.ts @@ -7,7 +7,6 @@ import reducer, { selectTab, collectionMetadataFetched, analyzeCollectionSchema, - SchemaAnalysisStatus, } from '../modules/collection-tab'; import type { Collection } from '@mongodb-js/compass-app-stores/provider'; import type { ActivateHelpers } from '@mongodb-js/compass-app-registry'; @@ -20,6 +19,7 @@ import { type PreferencesAccess, } from 'compass-preferences-model/provider'; import { ExperimentTestName } from '@mongodb-js/compass-telemetry/provider'; +import { SCHEMA_ANALYSIS_STATE_INITIAL } from '../schema-analysis-types'; export type CollectionTabOptions = { /** @@ -81,11 +81,7 @@ export function activatePlugin( metadata: null, editViewName, schemaAnalysis: { - status: SchemaAnalysisStatus.INITIAL, - schema: null, - sampleDocument: null, - schemaMetadata: null, - error: null, + status: SCHEMA_ANALYSIS_STATE_INITIAL, }, }, applyMiddleware( From cbbdb2052c602bae3164122b13c5d1320dc6d2fc Mon Sep 17 00:00:00 2001 From: Nataly Carbonell Date: Tue, 12 Aug 2025 16:40:47 -0400 Subject: [PATCH 5/7] remove export of CollectionActions --- packages/compass-collection/src/modules/collection-tab.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/compass-collection/src/modules/collection-tab.ts b/packages/compass-collection/src/modules/collection-tab.ts index 4ad96ac8ee5..92a208fe371 100644 --- a/packages/compass-collection/src/modules/collection-tab.ts +++ b/packages/compass-collection/src/modules/collection-tab.ts @@ -54,7 +54,7 @@ export type CollectionState = { schemaAnalysis: SchemaAnalysis; }; -export enum CollectionActions { +enum CollectionActions { CollectionMetadataFetched = 'compass-collection/CollectionMetadataFetched', SchemaAnalysisStarted = 'compass-collection/SchemaAnalysisStarted', SchemaAnalysisFinished = 'compass-collection/SchemaAnalysisFinished', From 63a43af078b05a99d4bb2e50c2ac537ce045ae09 Mon Sep 17 00:00:00 2001 From: Nataly Carbonell Date: Wed, 13 Aug 2025 11:32:43 -0400 Subject: [PATCH 6/7] use .sample and add more precise error typing and handling --- package-lock.json | 14 +++++---- packages/compass-collection/package.json | 1 + .../src/modules/collection-tab.ts | 29 +++++++++++++++---- .../src/schema-analysis-types.ts | 23 +++++++++------ 4 files changed, 47 insertions(+), 20 deletions(-) diff --git a/package-lock.json b/package-lock.json index e1c8f05e10c..368b008a5f4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -31224,9 +31224,9 @@ } }, "node_modules/mongodb": { - "version": "6.17.0", - "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-6.17.0.tgz", - "integrity": "sha512-neerUzg/8U26cgruLysKEjJvoNSXhyID3RvzvdcpsIi2COYM3FS3o9nlH7fxFtefTb942dX3W9i37oPfCVj4wA==", + "version": "6.18.0", + "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-6.18.0.tgz", + "integrity": "sha512-fO5ttN9VC8P0F5fqtQmclAkgXZxbIkYRTUi1j8JO6IYwvamkhtYDilJr35jOPELR49zqCJgXZWwCtW7B+TM8vQ==", "license": "Apache-2.0", "dependencies": { "@mongodb-js/saslprep": "^1.1.9", @@ -44123,6 +44123,7 @@ "@mongodb-js/mongodb-constants": "^0.12.2", "compass-preferences-model": "^2.49.0", "hadron-document": "^8.9.4", + "mongodb": "^6.18.0", "mongodb-collection-model": "^5.31.0", "mongodb-ns": "^2.4.2", "mongodb-schema": "^12.6.2", @@ -56778,6 +56779,7 @@ "electron-mocha": "^12.2.0", "hadron-document": "^8.9.4", "mocha": "^10.2.0", + "mongodb": "^6.18.0", "mongodb-collection-model": "^5.31.0", "mongodb-ns": "^2.4.2", "mongodb-schema": "^12.6.2", @@ -79716,9 +79718,9 @@ "integrity": "sha512-5LC9SOxjSc2HF6vO2CyuTDNivEdoz2IvyJJGj6X8DJ0eFyfszE0QiEd+iXmBvUP3WHxSjFH/vIsA0EN00cgr8w==" }, "mongodb": { - "version": "6.17.0", - "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-6.17.0.tgz", - "integrity": "sha512-neerUzg/8U26cgruLysKEjJvoNSXhyID3RvzvdcpsIi2COYM3FS3o9nlH7fxFtefTb942dX3W9i37oPfCVj4wA==", + "version": "6.18.0", + "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-6.18.0.tgz", + "integrity": "sha512-fO5ttN9VC8P0F5fqtQmclAkgXZxbIkYRTUi1j8JO6IYwvamkhtYDilJr35jOPELR49zqCJgXZWwCtW7B+TM8vQ==", "requires": { "@mongodb-js/saslprep": "^1.1.9", "bson": "^6.10.4", diff --git a/packages/compass-collection/package.json b/packages/compass-collection/package.json index 68b7186b639..968191b08d3 100644 --- a/packages/compass-collection/package.json +++ b/packages/compass-collection/package.json @@ -59,6 +59,7 @@ "@mongodb-js/mongodb-constants": "^0.12.2", "compass-preferences-model": "^2.49.0", "hadron-document": "^8.9.4", + "mongodb": "^6.18.0", "mongodb-collection-model": "^5.31.0", "mongodb-ns": "^2.4.2", "mongodb-schema": "^12.6.2", diff --git a/packages/compass-collection/src/modules/collection-tab.ts b/packages/compass-collection/src/modules/collection-tab.ts index 92a208fe371..223788fd300 100644 --- a/packages/compass-collection/src/modules/collection-tab.ts +++ b/packages/compass-collection/src/modules/collection-tab.ts @@ -17,9 +17,11 @@ import { SCHEMA_ANALYSIS_STATE_COMPLETE, SCHEMA_ANALYSIS_STATE_ERROR, SCHEMA_ANALYSIS_STATE_INITIAL, - type SchemaAnalysis, + type SchemaAnalysisError, + type SchemaAnalysisState, } from '../schema-analysis-types'; import { calculateSchemaDepth } from '../calculate-schema-depth'; +import type { MongoError } from 'mongodb'; const DEFAULT_SAMPLE_SIZE = 100; @@ -32,6 +34,24 @@ function isAction( return action.type === type; } +const ERROR_CODE_MAX_TIME_MS_EXPIRED = 50; + +function getErrorDetails(error: Error): SchemaAnalysisError { + const errorCode = (error as MongoError).code; + const errorMessage = error.message || 'Unknown error'; + let errorType: SchemaAnalysisError['errorType'] = 'general'; + if (errorCode === ERROR_CODE_MAX_TIME_MS_EXPIRED) { + errorType = 'timeout'; + } else if (error.message.includes('Schema analysis aborted: Fields count')) { + errorType = 'highComplexity'; + } + + return { + errorType, + errorMessage, + }; +} + type CollectionThunkAction = ThunkAction< R, CollectionState, @@ -51,7 +71,7 @@ export type CollectionState = { namespace: string; metadata: CollectionMetadata | null; editViewName?: string; - schemaAnalysis: SchemaAnalysis; + schemaAnalysis: SchemaAnalysisState; }; enum CollectionActions { @@ -154,7 +174,7 @@ const reducer: Reducer = ( ...state, schemaAnalysis: { status: SCHEMA_ANALYSIS_STATE_ERROR, - error: action.error, + error: getErrorDetails(action.error), }, }; } @@ -204,7 +224,7 @@ export const analyzeCollectionSchema = (): CollectionThunkAction< const driverOptions = { maxTimeMS: preferences.getPreferences().maxTimeMS, }; - const sampleCursor = dataService.sampleCursor( + const sampleDocuments = await dataService.sample( namespace, samplingOptions, driverOptions, @@ -212,7 +232,6 @@ export const analyzeCollectionSchema = (): CollectionThunkAction< fallbackReadPreference: 'secondaryPreferred', } ); - const sampleDocuments: Array = await sampleCursor.toArray(); if (sampleDocuments.length === 0) { logger.debug(NO_DOCUMENTS_ERROR); dispatch({ diff --git a/packages/compass-collection/src/schema-analysis-types.ts b/packages/compass-collection/src/schema-analysis-types.ts index d8a1f723a0c..3ca6b85e39a 100644 --- a/packages/compass-collection/src/schema-analysis-types.ts +++ b/packages/compass-collection/src/schema-analysis-types.ts @@ -11,20 +11,25 @@ export type SchemaAnalysisStatus = | typeof SCHEMA_ANALYSIS_STATE_COMPLETE | typeof SCHEMA_ANALYSIS_STATE_ERROR; -export type SchemaAnalysisInitial = { +export type SchemaAnalysisInitialState = { status: typeof SCHEMA_ANALYSIS_STATE_INITIAL; }; -export type SchemaAnalysisStarted = { +export type SchemaAnalysisStartedState = { status: typeof SCHEMA_ANALYSIS_STATE_ANALYZING; }; export type SchemaAnalysisError = { + errorMessage: string; + errorType: 'timeout' | 'highComplexity' | 'general'; +}; + +export type SchemaAnalysisErrorState = { status: typeof SCHEMA_ANALYSIS_STATE_ERROR; - error: Error; + error: SchemaAnalysisError; }; -export type SchemaAnalysisCompleted = { +export type SchemaAnalysisCompletedState = { status: typeof SCHEMA_ANALYSIS_STATE_COMPLETE; schema: Schema; sampleDocument: Document; @@ -34,8 +39,8 @@ export type SchemaAnalysisCompleted = { }; }; -export type SchemaAnalysis = - | SchemaAnalysisError - | SchemaAnalysisInitial - | SchemaAnalysisStarted - | SchemaAnalysisCompleted; +export type SchemaAnalysisState = + | SchemaAnalysisErrorState + | SchemaAnalysisInitialState + | SchemaAnalysisStartedState + | SchemaAnalysisCompletedState; From 0b0125e365c627355f69ed59738028482c12023f Mon Sep 17 00:00:00 2001 From: Nataly Carbonell Date: Wed, 13 Aug 2025 14:53:56 -0400 Subject: [PATCH 7/7] add reset schemaAnalysis and fix typing --- .../src/modules/collection-tab.ts | 25 +++++++++++++++---- .../src/schema-analysis-types.ts | 5 ++-- 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/packages/compass-collection/src/modules/collection-tab.ts b/packages/compass-collection/src/modules/collection-tab.ts index 223788fd300..21194821368 100644 --- a/packages/compass-collection/src/modules/collection-tab.ts +++ b/packages/compass-collection/src/modules/collection-tab.ts @@ -21,7 +21,7 @@ import { type SchemaAnalysisState, } from '../schema-analysis-types'; import { calculateSchemaDepth } from '../calculate-schema-depth'; -import type { MongoError } from 'mongodb'; +import type { Document, MongoError } from 'mongodb'; const DEFAULT_SAMPLE_SIZE = 100; @@ -79,6 +79,7 @@ enum CollectionActions { SchemaAnalysisStarted = 'compass-collection/SchemaAnalysisStarted', SchemaAnalysisFinished = 'compass-collection/SchemaAnalysisFinished', SchemaAnalysisFailed = 'compass-collection/SchemaAnalysisFailed', + SchemaAnalysisReset = 'compass-collection/SchemaAnalysisReset', } interface CollectionMetadataFetchedAction { @@ -86,6 +87,10 @@ interface CollectionMetadataFetchedAction { metadata: CollectionMetadata; } +interface SchemaAnalysisResetAction { + type: CollectionActions.SchemaAnalysisReset; +} + interface SchemaAnalysisStartedAction { type: CollectionActions.SchemaAnalysisStarted; } @@ -129,6 +134,20 @@ const reducer: Reducer = ( }; } + if ( + isAction( + action, + CollectionActions.SchemaAnalysisReset + ) + ) { + return { + ...state, + schemaAnalysis: { + status: SCHEMA_ANALYSIS_STATE_INITIAL, + }, + }; + } + if ( isAction( action, @@ -139,10 +158,6 @@ const reducer: Reducer = ( ...state, schemaAnalysis: { status: SCHEMA_ANALYSIS_STATE_ANALYZING, - error: null, - schema: null, - sampleDocument: null, - schemaMetadata: null, }, }; } diff --git a/packages/compass-collection/src/schema-analysis-types.ts b/packages/compass-collection/src/schema-analysis-types.ts index 3ca6b85e39a..0d63615f77a 100644 --- a/packages/compass-collection/src/schema-analysis-types.ts +++ b/packages/compass-collection/src/schema-analysis-types.ts @@ -1,3 +1,4 @@ +import type { Document } from 'mongodb'; import { type Schema } from 'mongodb-schema'; export const SCHEMA_ANALYSIS_STATE_INITIAL = 'initial'; @@ -40,7 +41,7 @@ export type SchemaAnalysisCompletedState = { }; export type SchemaAnalysisState = - | SchemaAnalysisErrorState | SchemaAnalysisInitialState | SchemaAnalysisStartedState - | SchemaAnalysisCompletedState; + | SchemaAnalysisCompletedState + | SchemaAnalysisErrorState;