From b0018b2a8a6254b44ddce1e27f8ca6b8149828ae Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Tue, 25 Mar 2025 21:19:07 -0600 Subject: [PATCH 1/6] separate sync as its own stream --- neo/rawio/openephysbinaryrawio.py | 35 +++++++++++++++---- .../rawiotest/test_openephysbinaryrawio.py | 22 ++++++++++++ 2 files changed, 51 insertions(+), 6 deletions(-) diff --git a/neo/rawio/openephysbinaryrawio.py b/neo/rawio/openephysbinaryrawio.py index 6059ec0b3..3ced72311 100644 --- a/neo/rawio/openephysbinaryrawio.py +++ b/neo/rawio/openephysbinaryrawio.py @@ -72,6 +72,12 @@ def __init__(self, dirname="", load_sync_channel=False, experiment_names=None): experiment_names = [experiment_names] self.experiment_names = experiment_names self.load_sync_channel = load_sync_channel + if load_sync_channel: + warn( + "The load_sync_channel=True option is deprecated and will be removed in a future version. " + "Use load_sync_channel=False instead, which will add sync channels as separate streams.", + DeprecationWarning, stacklevel=2 + ) self.folder_structure = None self._use_direct_evt_timestamps = None @@ -123,7 +129,7 @@ def _parse_header(self): # signals zone # create signals channel map: several channel per stream signal_channels = [] - + sync_stream_id_to_buffer = {} for stream_index, stream_name in enumerate(sig_stream_names): # stream_index is the index in vector stream names stream_id = str(stream_index) @@ -141,8 +147,9 @@ def _parse_header(self): # Special cases for stream if "SYNC" in chan_id and not self.load_sync_channel: - # the channel is removed from stream but not the buffer - stream_id = "" + # Every stream sync channel is added as its own stream + stream_id = chan_id + sync_stream_id_to_buffer[stream_id] = buffer_id if "ADC" in chan_id: # These are non-neural channels and their stream should be separated @@ -174,12 +181,18 @@ def _parse_header(self): signal_buffers = [] unique_streams_ids = np.unique(signal_channels["stream_id"]) + for stream_id in unique_streams_ids: - # Handle special case of Synch channel having stream_id empty - if stream_id == "": + + # Handle sync channel on a special way + if "SYNC" in stream_id: + # This is a sync channel and should not be added to the signal streams + sync_stream_buffer = sync_stream_id_to_buffer[stream_id] + signal_streams.append((stream_id, stream_id, sync_stream_buffer)) continue - stream_index = int(stream_id) + # Neural signal + stream_index = int(stream_id) if stream_index < self._num_of_signal_streams: stream_name = sig_stream_names[stream_index] buffer_id = stream_id @@ -254,7 +267,12 @@ def _parse_header(self): if num_adc_channels == 0: if has_sync_trace and not self.load_sync_channel: + # Exclude the sync channel from the main stream self._stream_buffer_slice[stream_id] = slice(None, -1) + + # Add a buffer slice for the sync channel + sync_channel_name = info["channels"][-1]["channel_name"] + self._stream_buffer_slice[sync_channel_name] = slice(-1, None) else: self._stream_buffer_slice[stream_id] = None else: @@ -264,7 +282,12 @@ def _parse_header(self): self._stream_buffer_slice[stream_id_neural] = slice(0, num_neural_channels) if has_sync_trace and not self.load_sync_channel: + # Exclude the sync channel from the non-neural stream self._stream_buffer_slice[stream_id_non_neural] = slice(num_neural_channels, -1) + + # Add a buffer slice for the sync channel + sync_channel_name = info["channels"][-1]["channel_name"] + self._stream_buffer_slice[sync_channel_name] = slice(-1, None) else: self._stream_buffer_slice[stream_id_non_neural] = slice(num_neural_channels, None) diff --git a/neo/test/rawiotest/test_openephysbinaryrawio.py b/neo/test/rawiotest/test_openephysbinaryrawio.py index 9a9704209..a245cbc2f 100644 --- a/neo/test/rawiotest/test_openephysbinaryrawio.py +++ b/neo/test/rawiotest/test_openephysbinaryrawio.py @@ -43,6 +43,28 @@ def test_sync(self): block_index=0, seg_index=0, i_start=0, i_stop=100, stream_index=stream_index ) assert chunk.shape[1] == 384 + + def test_sync_channel_access(self): + """Test that sync channels can be accessed as separate streams when load_sync_channel=False.""" + rawio = OpenEphysBinaryRawIO( + self.get_local_path("openephysbinary/v0.6.x_neuropixels_with_sync"), load_sync_channel=False + ) + rawio.parse_header() + + # Find sync channel streams + sync_stream_names = [s_name for s_name in rawio.header["signal_streams"]["name"] if "SYNC" in s_name] + assert len(sync_stream_names) > 0, "No sync channel streams found" + + # Get the stream index for the first sync channel + sync_stream_index = list(rawio.header["signal_streams"]["name"]).index(sync_stream_names[0]) + + # Check that we can access the sync channel data + chunk = rawio.get_analogsignal_chunk( + block_index=0, seg_index=0, i_start=0, i_stop=100, stream_index=sync_stream_index + ) + + # Sync channel should have only one channel + assert chunk.shape[1] == 1, f"Expected sync channel to have 1 channel, got {chunk.shape[1]}" def test_no_sync(self): # requesting sync channel when there is none raises an error From 1ca069009f65e094991043a8b4baac973ba9d426 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Tue, 25 Mar 2025 21:25:33 -0600 Subject: [PATCH 2/6] small refactoring --- neo/rawio/openephysbinaryrawio.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/neo/rawio/openephysbinaryrawio.py b/neo/rawio/openephysbinaryrawio.py index 3ced72311..7ba7b7301 100644 --- a/neo/rawio/openephysbinaryrawio.py +++ b/neo/rawio/openephysbinaryrawio.py @@ -129,7 +129,7 @@ def _parse_header(self): # signals zone # create signals channel map: several channel per stream signal_channels = [] - sync_stream_id_to_buffer = {} + sync_stream_id_to_buffer_id = {} for stream_index, stream_name in enumerate(sig_stream_names): # stream_index is the index in vector stream names stream_id = str(stream_index) @@ -149,7 +149,7 @@ def _parse_header(self): if "SYNC" in chan_id and not self.load_sync_channel: # Every stream sync channel is added as its own stream stream_id = chan_id - sync_stream_id_to_buffer[stream_id] = buffer_id + sync_stream_id_to_buffer_id[stream_id] = buffer_id if "ADC" in chan_id: # These are non-neural channels and their stream should be separated @@ -182,13 +182,16 @@ def _parse_header(self): unique_streams_ids = np.unique(signal_channels["stream_id"]) + # This is getting too complicated, we probably should just have a table which would be easier to read + # And for users to understand for stream_id in unique_streams_ids: # Handle sync channel on a special way if "SYNC" in stream_id: # This is a sync channel and should not be added to the signal streams - sync_stream_buffer = sync_stream_id_to_buffer[stream_id] - signal_streams.append((stream_id, stream_id, sync_stream_buffer)) + buffer_id = sync_stream_id_to_buffer_id[stream_id] + stream_name = stream_id + signal_streams.append((stream_name, stream_id, buffer_id)) continue # Neural signal From e9623180201d62e21edb5ca90755a5bf67e2064f Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Fri, 4 Apr 2025 09:27:49 -0600 Subject: [PATCH 3/6] Update neo/rawio/openephysbinaryrawio.py Co-authored-by: Alessio Buccino --- neo/rawio/openephysbinaryrawio.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/neo/rawio/openephysbinaryrawio.py b/neo/rawio/openephysbinaryrawio.py index 7ba7b7301..b14c0fee6 100644 --- a/neo/rawio/openephysbinaryrawio.py +++ b/neo/rawio/openephysbinaryrawio.py @@ -148,7 +148,7 @@ def _parse_header(self): # Special cases for stream if "SYNC" in chan_id and not self.load_sync_channel: # Every stream sync channel is added as its own stream - stream_id = chan_id + stream_id = f"{chan_id}-{str(stream_index)}" sync_stream_id_to_buffer_id[stream_id] = buffer_id if "ADC" in chan_id: From 8d32b9b03e737b1be6748d134572d5416c16daa4 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Fri, 4 Apr 2025 10:04:04 -0600 Subject: [PATCH 4/6] fix stream --- neo/rawio/openephysbinaryrawio.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/neo/rawio/openephysbinaryrawio.py b/neo/rawio/openephysbinaryrawio.py index b14c0fee6..9a9aacc92 100644 --- a/neo/rawio/openephysbinaryrawio.py +++ b/neo/rawio/openephysbinaryrawio.py @@ -275,7 +275,8 @@ def _parse_header(self): # Add a buffer slice for the sync channel sync_channel_name = info["channels"][-1]["channel_name"] - self._stream_buffer_slice[sync_channel_name] = slice(-1, None) + stream_name = f"{sync_channel_name}-{str(stream_id)}" + self._stream_buffer_slice[stream_name] = slice(-1, None) else: self._stream_buffer_slice[stream_id] = None else: From 405f8a01d85a021f09aa863922315570b937451e Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Sun, 6 Apr 2025 14:53:10 -0600 Subject: [PATCH 5/6] added specific deprecation date --- neo/rawio/openephysbinaryrawio.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/neo/rawio/openephysbinaryrawio.py b/neo/rawio/openephysbinaryrawio.py index 9a9aacc92..e5d51de76 100644 --- a/neo/rawio/openephysbinaryrawio.py +++ b/neo/rawio/openephysbinaryrawio.py @@ -74,7 +74,7 @@ def __init__(self, dirname="", load_sync_channel=False, experiment_names=None): self.load_sync_channel = load_sync_channel if load_sync_channel: warn( - "The load_sync_channel=True option is deprecated and will be removed in a future version. " + "The load_sync_channel=True option is deprecated and will be removed in version 0.15. " "Use load_sync_channel=False instead, which will add sync channels as separate streams.", DeprecationWarning, stacklevel=2 ) From 02bfcd3d46754c932a8f180749b535f133b0e641 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Tue, 8 Apr 2025 09:55:51 -0600 Subject: [PATCH 6/6] add sync to stream name --- neo/rawio/openephysbinaryrawio.py | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/neo/rawio/openephysbinaryrawio.py b/neo/rawio/openephysbinaryrawio.py index e5d51de76..d56d0f441 100644 --- a/neo/rawio/openephysbinaryrawio.py +++ b/neo/rawio/openephysbinaryrawio.py @@ -130,6 +130,7 @@ def _parse_header(self): # create signals channel map: several channel per stream signal_channels = [] sync_stream_id_to_buffer_id = {} + normal_stream_id_to_sync_stream_id = {} for stream_index, stream_name in enumerate(sig_stream_names): # stream_index is the index in vector stream names stream_id = str(stream_index) @@ -140,6 +141,7 @@ def _parse_header(self): chan_id = chan_info["channel_name"] units = chan_info["units"] + channel_stream_id = stream_id if units == "": # When units are not provided they are microvolts for neural channels and volts for ADC channels # See https://open-ephys.github.io/gui-docs/User-Manual/Recording-data/Binary-format.html#continuous @@ -148,14 +150,19 @@ def _parse_header(self): # Special cases for stream if "SYNC" in chan_id and not self.load_sync_channel: # Every stream sync channel is added as its own stream - stream_id = f"{chan_id}-{str(stream_index)}" - sync_stream_id_to_buffer_id[stream_id] = buffer_id + sync_stream_id = f"{stream_name}SYNC" + sync_stream_id_to_buffer_id[sync_stream_id] = buffer_id + + # We save this mapping for the buffer description protocol + normal_stream_id_to_sync_stream_id[stream_id] = sync_stream_id + # We then set the stream_id to the sync stream id + channel_stream_id = sync_stream_id if "ADC" in chan_id: # These are non-neural channels and their stream should be separated # We defined their stream_id as the stream_index of neural data plus the number of neural streams # This is to not break backwards compatbility with the stream_id numbering - stream_id = str(stream_index + len(sig_stream_names)) + channel_stream_id = str(stream_index + len(sig_stream_names)) gain = float(chan_info["bit_volts"]) sampling_rate = float(info["sample_rate"]) @@ -169,7 +176,7 @@ def _parse_header(self): units, gain, offset, - stream_id, + channel_stream_id, buffer_id, ) ) @@ -274,9 +281,8 @@ def _parse_header(self): self._stream_buffer_slice[stream_id] = slice(None, -1) # Add a buffer slice for the sync channel - sync_channel_name = info["channels"][-1]["channel_name"] - stream_name = f"{sync_channel_name}-{str(stream_id)}" - self._stream_buffer_slice[stream_name] = slice(-1, None) + sync_stream_id = normal_stream_id_to_sync_stream_id[stream_id] + self._stream_buffer_slice[sync_stream_id] = slice(-1, None) else: self._stream_buffer_slice[stream_id] = None else: @@ -290,8 +296,8 @@ def _parse_header(self): self._stream_buffer_slice[stream_id_non_neural] = slice(num_neural_channels, -1) # Add a buffer slice for the sync channel - sync_channel_name = info["channels"][-1]["channel_name"] - self._stream_buffer_slice[sync_channel_name] = slice(-1, None) + sync_stream_id = normal_stream_id_to_sync_stream_id[stream_id] + self._stream_buffer_slice[sync_stream_id] = slice(-1, None) else: self._stream_buffer_slice[stream_id_non_neural] = slice(num_neural_channels, None)