diff --git a/setup.py b/setup.py index 1e264b7..e79c0d3 100644 --- a/setup.py +++ b/setup.py @@ -32,7 +32,7 @@ 'pyOpenSSL==23.0.0', # Math - 'pyts>=0.10.0', + 'pyts>=0.11.0', # Flask 'Flask==1.0.2', diff --git a/sintel/data.py b/sintel/data.py index 78b8e9b..d217620 100644 --- a/sintel/data.py +++ b/sintel/data.py @@ -25,7 +25,7 @@ os.path.dirname(os.path.abspath(__file__)), 'data' ) -BUCKET = 'd3-ai-orion' +BUCKET = 'sintel-orion' S3_URL = 'https://{}.s3.amazonaws.com/{}' NASA_SIGNALS = ( diff --git a/sintel/db/utils.py b/sintel/db/utils.py index ed42e7f..6c73c6b 100644 --- a/sintel/db/utils.py +++ b/sintel/db/utils.py @@ -311,7 +311,7 @@ def _update_prediction(signalrun, v, stock=False): print(e) -def _update_period(signalrun, v, stock=False): +def _update_period(signalrun, v, my_interval=1440, stock=False): year_start = datetime.utcfromtimestamp(v['raw_index'][0]).year year_end = datetime.utcfromtimestamp(v['raw_index'][-1]).year @@ -320,7 +320,6 @@ def _update_period(signalrun, v, stock=False): # optimal interval for periodical description diff = (v['raw_index'][1] - v['raw_index'][0]) / 60 - my_interval = 1440 for interval in [6, 30, 60, 120, 180, 240, 360, 480, 720]: if diff <= interval: my_interval = interval @@ -447,7 +446,7 @@ def _update_raw(signal, interval=360, method=['mean'], stock=False): schema.SignalRaw.insert(**raw_doc) -def update_db(fs, exp_filter=None, stock=False): +def update_db(fs, interval=360, my_interval=1440, exp_filter=None, stock=False): global g_fs g_fs = fs @@ -463,7 +462,7 @@ def update_db(fs, exp_filter=None, stock=False): cc += 1 LOGGER.info('{}/{}: Processing signal {}'.format(cc, total, signal.name)) if not schema.SignalRaw.find_one(signal=signal): - _update_raw(signal, stock=stock) + _update_raw(signal, interval=interval, stock=stock) else: LOGGER.info('Skip - this signal data has been processed previously') except Exception as e: @@ -494,7 +493,7 @@ def update_db(fs, exp_filter=None, stock=False): if (schema.Period.find_one(signalrun=signalrun.id) is not None): continue else: - _update_period(signalrun, v, stock=stock) + _update_period(signalrun, v, my_interval=my_interval, stock=stock) except Exception as e: print(e) diff --git a/sintel/resources/datarun.py b/sintel/resources/datarun.py index 7df26ca..39fed5a 100644 --- a/sintel/resources/datarun.py +++ b/sintel/resources/datarun.py @@ -1,6 +1,6 @@ -import json +# import json import logging -import sys +# import sys from bson import ObjectId from flask_restful import Resource, reqparse diff --git a/tutorials/pipelines/orion_lstmdt.json b/tutorials/pipelines/orion_lstmdt.json index 5b22a40..75706fc 100644 --- a/tutorials/pipelines/orion_lstmdt.json +++ b/tutorials/pipelines/orion_lstmdt.json @@ -1,15 +1,15 @@ { "primitives": [ - "mlprimitives.custom.timeseries_preprocessing.time_segments_aggregate", + "mlstars.custom.timeseries_preprocessing.time_segments_aggregate", "sklearn.impute.SimpleImputer", "sklearn.preprocessing.MinMaxScaler", - "mlprimitives.custom.timeseries_preprocessing.rolling_window_sequences", + "mlstars.custom.timeseries_preprocessing.rolling_window_sequences", "keras.Sequential.LSTMTimeSeriesRegressor", "orion.primitives.timeseries_errors.regression_errors", "orion.primitives.timeseries_anomalies.find_anomalies" ], "init_params": { - "mlprimitives.custom.timeseries_preprocessing.time_segments_aggregate#1": { + "mlstars.custom.timeseries_preprocessing.time_segments_aggregate#1": { "time_column": "timestamp", "interval": 21600, "method": "mean" @@ -20,7 +20,7 @@ 1 ] }, - "mlprimitives.custom.timeseries_preprocessing.rolling_window_sequences#1": { + "mlstars.custom.timeseries_preprocessing.rolling_window_sequences#1": { "target_column": 0, "window_size": 250 }, @@ -57,7 +57,7 @@ }, { "name": "raw_index", - "variable": "mlprimitives.custom.timeseries_preprocessing.time_segments_aggregate#1.index" + "variable": "mlstars.custom.timeseries_preprocessing.time_segments_aggregate#1.index" }, { "name": "X_nm", @@ -65,11 +65,11 @@ }, { "name": "target_index", - "variable": "mlprimitives.custom.timeseries_preprocessing.rolling_window_sequences#1.target_index" + "variable": "mlstars.custom.timeseries_preprocessing.rolling_window_sequences#1.target_index" }, { "name": "y", - "variable": "mlprimitives.custom.timeseries_preprocessing.rolling_window_sequences#1.y" + "variable": "mlstars.custom.timeseries_preprocessing.rolling_window_sequences#1.y" }, { "name": "y_hat", diff --git a/tutorials/pipelines/orioin_tadgan.json b/tutorials/pipelines/orion_tadgan.json similarity index 81% rename from tutorials/pipelines/orioin_tadgan.json rename to tutorials/pipelines/orion_tadgan.json index de21544..4306276 100644 --- a/tutorials/pipelines/orioin_tadgan.json +++ b/tutorials/pipelines/orion_tadgan.json @@ -1,15 +1,15 @@ { "primitives": [ - "mlprimitives.custom.timeseries_preprocessing.time_segments_aggregate", + "mlstars.custom.timeseries_preprocessing.time_segments_aggregate", "sklearn.impute.SimpleImputer", "sklearn.preprocessing.MinMaxScaler", - "mlprimitives.custom.timeseries_preprocessing.rolling_window_sequences", + "mlstars.custom.timeseries_preprocessing.rolling_window_sequences", "orion.primitives.tadgan.TadGAN", "orion.primitives.tadgan.score_anomalies", "orion.primitives.timeseries_anomalies.find_anomalies" ], "init_params": { - "mlprimitives.custom.timeseries_preprocessing.time_segments_aggregate#1": { + "mlstars.custom.timeseries_preprocessing.time_segments_aggregate#1": { "time_column": "timestamp", "interval": 21600, "method": "mean" @@ -20,7 +20,7 @@ 1 ] }, - "mlprimitives.custom.timeseries_preprocessing.rolling_window_sequences#1": { + "mlstars.custom.timeseries_preprocessing.rolling_window_sequences#1": { "target_column": 0, "window_size": 100, "target_size": 1 @@ -49,7 +49,7 @@ "orion.primitives.tadgan.TadGAN#1": { "y": "y_hat" }, - "mlprimitives.custom.timeseries_preprocessing.rolling_window_sequences#1": { + "mlstars.custom.timeseries_preprocessing.rolling_window_sequences#1": { "index": "X_index" } },