Skip to content

[SKStore] Another set of minor preparatory changes #934

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
Jun 26, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions skiplang/prelude/src/skstore/Context.sk
Original file line number Diff line number Diff line change
Expand Up @@ -706,11 +706,11 @@ mutable class Context private {
| None() -> invariant_violation("Could not find parent")
| Some(f) -> f
};
EagerDir::update(
parent = this.unsafeGetEagerDir(parentName);
parent.update(
this,
this.dirty.maybeGet(parentName),
this.dirtyReaders.maybeGet(parentName),
parentName,
parentMaps,
child,
)
Expand Down
36 changes: 19 additions & 17 deletions skiplang/prelude/src/skstore/EagerDir.sk
Original file line number Diff line number Diff line change
Expand Up @@ -41,11 +41,11 @@ mutable class Writer{
/* The signature of a function used by apply. */
/*****************************************************************************/

type MapFun = (
type MapFun<K, F> = (
mutable Context,
mutable Writer,
Key,
mutable Iterator<File>,
K,
mutable Iterator<F>,
) ~> void;

/*****************************************************************************/
Expand Down Expand Up @@ -679,7 +679,7 @@ class EagerDir protected {
DirName,
Array<
(
MapFun,
MapFun<Key, File>,
?Array<KeyRange<Key>>,
?((mutable Context, SortedSet<Key>) ~> void),
),
Expand Down Expand Up @@ -960,7 +960,7 @@ class EagerDir protected {
parent: EagerDir,
childName: DirName,
timeStack: TimeStack,
f: MapFun,
f: MapFun<Key, File>,
acc: mutable Vector<FixedRow<Key, Array<File>>>,
current: mutable IntRef,
): void {
Expand Down Expand Up @@ -1020,7 +1020,7 @@ class EagerDir protected {
parent: EagerDir,
childName: DirName,
timeStack: TimeStack,
f: MapFun,
f: MapFun<Key, File>,
currentStart: Int,
currentEnd: Int,
start: ?Boundary<Key>,
Expand Down Expand Up @@ -1111,7 +1111,7 @@ class EagerDir protected {
parent: EagerDir,
childName: DirName,
timeStack: TimeStack,
f: MapFun,
f: MapFun<Key, File>,
acc: mutable Vector<FixedRow<Key, Array<File>>>,
rangeOpt: ?KeyRange<Key>,
): void {
Expand Down Expand Up @@ -1175,7 +1175,7 @@ class EagerDir protected {
timeStack: TimeStack,
key: Key,
valueIter: mutable Iterator<File>,
f: MapFun,
f: MapFun<Key, File>,
): void {
arrow = ArrowKey::create{parentName, childName, key};
context.enter(arrow, timeStack);
Expand Down Expand Up @@ -1231,7 +1231,7 @@ class EagerDir protected {
DirName,
Array<
(
MapFun,
MapFun<Key, File>,
?Array<KeyRange<Key>>,
?((mutable Context, SortedSet<Key>) ~> void),
),
Expand Down Expand Up @@ -1286,14 +1286,15 @@ class EagerDir protected {
context: mutable Context,
parentName: DirName,
childName: DirName,
f: MapFun,
f: MapFun<Key, File>,
reducerOpt: ?IReducer<File> = None(),
rangeOpt: ?Array<KeyRange<Key>> = None(),
onUpdate: ?((mutable Context, SortedSet<Key>) ~> void) = None(),
unsafeSkipInit: Bool = false,
): void {
parents = FixedSingle::create(
mutable Vector[(parentName, Array[(f, rangeOpt, onUpdate)])],
parents = FixedSingle::singleton(
parentName,
Array[(f, rangeOpt, onUpdate)],
);
static::applyMany(context, parents, childName, reducerOpt, unsafeSkipInit);
}
Expand All @@ -1304,7 +1305,7 @@ class EagerDir protected {
DirName,
Array<
(
MapFun,
MapFun<Key, File>,
?Array<KeyRange<Key>>,
?((mutable Context, SortedSet<Key>) ~> void),
),
Expand Down Expand Up @@ -1569,21 +1570,22 @@ class EagerDir protected {
}
}

static fun update(
fun update(
/* this is the parent */
context: mutable Context,
parentDirtyOpt: ?SortedSet<Key>,
contextDirtyReadersOpt: ?SortedMap<DirName, SortedSet<Key>>,
parentName: DirName,
parentMaps: Array<
(
MapFun,
MapFun<Key, File>,
?Array<KeyRange<Key>>,
?((mutable Context, SortedSet<Key>) ~> void),
),
>,
childRef: EagerDir,
): void {
parent = context.unsafeGetEagerDir(parentName);
parent = this;
parentName = parent.dirName;
childName = childRef.dirName;
timeStack = childRef.timeStack;

Expand Down
12 changes: 10 additions & 2 deletions skiplang/prelude/src/skstore/FixedDir.sk
Original file line number Diff line number Diff line change
Expand Up @@ -440,12 +440,20 @@ class FixedSingle<K: Orderable, +V: frozen> private (
FixedSingle()
}

static fun singleton(k: K, v: V): this {
static(Array[(k, v)])
}

static fun pair(kv1: (K, V), kv2: (K, V)): this {
static::create(mutable Vector[kv1, kv2])
}

fun items(): mutable Iterator<(K, V)> {
this.data.iterator()
}

static fun createFromItems(items: Sequence<(K, V)>): this {
static::create(Vector::mcreateFromItems(items));
static fun createFromIterator(it: mutable Iterator<(K, V)>): this {
static::create(Vector::mcreateFromIterator(it))
}

static fun create(
Expand Down
6 changes: 2 additions & 4 deletions skiplang/prelude/src/skstore/Handle.sk
Original file line number Diff line number Diff line change
Expand Up @@ -410,10 +410,8 @@ class EHandle<K: Key, V: File> extends Handle<K, V> {
};
dynParents[parentName].push((dynamicMapFun, rangeOpt, None()))
};
fixedParents = FixedSingle::create(
Vector::mcreateFromIterator(
dynParents.items().map(kv -> (kv.i0, kv.i1.toArray())),
),
fixedParents = FixedSingle::createFromIterator(
dynParents.items().map(kv -> (kv.i0, kv.i1.toArray())),
);
EagerDir::applyMany(context, fixedParents, dirName, reducerOpt);
EHandle(typeOutputKey, typeOutput, dirName)
Expand Down
102 changes: 45 additions & 57 deletions sql/src/SqlSelect.sk
Original file line number Diff line number Diff line change
Expand Up @@ -436,11 +436,9 @@ class SelectEvaluator{

SKStore.EagerDir::applyMany(
context,
SKStore.FixedSingle::create(
mutable Vector[
(leftHandle.dirName, Array[(noop, leftRanges, None())]),
(shardsDirName, Array[(noop, None(), None())]),
],
SKStore.FixedSingle::pair(
(leftHandle.dirName, Array[(noop, leftRanges, None())]),
(shardsDirName, Array[(noop, None(), None())]),
),
projLeftName,
);
Expand All @@ -449,11 +447,9 @@ class SelectEvaluator{

SKStore.EagerDir::applyMany(
context,
SKStore.FixedSingle::create(
mutable Vector[
(rightHandle.dirName, Array[(noop, rightRanges, None())]),
(shardsDirName, Array[(noop, None(), None())]),
],
SKStore.FixedSingle::pair(
(rightHandle.dirName, Array[(noop, rightRanges, None())]),
(shardsDirName, Array[(noop, None(), None())]),
),
projRightName,
);
Expand Down Expand Up @@ -494,27 +490,21 @@ class SelectEvaluator{

SKStore.EagerDir::applyMany(
context,
SKStore.FixedSingle::create(
mutable Vector[
(projLeftName, Array[(projLeftFun, None(), None())]),
(projRightName, Array[(projRightFun, None(), None())]),
],
SKStore.FixedSingle::pair(
(projLeftName, Array[(projLeftFun, None(), None())]),
(projRightName, Array[(projRightFun, None(), None())]),
),
projName,
);
SKStore.EagerDir::applyMany(
context,
SKStore.FixedSingle::create(
mutable Vector[
SKStore.FixedSingle::singleton(
projName,
Array[
(
projName,
Array[
(
makeJoinReducer(shards, joinKind, kinds, onlyLeftValues),
None(),
None(),
),
],
makeJoinReducer(shards, joinKind, kinds, onlyLeftValues),
None(),
None(),
),
],
),
Expand Down Expand Up @@ -712,37 +702,35 @@ class SelectEvaluator{

SKStore.EagerDir::applyMany(
context,
SKStore.FixedSingle::create(
mutable Vector[
(
selectName1Empty,
Array[
(
(context, writer, key, values) ~> {
dataDir = context.unsafeGetEagerDir(dataName);
if (dataDir.isEmpty(context)) {
writer.setArray(key, values.collect(Array))
}
},
None(),
None(),
),
],
),
(
dataName,
Array[
(
(_context, writer, key, values) ~> {
arr = values.collect(Array);
writer.setArray(key, arr)
},
None(),
None(),
),
],
),
],
SKStore.FixedSingle::pair(
(
selectName1Empty,
Array[
(
(context, writer, key, values) ~> {
dataDir = context.unsafeGetEagerDir(dataName);
if (dataDir.isEmpty(context)) {
writer.setArray(key, values.collect(Array))
}
},
None(),
None(),
),
],
),
(
dataName,
Array[
(
(_context, writer, key, values) ~> {
arr = values.collect(Array);
writer.setArray(key, arr)
},
None(),
None(),
),
],
),
),
this.selectName1,
)
Expand Down Expand Up @@ -1611,7 +1599,7 @@ fun makeJoinReducer(
joinKind: P.JoinKind,
kinds: Array<(Int, P.IKind, P.Type)>,
onlyLeftValues: Bool,
): SKStore.MapFun {
): SKStore.MapFun<SKStore.Key, SKStore.File> {
(context, writer, _, files) ~> {
leftRows = mutable Vector[];
rightRows = mutable Vector[];
Expand Down