|
2 | 2 | import logging
|
3 | 3 | from pathlib import Path
|
4 | 4 | from time import sleep
|
5 |
| -from typing import Generator, List, Tuple |
| 5 | +from typing import Dict, Generator, List, Optional, Tuple |
6 | 6 | from urllib.parse import urlparse
|
7 | 7 |
|
8 | 8 | import boto3 as boto
|
@@ -351,10 +351,10 @@ def _get_aws_cluster_info_from_s3(bucket: str, file_key: str, cluster_id):
|
351 | 351 |
|
352 | 352 | def save_cluster_report(
|
353 | 353 | cluster_id: str,
|
354 |
| - instance_timelines: list[dict], |
355 |
| - cluster_log_destination: tuple[str, ...] | None = None, |
356 |
| - cluster_report_destination_override: dict[str, str] | None = None, |
357 |
| - write_function=None |
| 354 | + instance_timelines: List[dict], |
| 355 | + cluster_log_destination: Optional[Tuple[str, ...]] = None, |
| 356 | + cluster_report_destination_override: Optional[Dict[str, str]] = None, |
| 357 | + write_function=None, |
358 | 358 | ) -> bool:
|
359 | 359 | cluster = get_default_client().get_cluster(cluster_id)
|
360 | 360 | spark_context_id = cluster.get("spark_context_id")
|
@@ -474,7 +474,10 @@ def _monitor_cluster(
|
474 | 474 | all_timelines = retired_timelines + list(active_timelines_by_id.values())
|
475 | 475 |
|
476 | 476 | save_cluster_report(
|
477 |
| - cluster_id, all_timelines, cluster_log_destination=cluster_log_destination, write_function=write_function |
| 477 | + cluster_id, |
| 478 | + all_timelines, |
| 479 | + cluster_log_destination=cluster_log_destination, |
| 480 | + write_function=write_function, |
478 | 481 | )
|
479 | 482 |
|
480 | 483 | if kill_on_termination:
|
|
0 commit comments