Update to use pandas v2.*

.github/workflows/core_tests.yml

@@ -275,11 +275,11 @@ jobs:
           - region: Standard 1-Zone Example (MTC)
             region-org: ActivitySim
             region-repo: activitysim-prototype-mtc
-            region-branch: extended
+            region-branch: pandas2
           - region: Standard 2-Zone Example (SANDAG)
             region-org: ActivitySim
             region-repo: sandag-abm3-example
-            region-branch: main
+            region-branch: pandas2
       fail-fast: false
     defaults:
       run:

activitysim/abm/models/disaggregate_accessibility.py

@@ -158,7 +158,7 @@ class DisaggregateAccessibilitySettings(PydanticReadable, extra="forbid"):
     """
     Disaggreate accessibility table is grouped by the "by" cols above and the KEEP_COLS are averaged
     across the group.  Initializing the below as NA if not in the auto ownership level, they are skipped
-    in the groupby mean and the values are correct. 
+    in the groupby mean and the values are correct.
     (It's a way to avoid having to update code to reshape the table and introduce new functionality there.)
     If none, will keep all of the columns with "accessibility" in the name.
     """
@@ -581,7 +581,7 @@ def expand_template_zones(self, tables):
         _expanded = pd.DataFrame(util.named_product(**index_params)).set_index("index")
 
         # Use result to join template onto expanded table of zones
-        ex_table = _expanded.join(master_template).reset_index()
+        ex_table = _expanded.join(master_template).sort_index().reset_index()
 
         # Concatenate a new unique set of ids
         cols = ["home_zone_id", "proto_household_id", "proto_person_id"]
@@ -654,7 +654,9 @@ def create_proto_pop(self):
                 .set_index("index")
                 .rename(columns={"hhid": hhid})
             )
-            persons = rep.join(persons).sort_values(hhid).reset_index(drop=True)
+            persons = (
+                rep.join(persons, sort=True).sort_values(hhid).reset_index(drop=True)
+            )
             persons[perid] = persons.index + 1
 
             # Assign persons to tours
@@ -730,6 +732,7 @@ def merge_persons(self):
 
         perid = self.params["proto_persons"]["index_col"]
         persons_merged.set_index(perid, inplace=True, drop=True)
+        persons_merged = persons_merged.sort_index()
         self.proto_pop["proto_persons_merged"] = persons_merged
 
         # Store in pipeline

activitysim/abm/models/input_checker.py

@@ -301,34 +301,42 @@ def report_errors(state, input_checker_settings, v_warnings, v_errors):
 
             for warn in warns:
                 if "dataframe validator" in str(warn.message):
-                    file_logger.warning(
-                        "Failed dataframe validator: "
-                        + str(warn.message).split("\n")[-1]
-                    )
-                elif "element-wise validator" in str(warn.message):
-                    if "DataFrameSchema" in str(warn.message):
-                        file_logger.warning(
-                            "Failed element-wise validator: <"
-                            + str(warn.message).split("\n")[0].split(" ")[1]
-                            + table_name
-                            + ")>\n\t"
-                            + str(warn.message)
-                            .split("failure cases:\n")[0]
-                            .split("\n")[-2]
-                            + "\n\tfailure cases:\n\t"
-                            + "\n\t".join(
-                                str(warn.message)
-                                .split("failure cases:\n")[1]
-                                .split("\n")
-                            )
-                        )
-                    else:
+                    try:
                         file_logger.warning(
-                            "Failed element-wise validator: <"
-                            + " ".join(str(warn.message).split("\n")[0].split(" ")[1:3])
-                            + "\n\t"
-                            + "\n\t".join(str(warn.message).split("\n")[1:])
+                            "Failed dataframe validator: "
+                            + str(warn.message).split("\n")[-1]
                         )
+                    except Exception:
+                        file_logger.warning(warn)
+                elif "element-wise validator" in str(warn.message):
+                    try:
+                        if "DataFrameSchema" in str(warn.message):
+                            file_logger.warning(
+                                "Failed element-wise validator: <"
+                                + str(warn.message).split("\n")[0].split(" ")[1]
+                                + table_name
+                                + ")>\n\t"
+                                + str(warn.message)
+                                .split("failure cases:\n")[0]
+                                .split("\n")[-2]
+                                + "\n\tfailure cases:\n\t"
+                                + "\n\t".join(
+                                    str(warn.message)
+                                    .split("failure cases:\n")[1]
+                                    .split("\n")
+                                )
+                            )
+                        else:
+                            file_logger.warning(
+                                "Failed element-wise validator: <"
+                                + " ".join(
+                                    str(warn.message).split("\n")[0].split(" ")[1:3]
+                                )
+                                + "\n\t"
+                                + "\n\t".join(str(warn.message).split("\n")[1:])
+                            )
+                    except Exception:
+                        file_logger.warning(warn)
                 else:
                     file_logger.warning(warn)
             file_logger.warning("\n")

activitysim/abm/models/school_escorting.py

@@ -634,7 +634,7 @@ def school_escorting(
     state.add_table("tours", tours)
     state.get_rn_generator().drop_channel("tours")
     state.get_rn_generator().add_channel("tours", tours)
-    state.add_table("escort_bundles", escort_bundles)
+    state.add_table("escort_bundles", escort_bundles.reset_index(drop=True))
     # save school escorting tours and trips in pipeline so we can overwrite results from downstream models
     state.add_table("school_escort_tours", school_escort_tours)
     state.add_table("school_escort_trips", school_escort_trips)

activitysim/abm/models/trip_departure_choice.py

@@ -404,7 +404,7 @@ def apply_stage_two_model(
     trace_label: str,
     compute_settings: ComputeSettings | None = None,
 ):
-    if not trips.index.is_monotonic:
+    if not trips.index.is_monotonic_increasing:
         trips = trips.sort_index()
 
     # Assign the duration of the appropriate leg to the trip

activitysim/abm/models/util/school_escort_tours_trips.py

@@ -353,7 +353,7 @@ def create_chauf_escort_trips(bundles):
             "outbound",
             "purpose",
         ]
-    ).reset_index()
+    ).reset_index(drop=True)
 
     # numbering trips such that outbound escorting trips must come first and inbound trips must come last
     outbound_trip_num = -1 * (
@@ -539,7 +539,7 @@ def create_escortee_trips(bundles):
     # create a new trip for each escortee destination
     escortee_trips = escortee_trips.explode(
         ["destination", "escort_participants", "school_escort_trip_num", "purpose"]
-    ).reset_index()
+    ).reset_index(drop=True)
 
     # numbering trips such that outbound escorting trips must come first and inbound trips must come last
     # this comes in handy when merging trips to others in the tour decided downstream

activitysim/abm/models/vehicle_allocation.py

@@ -261,6 +261,7 @@ def vehicle_allocation(
             ]
 
         # set choice for non-household vehicle option
+        choices["choice"] = choices["choice"].astype(veh_choice_dtype)
         choices.loc[
             choices["alt_choice"] == alts_from_spec[-1], "choice"
         ] = alts_from_spec[-1]

activitysim/cli/create.py

@@ -2,6 +2,7 @@
 
 import glob
 import hashlib
+import importlib.resources
 import logging
 import os
 import shutil
@@ -21,14 +22,15 @@
 
 def _example_path(resource):
     resource = os.path.join(EXAMPLES_DIR, resource)
-    path = pkg_resources.resource_filename(PACKAGE, resource)
-
-    return path
+    return importlib.resources.as_file(
+        importlib.resources.files(PACKAGE).joinpath(resource)
+    )
 
 
 def _load_manifest():
-    with open(_example_path(MANIFEST), "r") as f:
-        manifest = yaml.safe_load(f.read())
+    with _example_path(MANIFEST) as f_pth:
+        with open(f_pth, "r") as f:
+            manifest = yaml.safe_load(f.read())
 
     assert manifest, f"error: could not load {MANIFEST}"
     return {example["name"]: example for example in manifest}
@@ -177,8 +179,9 @@ def get_example(
             )
 
         else:
-            for asset_path in glob.glob(_example_path(assets)):
-                copy_asset(asset_path, target_path, dirs_exist_ok=True)
+            with _example_path(assets) as pth:
+                for asset_path in glob.glob(str(pth)):
+                    copy_asset(asset_path, target_path, dirs_exist_ok=True)
 
     print(f"copied! new project files are in {os.path.abspath(dest_path)}")
 

activitysim/core/assign.py

@@ -96,7 +96,36 @@ def read_assignment_spec(
     """
 
     try:
-        cfg = pd.read_csv(file_name, comment="#")
+        # we use an explicit list of na_values, these are the values that
+        # Pandas version 1.5 recognized as NaN by default.  Notably absent is
+        # 'None' which is used in some spec files to be the object `None` not
+        # the float value NaN.
+        cfg = pd.read_csv(
+            file_name,
+            comment="#",
+            na_values=[
+                "",
+                "#N/A",
+                "#N/A N/A",
+                "#NA",
+                "-1.#IND",
+                "-1.#QNAN",
+                "-NaN",
+                "-nan",
+                "1.#IND",
+                "1.#QNAN",
+                "<NA>",
+                "N/A",
+                "NA",
+                "NULL",
+                "NaN",
+                "n/a",
+                "nan",
+                "null",
+            ],
+            keep_default_na=False,
+        )
+
     except Exception as e:
         logger.error(f"Error reading spec file: {file_name}")
         logger.error(str(e))

activitysim/core/fast_eval.py

@@ -0,0 +1,106 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+import pandas as pd
+from pandas import eval as _eval
+
+if TYPE_CHECKING:
+    from collections.abc import Hashable, Iterator, Mapping, Sequence
+
+    from pandas._typing import ArrayLike
+
+
+def _get_cleaned_column_resolvers(
+    df: pd.DataFrame, raw: bool = True
+) -> dict[Hashable, ArrayLike | pd.Series]:
+    """
+    Return the special character free column resolvers of a dataframe.
+
+    Column names with special characters are 'cleaned up' so that they can
+    be referred to by backtick quoting.
+    Used in :meth:`DataFrame.eval`.
+    """
+    from pandas import Series
+    from pandas.core.computation.parsing import clean_column_name
+
+    if isinstance(df, pd.Series):
+        return {clean_column_name(df.name): df}
+
+    # CHANGED FROM PANDAS: do not even convert the arrays to pd.Series, just
+    # give the raw arrays to the compute engine. This is potentially a breaking
+    # change if any of the operations in the eval string require a pd.Series.
+    if raw:
+        # Performance tradeoff: in the dict below, we iterate over `df.items`,
+        # which yields tuples of (column_name, data as pd.Series). This is marginally
+        # slower than iterating over `df.columns` and `df._iter_column_arrays()`,
+        # but the latter is not in Pandas' public API, and may be removed in the future.
+        return {
+            clean_column_name(k): v for k, v in df.items() if not isinstance(k, int)
+        }
+
+    # CHANGED FROM PANDAS: do not call df.dtype inside the dict comprehension loop
+    # This update has been made in https://github.com/pandas-dev/pandas/pull/59573,
+    # but appears not to have been released yet as of pandas 2.2.3
+    dtypes = df.dtypes
+
+    return {
+        clean_column_name(k): Series(
+            v, copy=False, index=df.index, name=k, dtype=dtypes[k]
+        ).__finalize__(df)
+        for k, v in zip(df.columns, df._iter_column_arrays())
+        if not isinstance(k, int)
+    }
+
+
+def fast_eval(df: pd.DataFrame, expr: str, **kwargs) -> Any | None:
+    """
+    Evaluate a string describing operations on DataFrame columns.
+
+    Operates on columns only, not specific rows or elements.  This allows
+    `eval` to run arbitrary code, which can make you vulnerable to code
+    injection if you pass user input to this function.
+
+    This function is a wrapper that replaces :meth:`~pandas.DataFrame.eval`
+    with a more efficient version than in the default pandas library (as
+    of pandas 2.2.3). It is recommended to use this function instead of
+    :meth:`~pandas.DataFrame.eval` for better performance. However, if you
+    encounter issues with this function, you can switch back to the default
+    pandas eval by changing the function call from `fast_eval(df, ...)` to
+    `df.eval(...)`.
+
+    Parameters
+    ----------
+    expr : str
+        The expression string to evaluate.
+    **kwargs
+        See the documentation for  :meth:`~pandas.DataFrame.eval` for complete
+        details on the keyword arguments accepted.
+
+    Returns
+    -------
+    ndarray, scalar, or pandas object
+        The result of the evaluation.
+    """
+
+    inplace = False
+    kwargs["level"] = kwargs.pop("level", 0) + 1
+    index_resolvers = df._get_index_resolvers()
+    column_resolvers = _get_cleaned_column_resolvers(df)
+    resolvers = column_resolvers, index_resolvers
+    if "target" not in kwargs:
+        kwargs["target"] = df
+    kwargs["resolvers"] = tuple(kwargs.get("resolvers", ())) + resolvers
+
+    try:
+        return pd.Series(
+            _eval(expr, inplace=inplace, **kwargs), index=df.index, name=expr
+        ).__finalize__(df)
+    except Exception as e:
+        # Initially assume that the exception is caused by the potentially
+        # breaking change in _get_cleaned_column_resolvers, and try again
+        # TODO: what kind of exception should be caught here so it is less broad
+        column_resolvers = _get_cleaned_column_resolvers(df, raw=False)
+        resolvers = column_resolvers, index_resolvers
+        kwargs["resolvers"] = kwargs["resolvers"][:-2] + resolvers
+        return _eval(expr, inplace=inplace, **kwargs)

activitysim/core/interaction_simulate.py

@@ -14,6 +14,7 @@
 
 from activitysim.core import chunk, logit, simulate, tracing, util, workflow
 from activitysim.core.configuration.base import ComputeSettings
+from activitysim.core.fast_eval import fast_eval
 
 logger = logging.getLogger(__name__)
 
@@ -287,7 +288,7 @@ def to_series(x):
                         if expr.startswith("@"):
                             v = to_series(eval(expr[1:], globals(), locals_d))
                         else:
-                            v = df.eval(expr, resolvers=[locals_d])
+                            v = fast_eval(df, expr, resolvers=[locals_d])
 
                         if check_for_variability and v.std() == 0:
                             logger.info(
@@ -556,7 +557,7 @@ def to_series(x):
                             if expr.startswith("@"):
                                 v = to_series(eval(expr[1:], globals(), locals_d))
                             else:
-                                v = df.eval(expr, resolvers=[locals_d])
+                                v = fast_eval(df, expr, resolvers=[locals_d])
                             if check_for_variability and v.std() == 0:
                                 logger.info(
                                     "%s: no variability (%s) in: %s"

activitysim/core/los.py

@@ -780,7 +780,15 @@ def get_mazpairs(self, omaz, dmaz, attribute):
                 self.maz_ceiling
             ) + np.asanyarray(dmaz, dtype=np.int64)
         else:
-            i = np.asanyarray(omaz) * self.maz_ceiling + np.asanyarray(dmaz)
+            # if we have less than a 32-bit index, it will
+            # overflow so we need to upgrade to at least 32 bit
+            omaz_as_array = np.asanyarray(omaz)
+            if omaz_as_array.dtype not in (np.int32, np.int64):
+                omaz_as_array = omaz_as_array.astype(np.int32)
+            dmaz_as_array = np.asanyarray(dmaz)
+            if dmaz_as_array.dtype not in (np.int32, np.int64):
+                dmaz_as_array = dmaz_as_array.astype(np.int32)
+            i = omaz_as_array * self.maz_ceiling + dmaz_as_array
         s = util.quick_loc_df(i, self.maz_to_maz_df, attribute)
 
         # FIXME - no point in returning series?