Skip to content

seasonal_production_performance pipeline see HEA-169 #85

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 12 commits into
base: main
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
51 changes: 51 additions & 0 deletions apps/baseline/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -2104,6 +2104,24 @@ class Meta:
verbose_name_plural = _("Market Prices")


class SeasonalProductionPerformanceManager(common_models.IdentifierManager):
def get_by_natural_key(
self,
code: str,
reference_year_end_date: str,
community_full_name: str,
season_name: str,
performance_year_end_date: str,
):
return self.get(
community__livelihood_zone_baseline__livelihood_zone__code=code,
community__livelihood_zone_baseline__reference_year_end_date=reference_year_end_date,
community__full_name=community_full_name,
season__name_en=season_name,
performance_year_end_date=performance_year_end_date,
)


class SeasonalProductionPerformance(common_models.Model):
"""
Relative production performance experienced in a specific season / year.
Expand Down Expand Up @@ -2141,12 +2159,35 @@ class Performance(models.IntegerChoices):
verbose_name=_("Seasonal Performance"),
help_text=_("Rating of the seasonal production performance from Very Poor (1) to Very Good (5)"),
)
objects = SeasonalProductionPerformanceManager()

def natural_key(self):
return (
self.community.livelihood_zone_baseline.livelihood_zone.code,
self.community.livelihood_zone_baseline.reference_year_end_date.isoformat(),
self.community.full_name,
self.season.name_en,
self.performance_year_end_date.isoformat(),
)

class Meta:
verbose_name = _("Seasonal Production Performance")
verbose_name_plural = _("Seasonal Production Performance")


class HazardManager(common_models.IdentifierManager):
def get_by_natural_key(
self, code: str, reference_year_end_date: str, community_full_name: str, chronic_or_periodic: str, ranking: str
):
return self.get(
community__livelihood_zone_baseline__livelihood_zone__code=code,
community__livelihood_zone_baseline__reference_year_end_date=reference_year_end_date,
community__full_name=community_full_name,
chronic_or_periodic=chronic_or_periodic,
ranking=int(ranking),
)


class Hazard(common_models.Model):
"""
A shock such as drought, flood, conflict or market disruption which is likely
Expand Down Expand Up @@ -2185,11 +2226,21 @@ class HazardRanking(models.IntegerChoices):
description = common_models.DescriptionField(
max_length=255, verbose_name=_("Description of Event(s) and/or Response(s)")
)
objects = HazardManager()

class Meta:
verbose_name = _("Hazard")
verbose_name_plural = _("Hazards")

def natural_key(self):
return (
self.community.livelihood_zone_baseline.livelihood_zone.code,
self.community.livelihood_zone_baseline.reference_year_end_date.isoformat(),
self.community.full_name,
self.chronic_or_periodic,
str(self.ranking),
)


class Event(common_models.Model):
"""
Expand Down
5 changes: 5 additions & 0 deletions apps/metadata/lookups.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
from common.lookups import Lookup

from .models import (
HazardCategory,
LivelihoodCategory,
ReferenceData,
Season,
Expand Down Expand Up @@ -66,3 +67,7 @@ class SeasonNameLookup(SeasonLookup):
*translation_fields("description"),
"aliases",
]


class HazardCategoryLookup(ReferenceDataLookup):
model = HazardCategory
22 changes: 22 additions & 0 deletions pipelines/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,18 @@
other_cash_income_label_dataframe,
summary_other_cash_income_labels_dataframe,
)
from .assets.seasonal_production_performance import (
all_hazard_labels_dataframe,
all_seasonal_production_performance_labels_dataframe,
hazard_instances,
hazard_labels_dataframe,
hazards_dataframe,
seasonal_production_performance_dataframe,
seasonal_production_performance_instances,
seasonal_production_performance_label_dataframe,
summary_hazard_labels_dataframe,
summary_seasonal_production_performance_labels_dataframe,
)
from .assets.wealth_characteristic import (
all_wealth_characteristic_labels_dataframe,
summary_wealth_characteristic_labels_dataframe,
Expand Down Expand Up @@ -96,6 +108,16 @@
consolidated_fixtures,
uploaded_baselines,
imported_baselines,
seasonal_production_performance_dataframe,
hazards_dataframe,
seasonal_production_performance_label_dataframe,
all_seasonal_production_performance_labels_dataframe,
summary_seasonal_production_performance_labels_dataframe,
hazard_labels_dataframe,
all_hazard_labels_dataframe,
summary_hazard_labels_dataframe,
seasonal_production_performance_instances,
hazard_instances,
],
jobs=[update_metadata, update_external_assets, upload_baselines, extract_dataframes, import_baseline_from_fixture],
resources={
Expand Down
26 changes: 15 additions & 11 deletions pipelines/assets/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,10 @@
"groupe de richesse": "fr",
"groupe socio-economique": "fr",
"group socio-economique": "fr",
"year of harvest": "en",
"production year": "en",
"year": "en",
"année": "fr",
}

# List of labels that indicate the start of the summary columns from row 3 in the Data, Data, and Data3 worksheets
Expand All @@ -59,6 +63,8 @@
"range",
"interval",
"intervales", # 2023 Mali BSSs
"Average", # For timeline sheet
"MOYENNE",
]


Expand Down Expand Up @@ -309,7 +315,7 @@ def validate_previous_value(cell, expected_prev_value, prev_value):

def get_bss_dataframe(
config: BSSMetadataConfig,
filepath_or_buffer,
filepath_or_buffer: object,
bss_sheet: str,
start_strings: list[str],
end_strings: Optional[list[str]] = None,
Expand All @@ -327,26 +333,24 @@ def get_bss_dataframe(
# The requested worksheet does not exist in the file
return Output(
pd.DataFrame(),
metadata={
"worksheet": bss_sheet,
"row_count": "Worksheet not present in file",
},
metadata={"worksheet": bss_sheet, "row_count": "Worksheet not present in file", "datapoint_count": 0},
)

# Use a 1-based index to match the Excel Row Number
df.index += 1
# Set the column names to match Excel
df.columns = [get_column_letter(col + 1) for col in df.columns]

# Find the last column before the summary column, which is in row 3
end_col = get_index(SUMMARY_LABELS, df.loc[3], offset=-1)
if not end_col:
# Find the last column before the summary column, which is in row 3 for the WB, data sheets and 4 for timeline sheet
row = 4 if bss_sheet == "Timeline" else 3
end_col = get_index(SUMMARY_LABELS, df.loc[row], offset=-1)
if end_col == df.columns[-1]: # Need the last row because get_index has offset=-1
raise ValueError(f'No cell containing any of the summary strings: {", ".join(SUMMARY_LABELS)}')

if not num_summary_cols:
# If the number of summary columns wasn't specified, then assume that
# there is one summary column for each wealth category, from row 3.
num_summary_cols = df.loc[3, "B":end_col].dropna().nunique()
num_summary_cols = df.loc[row, "B":end_col].dropna().nunique()
end_col = df.columns[df.columns.get_loc(end_col) + num_summary_cols]

# Find the row index of the start of the Livelihood Activities or Wealth Group Characteristic Values
Expand All @@ -370,8 +374,8 @@ def get_bss_dataframe(
# for rows that rely on copying down the label in column A from a previous row.
end_row = df.index[-1]

# Find the language based on the value in cell A3
lang = LANGS[df.loc[3, "A"].strip().lower()]
# Find the language based on the value in cell A3, or A4
lang = LANGS[df.loc[row, "A"].strip().lower()]

# Filter to just the Wealth Group header rows and the Livelihood Activities
df = pd.concat([df.loc[header_rows, :end_col], df.loc[start_row:end_row, :end_col]])
Expand Down
5 changes: 5 additions & 0 deletions pipelines/assets/fixtures.py
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,8 @@ def consolidated_instances(
livelihood_activity_instances,
other_cash_income_instances,
wild_foods_instances,
seasonal_production_performance_instances,
hazard_instances,
) -> Output[dict]:
"""
Consolidated record instances from a BSS, ready to be validated.
Expand All @@ -109,7 +111,10 @@ def consolidated_instances(
# WealthGroup instances, which are needed as a foreign key from LivelihoodActivity, etc.
**wealth_characteristic_instances,
**livelihood_activity_instances,
**seasonal_production_performance_instances,
**hazard_instances,
}

# Add the wild foods and other cash income instances, if they are present
for model_name, instances in {**other_cash_income_instances, **wild_foods_instances}.items():
if instances:
Expand Down
Loading