Last fixes for top ten rus12 tables

Author: aesharpe

dbt/models/rus12/core_rus12__yearly_loans/schema.yml

@@ -2,9 +2,13 @@ version: 2
 sources:
   - name: pudl
     tables:
-      - name: core_rus12__yearly_renewable_plants
+      - name: core_rus12__yearly_loans
         data_tests:
           - expect_columns_not_all_null
+          - expect_col1_greater_or_equal_to_col2:
+              arguments:
+                col1: loan_original_amount
+                col2: loan_balance
         columns:
           - name: report_date
           - name: borrower_id_rus

dbt/tests/data_tests/generic_tests/expect_col1_greater_or_equal_to_col2.sql

@@ -0,0 +1,10 @@
+{% test expect_col1_greater_or_equal_to_col2(model, col1, col2) %}
+
+select *
+from {{ model }}
+where
+    {{ col1 }} is not null
+    and {{ col2 }} is not null
+    and {{ col1 }} < {{ col2 }}
+
+{% endtest %}

dbt/tests/data_tests/generic_tests/expect_positive_values.sql

@@ -0,0 +1,14 @@
+{% test expect_positive_values(model, positive_columns) %}
+
+select *
+from {{ model }}
+where
+    {% for col in positive_columns %}
+        (
+            {{ col }} is not null
+            and {{ col }} < 0
+        )
+        {% if not loop.last %} or {% endif %}
+    {% endfor %}
+
+{% endtest %}

dbt/tests/data_tests/generic_tests/schema.yml

@@ -289,3 +289,22 @@ macros:
       - name: max_differing_fuel_rate
         type: float
         description: Maximum acceptable fraction of units mapping to multiple fuels. (optional; default 0.01)
+
+  - name: test_expect_col1_greater_or_equal_to_col2
+    description: >
+      Check that values in one column are greater than or equal to values in another column.
+    arguments:
+      - name: col1
+        type: string
+        description: The name of the first column, which should have values greater than or equal to the second column.
+      - name: col2
+        type: string
+        description: The name of the second column, which should have values less than or equal to the first column.
+
+  - name: test_expect_positive_values
+    description: >
+      Check that values in a column are positive.
+    arguments:
+      - name: positive_columns
+        type: list
+        description: A list of the names of columns that should have positive values.

…ns/122a38524571_add_next_rus12_tables.py …142bdd083d_add_next_four_rus12_tables.pymigrations/versions/122a38524571_add_next_rus12_tables.py renamed to migrations/versions/0c142bdd083d_add_next_four_rus12_tables.py migrations/versions/122a38524571_add_next_rus12_tables.py renamed to migrations/versions/0c142bdd083d_add_next_four_rus12_tables.py

@@ -1,16 +1,16 @@
-"""add next rus12 tables
+"""Add next four rus12 tables
 
-Revision ID: 122a38524571
+Revision ID: 0c142bdd083d
 Revises: 86e3a75cf325
-Create Date: 2026-02-09 15:59:36.418333
+Create Date: 2026-02-10 18:10:58.410575
 
 """
 from alembic import op
 import sqlalchemy as sa
 
 
 # revision identifiers, used by Alembic.
-revision = '122a38524571'
+revision = '0c142bdd083d'
 down_revision = '86e3a75cf325'
 branch_labels = None
 depends_on = None
@@ -26,16 +26,16 @@ def upgrade() -> None:
     sa.Column('loan_maturity_date', sa.Date(), nullable=True, comment='The date on which a loan is scheduled to be fully paid.'),
     sa.Column('loan_original_amount', sa.Float(), nullable=True, comment='The original amount of a loan.'),
     sa.Column('loan_balance', sa.Float(), nullable=True, comment='The amount of money still owned on a loan at the end of the reporting year.'),
-    sa.Column('for_rural_development', sa.Boolean(), nullable=True, comment='Whether or not the investment is for rural development.')
+    sa.Column('for_rural_development', sa.Boolean(), nullable=True, comment='Whether or not the investment is for rural development. This includes investments in any/all types of projects or products that were made to improve the economy and/or quality of life in the specified area.')
     )
     op.create_table('core_rus12__yearly_plant_labor',
     sa.Column('report_date', sa.Date(), nullable=True, comment='Date reported.'),
     sa.Column('borrower_id_rus', sa.Text(), nullable=True, comment="Unique identifier of RUS (Rural Utilities Service) borrower. These ID's are structured as: two character state acronyms followed by four digits."),
     sa.Column('borrower_name_rus', sa.Text(), nullable=True, comment='The name of the RUS (Rural Utilities Service) borrower.'),
     sa.Column('plant_name_rus', sa.Text(), nullable=True, comment='Name of the plant as reported to RUS.'),
     sa.Column('plant_type', sa.Enum('Combined Cycle', 'Steam', 'Hydro', 'Internal Combustion', 'Nuclear'), nullable=True, comment='Type of plant.'),
-    sa.Column('employees_full_time_num', sa.Integer(), nullable=True, comment='Number of full time employees.'),
-    sa.Column('employees_part_time_num', sa.Integer(), nullable=True, comment='Number of part time employees.'),
+    sa.Column('employees_full_time_num', sa.Integer(), nullable=True, comment='Number of employees hired full-time for normal operations of the system.'),
+    sa.Column('employees_part_time_num', sa.Integer(), nullable=True, comment='Number employees regularly employed on a part-time basis. Exclude employees hired for short periods of time to complete special jobs.'),
     sa.Column('employee_hours_worked_total', sa.Float(), nullable=True, comment='Total number of hours worked by employees.'),
     sa.Column('payroll_maintenance', sa.Float(), nullable=True, comment='The amount of payroll spent on plant maintenance.'),
     sa.Column('payroll_operations', sa.Float(), nullable=True, comment='The amount of payroll spent on plant operations.'),
@@ -45,7 +45,7 @@ def upgrade() -> None:
     sa.Column('report_date', sa.Date(), nullable=False, comment='Date reported.'),
     sa.Column('borrower_id_rus', sa.Text(), nullable=False, comment="Unique identifier of RUS (Rural Utilities Service) borrower. These ID's are structured as: two character state acronyms followed by four digits."),
     sa.Column('borrower_name_rus', sa.Text(), nullable=True, comment='The name of the RUS (Rural Utilities Service) borrower.'),
-    sa.Column('source_of_energy', sa.Text(), nullable=False, comment='The source of energy (not plant type).'),
+    sa.Column('source_of_energy', sa.Enum('energy_available_for_sale', 'energy_used_by_borrower', 'purchased_power', 'total_energy_accounted', 'total_energy_losses', 'total_plant', 'total_sales', 'delivered_out_of_system_gross', 'net_interchange', 'received_into_system_gross', 'delivered_out_of_system_wheeling', 'net_energy_wheeled', 'received_into_system_wheeling', 'energy_furnished_without_charge'), nullable=False, comment='The source of energy (not plant type).'),
     sa.Column('net_energy_received_mwh', sa.Float(), nullable=True, comment='The net amount of energy received into the system.'),
     sa.Column('cost', sa.Float(), nullable=True, comment='Cost value.'),
     sa.PrimaryKeyConstraint('report_date', 'borrower_id_rus', 'source_of_energy', name=op.f('pk_core_rus12__yearly_sources_and_distribution'))

src/pudl/metadata/enums.py

@@ -981,7 +981,7 @@
     "Biomass - wood",
     "Methane - waste",
     "Hybrid",
-    "Solar - photvoltaic",
+    "Solar - photovoltaic",
     "Solar - thermal generation",
     "Other",
 ]

src/pudl/metadata/fields.py

@@ -50,7 +50,7 @@
     REVENUE_CLASSES_EIA176,
     REVENUE_CLASSES_EIA861,
     RTO_CLASSES,
-    # SOURCE_OF_ENERGY_RUS12,
+    SOURCE_OF_ENERGY_RUS12,
     SUBDIVISION_CODES_ISO3166,
     TECH_CLASSES,
     TECH_DESCRIPTIONS,
@@ -2663,11 +2663,11 @@
     },
     "employees_full_time_num": {
         "type": "integer",
-        "description": "Number of full time employees.",
+        "description": "Number of employees hired full-time for normal operations of the system.",
     },
     "employees_part_time_num": {
         "type": "integer",
-        "description": "Number of part time employees.",
+        "description": "Number employees regularly employed on a part-time basis. Exclude employees hired for short periods of time to complete special jobs.",
     },
     "employee_hours_worked_total": {
         "type": "number",
@@ -3320,7 +3320,12 @@
     },
     "for_rural_development": {
         "type": "boolean",
-        "description": "Whether or not the investment is for rural development.",
+        "description": (
+            "Whether or not the investment is for rural development. "
+            "This includes investments in any/all types of projects or "
+            "products that were made to improve the economy and/or quality "
+            "of life in the specified area."
+        ),
     },
     "forecast_year": {
         "type": "integer",
@@ -7092,7 +7097,7 @@
     "source_of_energy": {
         "type": "string",
         "description": "The source of energy (not plant type).",
-        # "constraints": {"enum": SOURCE_OF_ENERGY_RUS12},
+        "constraints": {"enum": SOURCE_OF_ENERGY_RUS12},
     },
     "source_url": {
         "type": "string",

src/pudl/metadata/resources/rus12.py

@@ -317,9 +317,7 @@
             "usage_warnings": ["experimental_wip"],
             "additional_source_text": "(Parts D, E, F, G - Section B)",
             "additional_details_text": (
-                "There are also some plants (at least one, Walter Scott) that are reported "
-                "by multiple borrowers (IA0084 and IA0083) with the same values. "
-                "Also note the lack of plant_type pre-2009 leading to a lack of "
+                "Note the lack of plant_type pre-2009 leading to a lack of "
                 "reliable primary keys."
             ),
         },

src/pudl/transform/rus12.py

@@ -93,6 +93,12 @@ def core_rus12__yearly_renewable_plants(raw_rus12__renewable_plants):
     df = rus.convert_units(
         df, old_unit="thousand_dollars", new_unit=None, converter=1000
     )
+
+    # Fix typo in primary_renewable_fuel_type values
+    df.primary_renewable_fuel_type = df.primary_renewable_fuel_type.replace(
+        {"Solar - photvoltaic": "Solar - photovoltaic"}
+    )
+
     # TODO: Make primary_renewable_fuel_type look like renewable fuels from other sources.
 
     return df
@@ -141,12 +147,6 @@ def core_rus12__yearly_loans(raw_rus12__loans):
     df.loan_maturity_date = pd.to_datetime(df.loan_maturity_date, format="mixed")
     df.for_rural_development = df.for_rural_development.astype("boolean")
 
-    # Make sure loan balance isn't more than original loan amount
-    loan_diff = df.loan_original_amount - df.loan_balance
-    assert len(loan_diff[loan_diff < 0]) == 0, (
-        "Loan balance exceeds original loan amount for some loans."
-    )
-
     return df
 
 
@@ -155,6 +155,18 @@ def core_rus12__yearly_plant_labor(raw_rus12__plant_labor):
     """Transform the raw_rus12__plant_labor table."""
     df = rus.early_transform(raw_df=raw_rus12__plant_labor)
 
+    # Remove duplicate Walter Scott plant entries.
+    exclude_cols = ["borrower_id_rus", "borrower_name_rus"]
+    dupe_mask = (
+        df["borrower_id_rus"].isin(["IA0083", "IA0084"])
+        & (df["plant_name_rus"] == "Walter Scott")
+        & df.drop(columns=exclude_cols).duplicated(keep=False)
+        & (
+            df["borrower_id_rus"] == "IA0083"
+        )  # dropping the IA0083 and keeping the IA0084 so both borrowers show up in the data.
+    )
+    df = df.loc[~dupe_mask]
+
     # Test payroll_total column so can remove it in the schema
     payroll_cols = [
         "payroll_maintenance",
@@ -212,9 +224,6 @@ def core_rus12__yearly_sources_and_distribution_by_plant_type(
         converter=0.001,
     )
 
-    # # Make sure plant num is only int values and then convert to integer
-    # assert (df.plant_num.dropna() % 1 == 0).all()
-    # df.plant_num = df.plant_num.astype("Int64")
     # TODO: add is_total column.
     return df