Test new code for pandas function

[rburghol]

Different commits:

• Note: Clean installs are a must
  • rm build -Rf from HSPsquared directory: There were old build directories in the hsp2 source tree that can really put some garbage in there.
  • vbuild p39test destroys and recreates the p39test environment saving about a half dozen keystrokes and confusion.
• Process:
  • Check out the test commit
  • re-do the p39test env with vbuild p39test
  • Activate the test environment . p39test
  • run the test for one without doing the reimport of hdf5
  • run test for one with the reimport of the hdf5
• Versions Tested:
  • 👎 merge update_uci git checkout 44c9ede67783e45c3214da32e5547c963ccb4d4b from respec
  • change name from test (last 11/11 test commit) git checkout 95d4203b41a056ced08c9f6132dd896facd7d246 from respec
  • IF block UCI and parser: git checkout 79fac7abad191b7f76bdaa5c6dfb4221feea0478 from respec
  • 1/5/2026: Add update_uci git checkout 07938cf760dac4beb8a9d33ff7b31c9e9ae9ac16 from respec
  • My state reorg from Oct 16: git checkout 8fda028257bcf2403841ca9c26d1073adee5f852 from respec
  • Tim's last UCI changes (shouldn't have done much, and DID pass all tests): git checkout b6f52191e370b67378b18342dd458b4c86433eb7
  • My step() PR from March 2025: git checkout fd78867e54136695df582c3c8f3f828baa39933d
    • This PR shows equivalent flow quantiles for specl and non specl

Create a blank environment

. py39 # activate an env
cd /usr/local/share/venv/
rm -Rf p39test 
python3.9 -m virtualenv p39test
# must pause between these for some reason
deactivate
# why do we needs spaces?  new ubuntu terminal is weird...
source /usr/local/share/venv/p39test/bin/activate

cd /opt/model/HSPsquared/
# check out confirmed commit and install hsp2
git checkout fd78867e54136695df582c3c8f3f828baa39933d
python3 -m pip install .

Main Testing Code to load harness and use it to view data

# Note: This code must be run from the tests dir due to importing
# the `convert` directory where the regression_base file lives
# todo: make this convert path passable by argument 
import numpy as np
from convert.regression_base import RegressTest

case = "test10specl"
base_case = "test10"
#case = "testcbp"
tdir = "/opt/model/HSPsquared/tests"
#import_uci(str(hsp2_specl_uci), str(temp_specl_h5file))
#run(temp_specl_h5file, saveall=True, compress=False)

############# RegressTest data loader
## TEST CASE
test = RegressTest(case, threads=1)
# test object hydr
rchres_hydr_hsp2_test_table = test.hsp2_data._read_table('RCHRES', '001', 'HYDR')
perlnd_pwat_hsp2_test_table = test.hsp2_data._read_table('PERLND', '001', 'PWATER')
rchres_hydr_hsp2_test = test.hsp2_data.get_time_series('RCHRES', '001', 'RO', 'HYDR')
rchres_hydr_hspf_test = test.get_hspf_time_series( ('RCHRES', 'HYDR', '001', 'RO', 2)) #Note: the order of arguments is wonky in Regressbase
rchres_hydr_hsp2_test_mo = rchres_hydr_hsp2_test.resample('MS').mean()
rchres_hydr_hspf_test_mo = rchres_hydr_hspf_test.resample('MS').mean()
## BASE CASE
base = RegressTest(base_case, threads=1)
# base object hydr
rchres_hydr_hsp2_base_table = base.hsp2_data._read_table('RCHRES', '001', 'HYDR')
perlnd_pwat_hsp2_base_table = base.hsp2_data._read_table('PERLND', '001', 'PWATER')
rchres_hydr_hsp2_base = base.hsp2_data.get_time_series('RCHRES', '001', 'RO', 'HYDR')
rchres_hydr_hspf_base = base.get_hspf_time_series( ('RCHRES', 'HYDR', '001', 'RO', 2))  #Note: the order of arguments is wonky in Regressbase
rchres_hydr_hsp2_base_mo = rchres_hydr_hsp2_base.resample('MS').mean()
rchres_hydr_hspf_base_mo = rchres_hydr_hspf_base.resample('MS').mean()

# Show quantiles
np.quantile(rchres_hydr_hsp2_test, [0,0.25,0.5,0.75,1.0])
np.quantile(rchres_hydr_hspf_test, [0,0.25,0.5,0.75,1.0])
np.quantile(rchres_hydr_hsp2_base, [0,0.25,0.5,0.75,1.0])
np.quantile(rchres_hydr_hspf_base, [0,0.25,0.5,0.75,1.0])
# Monthly mean value comparisons
rchres_hydr_hsp2_test_mo
rchres_hydr_hspf_test_mo
rchres_hydr_hsp2_base_mo
rchres_hydr_hspf_base_mo

# Compare ANY arbitrary timeseries, not just the ones coded into the RegressTest object
# 3rd argument is tolerance to use
tol = 10.0
test.compare_time_series(rchres_hydr_hsp2_base_table['RO'], rchres_hydr_hsp2_test_table['RO'], tol)
# Example:  (True, 8.7855425)

# Compare inflows and outflows
np.mean(perlnd_pwat_hsp2_test_table['PERO']) * 6000 * 0.0833
np.mean(rchres_hydr_hsp2_test_table['IVOL'])
np.mean(rchres_hydr_hsp2_test_table['ROVOL'])
np.mean(perlnd_pwat_hsp2_base_table['PERO']) * 6000 * 0.0833
np.mean(rchres_hydr_hsp2_base_table['IVOL'])
np.mean(rchres_hydr_hsp2_base_table['ROVOL'])

# now do a comparison
# HYDR diff should be almost nonexistent
test.check_con(params = ('RCHRES', 'HYDR', '001', 'ROVOL', 2))
# this is very large for PWTGAS
# git: test10specl ('PERLND', 'PWTGAS', '001', 'POHT', '2') 1163640%
test.check_con(params = ('PERLND', 'PWTGAS', '001', 'POHT', '2'))
# Other mismatches in PERLND
test.check_con(params = ('PERLND', 'PWATER', '001', 'AGWS', '2'))
test.check_con(params = ('PERLND', 'PWATER', '001', 'PERO', '2'))
# Now run the full test
results = test.run_test()
found = False
mismatches = []
for key, results in results.items():
    no_data_hsp2, no_data_hspf, match, diff = results
    if any([no_data_hsp2, no_data_hspf]):
        continue
    if not match:
        mismatches.append((case, key, results))
    found = True

print(mismatches)

if mismatches:
    for case, key, results in mismatches:
        diff = results
        print(case, key, f"{diff:0.00%}")

# checkout the commit you want to test

# install
python3 -m pip install .
# cd and run
cd tests/test10/HSPresults
hsp2 run test10.h5
# load the testignscrtips

# Note: This code must be run from the tests dir due to importing
# the `convert` directory where the regression_base file lives
# todo: make this convert path passable by argument 
from pathlib import Path
import os
import pandas as pd
from hsp2.hsp2tools.commands import import_uci, run
from hsp2.hsp2tools.HDF5 import HDF5

from convert.regression_base import RegressTest


case = "test10"
tdir = "/opt/model/HSPsquared/tests"
#import_uci(str(hsp2_specl_uci), str(temp_specl_h5file))
#run(temp_specl_h5file, saveall=True, compress=False)

test = RegressTest(case, threads=1, tests_root_dir = tdir)
test_dir = os.path.join(test.tests_root_dir, test.compare_case)
dstore_hsp2 = pd.HDFStore(str(test_dir) + '/HSP2results/' + case + '.h5', mode='r')
rchres_hydr_hsp2 = pd.read_hdf(dstore_hsp2, '/RESULTS/RCHRES_R005/HYDR')
dstore_hsp2.close() # tidy up
results = test.run_test()
test.temp_h5file.unlink()
rchres_hydr_test = test.hsp2_data.data[('RCHRES', '005', 'HYDR')]
np.quantile(rchres_hydr_hsp2['ROVOL'], [0,0.25,0.5,0.75,1.0])

# test object hydr

[rburghol]

p 3.9

cd /usr/local/share/venv/
python3.9 -m virtualenv p39test
deactivate
cd /opt/model/HSPsquared/
source /usr/local/share/venv/p39test/bin/activate

# check out a 1 year old confirmed commit and install hsp2
git checkout fd78867e54136695df582c3c8f3f828baa39933d
python3 -m pip install .

# run the base case
cd tests/test10specl/HSP2results
rm test10specl.h5
hsp2 import_uci test10specl.uci test10specl.h5

cd ../../ # into main tests directory
python3
# now run the code from thje issue top for quick comparisons

[rburghol]

• test conversion to new pandas forms for date resampling