From b9f528e7a321d46b1210ef0f4a1f23e1ef1928b9 Mon Sep 17 00:00:00 2001 From: "jia.zhou" <48254033+JiaZhou-PU@users.noreply.github.com> Date: Mon, 11 May 2026 16:27:59 -0500 Subject: [PATCH 01/34] sql test --- README.md | 4 +- docs/source/dev/styleguide.rst | 24 +- docs/source/examples/stellarator.rst | 3 +- docs/source/generate_sp_docs.py | 236 +- docs/source/inputsIndex/Main.py | 1915 ++++++++++----- docs/source/inputsIndex/utility_accert.py | 833 +++---- docs/source/user/build_your_own.rst | 18 +- docs/source/user/install.rst | 624 +---- requirements.txt | 1 - src/ACCERT_README.html | 30 +- src/Main.py | 26 +- src/accert_sqlite_procedures.py | 585 +++++ src/accertdb.sqlite | Bin 0 -> 598016 bytes ...db.sql => accertdb_sqlite_schema_data.sql} | 2043 +---------------- src/database_install.py | 123 +- src/database_install_sqlite.py | 43 + src/scripts/gen_user_defined.py | 10 +- src/scripts/run_sql.py | 99 +- src/setup_accert.sh | 33 +- src/sqlite_accert_connection.py | 112 + test/conftest.py | 35 +- test/gold/output.abr1000.out | 222 +- test/gold/output.heatpipe.out | 36 +- test/gold/output.pwr12be.out | 248 +- test/test_integration.py | 34 +- test/test_mysql.py | 139 -- test/test_sqlite.py | 134 ++ tutorial/accert.yml | 49 + 28 files changed, 3294 insertions(+), 4365 deletions(-) create mode 100644 src/accert_sqlite_procedures.py create mode 100644 src/accertdb.sqlite rename src/{accertdb.sql => accertdb_sqlite_schema_data.sql} (82%) create mode 100644 src/database_install_sqlite.py create mode 100644 src/sqlite_accert_connection.py delete mode 100644 test/test_mysql.py create mode 100644 test/test_sqlite.py create mode 100644 tutorial/accert.yml diff --git a/README.md b/README.md index d24e5a0..59ec2dc 100644 --- a/README.md +++ b/README.md @@ -18,11 +18,11 @@ The software comprises three major components: ACCERT is designed for integration with the [NEAMS Workbench](https://www.ornl.gov/project/neams-workbench) and relies on input -files using Workbench's SON format. Instructions for installing ACCERT both +files using Workbench's SON format. ACCERT uses the bundled SQLite database at +`src/accertdb.sqlite`; no MySQL server or root password is required. Instructions for installing ACCERT both with and without Workbench are provided in this README. ## Documentation Documentation for ACCERT can be found [__here__](https://accert.readthedocs.io/en/latest/index.html). - diff --git a/docs/source/dev/styleguide.rst b/docs/source/dev/styleguide.rst index cb04e2d..a5e49f3 100644 --- a/docs/source/dev/styleguide.rst +++ b/docs/source/dev/styleguide.rst @@ -3,7 +3,7 @@ Style Guide ----------- -ACCERT is written in **Python 3** and **MySQL 8**. +ACCERT is written in **Python 3** and uses **SQLite** for the bundled database. Style for Python code should follow `PEP 8`_. @@ -19,24 +19,8 @@ Python code should work with all currently `supported versions`_ of Python. .. _os: https://docs.python.org/3/library/os.html .. _Path: https://docs.python.org/3/library/pathlib.html#pathlib.Path -Style for MySQL code should follow Google coding style - -New MySQL code uses the `Google C++ coding style `_, with two exceptions: - -Member variable names: Do not use `foo_.` Instead, use `m_foo` (non-static) or `s_foo` (static). -Old projects and modifications to old code use an older MySQL-specific style for the time being. Since 8.0, MySQL style uses the same formatting rules as Google coding style (e.g., brace placement, indentation, line lengths, etc.), but differs in a few important aspects: - -Class names: Do not use MyClass. Instead, use My_class. - -Function names: Use snake_case(). - -Comment Style: Use either the // or /* */ syntax. // is much more common but both syntaxes are permitted for the time being. -Doxygen comments: Use /** ... */ syntax and not ///. - -Doxygen commands: Use '@' and not '\' for doxygen commands. - -You may see structs starting with st_ and being typedef-ed to some UPPERCASE (e.g. typedef struct st_foo { ... } FOO). However, this is legacy from when the codebase contained C. Do not make such new typedefs nor structs with st_ prefixes, and feel free to remove those that already exist, except in public header files that are part of libmysql (which need to be parseable as C99). - -Code formatting is enforced by use of clang-format throughout the code base. However, note that formatting is only one part of coding style; you are required to take care of non-formatting issues yourself, such as following naming conventions, having clear ownership of code or minimizing the use of macros. See the Google coding style guide for the entire list. +SQLite replacement logic for former stored procedures lives in +``src/accert_sqlite_procedures.py``. New database code should use parameterized +queries for values and validate table or column identifiers before interpolation. Consistent style is important for us, because everyone must know what to expect. Knowing our rules, you'll find it easier to read our code, and when you decide to contribute (which we hope you'll consider!) we'll find it easier to read and review your code. diff --git a/docs/source/examples/stellarator.rst b/docs/source/examples/stellarator.rst index 8e2061b..6a1607c 100644 --- a/docs/source/examples/stellarator.rst +++ b/docs/source/examples/stellarator.rst @@ -11,7 +11,7 @@ Prerequisites -------------- - ACCERT installation (see :doc:`Installation Guide `). -- MySQL database initialized with the updated schema containing the *stellarator* algorithms and variables. +- SQLite database initialized with the updated schema containing the *stellarator* algorithms and variables. - ACCERT main branch after PR #47. New Features @@ -66,4 +66,3 @@ The output will be generated in the ``tutorial/accert`` directory, as ``output.o - **stellarator_updated_account.xlsx**: Contains the updated cost breakdown for the stellarator model. - **stellarator_LCOE_results.xlsx**: Contains the LCOE analysis results specific to the stellarator configuration. - diff --git a/docs/source/generate_sp_docs.py b/docs/source/generate_sp_docs.py index 549bbda..5b5ee93 100644 --- a/docs/source/generate_sp_docs.py +++ b/docs/source/generate_sp_docs.py @@ -1,185 +1,95 @@ -import re +"""Generate reference pages for SQLite procedure replacements.""" + +from __future__ import annotations + +import ast import os -import textwrap - -current_dir = os.path.dirname(os.path.abspath(__file__)) -sql_file_path = os.path.join(current_dir, '../..', 'src', 'accertdb.sql') -# Path to your SQL file use absolute path -# ../../sql/your_sql_file.sql -# os.path.abspath('../src/accertdb.sql') -sql_file_path = os.path.abspath(sql_file_path) -# Output directory for Markdown files -# output_dir = 'source/reference/database/' - -output_dir = os.path.join(current_dir,'reference','database') -# Path to the Toctree file -# toctree_file_path = 'source/reference/database.rst' -toctree_file_path = os.path.join(current_dir,'reference','database.rst') - -def clean_output_directory(directory): - """ - Removes all .md files in the specified directory to clean up old documentation. - """ - if os.path.exists(directory): - for filename in os.listdir(directory): - if filename.endswith('.md'): - file_path = os.path.join(directory, filename) - os.remove(file_path) - print(f'Removed old Markdown file: {file_path}') - else: - os.makedirs(directory, exist_ok=True) - print(f'Created output directory: {directory}') - -def append_to_toctree(sp_name): - """ - Appends the generated Markdown file to the Toctree in 'source/reference/database.rst'. - """ - with open(toctree_file_path, 'a', encoding='utf-8') as toctree_file: - toctree_file.write(f' database/{sp_name}\n') - -def generate_markdown_table(parameters): - """ - Generates a Markdown table for the given parameters. - """ - if not parameters: - return """ -| None | No parameters. | -|------|-----------------| -""" +from pathlib import Path - # Define table header - table = """ -| **Name** | **Type** | -|----------|----------| -""" - # Add each parameter as a table row - for param in parameters: - name = param['name'] - input_type = param['type'] - # Escape pipe characters in descriptions to prevent table issues - input_type = input_type.replace('|', '\\|') - table += f"| {name} | {input_type} |\n" - - return table - -def main(): - print(f"Reading SQL file from path: {sql_file_path}") - print(f"Output directory: {output_dir}") - print(f"Toctree file path: {toctree_file_path}") - - # Step 1: Clean up old Markdown files + +current_dir = Path(__file__).resolve().parent +procedure_file_path = (current_dir / "../.." / "src" / "accert_sqlite_procedures.py").resolve() +output_dir = current_dir / "reference" / "database" +toctree_file_path = current_dir / "reference" / "database.rst" + + +def _procedure_names(tree: ast.Module) -> list[str]: + for node in tree.body: + if isinstance(node, ast.Assign): + for target in node.targets: + if isinstance(target, ast.Name) and target.id == "PROCEDURES": + if isinstance(node.value, ast.Dict): + return sorted( + key.value + for key in node.value.keys + if isinstance(key, ast.Constant) and isinstance(key.value, str) + ) + return [] + + +def clean_output_directory(directory: Path) -> None: + directory.mkdir(parents=True, exist_ok=True) + for filename in directory.iterdir(): + if filename.suffix == ".md": + filename.unlink() + print(f"Removed old Markdown file: {filename}") + + +def append_to_toctree(sp_name: str) -> None: + with toctree_file_path.open("a", encoding="utf-8") as toctree_file: + toctree_file.write(f" database/{sp_name}\n") + + +def main() -> None: + print(f"Reading SQLite procedure replacements from: {procedure_file_path}") clean_output_directory(output_dir) - # remove the old toctree file if it exists - if os.path.exists(toctree_file_path): - os.remove(toctree_file_path) - # create a new toctree file - with open(toctree_file_path, 'w', encoding='utf-8') as toctree_file: - toctree_file.write(f""" - -Database Stored Procedures -========================== + if toctree_file_path.exists(): + toctree_file_path.unlink() + with toctree_file_path.open("w", encoding="utf-8") as toctree_file: + toctree_file.write( + """ +Database Procedure Replacements +=============================== .. toctree:: :maxdepth: 1 :caption: Contents: - -""") - - # Step 2: Read the SQL file - try: - with open(sql_file_path, 'r', encoding='utf-8') as file: - sql_content = file.read() - except FileNotFoundError: - print(f"SQL file not found at path: {sql_file_path}") - return - - # Step 3: Define Regex to match stored procedures - # This regex accounts for: - # - Optional DEFINER clause - # - Procedure name with backticks - # - Multiline parameters - # - Custom delimiter ;; - sp_pattern = re.compile( - r'CREATE\s+DEFINER=`[^`]+`@`[^`]+`\s+PROCEDURE\s+`(\w+)`\s*\((.*?)\)\s*BEGIN\s*(.*?)END\s*;;', - re.IGNORECASE | re.DOTALL - ) - - # Step 4: Find all stored procedures - stored_procedures = sp_pattern.findall(sql_content) - - if not stored_procedures: - print("No stored procedures found in the SQL file.") - return - - # Step 5: Process each stored procedure - for sp_name, params, body in stored_procedures: - # Clean and process the SQL body - sp_sql = textwrap.dedent(body).strip() - - # Attempt to extract description from comments (if any) - # Assuming descriptions are provided in comments like /*! Description: ... */ - sp_description_match = re.search(r'/\*!\s*Description\s*:\s*(.*?)\s*\*/', sp_sql, re.DOTALL | re.IGNORECASE) - if sp_description_match: - sp_description = sp_description_match.group(1).strip() - # Remove the description comment from the SQL body - sp_sql = re.sub(r'/\*!\s*Description\s*:\s*.*?\s*\*/', '', sp_sql, flags=re.DOTALL | re.IGNORECASE).strip() - else: - sp_description = '\n' - - # Extract parameters: Assuming parameters are defined as IN param_name param_type - # This regex captures "IN param_name param_type", "OUT param_name param_type", etc. - # Additionally, it captures optional inline comments for descriptions - sp_parameters = re.findall(r'\b(IN|OUT|INOUT)\s+(\w+)\s+([^\s,]+)(?:\s*/\*\s*(.*?)\s*\*/)?', params, re.IGNORECASE) - - # Format parameters for the table - parameters = [] - for direction, param_name, param_type, param_desc in sp_parameters: - parameters.append({ - 'name': param_name, - 'type': param_type - }) - - # Generate the Markdown table for parameters - parameters_table = generate_markdown_table(parameters) - - # Prepare the full SQL definition - formatted_params = ', '.join([f'{dir} {pname} {ptype}' for dir, pname, ptype, _ in sp_parameters]) - full_sp_sql = f"CREATE PROCEDURE `{sp_name}`({formatted_params})\nBEGIN\n{textwrap.indent(sp_sql, ' ')}\nEND;;" - - # Create Markdown content using a multi-line f-string with proper formatting - rst_content = f"""--- + +""" + ) + + tree = ast.parse(procedure_file_path.read_text(encoding="utf-8")) + function_nodes = {node.name: node for node in tree.body if isinstance(node, ast.FunctionDef)} + for sp_name in _procedure_names(tree): + function_node = function_nodes.get(sp_name) + if function_node is None: + continue + params = [arg.arg for arg in function_node.args.args if arg.arg != "conn"] + params_table = "\n".join(f"| {param} |" for param in params) or "| None |" + content = f"""--- title: {sp_name} --- # {sp_name} -{sp_description} +SQLite/Python replacement for the former ACCERT database stored procedure. ## Parameters -{parameters_table} +| **Name** | +|----------| +{params_table} -## SQL Definition +## Implementation -```sql -{full_sp_sql} +See ``src/accert_sqlite_procedures.py``. """ - # Define the output Markdown file path - md_file_path = os.path.join(output_dir, f'{sp_name}.md') - - # Write Markdown content to file - try: - with open(md_file_path, 'w', encoding='utf-8') as md_file: - md_file.write(rst_content) - except Exception as e: - print(f'Error writing Markdown file for {sp_name}: {e}') - continue - # Append the Markdown file to the Toctree + (output_dir / f"{sp_name}.md").write_text(content, encoding="utf-8") append_to_toctree(sp_name) - - print("All stored procedures have been documented successfully.") -if __name__ == '__main__': + print("All SQLite procedure replacement docs have been generated successfully.") + + +if __name__ == "__main__": main() - \ No newline at end of file diff --git a/docs/source/inputsIndex/Main.py b/docs/source/inputsIndex/Main.py index a754307..4f6f9fa 100644 --- a/docs/source/inputsIndex/Main.py +++ b/docs/source/inputsIndex/Main.py @@ -1,10 +1,11 @@ -import mysql.connector +from sqlite_accert_connection import connect as accert_sqlite_connect import os from prettytable import PrettyTable import configparser import xml2obj from utility_accert import Utility_methods from Algorithm import Algorithm +import importlib import numpy as np import sys import pandas.io.sql as sql @@ -15,11 +16,111 @@ warnings.filterwarnings('ignore') PathLike = Union[str, bytes, os.PathLike] + +def _accert_sqlite_db_path() -> str: + """Return the SQLite database path for ACCERT.""" + return os.environ.get( + "ACCERT_SQLITE_DB", + os.path.join(os.path.dirname(os.path.abspath(__file__)), "accertdb.sqlite"), + ) + + class Accert: def __init__(self, input_path, accert_path): + """ + Initialize the Accert class. + + Parameters + ---------- + input_path : PathLike + Inputs file path. + accert_path: PathLike + ACCERT's repository path. + """ self.input_path = input_path self.accert_path = accert_path self.input = self.load_obj(self.input_path, self.accert_path) + self.ref_model = None + self.acc_tabl = None + self.cel_tabl = None + self.var_tabl = None + self.alg_tabl = None + self.esc_tabl = None + self.fac_tabl = None + self.use_gncoa = False + self.gncoa_map = 'gncoamapping' + + def setup_table_names(self,xml2obj): + """Setup different table names in the database. + + Parameters + ---------- + xml2obj : xml2obj + xml2obj class instantiates objects that can convert son file to xml stream and create python data structure. + + Returns + ------- + None + """ + if xml2obj.use_gncoa is not None: + self.use_gncoa = str(xml2obj.use_gncoa.value).lower() == 'true' + if "abr1000" in str(xml2obj.ref_model.value).lower(): + self.ref_model = 'abr1000' + self.acc_tabl = 'abr_account' + self.cel_tabl = 'abr_cost_element' + self.var_tabl = 'abr_variable' + self.alg_tabl = 'algorithm' + self.esc_tabl = 'escalation' + self.fac_tabl = 'facility' + elif "heatpipe" in str(xml2obj.ref_model.value).lower(): + self.ref_model = 'heatpipe' + self.acc_tabl = 'heatpipe_account' + self.cel_tabl = 'heatpipe_cost_element' + self.var_tabl = 'heatpipe_variable' + self.alg_tabl = 'algorithm' + self.esc_tabl = 'escalation' + self.fac_tabl = 'facility' + elif "pwr12-be" in str(xml2obj.ref_model.value).lower(): + self.ref_model = 'pwr12-be' + self.acc_tabl = 'account' + self.cel_tabl = 'cost_element' + self.var_tabl = 'variable' + self.alg_tabl = 'algorithm' + self.esc_tabl = 'escalation' + self.fac_tabl = 'facility' + elif "lfr" in str(xml2obj.ref_model.value).lower(): + self.ref_model = 'lfr' + self.acc_tabl = 'lfr_account' + self.cel_tabl = 'abr_cost_element' + self.var_tabl = 'abr_variable' + self.alg_tabl = 'algorithm' + self.esc_tabl = 'escalation' + self.fac_tabl = 'facility' + elif "fusion" in str(xml2obj.ref_model.value).lower(): + self.ref_model = 'fusion' + self.acc_tabl = 'fusion_acco' + self.cel_tabl = None + self.var_tabl = 'fusion_varv' + self.alg_tabl = 'fusion_alg' + self.esc_tabl = 'escalation' + self.fac_tabl = 'facility' + elif "stellarator" in str(xml2obj.ref_model.value).lower(): + self.ref_model = 'stellarator' + self.acc_tabl = 'ste_acc' + self.cel_tabl = None + self.var_tabl = 'ste_var' + self.alg_tabl = 'fusion_alg' + self.esc_tabl = 'escalation' + self.fac_tabl = 'facility' + elif "user_defined" in str(xml2obj.ref_model.value).lower(): + self.ref_model = 'user_defined' + self.acc_tabl = 'user_defined_account' + self.cel_tabl = None + self.var_tabl = 'user_defined_variable' + self.alg_tabl = 'user_defined_algorithm' + self.esc_tabl = 'escalation' + self.fac_tabl = 'facility' + return None def load_obj(self, input_path, accert_path): """Convert son file to xml stream and creates a python data structure. @@ -37,8 +138,8 @@ def load_obj(self, input_path, accert_path): """ import subprocess - sonvalidxml = accert_path + "/bin/sonvalidxml" - schema = accert_path + "/src/etc/accert.sch" + sonvalidxml = os.path.join(accert_path, "bin", "sonvalidxml") + schema = os.path.join(accert_path, "src", "etc", "accert.sch") cmd = ' '.join([sonvalidxml, schema, input_path]) xmlresult = subprocess.check_output(cmd, shell=True) ### obtain pieces of input by name for convenience @@ -62,37 +163,59 @@ def get_current_COAs(self, c, inp_id): coa_others List of a COA's other info, including ind, lft, rgt. """ - c.execute("""SELECT code_of_account, ind, rgt FROM account WHERE supaccount = '{}';""".format(inp_id)) - coa_info = c.fetchall() + # DELIMITER $$ + # CREATE DEFINER=`root`@`localhost` PROCEDURE `get_current_COAs`(IN table_name VARCHAR(50), + # IN inp_id VARCHAR(50)) + # BEGIN + # SET @stmt = CONCAT('SELECT code_of_account, + # ind FROM ', table_name, ' WHERE supaccount = ?'); + # PREPARE stmt FROM @stmt; + # SET @inp_id = inp_id; + # EXECUTE stmt USING @inp_id; + # DEALLOCATE PREPARE stmt; + # END$$ + # DELIMITER ; + c.callproc('get_current_COAs',(self.acc_tabl, inp_id)) + for row in c.stored_results(): + coa_info = row.fetchall() coa_lst = [] coa_other =[] for coa in coa_info: coa_lst.append(coa[0]) coa_other.append(coa[1:]) - return coa_lst , coa_other + return coa_lst, coa_other - def update_account_before_insert(self, c, max_ind, max_rgt): - """ Updates the current COAs ind, lft, rgt. + def update_account_before_insert(self, c, min_ind): + """Updates the current COAs ind. Parameters ---------- c : MySQLCursor MySQLCursor class instantiates objects that can execute MySQL statements. - max_ind : int + min_ind : int Original index of the account next to the inserted COA. - max_rgt : int - Original rgt of the account next to the inserted COA. + """ - c.execute("UPDATE accert_db.account SET ind = ind + 1 WHERE ind > {}".format(max_ind)) - c.execute("UPDATE accert_db.account SET lft = lft + 2 WHERE lft > {}".format(max_rgt)) - c.execute("UPDATE accert_db.account SET rgt = rgt + 2 WHERE rgt > {}".format(max_rgt)) + # DELIMITER $$ + # CREATE DEFINER=`root`@`localhost` PROCEDURE `update_account_before_insert`(IN table_name VARCHAR(50), + # IN min_ind INT) + # BEGIN + # SET @stmt = CONCAT('UPDATE ', table_name, + # ' SET ind = ind + 1 WHERE ind > ?'); + # PREPARE stmt FROM @stmt; + # SET @min_ind = min_ind-1; + # EXECUTE stmt USING @max_ind; + # DEALLOCATE PREPARE stmt; + # END$$ + # DELIMITER ; + + c.callproc('update_account_before_insert',(self.acc_tabl, min_ind-1)) return None - def insert_new_COA(self, c, ind, supaccount, level, lft, rgt, - code_of_account, account_description= None,var_value=None, - var_unit=None, total_cost=0, unit='dollar',main_subaccounts=None, - cost_elements=None, review_status='Added', prn='0'): - """Insert new COA in between an index in the account table. + def insert_new_COA(self, c, ind, supaccount, level, + code_of_account, account_description= None, + total_cost=0, review_status='Added', prn='0'): + """Insert a new COA in between an index in the account table. Parameters ---------- @@ -104,33 +227,65 @@ def insert_new_COA(self, c, ind, supaccount, level, lft, rgt, Super account of the new inserted COA. level : int Level of the new inserted COA. - lft : int - Lft index for nested model of the new inserted COA. - rgt : int - Rgt index for nested model of the new inserted COA. code_of_account : str, optional COA of the new inserted COA, by default "new" account_description : str, optional Account description of the new inserted COA. (By default none) total_cost : int, optional Total cost of the new inserted COA. (Set to 0 dollars by default) - unit : str, optional - Unit of the new inserted COA. (By default set to dollars) - main_subaccounts : List[str], optional - Main subaccounts of the new inserted COA. (By default none) - cost_elements : List[str], optional - Cost elements of the new inserted COA. (By default none) review_status : str, optional Review status of the new inserted COA. (By default 'Unchanged') prn : str(float), optional Percentage of the total cost of new inserted COA. (Set to 0% by default) """ - + # DELIMITER $$ + # CREATE DEFINER=`root`@`localhost` PROCEDURE `insert_new_COA`(IN table_name VARCHAR(50), + # IN ind INT, + # IN supaccount VARCHAR(50), + # IN level INT, + # IN lft INT, + # IN rgt INT, + # IN code_of_account VARCHAR(50), + # IN account_description VARCHAR(50), + # IN total_cost INT, + # IN unit VARCHAR(50), + # IN main_subaccounts VARCHAR(100), + # IN cost_elements VARCHAR(50), + # IN review_status VARCHAR(50), + # IN prn VARCHAR(50)) + # BEGIN + # SET @stmt = CONCAT('INSERT INTO ', table_name, + # ' (ind, supaccount, level, lft, rgt, code_of_account, account_description, + # total_cost, unit, main_subaccounts, cost_elements, review_status, prn) + # VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)'); + # PREPARE stmt FROM @stmt; + # SET @ind = ind; + # SET @supaccount = supaccount; + # SET @level = level; + # SET @lft = lft; + # SET @rgt = rgt; + # SET @code_of_account = code_of_account; + # SET @account_description = account_description; + # SET @total_cost = total_cost; + # SET @unit = unit; + # SET @main_subaccounts = main_subaccounts; + # SET @cost_elements = cost_elements; + # SET @review_status = review_status; + # SET @prn = prn; + # EXECUTE stmt USING @ind, @supaccount, @level, @lft, @rgt, @code_of_account, @account_description, + # @total_cost, @unit, @main_subaccounts, @cost_elements, @review_status, @prn; + # DEALLOCATE PREPARE stmt; + # END$$ + # DELIMITER ; + + c.callproc('insert_new_COA',(self.acc_tabl, ind, supaccount, level, + code_of_account, account_description, total_cost, + review_status, prn)) return None - def insert_COA(self, c, sup_coa): - """ - Insert a new COA into the account table. + def insert_COA(self, c, sup_coa,user_added_coa,user_added_coa_desc, + user_added_coa_total_cost): + """Insert a new COA into the account table. Parameters ---------- @@ -138,63 +293,58 @@ def insert_COA(self, c, sup_coa): MySQLCursor class instantiates objects that can execute MySQL statements. sup_coa : str Super account of the new inserted COA. + user_added_coa : str + COA of the new inserted COA. + user_added_coa_desc : str + Account description of the new inserted COA. + user_added_coa_total_cost : int + Total cost of the new inserted COA. """ # collect current COAs # current_COAs are list of current COAs' code_of_account # current_COA_others are list of current COAs' other info # include current COAs' ind lft rgt current_COAs, current_COA_others = self.get_current_COAs(c, sup_coa) - max_ind = max(current_COA_others,key=lambda item:item[0])[0] - max_rgt = max(current_COA_others,key=lambda item:item[1])[1] - # NOTE : if new COA is added, it will be added to the end of the current suplist + print('[Updating] Inserting new COA under COA',sup_coa) + # print current COAs wrapped word for easy reading + current_COAs = ', '.join(current_COAs) + print('[Updating] Current COAs under COA {}: {}'.format(sup_coa, current_COAs)) + print(' ') + + min_ind = min(current_COA_others,key=lambda item:item[0])[0] + # NOTE : if new COA is added, it will be added to the end of the top suplist # TODO : return a new COA id with the COA list as input # new_COA = get_new_COA_id(current_COAs) - new_COA = "new" - sup_coa_level = c.fetchone()[0] - coa_level = sup_coa_level + 1 - # before inserting new COA, update the current COAs' ind, lft, rgt - self.update_account_before_insert(c, max_ind, max_rgt) - # insert new COA - ## NOTE need to fix this for passing supaccount - self.insert_new_COA(c, ind=max_ind+1, supaccount=str(l1_inp.id), level = coa_level, lft=max_rgt+1, rgt=max_rgt+2, code_of_account= new_COA ) - return None - def add_new_alg(self, c,alg_name, alg_for, alg_description, - alg_python, alg_formulation, alg_units, variables, constants): - """ - Adds a new algorithm into algorithm table based of the following parameters: - - Parameters - ---------- - c : MySQLCursor - MySQLCursor class instantiates objects that can execute MySQL statements. - alg_name : str - Name of the algorithm. - alg_for : str - Whether the algorithm is for cost element or for variable. - alg_description : str - Description of the algorithm. - alg_python : str - Python code of the algorithm. (Variables should be in the form of 'var1', 'var2', 'var3'...) - alg_formulation : str - Prints info of the algorithm. - alg_units : str - Units of the output of algorithm. - variables : List[str] - Variables of the algorithm. - constants : List[str] - Constants of the algorithm. - """ + # DELIMITER $$ + # CREATE DEFINER=`root`@`localhost` PROCEDURE `sup_coa_level`(IN table_name VARCHAR(50), + # IN supaccount VARCHAR(50)) + # BEGIN + # SET @stmt = CONCAT('SELECT level FROM ', table_name, ' WHERE code_of_account = ?'); + # PREPARE stmt FROM @stmt; + # SET @supaccount = supaccount; + # EXECUTE stmt USING @supaccount; + # DEALLOCATE PREPARE stmt; + # END$$ + # DELIMITER ; + + c.callproc('sup_coa_level',(self.acc_tabl, sup_coa)) + + for row in c.stored_results(): + sup_coa_level = row.fetchone()[0] + coa_level = sup_coa_level + 1 - c.execute("""INSERT INTO - accert_db.algorithm ( alg_name, alg_for, alg_description, - alg_python, alg_formulation, alg_units, variables, constants) - VALUES (%(alg_name)s, %(alg_for)s, %(alg_description)s, - %(alg_python)s, %(alg_formulation)s, %(alg_units)s, - %(variables)s, %(constants)s)""", - { 'alg_name': alg_name, 'alg_for': alg_for, 'alg_description': alg_description, - 'alg_python': alg_python, 'alg_formulation': alg_formulation, 'alg_units': alg_units, - 'variables': variables, 'constants': constants}) + # before inserting new COA, update the current COAs' ind + # for example if the new COA is inserted between 1 and 2, + # then the min_ind is 2, so the current COAs' ind will be updated + # from 2 to n change to 3 to n+1 + # and the new COA will be inserted at 2 + self.update_account_before_insert(c, min_ind) + # insert new COA + self.insert_new_COA(c, ind=min_ind, supaccount=sup_coa, + level = coa_level, code_of_account=user_added_coa, + account_description=user_added_coa_desc, + total_cost= user_added_coa_total_cost) return None def extract_variable_info_on_name(self, c,var_id): @@ -213,6 +363,22 @@ def extract_variable_info_on_name(self, c,var_id): var_info : List[str] Variable info including variable name and variable unit """ + + # DELIMITER $$ + # CREATE DEFINER=`root`@`localhost` PROCEDURE `extract_variable_info_on_name`(IN table_name VARCHAR(50), + # IN var_name VARCHAR(50)) + # BEGIN + # SET @stmt = CONCAT('SELECT var_value, var_unit FROM ', table_name, ' WHERE var_name = ?'); + # PREPARE stmt FROM @stmt; + # SET @var_name = var_name; + # EXECUTE stmt USING @var_name; + # DEALLOCATE PREPARE stmt; + # END$$ + # DELIMITER ; + var_id = str(var_id).replace("'","").replace('"','') + c.callproc('extract_variable_info_on_name',(self.var_tabl, var_id)) + for row in c.stored_results(): + results = row.fetchall() var_info = results[0] return var_info @@ -232,7 +398,20 @@ def extract_super_val(self, c,var_id): sup_val : List[str] Super variable info including the name of the super variable. """ - + # DELIMITER $$ + # CREATE DEFINER=`root`@`localhost` PROCEDURE `extract_super_val`(IN table_name VARCHAR(50), + # IN var_name VARCHAR(50)) + # BEGIN + # SET @stmt = CONCAT('SELECT v_linked FROM ', table_name, ' WHERE var_name = ?'); + # PREPARE stmt FROM @stmt; + # SET @var_name = var_name; + # EXECUTE stmt USING @var_name; + # DEALLOCATE PREPARE stmt; + # END$$ + # DELIMITER ; + c.callproc('extract_super_val',(self.var_tabl, var_id)) + for row in c.stored_results(): + results = row.fetchone() if results is not None: sup_val = results[0] else: @@ -259,18 +438,26 @@ def update_input_variable(self, c,var_id,u_i_var_value, quite : bool, optional Whether or not to print the update info. (By default not, or false) """ - if not quite: print('[Updating] {}Variable {}'.format(var_type,var_id)) org_var_info = self.extract_variable_info_on_name(c,var_id) # NOTE: org_var_info is a tuple org_var_value = float(org_var_info[0]) org_var_unit = str(org_var_info[1]) + # remove the quotes from the string + u_i_var_unit = str(u_i_var_unit).replace("'","").replace('"','') + unit_convert = self.check_unit_conversion(org_var_unit,u_i_var_unit) if unit_convert: u_i_var_value = self.convert_unit(u_i_var_value,u_i_var_unit,org_var_unit) u_i_var_unit = org_var_unit # # DEBUG print + # do not print unit if unit is '1' or 'N/A' or 'none' or 'None' + if u_i_var_unit == '1' or u_i_var_unit == 'N/A' or u_i_var_unit == 'none' or u_i_var_unit == 'None': + u_i_var_unit = '' + if org_var_unit == '1' or org_var_unit == 'N/A' or org_var_unit == 'none' or org_var_unit == 'None': + org_var_unit = '' + var_id = str(var_id).replace("'","").replace('"','') self.update_variable_info_on_name(c,var_id,u_i_var_value,u_i_var_unit) if not quite: print('[Updated] Changed from {} {} to {} {}\n'.format(org_var_value,org_var_unit, u_i_var_value, u_i_var_unit)) @@ -291,6 +478,24 @@ def update_variable_info_on_name(self, c,var_id,var_value,var_unit): var_unit : str Variable unit. """ + # DELIMITER $$ + # CREATE DEFINER=`root`@`localhost` PROCEDURE `update_variable_info_on_name`(IN table_name VARCHAR(50), + # IN `u_i_var_name` VARCHAR(50), IN `value` FLOAT, IN `unit` VARCHAR(50)) + # BEGIN + # SET @stmt = CONCAT('UPDATE ', table_name, ' SET var_value = ?, + # var_unit = ?, + # user_input = ? WHERE var_name = ?'); + # PREPARE stmt FROM @stmt; + # SET @var_value = value; + # SET @var_unit = unit; + # SET @user_input = 1; + # SET @var_name = u_i_var_name; + # EXECUTE stmt USING @var_value, @var_unit, @user_input, @var_name; + # DEALLOCATE PREPARE stmt; + # END$$ + # DELIMITER ; + args = (self.var_tabl, var_id, float(var_value), var_unit) + c.callproc('update_variable_info_on_name', args) return None def update_super_variable(self, c,var_id): @@ -304,7 +509,25 @@ def update_super_variable(self, c,var_id): var_id : str Variable ID. """ - + # DELIMITER $$ + # CREATE DEFINER=`root`@`localhost` PROCEDURE `update_super_variable`(IN var_table_name VARCHAR(50), + # IN alg_table_name VARCHAR(50), IN `u_i_var_name` VARCHAR(50)) + # BEGIN + # SET @stmt = CONCAT('SELECT var.ind, var.var_name, var.var_value, + # var.var_alg, var.var_need, alg.ind, alg.alg_python, + # alg.alg_formulation, alg.alg_units, var.var_unit + # FROM ', var_table_name, ' as var JOIN ', alg_table_name, ' as alg + # ON var.var_alg=alg.alg_name + # WHERE var.var_name=?'); + # PREPARE stmt FROM @stmt; + # SET @var_name = u_i_var_name; + # EXECUTE stmt USING @var_name; + # DEALLOCATE PREPARE stmt; + # END$$ + # DELIMITER ; + c.callproc('update_super_variable',(self.var_tabl, self.alg_tabl, var_id)) + for row in c.stored_results(): + result = row.fetchone() ### results is a tuple sup_var_name = result[1] org_var_value = result[2] @@ -318,17 +541,24 @@ def update_super_variable(self, c,var_id): # # # create a value list for debugging # # var_value_lst = [] variables = {} - for var_ind, var_name in enumerate(var_name_lst): - # var_value_lst.append(get_var_value_by_name(c, var_name)) - variables['v_{}'.format(var_ind+1)] = self.get_var_value_by_name(c, var_name) - # print('variables',variables) - print('[Updating] Sup Variable {}, running algorithm: [{}], \n[Updating] with formulation: {}'.format(sup_var_name, alg_name, alg_form)) - alg_value = self.run_pre_alg(alg, **variables) + if self.cel_tabl: + for var_ind, var_name in enumerate(var_name_lst): + # var_value_lst.append(get_var_value_by_name(c, var_name)) + variables['v_{}'.format(var_ind+1)] = self.get_var_value_by_name(c, var_name) + print('[Updating] Sup Variable {}, running algorithm: [{}], \n[Updating] with formulation: {}'.format(sup_var_name, alg_name, alg_form)) + alg_value = self.run_pre_alg(alg, **variables) + else: + for var_ind, var_name in enumerate(var_name_lst): + variables[var_name] = self.get_var_value_by_name(c, var_name) + print('[Updating] Sup Variable {}, running algorithm: [{}], \n[Updating] with formulation: {}'.format(sup_var_name, alg_name, alg_form)) + alg_value= self.update_account_value(alg, alg_name, variables) self.update_input_variable(c,sup_var_name,alg_value,sup_var_unit,quite = True) if alg_unit == '1': alg_unit='' sup_var_unit='' - print('[Updated] Reference value is : {} {}, calculated value is: {} {}'.format(org_var_value,alg_unit,alg_value,sup_var_unit)) + # print formatting value for scientific notation + print('[Updated] Reference value is : {:,.2e} {}, calculated value is: {:,.2e} {}'.format(org_var_value,alg_unit,alg_value,sup_var_unit)) + # print('[Updated] Reference value is : {} {}, calculated value is: {} {}'.format(org_var_value,alg_unit,alg_value,sup_var_unit)) print(' ') return None @@ -346,22 +576,66 @@ def extract_total_cost_on_name(self, c,tc_id): ## Keep the note here for ref ## example with tuple # c.execute("""SELECT code_of_account, account_description, total_cost, unit - # FROM `accert_db`.`account` + # FROM account # WHERE code_of_account = %s;""",(tc_id,)) ## example with direct format string - c.execute("""SELECT code_of_account, account_description, total_cost, unit - FROM `accert_db`.`account` - WHERE code_of_account = "{}" ; - """.format(tc_id)) - ## example with direct format string - # c.execute("""SELECT code_of_account, account_description, total_cost, unit - # FROM `accert_db`.`account` - # WHERE code_of_account = %(u_i_tc_name)s ;""",{'u_i_tc_name': str(tc_id).replace('"','')}) - results = c.fetchall() - tc_info = results[0] + # c.execute("""SELECT code_of_account, account_description, total_cost + # FROM account + # WHERE code_of_account = "{}" ; + # """.format(tc_id)) + # Stored procedure + # DELIMITER $$ + # CREATE DEFINER=`root`@`localhost` PROCEDURE `extract_total_cost_on_name`(IN tc_id VARCHAR(50), + # IN table_name VARCHAR(50)) + # BEGIN + # SET @stmt = CONCAT('SELECT code_of_account, account_description, total_cost, unit + # FROM ', table_name, ' WHERE code_of_account = ?'); + # PREPARE stmt FROM @stmt; + # SET @tc_id = tc_id; + # EXECUTE stmt USING @tc_id; + # DEALLOCATE PREPARE stmt; + # END$$ + # DELIMITER ; + tc_id = str(tc_id).replace("'","").replace('"','') + # remove single quotes or double quotes from the string + # c.execute("""SELECT code_of_account, account_description, total_cost + # FROM account + # WHERE code_of_account = %u_i_tc_name;""",{'u_i_tc_name': str(tc_id).replace("'","").replace('"','')}) + c.callproc('extract_total_cost_on_name',(self.acc_tabl, tc_id)) + for row in c.stored_results(): + results = row.fetchall() + tc_info = results[0] return tc_info + def cal_LCOE(self, c, ut, accert): + """ + Calculates the Levelized Cost of Energy (LCOE) based on the input data. + + Parameters + ---------- + c : MySQLCursor + MySQLCursor class instantiates objects that can execute MySQL statements. + ut : Utility_methods + Utility_methods class instantiates objects that can perform utility methods. + accert : Accert + Accert class instantiates objects that can perform ACCERT methods. + + Returns + ------- + None + """ + if self.ref_model == 'fusion' or self.ref_model == 'stellarator': + # inport the LCOE module + module = importlib.import_module('Algorithm.LCOE') + LCOE_module = module.LCOE(c, ut, accert) + LCOE_module.setup_tables(Accert) + LCOE_module.quote_variable_values(c,Accert) + LCOE_module.coelc() + LCOE_module.generate_excel() + else: + pass + def check_unit_conversion(self, org_unit, new_unit): """ Checks if unit conversion is needed. @@ -375,6 +649,9 @@ def check_unit_conversion(self, org_unit, new_unit): """ if org_unit == new_unit: return False + elif org_unit == "N/A" or org_unit == "none" or org_unit == "None": + print('[Note] Original unit is not available, no conversion needed') + return False else: return True @@ -396,7 +673,6 @@ def convert_unit(self, current_value, current_unit, to_unit): to_value : float Converted value. """ - scale = float(self.convert_unit_scale(current_unit,to_unit)) to_value = current_value * scale if to_unit != 'dollar': @@ -417,7 +693,7 @@ def convert_unit_scale(self, current_unit, to_unit): Returns ------- scale : float - """ + """ if current_unit == to_unit: return 1 elif current_unit == 'KW': @@ -497,16 +773,21 @@ def update_total_cost(self, c,tc_id, u_i_tc_value, u_i_tc_unit): print('[Updating] Total cost of account {}'.format(tc_id)) org_tc_info = self.extract_total_cost_on_name(c,tc_id) org_tc_value = float(org_tc_info[2]) - org_tc_unit = org_tc_info[3] + org_tc_unit = "dollar" unit_convert = self.check_unit_conversion(org_tc_unit,u_i_tc_unit) if unit_convert: u_i_tc_value = self.convert_unit(u_i_tc_value,u_i_tc_unit,org_tc_unit) u_i_tc_unit = org_tc_unit - self.update_total_cost_on_name(c,tc_id,u_i_tc_value,u_i_tc_unit) - print('[Updated] Changed from {:,.2f} {} to {:,.2f} {}\n'.format( org_tc_value,org_tc_unit, int(u_i_tc_value), u_i_tc_unit)) + self.update_total_cost_on_name(c,tc_id,u_i_tc_value) + # do not print unit if unit is '1' or 'N/A' or 'none' or 'None' + if u_i_tc_unit == '1' or u_i_tc_unit == 'N/A' or u_i_tc_unit == 'none' or u_i_tc_unit == 'None': + u_i_tc_unit = '' + if org_tc_unit == '1' or org_tc_unit == 'N/A' or org_tc_unit == 'none' or org_tc_unit == 'None': + org_tc_unit = '' + print('[Updated] Changed from {:,.2f} {} to {:,.2f} {}\n'.format( org_tc_value,org_tc_unit, int(u_i_tc_value), org_tc_unit)) return None - def update_total_cost_on_name(self, c, tc_id, u_i_tc_value, u_i_tc_unit): + def update_total_cost_on_name(self, c, tc_id, u_i_tc_value): """ Updates the total cost based on a total cost ID, without checking for unit conversion. @@ -520,19 +801,34 @@ def update_total_cost_on_name(self, c, tc_id, u_i_tc_value, u_i_tc_unit): Total cost's value. u_i_tc_unit : str Total cost's unit. + + Returns + ------- + None """ ## NOTE I'm not sure if this is the best way to update the total cost ## Statement is not working as expected when passing in a string in a dictionary ## but it works when passing in the string directly in .format() method - # c.execute("""UPDATE `accert_db`.`account` - # SET `total_cost` = %(u_i_tc_value)s , - # `unit` = %(u_i_tc_unit)s , - # `review_status` = 'User Input' - # WHERE `code_of_account` = "%(u_i_tc_name)s";""", - # {'u_i_tc_value':int(u_i_tc_value), - # 'u_i_tc_unit':u_i_tc_unit, - # 'u_i_tc_name':tc_id}) + # DELIMITER $$ + # CREATE DEFINER=`root`@`localhost` PROCEDURE `update_total_cost_on_name`(IN table_name VARCHAR(50), + # IN `tc_id` VARCHAR(50), + # IN `u_i_tc_value` FLOAT, + # IN `u_i_tc_unit` VARCHAR(50)) + # BEGIN + # SET @stmt = CONCAT('UPDATE ', table_name, ' SET total_cost = ?, unit = ?, + # review_status = "User Input" WHERE code_of_account = ?'); + # PREPARE stmt FROM @stmt; + # SET @tc_id = tc_id; + # SET @u_i_tc_value = u_i_tc_value; + # SET @u_i_tc_unit = u_i_tc_unit; + # EXECUTE stmt USING @u_i_tc_value, @u_i_tc_unit, @tc_id; + # DEALLOCATE PREPARE stmt; + # END$$ + # DELIMITER ; + u_i_tc_value= float(u_i_tc_value) + c.callproc('update_total_cost_on_name',(self.acc_tabl,tc_id,u_i_tc_value)) + return None def get_var_value_by_name(self, c, var_name): @@ -551,6 +847,20 @@ def get_var_value_by_name(self, c, var_name): var_value : str Variable value. """ + # DELIMITER $$ + # CREATE DEFINER=`root`@`localhost` PROCEDURE `get_var_value_by_name`(IN table_name VARCHAR(50), + # IN `var_name` VARCHAR(50)) + # BEGIN + # SET @stmt = CONCAT('SELECT var_value FROM ', table_name, ' WHERE var_name = ?'); + # PREPARE stmt FROM @stmt; + # SET @var_name = var_name; + # EXECUTE stmt USING @var_name; + # DEALLOCATE PREPARE stmt; + # END$$ + # DELIMITER ; + c.callproc('get_var_value_by_name',(self.var_tabl,var_name)) + for row in c.stored_results(): + var_value = row.fetchone()[0] return var_value def run_pre_alg(self, alg, **kwargs): @@ -563,6 +873,11 @@ def run_pre_alg(self, alg, **kwargs): Pre-algorithm name. **kwargs : dict Keyword arguments. + + Returns + ------- + alg_value : float + Algorithm value """ # NOTE: comments below is the original note from Patrick, # I would want to keep the original note for future reference @@ -573,9 +888,56 @@ def run_pre_alg(self, alg, **kwargs): # report back the user algorithm # evaluate the algorithm alg_value = eval(alg) - # print('Value: {}'.format(alg_value)) return alg_value + def update_account_value(self, alg_py, alg_name, variables): + """ + Calls the specified algorithm with the given variables. Only called for fusion model now. + For PWR, ABR,LFR, HEATPIPE the alg_py is in the form of a string that will be + evaluated in the Algorithm table stored in database. For Fusion, the algorithm is in + the form of a python file name that stored in Algorithm folder. For example, in + Fusion model, the alg_py value is 'FusionFunc' then it should look for FusionFunc.py + in the Algorithm folder. + + + Parameters + ---------- + alg_py : str + Algorithm in python. + alg_name : str + Algorithm name.my + variables : dict + Variables to be passed to the algorithm. + + Returns + ------- + result : float + Algorithm result. + + + """ + # Dynamically import the module + module = importlib.import_module(f'Algorithm.{alg_py}') + + # Get the class from the module + class_ = getattr(module, alg_py) + + # Create an instance of the class + algorithm_instance = class_( + ind=1, # Dummy value, may be needed for future reference + alg_name=alg_name, + alg_for='test', # Dummy value + alg_description=f'Description of {alg_name}', # Dummy value + alg_formulation=f'Formulation of {alg_name}', # Dummy value + alg_units='units', # Dummy value + variables=','.join(variables.keys()), # Convert variable names to a comma-separated string + constants='' # Dummy value, replace as needed + ) + + # Run the algorithm and get the result + result = algorithm_instance.run(variables) + return result + def update_cost_element_on_name(self, c, ce_name, alg_value): """ Updates the cost element based on cost element name. (Turn off safe update mode) @@ -588,8 +950,45 @@ def update_cost_element_on_name(self, c, ce_name, alg_value): Cost element name starting with the COA of the account. alg_value : float Cost element value. + + Returns + ------- + None """ - c.execute("""SET SQL_SAFE_UPDATES = 0;""") + # Turn off safe update mode + # keep the original note for future reference + c.execute("""SET SQL_SAFE_UPDATES = 0;""") + # DELIMITER $$ + # CREATE DEFINER=`root`@`localhost` PROCEDURE `update_cost_element_on_name`( + # IN table_name VARCHAR(50), + # IN ce_name VARCHAR(50), + # IN alg_value DECIMAL(20,5) + # ) + # BEGIN + # -- Disable safe updates for this operation + # SET SQL_SAFE_UPDATES = 0; + + # -- Build the dynamic SQL query + # SET @stmt = CONCAT('UPDATE ', table_name, + # ' SET cost_2017 = ', alg_value, + # ', updated = 1 WHERE cost_element = ''', ce_name, ''''); + + # -- Prepare and execute the dynamic statement + # PREPARE stmt FROM @stmt; + # EXECUTE stmt; + + # -- Deallocate the prepared statement + # DEALLOCATE PREPARE stmt; + + # END;$$ + # DELIMITER ; + + # NOTE, float is used for alg_value, but it can be changed to DECIMAL(20,15) in the + # stored procedure, since float in python is equivalent to double in MySQL, tested + # for several values but using float in stored procedure is not recommended since + # the rolled up value may not be accurate. + c.callproc('update_cost_element_on_name',(self.cel_tabl,ce_name,float(alg_value))) + return None def update_new_cost_elements(self, c): @@ -603,6 +1002,31 @@ def update_new_cost_elements(self, c): """ print(' Updating cost elements '.center(100,'=')) print('\n') + c.callproc('update_new_cost_elements',(self.cel_tabl,self.var_tabl,self.alg_tabl)) + for row in c.stored_results(): + results = row.fetchall() + # DELIMITER $$ + # CREATE DEFINER=`root`@`localhost` PROCEDURE `update_new_cost_elements`(IN cel_tabl_name VARCHAR(50), + # IN var_tabl_name VARCHAR(50), + # IN alg_tabl_name VARCHAR(50)) + # BEGIN + # SET SQL_SAFE_UPDATES = 0; + # SET @stmt = CONCAT("SELECT ce.ind, ce.cost_element, + # ce.cost_2017, ce.alg_name, + # ce.variables, ce.algno, + # alg.alg_python, alg.alg_formulation, alg.alg_units + # FROM ", cel_tabl_name, " AS ce + # JOIN ", alg_tabl_name, " AS alg ON ce.alg_name = alg.alg_name + # WHERE EXISTS ( + # SELECT 1 + # FROM ", var_tabl_name, " AS va + # WHERE va.user_input = 1 + # AND FIND_IN_SET(va.var_name, REPLACE(ce.variables, ' ', '')) > 0);"); + # PREPARE stmt FROM @stmt; + # EXECUTE stmt; + # DEALLOCATE PREPARE stmt; + # END$$ + # DELIMITER ; for row in results: ce_name = row[1] org_ce_value = row[2] @@ -616,7 +1040,7 @@ def update_new_cost_elements(self, c): # maybe the unit for cost element can be added to the cost_element table later??? # # create a value list for debugging # var_value_lst = [] - variables = {} + variables = {} for var_ind, var_name in enumerate(var_name_lst): # var_value_lst.append(get_var_value_by_name(c, var_name)) variables['v_{}'.format(var_ind+1)] = self.get_var_value_by_name(c, var_name) @@ -630,6 +1054,67 @@ def update_new_cost_elements(self, c): print(' ') return None + def update_new_accounts(self, c): + """ + Updates the affected accounts based on the variables. This function is called + when there is no cost element table. + + Parameters + ---------- + c : MySQLCursor + MySQLCursor class instantiates objects that can execute MySQL statements. + """ + # for fusion or user defined table, there is no cost_element table + # so the update_new_cost_elements will not be executed + # instead, the update_new_accounts will be executed + print(' Updating accounts '.center(100,'=')) + # DELIMITER $$ + # CREATE DEFINER=`root`@`localhost` PROCEDURE `update_new_accounts`(IN acc_tabl_name VARCHAR(50), + # IN var_tabl_name VARCHAR(50), + # IN alg_tabl_name VARCHAR(50)) + # BEGIN + # SET SQL_SAFE_UPDATES = 0; + # SET @stmt = CONCAT("SELECT ac.ind, ac.code_of_account, + # ac.total_cost, ac.alg_name, + # ac.variables, + # alg.alg_python, alg.alg_formulation, alg.alg_units + # FROM ", acc_tabl_name, " AS ac + # JOIN ", alg_tabl_name, " AS alg ON ac.alg_name = alg.alg_name + # WHERE EXISTS ( + # SELECT 1 + # FROM ", var_tabl_name, " AS va + # WHERE va.user_input = 1 + # AND FIND_IN_SET(va.var_name, REPLACE(ac.variables, ' ', '')) > 0);"); + # PREPARE stmt FROM @stmt; + # EXECUTE stmt; + # DEALLOCATE PREPARE stmt; + # END$$ + c.callproc('update_new_accounts',(self.acc_tabl,self.var_tabl,self.alg_tabl)) + for row in c.stored_results(): + results = row.fetchall() + for row in results: + acc_name = row[1] + # NOTE only for debugging + org_acc_value = row[2] + alg_name = row[3] + var_name_lst = [x.strip() for x in row[4].split(',')] + alg_py = row[5] + alg_form = row[6] + alg_unit = row[7] + variables = {} + for var_ind, var_name in enumerate(var_name_lst): + variables[var_name] = self.get_var_value_by_name(c, var_name) + print('[Updating] Account [{}], running algorithm: [{}], \n[Updating] with formulation: {}'.format(acc_name, alg_name, alg_form)) + # alg_py is the algorithm python file name in Algorithm folder + # alg_name is the function name in the alg_py file + # now pass the variables and run the algorithm + alg_value = self.update_account_value(alg_py, alg_name, variables) + unit_convert = self.check_unit_conversion('dollar',alg_unit) + if unit_convert: + alg_value = self.convert_unit(alg_value,alg_unit,'dollar') + self.update_total_cost(c, acc_name, alg_value, 'dollar') + print(' ') + def update_account_table_by_cost_elements(self, c): """ Updates the account table based on the sum of the cost elements. @@ -642,6 +1127,33 @@ def update_account_table_by_cost_elements(self, c): print(' Updating account table '.center(100,'=')) print('\n') print('[Updating] Updating account table by cost elements') + # DELIMITER $$ + # CREATE DEFINER=`root`@`localhost` PROCEDURE `update_account_table_by_cost_elements`(IN acc_tabl_name varchar(50), + # IN cel_tabl_name varchar(50)) + # BEGIN + # SET @stmt = CONCAT('UPDATE ', acc_tabl_name, ',', + # '(SELECT ', acc_tabl_name, '.code_of_account, + # ce.total_cost as cost, + # ce.updated as updated, + # ', acc_tabl_name, '.unit + # FROM ', acc_tabl_name, ' + # JOIN (SELECT account, + # sum(cost_2017) as total_cost, + # sum(updated) as updated + # FROM ', cel_tabl_name, ' + # GROUP BY ', cel_tabl_name, '.account ) as ce + # on ', acc_tabl_name, '.code_of_account = ce.account + # ORDER BY ', acc_tabl_name, '.ind) as updated_account + # SET ', acc_tabl_name, '.total_cost = updated_account.cost, + # review_status = \'Ready for Review\' + # WHERE updated_account.updated > 0 + # and ', acc_tabl_name, '.code_of_account = updated_account.code_of_account;'); + # PREPARE stmt FROM @stmt; + # EXECUTE stmt; + # DEALLOCATE PREPARE stmt; + # END$$ + # DELIMITER ; + c.callproc('update_account_table_by_cost_elements', (self.acc_tabl, self.cel_tabl)) print('[Updated] Account table updated from cost elements\n') return None @@ -657,8 +1169,9 @@ def roll_up_cost_elements(self, c): print(' Roll up cost elements '.center(100,'=')) print('\n') self.roll_up_cost_elements_by_level(c,3,2) - self.roll_up_cost_elements_by_level(c,2,1) - self.roll_up_cost_elements_by_level(c,1,0) + if self.ref_model=="pwr12-be": + self.roll_up_cost_elements_by_level(c,2,1) + self.roll_up_cost_elements_by_level(c,1,0) print('[Updated] Cost elements rolled up\n') return None @@ -671,15 +1184,39 @@ def roll_up_cost_elements_by_level(self, c,from_level,to_level): c : MySQLCursor MySQLCursor class instantiates objects that can execute MySQL statements. from_level : int - Roll up from level. + Roll up from a given level. to_level : int - Roll up to level + Roll up to a given level. """ - + # DELIMITER $$ + # CREATE DEFINER=`root`@`localhost` PROCEDURE `roll_up_cost_elements_by_level`(IN table_name varchar(50), + # IN from_level int, IN to_level int) + # BEGIN + # SET @stmt = CONCAT('UPDATE ', table_name, ',', + # '(SELECT c',to_level,'.cost_element as ce',to_level,'_ce, ', + # 'sum(uc',from_level,'.cost_2017) as c',to_level,'_cal_total_cost ', + # 'FROM ', table_name, ' as uc',from_level, + # ' JOIN ', table_name, ' as c',to_level, + # ' on uc',from_level,'.sup_cost_ele=c',to_level,'.cost_element ', + # 'join account as ac',to_level, + # ' on c',to_level,'.account = ac',to_level,'.code_of_account ', + # 'where ac',to_level,'.level=',to_level, + # ' group by c',to_level,'.cost_element) as updated_ce',to_level, + # ' SET ', + # table_name,'.cost_2017 = updated_ce',to_level,'.c',to_level,'_cal_total_cost,', + # table_name,'.updated = 1 ', + # 'WHERE ', + # table_name,'.cost_element = updated_ce',to_level,'.ce',to_level,'_ce'); + # PREPARE stmt FROM @stmt; + # EXECUTE stmt + # DEALLOCATE PREPARE stmt; + # END$$ + # DELIMITER ; + c.callproc('roll_up_cost_elements_by_level',(self.cel_tabl,from_level,to_level)) print('[Updating] Roll up cost elements from level {} to level {}'.format(from_level,to_level)) return None - def roll_up_account_table(self, c): + def roll_up_account_table(self, c, from_level=3, to_level=0, gncoa=False): """ Rolls up the account table from level 3 to 0. @@ -690,13 +1227,12 @@ def roll_up_account_table(self, c): """ print(' Rolling up account table '.center(100,'=')) print('\n') - self.roll_up_account_table_by_level(c,from_level=3,to_level=2) - + for i in range(from_level, to_level, -1): + self.roll_up_account_table_by_level(c,i,i-1,gncoa=gncoa) print('[Updated] Account table rolled up\n') - return None - def roll_up_account_table_by_level(self, c, from_level, to_level): + def roll_up_account_table_by_level(self, c, from_level, to_level, gncoa=False): """ Rolls up the account table from an input lower level to a higher level. @@ -705,60 +1241,37 @@ def roll_up_account_table_by_level(self, c, from_level, to_level): c : MySQLCursor MySQLCursor class instantiates objects that can execute MySQL statements. from_level : int - Roll up from level. + Roll up from a given level. to_level : int - Roll up to level. + Roll up to a given level. """ print('[Updating] Rolling up account table from level {} to level {} '.format(from_level,to_level)) - c.execute(""" - UPDATE account, - (SELECT a%(to)s.code_of_account as ac%(to)s_coa, - sum(ua%(from)s.total_cost) as a%(to)s_cal_total_cost - FROM account as ua%(from)s - JOIN account as a%(to)s on ua%(from)s.supaccount=a%(to)s.code_of_account - where ua%(from)s.level=%(from)s and a%(to)s.level=%(to)s - group by a%(to)s.code_of_account) as updated_ac%(to)s - SET - account.total_cost = updated_ac%(to)s.a%(to)s_cal_total_cost, - account.review_status = 'Updated' - WHERE - account.code_of_account = updated_ac%(to)s.ac%(to)s_coa - """,{'from':from_level,'to':to_level}) + if gncoa: + c.callproc('roll_up_account_table_by_gn_level',(self.acc_tabl,from_level,to_level)) + else: + c.callproc('roll_up_account_table_by_level',(self.acc_tabl,from_level,to_level)) return None - - def roll_up_abr_account(self, c): + + def roll_up_account_table_GNCOA(self, c): """ - Rolls up the account table for ABR from level 3 to 2. + Rolls up the account table for the reactor model that only has limited accounts. Parameters ---------- c : MySQLCursor MySQLCursor class instantiates objects that can execute MySQL statements. """ - - print('ABR1000 model only roll up level 3 to 2') - ##NOTE inner join only update 222 - print('[Updating] Rolling up account table from level {} to level {} '.format(3,2)) - c.execute(""" - UPDATE abr_account, - (SELECT a2.code_of_account as ac2_coa, - sum(ua3.total_cost) as a2_cal_total_cost - FROM abr_account as ua3 - JOIN abr_account as a2 on ua3.supaccount=a2.code_of_account - where ua3.level=3 and a2.level=2 - group by a2.code_of_account) as updated_ac2 - SET - abr_account.total_cost = updated_ac2.a2_cal_total_cost, - abr_account.review_status = 'Updated' - WHERE - abr_account.code_of_account = updated_ac2.ac2_coa - """) + print(' Rolling up account table by GNCOA '.center(100,'=')) + # remove 220A first + c.callproc('remove_specific_row',(self.acc_tabl,'220A')) + self.roll_up_account_table(c, from_level=4, to_level=0, gncoa=True) + # print('[Updated] Account table rolled up\n') return None - + def sum_cost_elements_2C(self, c): """ - Sums the cost element for ABR COA 2c. (Calculated cost) + Sums the cost elements for COA 2C (Calculated cost). Parameters ---------- @@ -766,68 +1279,44 @@ def sum_cost_elements_2C(self, c): MySQLCursor class instantiates objects that can execute MySQL statements. """ - print(' Summing cost elements for direct cost '.center(100,'=')) - print('\n') - print('[Updating] Summing cost elements') - c.execute("""SELECT sum(cef.cost_2017) from - (SELECT t1.code_of_account, - SUBSTRING_INDEX(SUBSTRING_INDEX(t1.cost_elements, ',', 1), ',', -1) as fac_name - FROM accert_db.abr_account AS t1 - LEFT JOIN abr_account as t2 - ON t1.code_of_account = t2.supaccount - WHERE t2.code_of_account IS NULL - and t1.code_of_account!='2' - and t1.code_of_account!='2C' )as ac - join accert_db.abr_cost_element as cef - on cef.cost_element = ac.fac_name - where ac.code_of_account!='2C'""") - sum_2c_fac = c.fetchone()[0] - c.execute("""UPDATE abr_cost_element - SET cost_2017 = %(sum_2c_fac)s, - updated = %(updated)s - WHERE cost_element = '2C_fac'""",{'sum_2c_fac':float(sum_2c_fac),'updated':1}) - - c.execute("""SELECT sum(cef.cost_2017) from - (SELECT t1.code_of_account, - SUBSTRING_INDEX(SUBSTRING_INDEX(t1.cost_elements, ',', 2), ',', -1) as lab_name - FROM accert_db.abr_account AS t1 - LEFT JOIN abr_account as t2 - ON t1.code_of_account = t2.supaccount - WHERE t2.code_of_account IS NULL - and t1.code_of_account!='2' - and t1.code_of_account!='2C' )as ac - join accert_db.abr_cost_element as cef - on cef.cost_element = ac.lab_name - where ac.code_of_account!='2C'""") - sum_2c_lab = c.fetchone()[0] - c.execute("""UPDATE abr_cost_element - SET cost_2017 = %(sum_2c_lab)s, - updated = %(updated)s - WHERE cost_element = '2C_lab'""",{'sum_2c_lab':float(sum_2c_lab),'updated':1}) - c.execute("""SELECT sum(cef.cost_2017) from - (SELECT t1.code_of_account, - SUBSTRING_INDEX(SUBSTRING_INDEX(t1.cost_elements, ',', 3), ',', -1) as mat_name - FROM accert_db.abr_account AS t1 - LEFT JOIN abr_account as t2 - ON t1.code_of_account = t2.supaccount - WHERE t2.code_of_account IS NULL - and t1.code_of_account!='2' - and t1.code_of_account!='2C' )as ac - join accert_db.abr_cost_element as cef - on cef.cost_element = ac.mat_name - where ac.code_of_account!='2C'""") - sum_2c_mat = c.fetchone()[0] - c.execute("""UPDATE abr_cost_element - SET cost_2017 = %(sum_2c_mat)s, - updated = %(updated)s - WHERE cost_element = '2C_mat'""",{'sum_2c_mat':float(sum_2c_mat),'updated':1}) + def fetch_sum_and_update(cost_type, proc_name): + """ + Fetches the sum of the cost elements and updates the cost element. + + Parameters + ---------- + cost_type : str + Cost type. + proc_name : str + Procedure name. + """ + # Call stored procedure and fetch results + print(f'[Updating] Summing cost element for {cost_type}') + c.callproc(proc_name, (self.cel_tabl, self.acc_tabl)) + for row in c.stored_results(): + results = row.fetchall() + sum_value = results[0][0] + + # Update cost element + self.update_cost_element_on_name(c, cost_type, sum_value) + return sum_value + + print(' Summing cost elements for direct cost '.center(100, '=')) + # Fetch and update cost elements + sum_2c_fac = fetch_sum_and_update('2c_fac', 'sum_cost_elements_2C_fac') + sum_2c_lab = fetch_sum_and_update('2c_lab', 'sum_cost_elements_2C_lab') + sum_2c_mat = fetch_sum_and_update('2c_mat', 'sum_cost_elements_2C_mat') + + print('[Updated] Cost elements 2c_fac, 2c_lab, 2c_mat are: ' + f'${sum_2c_fac:<11,.0f}, ${sum_2c_lab:<11,.0f}, ${sum_2c_mat:<11,.0f}') print('[Updated] Cost elements summed\n') + return None - def sum_up_abr_account_2C(self, c): + def roll_up_lmt_account_2C(self, c): """ - Sums up total cost of account 2C for ABR. + Sums up total cost of account 2C for the reactor model that only has limited accounts. Parameters ---------- @@ -835,41 +1324,27 @@ def sum_up_abr_account_2C(self, c): MySQLCursor class instantiates objects that can execute MySQL statements. """ - print(' Summing up account table '.center(100,'=')) + print(' Rolling up account table '.center(100,'=')) print('\n') - c.execute("""UPDATE abr_account, - (SELECT sum(t1.total_cost) as tc, sum(t1.prn) as tprn FROM - abr_account AS t1 LEFT JOIN abr_account as t2 - ON t1.code_of_account = t2.supaccount - WHERE t2.code_of_account IS NULL - and t1.code_of_account!='2' - and t1.code_of_account!='2C') as dircost - SET abr_account.total_cost = dircost.tc, - abr_account.prn=dircost.tprn, - review_status = 'Ready for Review' - WHERE abr_account.code_of_account = '2C';""") - print('[Updated] Account table summed up for calculated direct cost.\n') - return None + c.callproc('roll_up_lmt_account_2C', (self.acc_tabl,)) - def sum_up_abr_direct_cost(self, c): + print('[Updated] Account table summed up for calculated direct cost.') + return None + + def roll_up_lmt_direct_cost(self, c): """ - Sums up the total cost of account 2 from account 2c for ABR. + Sums up the total cost of account 2 from account 2C for + the reactor model that only has limited accounts. Parameters ---------- c : MySQLCursor MySQLCursor class instantiates objects that can execute MySQL statements. """ - c.execute("""UPDATE abr_account, - (SELECT (total_cost/prn) as talcost - FROM abr_account as pre_abr - WHERE pre_abr.code_of_account ='2C') as calcost - SET abr_account.total_cost = calcost.talcost, - review_status = 'Ready for Review' - WHERE abr_account.code_of_account = '2';""") - print('[Updated] Account table summed up for direct cost.\n') + c.callproc('roll_up_lmt_direct_cost',(self.acc_tabl,)) + print('[Updated] Account table rolled up for direct cost.\n') return None - + def cal_direct_cost_elements(self, c): """ Calculates the direct cost elements for the ABR including the factory, labor, and material costs. (2C_fac, 2C_lab, 2C_mat) @@ -879,51 +1354,34 @@ def cal_direct_cost_elements(self, c): c : MySQLCursor MySQLCursor class instantiates objects that can execute MySQL statements. """ - c.execute("""SELECT sum(t1.prn) as tprn FROM - abr_account AS t1 LEFT JOIN abr_account as t2 - ON t1.code_of_account = t2.supaccount - WHERE t2.code_of_account IS NULL - and t1.code_of_account!='2' - and t1.code_of_account!='2C';""") - tprn = c.fetchone()[0] - c.execute("""SELECT cost_2017 FROM accert_db.abr_cost_element - where account='2' - and cost_element='2c_fac' """) - fac = c.fetchone()[0]/tprn - c.execute("""SELECT cost_2017 FROM accert_db.abr_cost_element - where account='2' - and cost_element='2c_lab' """) - lab = c.fetchone()[0]/tprn - c.execute("""SELECT cost_2017 FROM accert_db.abr_cost_element - where account='2' - and cost_element='2c_mat' """) - mat = c.fetchone()[0]/tprn - # print(' Direct cost calculation '.center(100,'=')) - # print(fac, lab,mat) - # print('[Updated] Account table summed up for direct cost.\n') - return fac,lab,mat + c.callproc('cal_direct_cost_elements', (self.acc_tabl, self.cel_tabl)) - def roll_up_abr_account_table(self, c): + # After the procedure execution, fetch the OUT parameters from the cursor + # The stored procedure call doesn't return results, but the OUT parameters are updated + for row in c.stored_results(): + results = row.fetchall() + fac, lab, mat = results[0] + return fac,lab,mat + + def roll_up_lmt_account_table(self, c): """ - Rolls up the account table for the ABR. + Rolls up the account table for the reactor model that only has limited accounts. Parameters ---------- c : MySQLCursor - MySQLCursor class instantiates objects that can execute MySQL statements. + MySQLCursor class instantiates objects that can execute MySQL statements """ - print(' Rolling up account table '.center(100,'=')) - print('\n') ### only update account 222 and account 2C - self.roll_up_abr_account(c) - print('[Updated] Account table rolled up\n') - self.sum_up_abr_account_2C(c) - self.sum_up_abr_direct_cost(c) + self.roll_up_account_table(c, from_level=3, to_level=2) + # print('[Updated] Account table rolled up\n') + self.roll_up_lmt_account_2C(c) + self.roll_up_lmt_direct_cost(c) return None - + def print_logo(self): """ - Prints the logo. + Prints the ACCERT logo. """ print('\n') print("::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::") @@ -937,365 +1395,602 @@ def print_logo(self): print("..:::::..:::.......::::......::........::..:::::..:::::..:::::") print('\n') - def generate_results_table(self, c, conn, level=3): + def execute_accert(self, c, ut): """ - generates the results tables. + Executes the ACCERT program. Parameters ---------- c : MySQLCursor MySQLCursor class instantiates objects that can execute MySQL statements. - level : int - Level of detail in the results table. (How many levels) + ut : UserTable + UserTable class instantiates objects that can execute user input statements. """ + self.print_logo() - statement="SELECT rankedcoa.code_of_account, account.account_description, account.total_cost, account.unit, account.level, account.review_status FROM account JOIN (SELECT node.code_of_account AS COA, CONCAT( REPEAT(' ', COUNT(parent.code_of_account) - 1), node.code_of_account) AS code_of_account FROM account AS node, account AS parent WHERE node.lft BETWEEN parent.lft AND parent.rgt GROUP BY node.code_of_account) as rankedcoa ON account.code_of_account=rankedcoa.COA WHERE account.level <={} ORDER BY account.lft;".format(level) - filename = 'ACCERT_updated_account.xlsx' - # filename = 'ACCERT_updated_account.csv' - self.write_to_excel(statement, filename,conn) + accert = self.load_obj(input_path, accert_path).accert + c.execute("USE accert_db") + print(' Reading user input '.center(100, '=')) + print('\n') - statement="SELECT va.var_name, va.var_description, affectv.ce_affected FROM accert_db.variable as va JOIN (SELECT variable,group_concat(ce) as ce_affected FROM accert_db.variable_links group by variable) as affectv on va.var_name = affectv.variable WHERE va.user_input = 1 order by va.ind" - filename = 'ACCERT_variable_affected_cost_elements.xlsx' - # filename = 'ACCERT_variable_affected_cost_elements.csv' + if accert.ref_model: + self.process_reference_model(c, ut, accert) + else: + print('ERROR: model not found ') + self.exit_with_error(accert) + self.process_power_inputs(c, accert) + self.process_variables(c, accert) + self.process_COA(c, accert) + self.finalize_process(c, ut, accert) + self.generate_results(c, ut, accert) + self.cal_LCOE(c, ut, accert) + conn.close() + sys.stdout.close() + sys.stdout = stdoutOrigin - self.write_to_excel(statement, filename,conn) + def process_reference_model(self, c, ut, accert): + """ + Processes the reference model. - statement="SELECT ce.cost_element, ce.cost_2017 as cost, ce.sup_cost_ele, ce.alg_name, ce.account FROM accert_db.cost_element as ce WHERE ce.updated != 0 order by ce.ind" - filename = 'ACCERT_updated_cost_element.xlsx' - # filename = 'ACCERT_updated_cost_element.csv' - self.write_to_excel(statement, filename,conn) + Parameters + ---------- + c : MySQLCursor + MySQLCursor class instantiates objects that can execute MySQL statements. + ut : Utility class + Utility class for processing user input. + accert : ACCERT + xml2obj class instantiates objects that can parse the ACCERT XML file. + """ - def generate_abr_results_table(self, c, conn, level=3): + print('[USER_INPUT]', 'Reference model is', str(accert.ref_model.value), '\n') + self.setup_table_names(accert) + ut.setup_table_names(c, Accert) + # if ref.model is not fusion or user defined then process cost elements: + if self.cel_tabl: + ut.print_user_request_parameter(c) + else: + pass + + def process_power_inputs(self, c, accert): """ - Generates results tables for the ABR-1000. + Processes the power inputs. Parameters ---------- c : MySQLCursor MySQLCursor class instantiates objects that can execute MySQL statements. - level : int - Level of detail in the results table. (How many levels) + accert : ACCERT + xml2obj class instantiates objects that can parse the ACCERT XML file. + """ + + if accert.power: + for inp in accert.power: + print('[USER_INPUT]', str(inp.id), 'power is', str(inp.value.value), str(inp.unit.value), '\n') + var_id = 'mwth' if str(inp.id) == 'Thermal' else 'mwe' if str(inp.id) == 'Electric' else None + if var_id: + self.update_variable_info_on_name(c, var_id, str(inp.value.value), str(inp.unit.value)) + self.process_super_values(c, var_id) + else: + # warning + print('WARNING: No power input found in the user input file\n') + + def process_variables(self, c, accert): + """ + Processes the variables. + + Parameters + ---------- + c : MySQLCursor + MySQLCursor class instantiates objects that can execute MySQL statements. + accert : ACCERT + xml2obj class instantiates objects that can parse the ACCERT XML file. + """ + if accert.var: + for var_inp in accert.var: + u_i_var_value = float(str(var_inp.value.value)) + u_i_var_unit = str(var_inp.unit.value) + var_id = str(var_inp.id).replace('"', '') + self.update_input_variable(c, var_id, u_i_var_value, u_i_var_unit) + self.process_super_values(c, var_id) + + def process_super_values(self, c, var_id): + """ + Processes the super variables. + + Parameters + ---------- + c : MySQLCursor + MySQLCursor class instantiates objects that can execute MySQL statements. + var_id : str + Variable ID. + """ + var_id = str(var_id).replace('"', '').replace("'", "") + sup_val_lst = self.extract_super_val(c, var_id) + if sup_val_lst: + sup_val_lst = sup_val_lst.split(',') + # also remove the space after the comma + sup_val_lst = [x.strip() for x in sup_val_lst] + if sup_val_lst: + print('[Updating] Other variable(s) should be updated based on {} are {} \n'.format(var_id, sup_val_lst)) + while sup_val_lst: + sup_val = sup_val_lst.pop(0) + if sup_val: + self.update_super_variable(c, sup_val) + new_sup_val = self.extract_super_val(c, sup_val) + if new_sup_val: + sup_val_lst.extend(new_sup_val.split(',')) + + def process_COA(self, c, accert): + """ + Change the total cost of the account table by user inputs. + This function is called before processing the calculation. + + Parameters + ---------- + c : MySQLCursor + MySQLCursor class instantiates objects that can execute MySQL statements. + accert : ACCERT + xml2obj class instantiates objects that can parse the ACCERT XML file. """ - statement="SELECT rankedcoa.code_of_account, abr_account.account_description, abr_account.total_cost, abr_account.unit, abr_account.level,abr_account.prn as pct,abr_account.review_status FROM abr_account JOIN (SELECT node.code_of_account AS COA, CONCAT( REPEAT(' ', COUNT(parent.code_of_account) - 1), node.code_of_account) AS code_of_account FROM abr_account AS node, abr_account AS parent WHERE node.lft BETWEEN parent.lft AND parent.rgt GROUP BY node.code_of_account) as rankedcoa ON abr_account.code_of_account=rankedcoa.COA WHERE abr_account.level <=3 ORDER BY abr_account.lft;".format(level) - filename = 'ACCERT_updated_account.xlsx' - # filename = 'ACCERT_updated_account.csv' + if accert.l0COA and accert.l0COA.l1COA: + for l1_inp in accert.l0COA.l1COA: + if l1_inp.l2COA: + self.process_level_accounts(c, l1_inp.l2COA, accert, l1_inp.id) - self.write_to_excel(statement, filename,conn) + def process_level_accounts(self, c, level_accounts, accert, parent_id=None): + """ + Processes the level accounts and begins the calculation. - statement="SELECT va.var_name, va.var_description, affectv.ce_affected FROM accert_db.abr_variable as va JOIN (SELECT variable,group_concat(ce) as ce_affected FROM accert_db.abr_variable_links group by variable) as affectv on va.var_name = affectv.variable WHERE va.user_input = 1 order by va.ind" - filename = 'ACCERT_variable_affected_cost_elements.xlsx' - # filename = 'ACCERT_variable_affected_cost_elements.csv' + Parameters + ---------- + c : MySQLCursor + MySQLCursor class instantiates objects that can execute MySQL statements. + level_accounts : list + List of level accounts. + accert : ACCERT + xml2obj class instantiates objects that can parse the ACCERT XML file. + parent_id : str + Parent ID. + """ + for account in level_accounts: + if "new" in str(account.id): + user_added_coa = str(account.newCOA.id) + user_added_coa_desc = str(account.newCOA.descr.value) + if account.total_cost: + user_added_coa_total_cost = float(str(account.total_cost.value.value)) + user_added_coa_total_cost_unit = str(account.total_cost.unit.value) + org_tc_unit = "dollar" + unit_convert = self.check_unit_conversion(org_tc_unit,user_added_coa_total_cost_unit) + if unit_convert: + user_added_coa_total_cost = self.convert_unit(user_added_coa_total_cost,user_added_coa_total_cost_unit,org_tc_unit) + else: + user_added_coa_total_cost = 0 + print('[USER_INPUT]', 'New account', user_added_coa, user_added_coa_desc, user_added_coa_total_cost, '\n') + self.insert_COA(c, str(parent_id),user_added_coa,user_added_coa_desc,user_added_coa_total_cost) + # if ref.model is not fusion then process cost elements: + if self.cel_tabl: + self.process_ce(c, account) + else: + if account.alg: + for alg in account.alg: + if alg.var: + for var in alg.var: + if var.alg is None: + self.process_var(c, var) + else: + self.process_alg(c, var) + elif account.var: + for var in account.var: + self.process_var(c, var) + for i in range(3, 7): + next_level = getattr(account, f'l{i}COA', None) + if next_level: + self.process_level_accounts(c, next_level, accert, account.id) + + def process_ce(self, c, account): + """ + Processes the cost elements, either by changing variables or algorithms. - self.write_to_excel(statement, filename,conn) + Parameters + ---------- + c : MySQLCursor + MySQLCursor class instantiates objects that can execute MySQL statements. + account : Account + Account class instantiates objects that can parse the account. + """ + if account.ce: + for ce in account.ce: + if ce.alg: + for alg in ce.alg: + if alg.var: + for var in alg.var: + if var.alg is None: + self.process_var(c, var) + else: + self.process_alg(c, var) + elif ce.var: + for var in ce.var: + self.process_var(c, var) + + def process_var(self, c, var_inp): + """ + Processes the variables, changing the variables by user inputs, and updating the super values. - statement="SELECT ce.cost_element, ce.cost_2017 as cost, ce.sup_cost_ele, ce.alg_name, ce.account FROM accert_db.abr_cost_element as ce WHERE ce.updated != 0 order by ce.ind" - filename = 'ACCERT_updated_cost_element.xlsx' - # filename = 'ACCERT_updated_cost_element.csv' + Parameters + ---------- + c : MySQLCursor + MySQLCursor class instantiates objects that can execute MySQL statements. + var_inp : Variable - self.write_to_excel(statement, filename,conn) + """ + u_i_var_value = float(str(var_inp.value.value)) + u_i_var_unit = str(var_inp.unit.value) + var_id = str(var_inp.id).replace('"', '') + self.update_input_variable(c, var_id, u_i_var_value, u_i_var_unit) + self.process_super_values(c, var_id) - def write_to_excel(self, statement, filename,conn): + def process_alg(self, c, alg_inp): """ - Writes the results to an excel file. + Processes the variables with algorithm. Parameters ---------- - statement : str - SQL statement. - filename : str - Filename of the excel file - conn : MySQLConnection - MySQLConnection class instantiates objects that represent a connection to the MySQL database server. + c : MySQLCursor + MySQLCursor class instantiates objects that can execute MySQL statements. + alg_inp : Algorithm """ - df=sql.read_sql(statement,conn) - df.to_excel(filename,index=False) - print("Successfully created excel file {}".format(filename)) - def execute_accert(self, c, ut): - self.print_logo() - ut.print_user_request_parameter(c) - accert = self.load_obj(input_path, accert_path).accert + for alg_var in alg_inp.alg: + if alg_var.var: + for var in alg_var.var: + var_id = str(var.id).replace('"', '') + u_i_var_value = float(str(var.value.value)) + u_i_var_unit = str(var.unit.value) + self.update_input_variable(c, var_id, u_i_var_value, u_i_var_unit, var_type='Sub ') + var_id = str(alg_inp.id).replace('"', '') + + self.update_super_variable(c, var_id) - print(' Reading user input '.center(100,'=')) - print('\n') - if accert.ref_model is not None: - print('[USER_INPUT]', 'Reference model is',str(accert.ref_model.value),'\n') - if accert.power is not None: - for ind, inp in enumerate(accert.power): - print('[USER_INPUT]', str(inp.id),'power is',str(inp.value.value),str(inp.unit.value),'\n') - if str(inp.id)=='Thermal': - var_id = 'mwth' - if str(inp.id)=='Electric': - var_id = 'mwe' - var_value = str(inp.value.value) - var_unit = str(inp.unit.value) - if "abr1000" in str(accert.ref_model.value).lower(): - self.update_abr_variable_info_on_name(c,var_id,var_value,var_unit) - sup_val_lst= self.extract_abr_super_val(c,var_id) - if "pwr12-be" in str(accert.ref_model.value).lower(): - self.update_variable_info_on_name(c,var_id,var_value,var_unit) - sup_val_lst= self.extract_super_val(c,var_id) - if sup_val_lst: - sup_val_lst= sup_val_lst.split(',') - while sup_val_lst: - sup_val = sup_val_lst.pop(0) - if sup_val: - if "pwr12-be" in str(accert.ref_model.value).lower(): - self.update_super_variable(c,sup_val) - new_sup_val = self.extract_super_val(c,sup_val) - if new_sup_val: - sup_val_lst.extend(new_sup_val.split(',')) - elif "abr1000" in str(accert.ref_model.value).lower(): - self.update_abr_super_variable(c,sup_val) - new_sup_val = self.extract_abr_super_val(c,sup_val) - if new_sup_val: - sup_val_lst.extend(new_sup_val.split(',')) - - - if accert.var is not None: - for var_ind, var_inp in enumerate(accert.var): - u_i_var_value = float(str(var_inp.value.value)) - u_i_var_unit = str(var_inp.unit.value) - var_id = str(var_inp.id).replace('"','') - if "pwr12-be" in str(accert.ref_model.value).lower(): - self.update_input_variable(c,var_id,u_i_var_value,u_i_var_unit) - # sup_val = extract_super_val(c,var_id) - sup_val_lst= self.extract_super_val(c,var_id) - if sup_val_lst: - sup_val_lst= sup_val_lst.split(',') - while sup_val_lst: - sup_val = sup_val_lst.pop(0) - if sup_val: - self.update_super_variable(c,sup_val) - new_sup_val = self.extract_super_val(c,sup_val) - if new_sup_val: - sup_val_lst.extend(new_sup_val.split(',')) - if "abr1000" in str(accert.ref_model.value).lower(): - self.update_abr_input_variable(c,var_id,u_i_var_value,u_i_var_unit) - sup_val_lst= self.extract_abr_super_val(c,var_id) - if sup_val_lst: - sup_val_lst= sup_val_lst.split(',') - while sup_val_lst: - sup_val = sup_val_lst.pop(0) - if sup_val: - self.update_abr_super_variable(c,sup_val) - new_sup_val = self.extract_abr_super_val(c,sup_val) - if new_sup_val: - sup_val_lst.extend(new_sup_val.split(',')) - if accert.l0COA is not None: - if accert.l0COA.l1COA is not None: - # TODO: check if print info can be verbose or not - # # DEBUG print - # print('l1') - for l1_ind, l1_inp in enumerate(accert.l0COA.l1COA): - # # DEBUG print - # print('',l1_ind, l1_inp.id) - if l1_inp.l2COA is not None: - # # DEBUG print - # print(' l2') - for l2_ind, l2_inp in enumerate(l1_inp.l2COA): - if "new" in str(l2_inp.id): - self.insert_COA(c, str(l1_inp.id)) - # # DEBUG print - # print(' ', l2_inp.id) - if l2_inp.ce is not None: - # # DEBUG print - # print(' l2ce') - for l2ce_ind, l2ce_inp in enumerate(l2_inp.ce): - # # DEBUG print - # print(' ', l2ce_inp.id) - if l2ce_inp.alg is not None: - # # DEBUG print - # print(' l2ce alg') - for l2ce_alg_ind, l2ce_alg_inp in enumerate(l2ce_inp.alg): - # # DEBUG print - # print(' ', l2ce_alg_inp.id) - if l2ce_alg_inp.var is not None: - # # DEBUG print - # print(' l2ce alg var') - for l2ce_alg_var_ind, l2ce_alg_var_inp in enumerate(l2ce_alg_inp.var): - if l2ce_alg_var_inp.alg is None: - ### NOTE variable will be user input values - u_i_var_value = float(str(l2ce_alg_var_inp.value.value)) - u_i_var_unit = str(l2ce_alg_var_inp.unit.value) - var_id = str(l2ce_alg_var_inp.id).replace('"','') - self.update_input_variable(c,var_id,u_i_var_value,u_i_var_unit) - sup_val_lst= self.extract_super_val(c,var_id) - if sup_val_lst: - sup_val_lst= sup_val_lst.split(',') - while sup_val_lst: - sup_val = sup_val_lst.pop(0) - if sup_val: - self.update_super_variable(c,sup_val) - new_sup_val = self.extract_super_val(c,sup_val) - if new_sup_val: - sup_val_lst.extend(new_sup_val.split(',')) - - - # sup_val = extract_super_val(c,var_id) - # if sup_val is not None: - # update_super_variable(c,sup_val) - else: - ### NOTE variable need to be calculated - for l2ce_alg_var_alg_ind, l2ce_alg_var_alg_inp in enumerate(l2ce_alg_var_inp.alg): - if l2ce_alg_var_alg_inp.var is not None: - ### update sub_variable info for each sup_variable in the algorithm(for sup_variable) - for l2ce_alg_var_alg_var_ind, l2ce_alg_var_alg_var_inp in enumerate(l2ce_alg_var_alg_inp.var): - var_id = str(l2ce_alg_var_alg_var_inp.id).replace('"','') - u_i_var_value = float(str(l2ce_alg_var_alg_var_inp.value.value)) - u_i_var_unit = str(l2ce_alg_var_alg_var_inp.unit.value) - self.update_input_variable(c,var_id,u_i_var_value,u_i_var_unit,var_type='Sub ') - ### updating sup_variable - var_id = str(l2ce_alg_var_inp.id).replace('"','') - self.update_super_variable(c,var_id) - if l2_inp.l3COA is not None: - # # DEBUG print - # print(' l3') - for l3_ind, l3_inp in enumerate(l2_inp.l3COA): - ## NOTE this is not a great way to check if the 'new' - ## is in the string, but it works for now - if "new" in str(l3_inp.id): - self.insert_COA(c, str(l2_inp.id)) - # # DEBUG print - # print(' ', l3_inp.id) - if l3_inp.ce is not None: - # # DEBUG print - # print(' l3ce') - for l3ce_ind, l3ce_inp in enumerate(l3_inp.ce): - # # DEBUG print - # print(' ', l3ce_inp.id) - if l3ce_inp.alg is not None: - # # DEBUG print - # print(' l3ce alg') - for l3ce_alg_ind, l3ce_alg_inp in enumerate(l3ce_inp.alg): - # # DEBUG print - # print(' ', l3ce_alg_inp.id) - if l3ce_alg_inp.var is not None: - # # DEBUG print - # print(' l3ce alg var') - for l3ce_alg_var_ind, l3ce_alg_var_inp in enumerate(l3ce_alg_inp.var): - if l3ce_alg_var_inp.alg is None: - ### NOTE variable will be user input values - u_i_var_value = float(str(l3ce_alg_var_inp.value.value)) - u_i_var_unit = str(l3ce_alg_var_inp.unit.value) - var_id = str(l3ce_alg_var_inp.id).replace('"','') - self.update_input_variable(c,var_id,u_i_var_value,u_i_var_unit) - sup_val_lst= self.extract_super_val(c,var_id) - if sup_val_lst: - sup_val_lst= sup_val_lst.split(',') - while sup_val_lst: - sup_val = sup_val_lst.pop(0) - if sup_val: - self.update_super_variable(c,sup_val) - new_sup_val = self.extract_super_val(c,sup_val) - if new_sup_val: - sup_val_lst.extend(new_sup_val.split(',')) - - - # sup_val = extract_super_val(c,var_id) - # if sup_val is not None: - # update_super_variable(c,sup_val) - - else: - ### NOTE variable need to be calculated - for l3ce_alg_var_alg_ind, l3ce_alg_var_alg_inp in enumerate(l3ce_alg_var_inp.alg): - if l3ce_alg_var_alg_inp.var is not None: - ### update sub_variable info for each sup_variable in the algorithm(for sup_variable) - for l3ce_alg_var_alg_var_ind, l3ce_alg_var_alg_var_inp in enumerate(l3ce_alg_var_alg_inp.var): - var_id = str(l3ce_alg_var_alg_var_inp.id).replace('"','') - u_i_var_value = float(str(l3ce_alg_var_alg_var_inp.value.value)) - u_i_var_unit = str(l3ce_alg_var_alg_var_inp.unit.value) - self.update_input_variable(c,var_id,u_i_var_value,u_i_var_unit,var_type='Sub ') - ### updating sup_variable - var_id = str(l3ce_alg_var_inp.id).replace('"','') - self.update_super_variable(c,var_id) - if l3_inp.total_cost is not None: - # # DEBUG print - # print(' l3 total cost') - for l3_total_cost_ind, l3_total_cost_inp in enumerate(l3_inp.total_cost): - # # DEBUG print - # print(' ', l3_inp.id, l3_total_cost_inp.value.value, l3_total_cost_inp.unit.value) - tc_id = str(l3_inp.id).replace('"','') - u_i_tc_value = float(str(l3_total_cost_inp.value.value)) - u_i_tc_unit = str(l3_total_cost_inp.unit.value) - self.update_total_cost(c, tc_id, u_i_tc_value, u_i_tc_unit) - if l2_inp.total_cost is not None: - # # DEBUG print - # print(' l2 total cost') - for l2_total_cost_ind, l2_total_cost_inp in enumerate(l2_inp.total_cost): - # # DEBUG print - # print(' ', l2_inp.id, l2_total_cost_inp.value.value, l2_total_cost_inp.unit.value) - tc_id = str(l2_inp.id).replace('"','') - u_i_tc_value = float(str(l2_total_cost_inp.value.value)) - u_i_tc_unit = str(l2_total_cost_inp.unit.value) - if "abr1000" in str(accert.ref_model.value).lower(): - self.update_abr_total_cost(c,tc_id,u_i_tc_value,u_i_tc_unit) - elif "pwr12-be" in str(accert.ref_model.value).lower(): - self.update_total_cost(c,tc_id,u_i_tc_value,u_i_tc_unit) - else: - print("ERROR: model not found ") - print(accert.ref_model.value) - print("Exiting") - sys.exit(1) - # update_total_cost(c,tc_id,u_i_tc_value,u_i_tc_unit) - ###################### - - if "abr1000" in str(accert.ref_model.value).lower(): - ### print changed variables - ut.extract_user_changed_abr_variables(c) - ### print changed total cost_elements - ut.extract_affected_abr_cost_elements(c) - ### calculate and new cost_elements value update to the database in table cost_elements and also update the account table: - self.update_new_abr_cost_elements(c) - ###NOTE: cost elements should be rolled up as well - # ut.print_updated_abr_cost_elements(c) - ### update the account table: - self.roll_up_abr_cost_elements(c) - - self.sum_cost_elements_2C(c) - self.update_abr_account_table_by_cost_elements(c) - - ### roll up the account table: - self.roll_up_abr_account_table(c) - abr_fac,abr_lab,abr_mat = self.cal_direct_cost_elements(c) - print(' Generating results table for review '.center(100,'=')) - print('\n') - ut.print_leveled_abr_accounts(c, abr_fac,abr_lab,abr_mat,all=False,cost_unit='million',level=3) - - self.generate_abr_results_table(c, conn,level=3) - - elif "pwr12-be" in str(accert.ref_model.value).lower(): - ### print changed variables - ut.extract_user_changed_variables(c) - ### print changed total cost_elements + def check_and_process_total_cost(self, c, accert): + """ + Checks and processes the total cost at the end of the calculation, + if the total cost has changed by user inputs. It is important to note that + the total cost may not be reflected correctly in the cost elements table. Since + the total cost is a sum of the cost elements, the cost elements may have changed + by user inputs. + + Parameters + ---------- + c : MySQLCursor + MySQLCursor class instantiates objects that can execute MySQL statements. + accert : ACCERT + xml2obj class instantiates objects that can parse the ACCERT XML file. + """ + if self.check_total_cost_changed(c, accert): + print(" IMPORTANT NOTE ".center(100, '=')) + print("Some cost have changed by user inputs and may not be reflected correctly in the cost elements table.\n") + self.process_total_cost(c, accert) + + def check_total_cost_changed(self, c, accert): + """ + Checks if the total cost has changed. + + Parameters + ---------- + c : MySQLCursor + MySQLCursor class instantiates objects that can execute MySQL statements. + accert : ACCERT + xml2obj class instantiates objects that can parse the ACCERT XML file. + """ + changed = False + if accert.l0COA and accert.l0COA.l1COA: + for l1_inp in accert.l0COA.l1COA: + if l1_inp.l2COA: + changed |= self.check_total_cost_accounts(c, l1_inp.l2COA, accert) + return changed + + def check_total_cost_accounts(self, c, level_accounts, accert): + """ + Checks if the total cost has changed for the accounts. + + Parameters + ---------- + c : MySQLCursor + MySQLCursor class instantiates objects that can execute MySQL statements. + level_accounts : list + List of level accounts. + accert : ACCERT + xml2obj class instantiates objects that can parse the ACCERT XML file. + """ + changed = False + for account in level_accounts: + if account.total_cost: + changed = True + for i in range(3, 7): + next_level = getattr(account, f'l{i}COA', None) + if next_level: + changed |= self.check_total_cost_accounts(c, next_level, accert) + return changed + + def process_total_cost(self, c, accert): + """ + Changes the total cost for the accounts using the user inputs. + + Parameters + ---------- + c : MySQLCursor + MySQLCursor class instantiates objects that can execute MySQL statements. + accert : ACCERT + xml2obj class instantiates objects that can parse the ACCERT XML file. + """ + if accert.l0COA and accert.l0COA.l1COA: + for l1_inp in accert.l0COA.l1COA: + if l1_inp.l2COA: + self.process_total_cost_accounts(c, l1_inp.l2COA, accert) + + def process_total_cost_accounts(self, c, level_accounts, accert): + """ + Changes the total cost for the accounts using the user inputs for different levels. + This function is called after calculation is done. + + Parameters + ---------- + c : MySQLCursor + MySQLCursor class instantiates objects that can execute MySQL statements. + level_accounts : list + List of level accounts. + accert : ACCERT + xml2obj class instantiates objects that can parse the ACCERT XML file. + """ + for account in level_accounts: + if account.total_cost: + for total_cost_inp in account.total_cost: + tc_id = str(account.id).replace('"', '') + u_i_tc_value = float(str(total_cost_inp.value.value)) + u_i_tc_unit = str(total_cost_inp.unit.value) + if accert.ref_model: + # TODO: change the new into any thing else + # check if the total cost is a new added account in account table check if + # the revivew status is added + if "new" in tc_id: + user_add_coa_name = str(account.newCOA.id) + self.update_total_cost(c, user_add_coa_name, u_i_tc_value, u_i_tc_unit) + else: + self.update_total_cost(c, tc_id, u_i_tc_value, u_i_tc_unit) + else: + self.exit_with_error(accert) + for i in range(3, 7): + next_level = getattr(account, f'l{i}COA', None) + if next_level: + self.process_total_cost_accounts(c, next_level, accert) + + def exit_with_error(self, accert): + """ + Exits the program with an error message. + + Parameters + ---------- + accert : ACCERT + xml2obj class instantiates objects that can parse the ACCERT XML file. + """ + print("ERROR: model not found ") + print(accert.ref_model.value) + print("Exiting") + sys.exit(1) + + def finalize_process(self, c, ut, accert): + """ + Finalizes the process by extracting the affected variables, cost elements, and accounts. + + Parameters + ---------- + c : MySQLCursor + MySQLCursor class instantiates objects that can execute MySQL statements. + ut : Utility class + Utility class for processing user input. + accert : ACCERT + xml2obj class instantiates objects that can parse the ACCERT XML file. + """ + + ut.extract_user_changed_variables(c) + # if the model is not fusion or user assigned then process the cost elements + # NOTE: Accert is the instance of the Accert class use Capital A + if self.cel_tabl: + # NOTE the extract_affected_cost_elements will not be executed for fusion model ut.extract_affected_cost_elements(c) - ### NOTE: uncomment to print original and cost element values - # ### print original cost_elements value: - # extract_original_cost_elements(c) - ### calculate and new cost_elements value update to the database in table cost_elements and also update the account table: self.update_new_cost_elements(c) - # ut.extract_changed_cost_elements(c) + ut.print_updated_cost_elements(c) self.roll_up_cost_elements(c) - ### NOTE uncomment below to print new cost_elements value - # print_updated_cost_elements(c) - ### update the account table: - self.update_account_table_by_cost_elements(c) - ### roll up the account table: - self.roll_up_account_table(c) - ### print the account table: - print(' Generating results table for review '.center(100,'=')) + else: + # if the model is fusion or user assigned model without cost elements + # then the update_new_accounts will be executed otherwise the update_new_cost_elements should be executed + ut.extract_affected_accounts(c) + self.update_new_accounts(c) + + def generate_results(self, c, ut, accert): + """ + Generates the results. + + Parameters + ---------- + c : MySQLCursor + MySQLCursor class instantiates objects that can execute MySQL statements. + ut : Utility class + Utility class for processing user input. + accert : ACCERT + xml2obj class instantiates objects that can parse the ACCERT XML file. + """ + model = Accert.ref_model + if model: + # generate results for the models in the future we can add more models + self._generate_common_results(c, ut, accert, model) + if self.cel_tabl: + self.generate_results_table_with_cost_elements(c, conn, level=3) + self.generate_results_table(c, conn, level=3) + + def _generate_common_results(self, c, ut, accert, model): + """ + Generates the common results for the models. + + Parameters + ---------- + c : MySQLCursor + MySQLCursor class instantiates objects that can execute MySQL statements. + ut : Utility class + Utility class for processing user input. + accert : ACCERT + xml2obj class instantiates objects that can parse the ACCERT XML file. + model : str + + """ + if model == "abr1000" or model == "heatpipe" or model == "lfr": + self._common_cost_processing(c, accert) + fac, lab, mat = self.cal_direct_cost_elements(c) + all_flag = model != "lfr" + self._print_results(ut, c, fac, lab, mat, all_flag) + elif model == "pwr12-be": + self._pwr12be_processing(c, ut, accert) + else: + self._no_cost_element_processing(c, ut, accert) + + def _common_cost_processing(self, c, accert): + """ + Common cost processing for the models with limited accounts. + + Parameters + ---------- + c : MySQLCursor + MySQLCursor class instantiates objects that can execute MySQL statements. + accert : ACCERT + xml2obj class instantiates objects that can parse the ACCERT XML file. + """ + + self.sum_cost_elements_2C(c) + self.update_account_table_by_cost_elements(c) + self.check_and_process_total_cost(c, accert) + self.roll_up_lmt_account_table(c) + + def _print_results(self, ut, c, fac, lab, mat, all_flag): + """ + Prints the results. + + Parameters + ---------- + ut : Utility class + Utility class for processing user input. + c : MySQLCursor + MySQLCursor class instantiates objects that can execute MySQL statements. + fac : float + Factory cost. + lab : float + Labor cost. + mat : float + Material cost. + all_flag : bool + Flag to print all accounts. + """ + print(' Generating results table for review '.center(100, '=')) + print('\n') + if self.use_gncoa: + ut.print_leveled_accounts_gncoa(c, all=False, cost_unit='million', level=3) + else: + ut.print_leveled_accounts(c, all=all_flag, tol_fac=fac, tol_lab=lab, tol_mat=mat, cost_unit='million', level=3) + + def _pwr12be_processing(self, c, ut, accert): + """ + Processing for the pwr12-be model. + + Parameters + ---------- + c : MySQLCursor + MySQLCursor class instantiates objects that can execute MySQL statements. + ut : Utility class + Utility class for processing user input. + accert : ACCERT + xml2obj class instantiates objects that can parse the ACCERT XML file. + """ + self.update_account_table_by_cost_elements(c) + self.check_and_process_total_cost(c, accert) + if self.use_gncoa: + self.roll_up_account_table_GNCOA(c) + print(' Generating results table for review '.center(100, '=')) print('\n') - ut.print_leveled_accounts(c, all=False,cost_unit='million',level=3) - self.generate_results_table(c, conn,level=3) + ut.print_leveled_accounts_gncoa(c, all=False, cost_unit='million', level=3) + else: + self.roll_up_account_table(c, from_level=3, to_level=0) + print(' Generating results table for review '.center(100, '=')) + print('\n') + ut.print_leveled_accounts(c, all=True, cost_unit='million', level=3) + def _no_cost_element_processing(self, c, ut, accert): + """ + Processing for the fusion model. - ### close the connection: + Parameters + ---------- + c : MySQLCursor + MySQLCursor class instantiates objects that can execute MySQL statements. + ut : Utility class + Utility class for processing user input. + accert : ACCERT + xml2obj class instantiates objects that can parse the ACCERT XML file. + """ + self.check_and_process_total_cost(c, accert) + self.roll_up_account_table(c, from_level=4, to_level=0) + print(' Generating results table for review '.center(100, '=')) + print('\n') + ut.print_leveled_accounts(c, all=False, cost_unit='million', level=4) - conn.close() + def generate_results_table_with_cost_elements(self, c, conn, level=3): + """ + Generates the results table with cost elements. - sys.stdout.close() - sys.stdout=stdoutOrigin + Parameters + ---------- + c : MySQLCursor + MySQLCursor class instantiates objects that can execute MySQL statements. + conn : MySQLConnection + MySQLConnection class instantiates objects that represent a connection to the MySQL database server. + level : int + Level of the account. + """ + self._generate_excel(c, '_variable_affected_cost_elements.xlsx', 'extract_affected_cost_elements_w_dis', self.cel_tabl, self.var_tabl) + self._generate_excel(c, '_updated_cost_element.xlsx', 'print_updated_cost_elements', self.cel_tabl, remove_last_col=True) + + def _generate_excel(self, c, filename_suffix, proc_name, *args, remove_last_col=False): + """ + Generate an Excel file from stored procedure results. + + Parameters: + c : MySQLCursor + MySQLCursor class instantiates objects that can execute MySQL statements. + proc_name : str + Name of the stored procedure. + filename_suffix : str + Suffix of the filename. + args : tuple + Arguments for the stored procedure. + remove_last_col : bool + Remove the last column if required. + """ + c.callproc(proc_name, args) + for itered in c.stored_results(): + results = itered.fetchall() + field_names = [i[0] for i in itered.description] + df = pd.DataFrame(results, columns=field_names) + if remove_last_col: + df = df.iloc[:, :-1] # Remove the last column if required + filename = str(self.ref_model) + filename_suffix + df.to_excel(filename, index=False) + print(f"Successfully created excel file {filename}") + + + def generate_results_table(self, c, conn, level=3): + """ + Generates the results tables. + """ + self._generate_excel(c, '_updated_account.xlsx', 'print_leveled_accounts_simple', self.acc_tabl, level) if __name__ == "__main__": """ @@ -1304,26 +1999,15 @@ def execute_accert(self, c, ut): stdoutOrigin=sys.stdout sys.stdout = open("output.out", "w") - # print_logo() - if len(sys.argv) == 1: print("PLEASE ADD [Input_file_for_ACCERT]") sys.exit(-1) code_folder = os.path.dirname(os.path.abspath(__file__)) - initfile = os.path.join(code_folder, 'install.conf') - ins = configparser.ConfigParser() - ins.read(initfile) - passwd = ins.get("INSTALL","PASSWD") - - conn = mysql.connector.connect( - host="localhost", - user="root", - password=passwd, - database="accert_db", - auth_plugin="mysql_native_password" - ) + conn = accert_sqlite_connect(db_path=_accert_sqlite_db_path()) # conn.commit() + # NOTE: cursor is a class that instantiates objects that can execute MySQL statements + # only commit when you are sure that the transaction is complete c = conn.cursor() ut = Utility_methods() accert_path = os.path.abspath(os.path.join(code_folder, os.pardir)) @@ -1335,4 +2019,3 @@ def execute_accert(self, c, ut): raise SystemExit Accert = Accert(input_path,accert_path) Accert.execute_accert(c,ut) - diff --git a/docs/source/inputsIndex/utility_accert.py b/docs/source/inputsIndex/utility_accert.py index d0e2f1d..9dcac49 100644 --- a/docs/source/inputsIndex/utility_accert.py +++ b/docs/source/inputsIndex/utility_accert.py @@ -2,14 +2,33 @@ import textwrap class Utility_methods: - + """ + Utility class + """ def __init__(self): + self.acc_tabl = None + self.cel_tabl = None + self.var_tabl = None + self.vlk_tabl = None + self.alg_tabl = None + self.esc_tabl = None + self.fac_tabl = None pass + + def setup_table_names(self,c,Accert): + self.acc_tabl = Accert.acc_tabl + self.cel_tabl = Accert.cel_tabl + self.var_tabl = Accert.var_tabl + self.alg_tabl = Accert.alg_tabl + self.esc_tabl = Accert.esc_tabl + self.fac_tabl = Accert.fac_tabl + self.gncoa_map = Accert.gncoa_map + + return None def print_table(self, c, align_key=None,align=None,format_col=None): - """ - Prints the table in an organized format via the PrettyTable library. + """Prints the table in an organized format via the PrettyTable library. Parameters ---------- @@ -21,10 +40,14 @@ def print_table(self, c, align_key=None,align=None,format_col=None): List of alignments. Left, right or center. (By default none) format_col : list[str], optional List of column names to format. (By default none) - """ - results = c.fetchall() - columns = c.description - field_names = [i[0] for i in c.description] + """ + for itered in c.stored_results(): + results = itered.fetchall() + field_names = [i[0] for i in itered.description] + + # results = c.fetchall() + # columns = c.description + # field_names = [i[0] for i in c.description] x = PrettyTable(field_names) for row in results: row = list(row) @@ -41,8 +64,7 @@ def print_table(self, c, align_key=None,align=None,format_col=None): return None def print_account(self, c, all=False, cost_unit='dollar',level=3): - """ - Prints the account table. + """Prints the account table. Parameters ---------- @@ -56,26 +78,63 @@ def print_account(self, c, all=False, cost_unit='dollar',level=3): Level of account. (By default 3) """ if all: - c.execute("""SELECT * - FROM account - WHERE level <= %(u_i_level)s;""",{'u_i_level': str(level)}) + # DELIMITER $$ + # CREATE DEFINER=`root`@`localhost` PROCEDURE `print_account_all`(IN table_name varchar(50), + # IN level int) + # BEGIN + # SET @stmt=CONCAT('SELECT * FROM ',table_name,' WHERE level <= ?'); + # PREPARE stmt FROM @stmt; + # SET @level=level; + # EXECUTE stmt USING @level; + # DEALLOCATE PREPARE stmt; + # END$$ + # DELIMITER ; + + c.callproc('print_account_all', (self.acc_tabl,level)) + + # c.execute("""SELECT * + # FROM account + # WHERE level <= %(u_i_level)s;""",{'u_i_level': str(level)}) else: - c.execute("""SELECT ind, - code_of_account, - account_description, - total_cost, - unit, - level, - review_status - FROM account - WHERE level <= %(u_i_level)s;""",{'u_i_level': str(level)}) + # DELIMITER $$ + # CREATE DEFINER=`root`@`localhost` PROCEDURE `print_account_simple`(IN table_name varchar(50), + # IN level int) + # BEGIN + # SET @stmt=CONCAT('SELECT ind, + # code_of_account, + # account_description, + # total_cost, + # unit, + # level, + # review_status + # FROM ',table_name,' WHERE level <= ?'); + # PREPARE stmt FROM @stmt; + # SET @level=level; + # EXECUTE stmt USING @level; + # DEALLOCATE PREPARE stmt; + # END$$ + # DELIMITER ; + + c.callproc('print_account_simple', (self.acc_tabl,level)) + # c.execute("""SELECT ind, + # code_of_account, + # account_description, + # total_cost, + # unit, + # level, + # review_status + # FROM account + # WHERE level <= %(u_i_level)s;""",{'u_i_level': str(level)}) align_key=["code_of_accout", "account_description", "total_cost"] align=[ "l", "l", "r"] if cost_unit=='million': - results = c.fetchall() - columns = c.description - field_names = [i[0] for i in c.description] + for row in c.stored_results(): + results = row.fetchall() + field_names = [i[0] for i in row.description] + # results = c.fetchall() + # columns = c.description + # field_names = [i[0] for i in c.description] x = PrettyTable(field_names) for row in results: row = list(row) @@ -92,9 +151,8 @@ def print_account(self, c, all=False, cost_unit='dollar',level=3): self.print_table(c, align_key, align) return None - def print_leveled_accounts(self, c, all=False, cost_unit='dollar',level=3): - """ - Prints the output account table with COA line up as a nested list. + def print_leveled_accounts(self, c, all=False,tol_fac=None,tol_lab= None,tol_mat=None, cost_unit='dollar',level=3): + """Prints the output account table with COA line up as a nested list. Parameters ---------- @@ -108,91 +166,199 @@ def print_leveled_accounts(self, c, all=False, cost_unit='dollar',level=3): Level of account. (By default 3) """ if all: - c.execute("""select account.level, - rankedcoa.COA as code_of_account, - account.account_description, - sorted_ce.fac_cost, - sorted_ce.lab_cost, - sorted_ce.mat_cost, - account.total_cost, - account.unit, - account.review_status - FROM account - JOIN - (SELECT node.code_of_account, - CONCAT( REPEAT(' ', COUNT(parent.code_of_account) - 1), node.code_of_account) AS COA - FROM account AS node, - account AS parent - WHERE node.lft BETWEEN parent.lft AND parent.rgt - GROUP BY node.code_of_account) as rankedcoa - ON account.code_of_account=rankedcoa.code_of_account - JOIN (SELECT splt_act.code_of_account, - cef.cost_2017 as fac_cost, - cel.cost_2017 as lab_cost, - cem.cost_2017 as mat_cost - FROM - (SELECT code_of_account,total_cost, - SUBSTRING_INDEX(SUBSTRING_INDEX(cost_elements, ',', 1), ',', -1) as fac_name, - SUBSTRING_INDEX(SUBSTRING_INDEX(cost_elements, ',', 2), ',', -1) as lab_name, - SUBSTRING_INDEX(SUBSTRING_INDEX(cost_elements, ',', 3), ',', -1) as mat_name - FROM accert_db.account) as splt_act - LEFT JOIN cost_element as cef - ON cef.cost_element = splt_act.fac_name - LEFT JOIN cost_element as cel - ON cel.cost_element = splt_act.lab_name - LEFT JOIN cost_element as cem - ON cem.cost_element = splt_act.mat_name - ) as sorted_ce - ON sorted_ce.code_of_account = account.code_of_account - WHERE account.level <= %(u_i_level)s - ORDER BY account.lft;""",{'u_i_level': str(level)}) + # DELIMITER $$ + # CREATE DEFINER=`root`@`localhost` PROCEDURE `print_leveled_accounts_all`(IN acc_table varchar(50), + # IN cel_table varchar(50), + # IN level int) + # BEGIN + # SET @stmt=CONCAT('SELECT acc.level, + # rankedcoa.COA as code_of_account, + # acc.account_description, + # sorted_ce.fac_cost, + # sorted_ce.lab_cost, + # sorted_ce.mat_cost, + # acc.total_cost, + # acc.unit, + # acc.review_status + # FROM ',acc_table,' as acc + # JOIN + # (SELECT node.code_of_account, + # CONCAT( REPEAT(" ", COUNT(parent.code_of_account) - 1), node.code_of_account) AS COA + # FROM ',acc_table,' AS node, + # ',acc_table,' AS parent + # WHERE node.lft BETWEEN parent.lft AND parent.rgt + # GROUP BY node.code_of_account) as rankedcoa + # ON acc.code_of_account=rankedcoa.code_of_account + # JOIN (SELECT splt_act.code_of_account, + # cef.cost_2017 as fac_cost, + # cel.cost_2017 as lab_cost, + # cem.cost_2017 as mat_cost + # FROM + # (SELECT code_of_account,total_cost, + # SUBSTRING_INDEX(SUBSTRING_INDEX(cost_elements, ",", 1), ",", -1) AS fac_name, + # SUBSTRING_INDEX(SUBSTRING_INDEX(cost_elements, ",", 2), ",", -1) AS lab_name, + # SUBSTRING_INDEX(SUBSTRING_INDEX(cost_elements, ",", 3), ",", -1) AS mat_name + # FROM ',acc_table,') as splt_act + # LEFT JOIN ',cel_table,' as cef + # ON cef.cost_element= splt_act.fac_name + # LEFT JOIN ',cel_table,' as cel + # ON cel.cost_element= splt_act.lab_name + # LEFT JOIN ',cel_table,' as cem + # ON cem.cost_element= splt_act.mat_name) as sorted_ce + # ON sorted_ce.code_of_account=acc.code_of_account + # WHERE acc.level <= ? + # ORDER BY acc.lft;'); + # PREPARE stmt FROM @stmt; + # SET @level=level; + # EXECUTE stmt USING @level; + # DEALLOCATE PREPARE stmt; + # END$$ + # DELIMITER ; + + c.callproc('print_leveled_accounts_all', (self.acc_tabl,self.cel_tabl,level)) align_key=["code_of_account", "account_description", "fac_cost", "lab_cost", "mat_cost", "total_cost"] align=[ "l", "l", "r", "r", "r", "r"] else: - c.execute("""SELECT rankedcoa.code_of_account, - account.account_description, - account.total_cost, - account.unit, - account.level, - account.review_status - FROM account - JOIN - ( - SELECT node.code_of_account AS COA, CONCAT( REPEAT(' ', COUNT(parent.code_of_account) - 1), node.code_of_account) AS code_of_account - FROM account AS node, - account AS parent - WHERE node.lft BETWEEN parent.lft AND parent.rgt - GROUP BY node.code_of_account) as rankedcoa - ON account.code_of_account=rankedcoa.COA - WHERE account.level <= %(u_i_level)s - ORDER BY account.lft;""",{'u_i_level': str(level)}) + # DELIMITER $$ + # CREATE DEFINER=`root`@`localhost` PROCEDURE `print_leveled_accounts_simple`(IN acc_table VARCHAR(255), IN level INT) + # BEGIN + # SET @stmt = CONCAT('SELECT rankedcoa.code_of_account, + # acc.account_description, + # acc.total_cost, + # acc.unit, + # acc.level, + # acc.review_status + # FROM ',acc_table,' as acc + # JOIN + # (SELECT node.code_of_account AS COA , CONCAT( REPEAT(" ", COUNT(parent.code_of_account) - 1), node.code_of_account) AS code_of_account + # FROM ',acc_table,' AS node, + # ',acc_table,' AS parent + # WHERE node.lft BETWEEN parent.lft AND parent.rgt + # GROUP BY node.code_of_account) as rankedcoa + # ON acc.code_of_account=rankedcoa.COA + # WHERE acc.level <= ?'); + # PREPARE stmt FROM @stmt; + # SET @level=level; + # EXECUTE stmt USING @level; + # DEALLOCATE PREPARE stmt; + # END$$ + # DELIMITER ; + c.callproc('print_leveled_accounts_simple', (self.acc_tabl,level)) + # c.execute("""SELECT rankedcoa.code_of_account, + # account.account_description, + # account.total_cost, + # account.unit, + # account.level, + # account.review_status + # FROM account + # JOIN + # ( + # SELECT node.code_of_account AS COA, CONCAT( REPEAT(" ", COUNT(parent.code_of_account) - 1), node.code_of_account) AS code_of_account + # FROM account AS node, + # account AS parent + # WHERE node.lft BETWEEN parent.lft AND parent.rgt + # GROUP BY node.code_of_account) as rankedcoa + # ON account.code_of_account=rankedcoa.COA + # WHERE account.level <= %(u_i_level)s + # ORDER BY account.lft;""",{'u_i_level': str(level)}) align_key=["code_of_account", "account_description", "total_cost"] align=[ "l", "l", "r"] if cost_unit=='million': - results = c.fetchall() - columns = c.description - field_names = [i[0] for i in c.description] + for row in c.stored_results(): + results = row.fetchall() + field_names = [i[0] for i in row.description] + # results = c.fetchall() + # columns = c.description + # field_names = [i[0] for i in c.description] x = PrettyTable(field_names) - for row in results: + for idx, row in enumerate(results): row = list(row) - # NOTE the index of the row need to have a function - # just place this as a temporary solution if all: - row[3:7] = list(map(lambda x: '{:,.2f}'.format(x/1000000), row[3:7])) - row[7] = 'million' + # if index is 0, and tol_fac, tol_lab, tol_mat are not None, format the values + if idx == 0 and tol_fac and tol_lab and tol_mat: + # First row special formatting + row[3] = "{:,.2f}".format(tol_fac / 1000000) + row[4] = "{:,.2f}".format(tol_lab / 1000000) + row[5] = "{:,.2f}".format(tol_mat / 1000000) + row[6] = "{:,.2f}".format(row[6] / 1000000) + else: + # Format other rows or print 0 if value is None + row[3:7] = ['{:,.2f}'.format(x / 1000000) if x else '0' for x in row[3:7]] else: - row[2] = '{:,.2f}'.format(row[2]/1000000) - row[3] = 'million' + # Format only the third column for other cases + row[2] = '{:,.2f}'.format(row[2] / 1000000) + x.add_row(row) + if align_key: for i,k in enumerate(align_key): x.align[k] = align[i] - print (x) + print(x) else: self.print_table(c, align_key, align) return None - + + def print_leveled_accounts_gncoa(self, c, all=False, cost_unit='dollar',level=3): + """Prints the output account table with GNCOA line up as a nested list. + + Parameters + ---------- + c : MySQLCursor + MySQLCursor class instantiates objects that can execute MySQL statements. + all : bool, optional + If True, print all the accounts columns. (By default False) + cost_unit : str, optional + Unit of the total cost. (By default 'dollar') + level : int, optional + Level of account. (By default 3) + """ + # place holder for printing the GNCOA cost elements in a nested list + # this needs to be implemented in the future since the GNCOA cost elements + # are not rolled up in the current database + # c.callproc('print_leveled_accounts_gn_all', (self.acc_tabl,self.cel_tabl,level)) + # all=True + # tol_fac=None + # tol_lab=None + # tol_mat=None + + c.callproc('print_leveled_accounts_gn', (self.acc_tabl,self.gncoa_map,level)) + align_key=["gncoa","gncoa_description", "total_cost"] + align=[ "l","l", "r"] + if cost_unit=='million': + for row in c.stored_results(): + results = row.fetchall() + field_names = [i[0] for i in row.description] + x = PrettyTable(field_names) + for idx, row in enumerate(results): + row = list(row) + if all: + # if index is 0, and tol_fac, tol_lab, tol_mat are not None, format the values + if idx == 0 and tol_fac and tol_lab and tol_mat: + # First row special formatting + row[3] = "{:,.2f}".format(tol_fac / 1000000) + row[4] = "{:,.2f}".format(tol_lab / 1000000) + row[5] = "{:,.2f}".format(tol_mat / 1000000) + row[6] = "{:,.2f}".format(row[6] / 1000000) + else: + # Format other rows or print 0 if value is None + row[3:7] = ['{:,.2f}'.format(x / 1000000) if x else '0' for x in row[3:7]] + else: + # Format only the third column for other cases + if row[2]: + row[2] = '{:,.2f}'.format(row[2] / 1000000) + else: + row[2] = 0 + + x.add_row(row) + if align_key: + for i,k in enumerate(align_key): + x.align[k] = align[i] + print(x) + else: + self.print_table(c) + return None + def print_algorithm(self, c): """Prints the output algorithm table. @@ -201,9 +367,16 @@ def print_algorithm(self, c): c : MySQLCursor MySQLCursor class instantiates objects that can execute MySQL statements. """ - c.execute("""SELECT * - FROM algorithm; - """) + # DELIMITER $$ + # CREATE DEFINER=`root`@`localhost` PROCEDURE `print_table`(IN table_name VARCHAR(255)) + # BEGIN + # SET @stmt = CONCAT('SELECT * FROM ',table_name); + # PREPARE stmt FROM @stmt; + # EXECUTE stmt; + # DEALLOCATE PREPARE stmt; + # END$$ + # DELIMITER ; + c.callproc('print_table', (self.alg_tabl,)) self.print_table(c) return None @@ -215,9 +388,7 @@ def print_cost_element(self, c): c : MySQLCursor MySQLCursor class instantiates objects that can execute MySQL statements. """ - c.execute("""SELECT * - FROM cost_element; - """) + c.callproc('print_table', (self.cel_tabl,)) self.print_table(c) return None @@ -229,9 +400,10 @@ def print_facility(self, c): c : MySQLCursor MySQLCursor class instantiates objects that can execute MySQL statements. """ - c.execute("""SELECT * - FROM facility; - """) + c.callproc('print_table', (self.fac_tabl,)) + # c.execute("""SELECT * + # FROM facility; + # """) self.print_table(c) return None @@ -243,9 +415,10 @@ def print_escalation(self, c): c : MySQLCursor MySQLCursor class instantiates objects that can execute MySQL statements. """ - c.execute("""SELECT * - FROM escalation; - """) + c.callproc('print_table', (self.esc_tabl,)) + # c.execute("""SELECT * + # FROM escalation; + # """) self.print_table(c) return None @@ -257,65 +430,10 @@ def print_variable(self, c): c : MySQLCursor MySQLCursor class instantiates objects that can execute MySQL statements. """ - c.execute("""SELECT * - FROM variable; - """) - self.print_table(c) - return None - - def print_abr_account(self, c): - """Prints the output abr account table. - - Parameters - ---------- - c : MySQLCursor - MySQLCursor class instantiates objects that can execute MySQL statements. - """ - c.execute("""SELECT * - FROM abr_account; - """) - self.print_table(c) - return None - - def print_abr_cost_element(self, c): - """Prints the output abr cost element table. - - Parameters - ---------- - c : MySQLCursor - MySQLCursor class instantiates objects that can execute MySQL statements. - """ - c.execute("""SELECT * - FROM abr_cost_element; - """) - self.print_table(c) - return None - - def print_abr_variable(self, c): - """Prints the output abr variable table. - - Parameters - ---------- - c : MySQLCursor - MySQLCursor class instantiates objects that can execute MySQL statements. - """ - c.execute("""SELECT * - FROM abr_variable; - """) - self.print_table(c) - return None - - def print_abr_variable_links(self, c): - """Prints the output abr variable links table. - - Parameters - ---------- - c : MySQLCursor - MySQLCursor class instantiates objects that can execute MySQL statements. - """ - c.execute("""SELECT * - FROM abr_variable_links; - """) + c.callproc('print_table', (self.var_tabl,)) + # c.execute("""SELECT * + # FROM variable; + # """) self.print_table(c) return None @@ -329,26 +447,47 @@ def print_user_request_parameter(self,c, all=False): all : bool, optional If True, prints all columns. (By default False) """ + # DELIMITER $$ + # CREATE DEFINER=`root`@`localhost` PROCEDURE `print_user_request_parameter`(IN all_col BOOLEAN, + # IN var_table VARCHAR(50), + # IN vlk_table VARCHAR(50)) + # BEGIN + # IF all_col THEN + # SET @stmt = CONCAT('SELECT va.ind, va.var_name, affectv.ce_affected FROM ',var_table,' as va JOIN + # (SELECT variable, group_concat(ce) as ce_affected + # FROM ',vlk_table,' as vlk + # group by variable) as affectv on va.var_name = affectv.variable + # where va.var_value IS NULL + # order by va.ind'); + # ELSE + # SET @stmt = CONCAT('SELECT va.var_name, affectv.ce_affected FROM ',var_table,' as va JOIN + # (SELECT variable, group_concat(ce) as ce_affected + # FROM ',vlk_table,' as vlk + # group by variable) as affectv on va.var_name = affectv.variable + # where va.var_value IS NULL + # order by va.ind;'); + # END IF; + # PREPARE stmt FROM @stmt; + # EXECUTE stmt; + # DEALLOCATE PREPARE stmt; + # END$$ + # DELIMITER ; if all: - c.execute("""SELECT va.ind, va.var_name, affectv.ce_affected - FROM accert_db.variable as va JOIN - (SELECT variable,group_concat(ce) as ce_affected - FROM accert_db.variable_links - group by variable) as affectv - on va.var_name = affectv.variable - where va.var_value IS NULL - order by va.ind;""") + c.callproc('print_user_request_parameter', (True, self.var_tabl, self.cel_tabl)) self.print_table(c) else: - c.execute("""SELECT va.var_name,affectv.ce_affected - FROM accert_db.variable as va JOIN - (SELECT variable,group_concat(ce) as ce_affected - FROM accert_db.variable_links - group by variable) as affectv - on va.var_name = affectv.variable - where va.var_value IS NULL - order by va.ind;""") - results = c.fetchall() + c.callproc('print_user_request_parameter', (False, self.var_tabl, self.cel_tabl)) + # c.execute("""SELECT va.var_name,affectv.ce_affected + # FROM accert_db_test.variable as va JOIN + # (SELECT variable,group_concat(ce) as ce_affected + # FROM accert_db_test.variable_links + # group by variable) as affectv + # on va.var_name = affectv.variable + # where va.var_value IS NULL + # order by va.ind;""") + for row in c.stored_results(): + results = row.fetchall() + for row in results: print('Parameter "{}" is required for cost elements:'.format(row[0])) # print(row[1]) @@ -364,37 +503,26 @@ def print_updated_cost_elements(self, c): c : MySQLCursor MySQLCursor class instantiates objects that can execute MySQL statements. """ - c.execute("""SELECT ind, - cost_element, - cost_2017, - sup_cost_ele, - account, - updated - FROM cost_element - WHERE updated = 1""") + # DELIMITER $$ + # CREATE DEFINER=`root`@`localhost` PROCEDURE `print_updated_cost_elements`(IN cel_table VARCHAR(50)) + # BEGIN + # SET @stmt = CONCAT('SELECT ind, + # cost_element, + # cost_2017, + # sup_cost_ele, + # account, + # updated + # FROM ',cel_table,' + # WHERE updated = 1'); + # PREPARE stmt FROM @stmt; + # EXECUTE stmt; + # DEALLOCATE PREPARE stmt; + # END$$ + # DELIMITER ; + + c.callproc('print_updated_cost_elements', (self.cel_tabl,)) self.print_table(c) - - def extract_original_cost_elements(self, c): - """Extracts the original affected cost elements from the cost element table. (This function is ONLY used for debugging) - - Parameters - ---------- - c : MySQLCursor - MySQLCursor class instantiates objects that can execute MySQL statements. - """ - # # ce.updated, ce.algno - c.execute(""" SELECT ce.cost_element, ce.cost_2017 as orignal_cost - FROM cost_element as ce JOIN - (SELECT vl.ce - FROM - (SELECT * FROM variable - WHERE user_input = 1) as va - JOIN variable_links as vl - on va.var_name = vl.variable) as ce_affected - on ce.cost_element = ce_affected.ce""") - self.print_table(c) - return None - + def extract_affected_cost_elements(self,c): """Extracts affected cost elements from cost element table and groups them by changed variables. @@ -405,224 +533,123 @@ def extract_affected_cost_elements(self,c): """ print(' Extracting affected cost elements '.center(100,'=')) print('\n') - c.execute(""" SELECT vl.variable, group_concat(vl.ce) - FROM - (SELECT * FROM variable - WHERE user_input = 1) as va - JOIN variable_links as vl - on va.var_name = vl.variable - GROUP BY vl.variable""") - results = c.fetchall() + # DELIMITER $$ + # CREATE DEFINER=`root`@`localhost` PROCEDURE `extract_affected_cost_elements`(IN cel_table varchar(50), + # IN var_table varchar(50)) + # BEGIN + # SET @stmt = CONCAT("SELECT va.var_name, (SELECT GROUP_CONCAT(ce.cost_element SEPARATOR ', ') + # FROM ", cel_table, " ce + # WHERE FIND_IN_SET(va.var_name, REPLACE(ce.variables, ' ', '')) > 0) AS ce_affected + # FROM + # (SELECT * FROM ",var_table," + # WHERE user_input = 1) as va + # WHERE (SELECT GROUP_CONCAT(ce.cost_element SEPARATOR ', ') + # FROM ", cel_table, " ce + # WHERE FIND_IN_SET(va.var_name, REPLACE(ce.variables, ' ', '')) > 0) IS NOT NULL + # "); + # PREPARE stmt FROM @stmt; + # EXECUTE stmt; + # DEALLOCATE PREPARE stmt; + # END + # DELIMITER ; + + c.callproc('extract_affected_cost_elements',(self.cel_tabl,self.var_tabl)) + for row in c.stored_results(): + results = row.fetchall() for row in results: print('variable "{}" affects cost element(s):'.format(row[0])) print('{}\n'.format(textwrap.fill(row[1], 100))) return None - def extract_user_changed_variables(self,c): - """Extracts user changed variables from variable table. + def extract_affected_accounts(self,c): + """ Extracts affected accounts from account table. Parameters ---------- c : MySQLCursor MySQLCursor class instantiates objects that can execute MySQL statements. """ - print('Extracting user changed variables'.center(100,'=')) - c.execute("""SELECT var_name,var_description, var_value, var_unit - FROM `accert_db`.`variable` - WHERE user_input = 1 - ORDER BY var_name;""") - self.print_table(c,format_col=[3]) - return None - - def extract_changed_cost_elements(self,c): - """Extracts changed cost elements from cost element table. - - Parameters - ---------- - c : MySQLCursor - MySQLCursor class instantiates objects that can execute MySQL statements. - """ - print('Extracting changed cost elements'.center(100,'=')) - c.execute("""SELECT cost_element, cost_2017 - FROM `accert_db`.`cost_element` - WHERE updated != 0 - ORDER BY account, cost_element;""") - self.print_table(c,format_col=[2]) - return None - - def extract_user_changed_abr_variables(self,c): - """Extracts user changed abr variables from abr variable table. - - Parameters - ---------- - c : MySQLCursor - MySQLCursor class instantiates objects that can execute MySQL statements. - """ - print('Extracting user changed variables'.center(100,'=')) - c.execute("""SELECT var_name, var_description, var_value, var_unit - FROM `accert_db`.`abr_variable` - WHERE user_input = 1 - ORDER BY var_name;""") - self.print_table(c,format_col=[3]) - return None - - def extract_affected_abr_cost_elements(self,c): - """Extracts the affected abr cost elements from the abr cost element table group by changed variables. - - Parameters - ---------- - c : MySQLCursor - MySQLCursor class instantiates objects that can execute MySQL statements. - """ - print(' Extracting affected cost elements '.center(100,'=')) - print('\n') - c.execute(""" SELECT vl.variable, group_concat(vl.ce) - FROM - (SELECT * FROM abr_variable - WHERE user_input = 1) as va - JOIN abr_variable_links as vl - on va.var_name = vl.variable - GROUP BY vl.variable""") - results = c.fetchall() + print('Extracting affected accounts'.center(100,'=')) + # DELIMITER $$ + # CREATE DEFINER=`root@`localhost` PROCEDURE `extract_affected_accounts`(IN acc_table VARCHAR(50), + # IN var_table VARCHAR(50)) + # BEGIN + # SET @stmt = CONCAT('SELECT va.var_name, + # (SELECT GROUP_CONCAT(ac.code_of_account SEPARATOR ", ") + # FROM ',acc_table,' ac + # WHERE FIND_IN_SET(va.var_name, REPLACE(ac.variables, " ", "")) > 0) AS ac_affected + # FROM + # (SELECT * FROM ',var_table,' + # WHERE user_input = 1) as va + # WHERE (SELECT GROUP_CONCAT(ac.code_of_account SEPARATOR ", ") + # FROM ',acc_table,' ac + # WHERE FIND_IN_SET(va.var_name, REPLACE(ac.variables, " ", "")) > 0) IS NOT NULL;'); + # PREPARE stmt FROM @stmt; + # EXECUTE stmt; + # DEALLOCATE PREPARE stmt; + # END$$ + # DELIMITER ; + c.callproc('extract_affected_accounts',(self.acc_tabl,self.var_tabl)) + for row in c.stored_results(): + results = row.fetchall() for row in results: - print('variable "{}" affects cost element(s):'.format(row[0])) + print('variable "{}" affects account(s):'.format(row[0])) print('{}\n'.format(textwrap.fill(row[1], 100))) return None - def print_updated_abr_cost_elements(self,c): - """Prints the output updated abr cost elements table. + def extract_user_changed_variables(self,c): + """Extracts user changed variables from variable table. Parameters ---------- c : MySQLCursor MySQLCursor class instantiates objects that can execute MySQL statements. """ - c.execute("""SELECT ind, - cost_element, - cost_2017, - sup_cost_ele, - account, - updated - FROM abr_cost_element - WHERE updated = 1""") - self.print_table(c) + print('Extracting user changed variables'.center(100,'=')) + # DELIMITER $$ + # CREATE DEFINER=`root`@`localhost` PROCEDURE `extract_user_changed_variables`(IN table_name VARCHAR(50)) + # BEGIN + # SET @stmt = CONCAT('SELECT var_name,var_description, var_value, var_unit + # FROM ', table_name, ' WHERE user_input = 1 ORDER BY var_name;'); + # PREPARE stmt FROM @stmt; + # EXECUTE stmt; + # DEALLOCATE PREPARE stmt; + # END$$ + # DELIMITER ; + c.callproc('extract_user_changed_variables',(self.var_tabl,)) + # c.execute("""SELECT var_name,var_description, var_value, var_unit + # FROM `accert_db_test`.`variable` + # WHERE user_input = 1 + # ORDER BY var_name;""") + self.print_table(c,format_col=[3]) return None - def print_leveled_abr_accounts(self, c, abr_fac,abr_lab,abr_mat,all=False, - cost_unit='dollar',level=3): - """Prints the output leveled abr accounts table. + def extract_changed_cost_elements(self,c): + """Extracts changed cost elements from the cost element table. Parameters ---------- c : MySQLCursor MySQLCursor class instantiates objects that can execute MySQL statements. - abr_fac : float - Abr_fac is the ABR-1000 factor for factory cost. - abr_lab : float - Abr_lab is the ABR-1000 factor for labor cost. - abr_mat : float - Abr_mat is the ABR-1000 factor for material cost. - all : bool, optional - All is the flag to print all accounts or not. (By default False, or not) - cost_unit : str, optional - Cost_unit is the cost unit. (By default 'dollar') - level : int, optional - Level is the level of the account. (By default 3) """ - if all: - c.execute("""SELECT abr_account.level, - rankedcoa.COA as code_of_account, - abr_account.account_description, - sorted_ce.fac_cost, - sorted_ce.lab_cost, - sorted_ce.mat_cost, - abr_account.total_cost, - abr_account.unit, - abr_account.review_status - FROM abr_account - JOIN - (SELECT node.code_of_account, - CONCAT( REPEAT(' ', COUNT(parent.code_of_account) - 1), node.code_of_account) AS COA - FROM abr_account AS node, - abr_account AS parent - WHERE node.lft BETWEEN parent.lft AND parent.rgt - GROUP BY node.code_of_account) as rankedcoa - ON abr_account.code_of_account=rankedcoa.code_of_account - JOIN (SELECT splt_act.code_of_account, - cef.cost_2017 as fac_cost, - cel.cost_2017 as lab_cost, - cem.cost_2017 as mat_cost - FROM - (SELECT code_of_account,total_cost,supaccount, - SUBSTRING_INDEX(SUBSTRING_INDEX(cost_elements, ',', 1), ',', -1) as fac_name, - SUBSTRING_INDEX(SUBSTRING_INDEX(cost_elements, ',', 2), ',', -1) as lab_name, - SUBSTRING_INDEX(SUBSTRING_INDEX(cost_elements, ',', 3), ',', -1) as mat_name - FROM accert_db.abr_account) as splt_act - LEFT JOIN abr_cost_element as cef - ON cef.cost_element = splt_act.fac_name - LEFT JOIN abr_cost_element as cel - ON cel.cost_element = splt_act.lab_name - LEFT JOIN abr_cost_element as cem - ON cem.cost_element = splt_act.mat_name) as sorted_ce - ON sorted_ce.code_of_account = abr_account.code_of_account - WHERE abr_account.level <= 3 - ORDER BY abr_account.lft;""") - align_key=["code_of_account", "account_description", "fac_cost", "lab_cost", "mat_cost", "total_cost"] - align=[ "l", "l", "r", "r", "r", "r"] - else: - c.execute("""SELECT rankedcoa.code_of_account, - abr_account.account_description, - abr_account.total_cost, - abr_account.unit, - abr_account.level, - abr_account.review_status - FROM abr_account - JOIN - ( - SELECT node.code_of_account AS COA, CONCAT( REPEAT(' ', COUNT(parent.code_of_account) - 1), node.code_of_account) AS code_of_account - FROM abr_account AS node, - abr_account AS parent - WHERE node.lft BETWEEN parent.lft AND parent.rgt - GROUP BY node.code_of_account) as rankedcoa - ON abr_account.code_of_account=rankedcoa.COA - WHERE abr_account.level <= %(u_i_level)s - ORDER BY abr_account.lft;""",{'u_i_level': str(level)}) - align_key=["code_of_account", "account_description", "total_cost"] - align=[ "l", "l", "r"] + # DELIMITER $$ + # CREATE DEFINER=`root`@`localhost` PROCEDURE `extract_changed_cost_elements`(IN cel_table VARCHAR(50)) + # BEGIN + # SET @stmt = CONCAT('SELECT cost_element, cost_2017 + # FROM ',cel_table,' + # WHERE updated != 0 + # ORDER BY account, cost_element;'); + # PREPARE stmt FROM @stmt; + # EXECUTE stmt; + # DEALLOCATE PREPARE stmt; + # END$$ + # DELIMITER ; + print('Extracting changed cost elements'.center(100,'=')) + c.callproc('extract_changed_cost_elements',(self.cel_tabl,)) - if cost_unit=='million': - results = c.fetchall() - columns = c.description - # print() - # field_names = [i[0] for i in c.description] - field_names = [i[0] for i in columns] - x = PrettyTable(field_names) - row0=list(results[0]) - if all: - row0[3]="{:,.2f}".format(abr_fac/1000000) - row0[4]="{:,.2f}".format(abr_lab/1000000) - row0[5]="{:,.2f}".format(abr_mat/1000000) - row0[6]="{:,.2f}".format(row0[6]/1000000) - row0[7]="million" - else: - row0[2] = '{:,.2f}'.format(row0[2]/1000000) - row0[3] = 'million' - x.add_row(row0) - for row in results[1:]: - row = list(row) - # NOTE the index of the row need to have a function - # just place this as a temporary solution - if all: - row[3:7] = list(map(lambda x: '{:,.2f}'.format(x/1000000), row[3:7])) - row[7] = 'million' - else: - row[2] = '{:,.2f}'.format(row[2]/1000000) - row[3] = 'million' - x.add_row(row) - if align_key: - for i,k in enumerate(align_key): - x.align[k] = align[i] - print (x) - else: - self.print_table(c, align_key, align) - return None \ No newline at end of file + # c.execute("""SELECT cost_element, cost_2017 + # FROM `accert_db_test`.`cost_element` + # WHERE updated != 0 + # ORDER BY account, cost_element;""") + self.print_table(c,format_col=[2]) + return None diff --git a/docs/source/user/build_your_own.rst b/docs/source/user/build_your_own.rst index 406d1b5..43ef4c7 100644 --- a/docs/source/user/build_your_own.rst +++ b/docs/source/user/build_your_own.rst @@ -114,25 +114,11 @@ In the same folder a `user_defined_algorithm.sql` file will be generated. Run th The script will create the database table and insert the data from the `user_defined_account.csv`, `user_defined_variable.csv` and `user_defined_algorithm.csv` files into the database. -You can also run mysql command and source the `user_defined_algorithm.sql` file to create the database table. +To apply the SQL to a different SQLite database file, pass ``--db``: .. code-block:: bash - $ mysql -h localhost -u root -p - Enter password: - Welcome to the MySQL monitor. Commands end with ; or \g. - Your MySQL connection id is 2475 - Server version: 8.0.27 MySQL Community Server - GPL - - Copyright (c) 2000, 2018, Oracle and/or its affiliates. All rights reserved. - - Oracle is a registered trademark of Oracle Corporation and/or its - affiliates. Other names may be trademarks of their respective - owners. - - Type 'help;' or '\h' for help. Type '\c' to clear the current input statement. - - mysql>source user_defined_algorithm.sql + python ../../../src/scripts/run_sql.py user_defined_algorithm.sql --db /path/to/accertdb.sqlite Prepare the input file diff --git a/docs/source/user/install.rst b/docs/source/user/install.rst index b37eaee..070a3d3 100644 --- a/docs/source/user/install.rst +++ b/docs/source/user/install.rst @@ -1,608 +1,110 @@ - Installation Guide ================== -This guide provides step-by-step instructions to install ACCERT on **Windows**, **macOS**, and **Linux** systems. Please ensure you have administrative privileges on your device, as some steps require it. - +This guide describes how to install ACCERT on Windows, macOS, and Linux. +ACCERT now uses a bundled SQLite database, so no MySQL server, root password, +or ``install.conf`` file is required. Prerequisites ------------- -- **Git**: Ensure Git is installed on your system. - - - Download from `Git - Downloads `_ if not already installed. - - Verify installation by typing ``git`` in your terminal; if commands appear, Git is installed. -- **Python Coding Environment**: While not required, using an IDE like Visual Studio Code is recommended. -- **MySQL Community Server**: Required for database functionalities. -- **NEAMS Workbench**: Necessary for integrating ACCERT with the Workbench interface. - -Installation Instructions -------------------------- - -Choose your operating system to proceed with the installation: - -- `Installation on Windows`_ -- `Installation on macOS`_ -- `Installation on Linux`_ - -Installation on Windows ------------------------ - -.. _Installation on Windows: - -For certain aspects of this process, you will need administrative privileges. - -Cloning ACCERT -~~~~~~~~~~~~~~ - -1. **Create a Directory for ACCERT**: - - - Open your file explorer and create a folder where you want to install ACCERT (e.g., ``CODE``). - -2. **Open Git Bash or Terminal**: - - - Navigate to the directory you just created. - -3. **Clone the ACCERT Repository**: - - .. code-block:: shell - - $ cd /path/to/CODE - $ git clone https://github.com/accert-dev/ACCERT.git - $ cd ACCERT - - - Replace ``/path/to/CODE`` with the actual path to your ``CODE`` directory. - -4. **Verify Cloning**: - - - Ensure the ACCERT folder contains files similar to the ACCERT Git repository. - -Install MySQL Community Server -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -1. **Download MySQL**: - - - Visit `MySQL Community Server `_ and download the recommended version for Windows. - -2. **Install MySQL**: - - - Follow the installation prompts, selecting the **Developer Default** setup when available. - - .. admonition:: Important - :class: important - - During installation, you will be prompted to create a **root password** for MySQL. **Keep this password secure and accessible**; resetting it can be time-consuming and difficult. - - .. image:: ../_static/password1.png - :width: 600 - :alt: MySQL Password Setup +- **Git**: Install from `Git Downloads `_ if needed. +- **Python**: Python 3.9 or newer is recommended. +- **NEAMS Workbench**: Required for the Workbench interface and SON validation tools. - .. image:: ../_static/password2.png - :width: 600 - :alt: MySQL Configuration +Clone ACCERT +------------ -Install NEAMS Workbench -~~~~~~~~~~~~~~~~~~~~~~~ - -1. **Download NEAMS Workbench**: - - - Go to `NEAMS Workbench Downloads `_. - - Download the `.exe` file for Windows. - -2. **Run the Installer**: - - - Execute the downloaded file and follow the installation instructions. - - **Note**: Your system may flag the installer as unsafe. The file is safe; proceed by selecting the option to keep or run the file. - -3. **Launch NEAMS Workbench**: - - - Open the Workbench application before proceeding to the next steps. - -Setting Up ACCERT -~~~~~~~~~~~~~~~~~ - -1. **Navigate to the `src` Directory**: +1. Create a directory for ACCERT. +2. Open Git Bash, Terminal, or another shell. +3. Clone the repository: .. code-block:: shell - $ cd src - -2. **Edit the `workbench.sh` File**: - - - Open `workbench.sh` in your Python coding application. - - Set the `workbench_path` variable to point to your NEAMS Workbench installation directory. - - For example: - - .. code-block:: shell - - workbench_path="C:/Path/To/Workbench-" - - - Replace `` with the actual version number (e.g., `Workbench-5.3.1`). - - **Ensure there are no spaces in the folder path**, as this may cause issues. - - Save the file. - -3. **Run the Setup Script**: - - - Open a terminal in the `src` directory. - - Execute the setup script: - - .. code-block:: shell - - $ ./setup_accert.sh - - - **Note**: If you encounter issues running the script, ensure that Git Bash or a Unix-compatible terminal is used. - -4. **Copy Executables to ACCERT `bin` Directory**: - - - Manually create a `bin` directory inside your `ACCERT` folder if it doesn't exist. - - Navigate to the `bin` folder inside your NEAMS Workbench installation. - - Copy `sonvalidxml` from the Workbench `bin` folder to the `ACCERT/bin` folder. - -Create and Edit `install.conf` -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -1. **Create `install.conf`**: - - - In the `src` directory, create a new file named `install.conf`. - -2. **Add the Following Content**: - - .. code-block:: ini - - [INSTALL] - PASSWD = yourpassword - - # NOTE: ALL OTHER information should be set up later - # INSTALL_PATH = /usr/local - # DATADIR = /mysql/data - # INSTALL_PACKAGE = - # EXP_DIR = - - - Replace ``yourpassword`` with your MySQL root password. - - Save the file with the exact name `install.conf`. - - **Ensure file extensions are visible**: - - In File Explorer, go to `View` > `Show` > `File name extensions`. - - Verify that the file is not saved as `install.conf.txt`. - -Install database -~~~~~~~~~~~~~~~~ - -1. **Navigate to ACCERT/src folder**: - - type "cmd" in the address bar of the file explorer and press enter. - - .. code-block:: shell - - $ mysql -h localhost -u root -p - - - Enter your MySQL root password when prompted. - - Run the following command to create the ACCERT database: - - .. code-block:: shell - - mysql> source accertdb.sql - - - Verify that the database has been created by running the following command: - - .. code-block:: shell - - mysql> show databases; - - - You should see `accert_db` in the list of databases. Then exit the MySQL shell by typing: - - .. code-block:: shell - - mysql> \q - - - - - -Installation on macOS ---------------------- - -.. _Installation on macOS: - -For certain aspects of this process, you will need administrative privileges. - -Cloning ACCERT -~~~~~~~~~~~~~~ - -1. **Create a Directory for ACCERT**: - - - Open your file explorer and create a folder where you want to install ACCERT (e.g., ``CODE``). - -2. **Open Git Bash or Terminal**: - - - Navigate to the directory you just created. - -3. **Clone the ACCERT Repository**: - - .. code-block:: shell - - $ cd /path/to/CODE - $ git clone https://github.com/accert-dev/ACCERT.git - $ cd ACCERT - - - Replace ``/path/to/CODE`` with the actual path to your ``CODE`` directory. - -4. **Verify Cloning**: - - - Ensure the ACCERT folder contains files similar to the ACCERT Git repository. - -Install MySQL Community Server -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -1. **Download MySQL**: - - - Visit `MySQL Community Server `_ and download the recommended version for macOS. - -2. **Install MySQL**: - - - Follow the installation prompts, selecting the **Developer Default** setup when available. - - Remember to keep your MySQL root password secure. - -Install NEAMS Workbench -~~~~~~~~~~~~~~~~~~~~~~~ - -1. **Download NEAMS Workbench**: - - - Go to `NEAMS Workbench Downloads `_. - - Download the `.dmg` file for macOS. - -2. **Run the Installer**: - - - Open the downloaded `.dmg` file and follow the installation instructions. - - **Note**: Your system may flag the installer as unsafe. The file is safe; proceed accordingly. - -3. **Launch NEAMS Workbench**: - - - Open the Workbench application before proceeding to the next steps. - -Setting Up ACCERT -~~~~~~~~~~~~~~~~~ - -1. **Navigate to the `src` Directory**: - - .. code-block:: shell - - $ cd src - -2. **Edit the `workbench.sh` File**: - - - Open `workbench.sh` in your Python coding application. - - Set the `workbench_path` variable to point to your NEAMS Workbench installation directory: - - .. code-block:: shell - - workbench_path="/Applications/Workbench-.app/Contents" - - - Replace `` with the actual version number (e.g., `Workbench-5.3.1`). - - Save the file. - -3. **Run the Setup Script**: - - - Make the setup script executable: - - .. code-block:: shell - - $ chmod +x setup_accert.sh - - - Execute the setup script: - - .. code-block:: shell - - $ ./setup_accert.sh - -Create and Edit `install.conf` -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -1. **Create `install.conf`**: - - - In the `src` directory, create a new file named `install.conf`. - -2. **Add the Following Content**: - - .. code-block:: ini - - [INSTALL] - PASSWD = yourpassword - - # NOTE: ALL OTHER information should be set up later - # INSTALL_PATH = /usr/local - # DATADIR = /mysql/data - # INSTALL_PACKAGE = - # EXP_DIR = - - - Replace ``yourpassword`` with your MySQL root password. - - Save the file with the exact name `install.conf`. - - **Ensure file extensions are visible**: - - In File Explorer, go to `View` > `Show` > `File name extensions`. - - Verify that the file is not saved as `install.conf.txt`. - -Install database -~~~~~~~~~~~~~~~~ - -1. ** connect to MySQL**: - .. code-block:: shell - - $ mysql -h localhost -u root -p - - - Enter your MySQL root password when prompted. - - Run the following command to create the ACCERT database: - - .. code-block:: shell - - mysql> source accertdb.sql - - - Verify that the database has been created by running the following command: - - .. code-block:: shell - - mysql> show databases; - - - You should see `accert_db` in the list of databases. Then exit the MySQL shell by typing: - - .. code-block:: shell - - mysql> \q - - - - - -Installation on Linux ---------------------- - -.. _Installation on Linux: - -For certain aspects of this process, you will need administrative privileges. - -Cloning ACCERT -~~~~~~~~~~~~~~ - -1. **Create a Directory for ACCERT**: - - - Open your file explorer and create a folder where you want to install ACCERT (e.g., ``CODE``). - -2. **Open Git Bash or Terminal**: - - - Navigate to the directory you just created. - -3. **Clone the ACCERT Repository**: - - .. code-block:: shell - - $ cd /path/to/CODE - $ git clone https://github.com/accert-dev/ACCERT.git - $ cd ACCERT - - - Replace ``/path/to/CODE`` with the actual path to your ``CODE`` directory. - -4. **Verify Cloning**: - - - Ensure the ACCERT folder contains files similar to the ACCERT Git repository. - -Install MySQL Community Server -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -1. **Download and Install MySQL**: - - - Install MySQL using your distribution's package manager, or download it from `MySQL Community Server `_. - - .. code-block:: shell - - # For Debian/Ubuntu - $ sudo apt-get update - $ sudo apt-get install mysql-server - - # For CentOS/RHEL - $ sudo yum install mysql-server - -2. **Secure MySQL Installation**: - - .. code-block:: shell - - $ sudo mysql_secure_installation - - - Set the root password and follow the prompts. + $ cd /path/to/CODE + $ git clone https://github.com/accert-dev/ACCERT.git + $ cd ACCERT Install NEAMS Workbench -~~~~~~~~~~~~~~~~~~~~~~~ - -1. **Download NEAMS Workbench**: - - - Go to `NEAMS Workbench Downloads `_. - - Download the `.tar.gz` file for Linux. - -2. **Extract and Install**: - - .. code-block:: shell - - $ tar -xzvf Workbench-.tar.gz - $ cd Workbench- - -3. **Run the Installer**: - - - Follow any additional installation instructions provided. - -4. **Launch NEAMS Workbench**: - - - Run the Workbench application before proceeding to the next steps. - -Setting Up ACCERT -~~~~~~~~~~~~~~~~~ - -1. **Navigate to the `src` Directory**: - - .. code-block:: shell - - $ cd src - -2. **Edit the `workbench.sh` File**: - - - Open `workbench.sh` in your Python coding application. - - Set the `workbench_path` variable to point to your NEAMS Workbench installation directory: - - .. code-block:: shell - - workbench_path="/path/to/Workbench-" - - - Replace `` with the actual version number. - - Save the file. - -3. **Run the Setup Script**: +----------------------- - - Make the setup script executable: +Download NEAMS Workbench from +`NEAMS Workbench Downloads `_. - .. code-block:: shell +- Windows: download and run the ``.exe`` installer. +- macOS: download and open the ``.dmg`` installer. +- Linux: download and extract the ``.tar.gz`` package, then follow the bundled instructions. - $ chmod +x setup_accert.sh +Launch Workbench once before running the ACCERT setup script. - - Execute the setup script: +Configure Workbench Path +------------------------ - .. code-block:: shell +Edit ``src/workbench.sh`` and set ``workbench_path`` to your Workbench +installation. Examples: - $ ./setup_accert.sh +.. code-block:: shell -Create and Edit `install.conf` -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + # Windows, from Git Bash + workbench_path="C:/Path/To/Workbench-5.3.1" -1. **Create `install.conf`**: + # macOS + workbench_path="/Applications/Workbench-5.3.1.app/Contents" - - In the `src` directory, create a new file named `install.conf`. + # Linux + workbench_path="/path/to/Workbench-5.3.1" -2. **Add the Following Content**: +Run Setup +--------- - .. code-block:: ini +From the ``src`` directory, run: - [INSTALL] - PASSWD = yourpassword +.. code-block:: shell - # NOTE: ALL OTHER information should be set up later - # INSTALL_PATH = /usr/local - # DATADIR = /mysql/data - # INSTALL_PACKAGE = - # EXP_DIR = + $ cd src + $ chmod +x setup_accert.sh + $ ./setup_accert.sh - - Replace ``yourpassword`` with your MySQL root password. - - Save the file with the exact name `install.conf`. +The setup script installs Python requirements, refreshes +``src/accertdb.sqlite`` from ``src/accertdb_sqlite_schema_data.sql``, copies the +Workbench integration file, and creates the local ``bin/sonvalidxml`` link. -Install database -~~~~~~~~~~~~~~~~ +SQLite Database +--------------- -1. ** connect to MySQL**: - .. code-block:: shell +The default database file is: - $ mysql -h localhost -u root -p +.. code-block:: shell - - Enter your MySQL root password when prompted. - - Run the following command to create the ACCERT database: - - .. code-block:: shell + src/accertdb.sqlite - mysql> source accertdb.sql - - - Verify that the database has been created by running the following command: - - .. code-block:: shell +To rebuild it manually, run: - mysql> show databases; +.. code-block:: shell - - You should see `accert_db` in the list of databases. Then exit the MySQL shell by typing: + $ python src/database_install.py - .. code-block:: shell +To use a different database file for a run, set ``ACCERT_SQLITE_DB``: - mysql> \q +.. code-block:: shell + $ ACCERT_SQLITE_DB=/path/to/accertdb.sqlite python src/Main.py tutorial/accert/ABR1000.son Testing the Installation -------------------------- -1. **Navigate to the Test Directory**: - - .. code-block:: shell - - $ cd ../test - -2. **Run Tests Using Pytest**: - - .. code-block:: shell - - $ pytest +------------------------ - - This will run the test suite to verify that ACCERT is installed correctly. +Run the test suite from the repository root: -Configuration with NEAMS Workbench ----------------------------------- +.. code-block:: shell -1. **Open NEAMS Workbench**. - -2. **Add ACCERT Configuration**: - - - Go to `Workbench` > `Configurations` and click `Add`. - - Select `Accert` from the drop-down menu and click `OK`. - -3. **Set Executable Path**: - - - In the configuration settings, set the **Executable** field to the full path of `Main.py` in the `ACCERT/src/` directory. - -4. **Load Grammar**: - - - In the configuration, click `Load Grammar` to load ACCERT's input grammar into Workbench. - -ACCERT Execution ----------------- - -**Through NEAMS Workbench** - - -- Press the `Run` button within the Workbench interface to execute ACCERT with your selected input file. - -**Through Command Line** - - -- Execute ACCERT using Python from the repository root: - - .. code-block:: shell - - $ cd ACCERT - $ python src/Main.py -i tutorial/accert/PWR12-BE.son - - - Replace ``tutorial/accert/PWR12-BE.son`` with your input file, such as - ``tutorial/accert/ABR1000.son`` or another ``.son`` file. + $ pytest Troubleshooting --------------- -- **Conda Errors**: - - - Ensure that Conda is correctly installed and accessible in your system's PATH. - - Running ``conda install -r requirements.txt`` should be done in the environment where ACCERT will run. - - If you encounter an error like ``bash: ./conda: Is a directory``, ensure you're referencing the correct path to the Conda executable. - -- **Workbench Connection Issues**: - - - If ACCERT cannot connect to Workbench: - - Verify that the `workbench_path` in `workbench.sh` is correct and does not contain spaces. - - Ensure that you have the necessary permissions to execute scripts. - -- **Password Issues**: - - - If you forget your MySQL root password, refer to MySQL's official documentation on how to reset the password. - - It's crucial to keep your password secure and accessible. - -- **File Extensions on Windows**: - - - Ensure that file extensions are visible: - - In File Explorer, go to `View` > `Show` > `File name extensions`. - - Verify that `install.conf` is not mistakenly saved as `install.conf.txt`. - -Additional Resources --------------------- - -- **ACCERT GitHub Repository**: - - `https://github.com/accert-dev/ACCERT `_ -- **NEAMS Workbench Documentation**: - - `NEAMS Workbench User Guide `_ - - - +- If setup cannot find Workbench, check the ``workbench_path`` value in + ``src/workbench.sh``. +- If ``sonvalidxml`` is missing, verify that Workbench has a ``bin`` directory + and rerun ``src/setup_accert.sh``. +- If the SQLite database is missing or stale, rerun + ``python src/database_install.py``. diff --git a/requirements.txt b/requirements.txt index 1b979ca..b8cf399 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,3 @@ -mysql.connector-python==8.0.24 prettytable==3.4.0 configparser openpyxl diff --git a/src/ACCERT_README.html b/src/ACCERT_README.html index 092e617..348eea3 100644 --- a/src/ACCERT_README.html +++ b/src/ACCERT_README.html @@ -338,7 +338,9 @@

Getting Started

href="https://www.ornl.gov/project/neams-workbench">NEAMS Workbench and relies on input files using Workbench’s SON format. Instructions for installing ACCERT both with and without Workbench are provided in this -README.

+README. ACCERT uses the bundled SQLite database at +src/accertdb.sqlite; no MySQL server or root password is +required.

Installation

Clone ACCERT