Add test-suite to run compiler and html generator

.gitignore

@@ -54,3 +54,5 @@ book/pp.tex
 
 # Auto-generated by Makefile's [gen] target
 harmony_model_checker/__init__.py
+
+.coverage

Makefile

@@ -42,8 +42,25 @@ upload-test: dist
 upload: dist
 	twine upload dist/*
 
+test-unit:
+	coverage run -m unittest discover tests/harmony
+
+test-e2e:
+	coverage run -m unittest discover tests/e2e
+
+test: test-unit test-e2e
+
 clean:
+	# Harmony outputs in `code` directory
 	rm -f code/*.htm code/*.hvm code/*.hco code/*.png code/*.hfa code/*.tla code/*.gv *.html
+
+	# Harmony outputs in `modules` directory
 	(cd harmony_model_checker/modules; rm -f *.htm *.hvm *.hco *.png *.hfa *.tla *.gv *.html)
+
 	rm -rf compiler_integration_results.md compiler_integration_results/
+
+	# Package publication related outputs
 	rm -rf build/ dist/ harmony_model_checker.egg-info/
+
+	# Test coverage related outputs
+	rm -rf .coverage htmlcov

code/BBsema.hny

@@ -1,7 +1,15 @@
-import synch;
+from synch import Semaphore, P, V;
 
 const NSLOTS = 2;     # size of bounded buffer
 
+buf = { x:() for x in {1..NSLOTS} };
+b_in = 1;
+b_out = 1;
+l_in = Semaphore(1);
+l_out = Semaphore(1);
+n_full = Semaphore(0);
+n_empty = Semaphore(NSLOTS);
+
 def produce(item):
     P(?n_empty);
     P(?l_in);
@@ -18,10 +26,3 @@ def consume():
     V(?l_out);
     V(?n_empty);
 ;
-buf = { x:() for x in {1..NSLOTS} };
-b_in = 1;
-b_out = 1;
-l_in = Semaphore(1);
-l_out = Semaphore(1);
-n_full = Semaphore(0);
-n_empty = Semaphore(NSLOTS);

code/BBsematest.hny

@@ -1,4 +1,4 @@
-import BBsema;
+from BBsema import produce, consume;
 
 const NPRODS = 3;     # number of producers
 const NCONSS = 3;     # number of consumers

harmony_model_checker/harmony/ast.py

@@ -1270,9 +1270,9 @@ def compile(self, scope, code, stmt):
         for var_or_cond in self.vars_and_conds:
             if var_or_cond[0] == 'var':
                 (_, var, expr, tkn, endtkn, op) = var_or_cond
-                stmt = self.range(token, endtkn)
+                stmt = self.range(tkn, endtkn)
                 expr.compile(ns, code, stmt)
-                code.append(StoreVarOp(var), token, op, stmt=stmt)
+                code.append(StoreVarOp(var), tkn, op, stmt=stmt)
                 self.define(ns, var)
             elif var_or_cond[0] == 'cond':
                 (_, cond, token, endtkn) = var_or_cond

requirements.txt

@@ -1,6 +1,7 @@
 antlr-denter==1.3.1
 antlr4-python3-runtime==4.9.3
 automata-lib==5.0.0
+coverage==6.3.2
 pydot==1.4.2
 requests==2.27.1
 twine==3.7.1

tests/README.md

@@ -0,0 +1,11 @@
+# Harmony Tests
+
+Types of tests:
+
+- Compilation/Performance Tests
+- Unit Tests
+
+## Compilation/Performance Tests
+
+## Unit Tests
+

tests/e2e/constants/base.hny

@@ -0,0 +1,3 @@
+
+const C = 23
+print C

tests/e2e/errors/bad_elif_stmt.hny

@@ -0,0 +1,5 @@
+
+if True:
+    pass
+else if:
+    pass

tests/e2e/errors/def_label.hny

@@ -0,0 +1,2 @@
+
+def: x = 3

tests/e2e/errors/eof_string.hny

@@ -0,0 +1,2 @@
+
+v = "Hello World

tests/e2e/errors/missing_indent.hny

@@ -0,0 +1,3 @@
+
+def ok():
+pass

tests/e2e/errors/unclosed_paren.hny

@@ -0,0 +1,2 @@
+
+a, b = (1, 5,

tests/e2e/load_test_files.py

@@ -0,0 +1,49 @@
+from datetime import timedelta
+import pathlib
+from typing import List, NamedTuple
+
+
+class Params(NamedTuple):
+    filename: pathlib.Path
+    max_time: timedelta
+    modules: List[str]
+    constants: List[str]
+
+def load_dir(dir: pathlib.Path, modules=None, constants=None):
+    return [
+        Params(
+            filename=f,
+            max_time=timedelta(seconds=3),
+            modules=modules or [],
+            constants=constants or [],
+        ) for f in dir.glob("*.hny")
+    ]
+
+_DIR = pathlib.Path(__file__).parent
+
+def load_public_harmony_files():
+    code_dir = _DIR.parent.parent / "code"
+    return load_dir(code_dir)
+
+def load_failing_harmony_files():
+    code_dir = _DIR / "errors"
+    return load_dir(code_dir)
+
+def load_constant_defined_harmony_files():
+    code_dir = _DIR / "constants"
+    return load_dir(code_dir, constants=['C=42'])
+
+def load_failing_constant_defined_harmony_files():
+    code_dir = _DIR / "constants"
+    return load_dir(code_dir, constants=['C=']) \
+         + load_dir(code_dir, constants=['C=42', 'A=12'])   # unused constants
+
+def load_module_defined_harmony_files():
+    code_dir = _DIR / "modules"
+    return load_dir(code_dir, modules=['math=resources/math'])
+
+def load_failing_module_defined_harmony_files():
+    code_dir = _DIR / "modules"
+    return load_dir(code_dir, modules=['math=resources/matt']) \
+         + load_dir(code_dir, modules=['math=resources/math',
+                                       'numpy=resources/numpy']) # unused modules

tests/e2e/modules/base.hny

@@ -0,0 +1,4 @@
+
+import math
+
+print math.sin(10)

tests/e2e/modules/resources/math.hny

@@ -0,0 +1,3 @@
+
+def sin(x):
+    result = "0.12423"

tests/e2e/test_compilation.py

@@ -0,0 +1,49 @@
+import unittest
+from tests.e2e.load_test_files import *
+
+import time
+from harmony_model_checker.exception import HarmonyCompilerError, HarmonyCompilerErrorCollection
+
+import harmony_model_checker.harmony.harmony as legacy_harmony
+from harmony_model_checker.compile import do_compile
+
+class CompilationTestCase(unittest.TestCase):
+    def run_before_tests(self):
+        legacy_harmony.files.clear()           # files that have been read already
+        legacy_harmony.modules.clear()         # modules modified with -m
+        legacy_harmony.used_modules.clear()    # modules modified and used
+        legacy_harmony.namestack.clear()       # stack of module names being compiled
+
+        legacy_harmony.imported.clear()
+        legacy_harmony.constants.clear()
+        legacy_harmony.used_constants.clear()
+
+    def test_compilation_success(self):
+        params = load_public_harmony_files() \
+            + load_constant_defined_harmony_files() \
+            + load_module_defined_harmony_files()
+        for param in params:
+            f = str(param.filename)
+            self.run_before_tests()
+            with self.subTest(f"Success compilation test: {f}"):
+                start_time = time.perf_counter_ns()
+                do_compile(f, consts=param.constants, mods=param.modules, interface=None)
+                end_time = time.perf_counter_ns()
+                duration = end_time - start_time
+                self.assertLessEqual(duration, param.max_time.total_seconds() * 1e9)
+
+    def test_compilation_error(self):
+        params = load_failing_harmony_files() \
+            + load_failing_constant_defined_harmony_files() \
+            + load_failing_module_defined_harmony_files()
+        for param in params:
+            f = str(param.filename)
+            self.run_before_tests()
+            with self.subTest(f"Failure compilation test: {f}"):
+                start_time = time.perf_counter_ns()
+                self.assertRaises(
+                    (HarmonyCompilerErrorCollection, HarmonyCompilerError),
+                    lambda: do_compile(f, consts=param.constants, mods=param.modules, interface=None))
+                end_time = time.perf_counter_ns()
+                duration = end_time - start_time
+                self.assertLessEqual(duration, param.max_time.total_seconds() * 1e9)

tests/e2e/test_gen_html.py

@@ -0,0 +1,67 @@
+import logging
+import subprocess
+import unittest
+from harmony_model_checker.main import handle_hco
+import harmony_model_checker.harmony.harmony as legacy_harmony
+
+from tests.e2e.load_test_files import *
+
+logger = logging.Logger(__file__)
+
+class MockNS:
+    B = None
+    noweb = True
+    const = None
+    mods = None
+    intf = None
+    module = None
+    cf = []
+    suppress = False
+
+_HARMONY_SCRIPT = pathlib.Path(__file__).parent.parent.parent / "harmony"
+
+def _replace_ext(p: pathlib.Path, ext: str):
+    p_ext = p.suffix
+    return str(p)[:-len(p_ext)] + '.' + ext
+
+class GenHtmlTestCase(unittest.TestCase):
+    def run_before_tests(self):
+        legacy_harmony.files.clear()           # files that have been read already
+        legacy_harmony.modules.clear()         # modules modified with -m
+        legacy_harmony.used_modules.clear()    # modules modified and used
+        legacy_harmony.namestack.clear()       # stack of module names being compiled
+
+        legacy_harmony.imported.clear()
+        legacy_harmony.constants.clear()
+        legacy_harmony.used_constants.clear()
+
+    def test_gen_html(self):
+        params = load_public_harmony_files() \
+            + load_constant_defined_harmony_files() \
+            + load_module_defined_harmony_files()
+        mock_ns = MockNS()
+        for param in params:
+            output_files = {
+                "hfa": None,
+                "htm": _replace_ext(param.filename, 'htm'),
+                "hco": _replace_ext(param.filename, 'hco'),
+                "hvm": _replace_ext(param.filename, 'hvm'),
+                "hvb": _replace_ext(param.filename, 'hvb'),
+                "png": None,
+                "tla": None,
+                "gv":  None
+            }
+            with self.subTest():
+                try:
+                    # If it takes longer than 3 seconds, just skip.
+                    r = subprocess.run(
+                        args=[_HARMONY_SCRIPT,
+                              str(param.filename), '--noweb'] +
+                              [('-c' + c) for c in param.constants] +
+                              [('-m' + m) for m in param.modules],
+                              timeout=3)
+                    self.assertEqual(r.returncode, 0)
+                except subprocess.TimeoutExpired:
+                    logger.warning("TimeoutExpired for file %s.", str(param.filename))
+                    continue
+                handle_hco(mock_ns, output_files)