diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..21476c34 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,67 @@ +# Git +.git +.gitignore + +# Python +__pycache__/ +*.py[cod] +*$py.class +.Python +env/ +venv/ +ENV/ +.venv/ +Pipfile.lock + +# Python Build / Distribution +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# C/C++ Build Artifacts +target/ +*.o +*.out +*.so +*.dylib +*.dll +CMakeCache.txt +CMakeFiles/ + +# Testing and Coverage +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# IDE and OS files +.vscode/ +.idea/ +*.swp +*.swo +.DS_Store +Thumbs.db + +Dockerfile* +docker-compose.yml +.dockerignore \ No newline at end of file diff --git a/.github/workflows/PR-review.yaml b/.github/workflows/PR-review.yaml new file mode 100644 index 00000000..b7dc2752 --- /dev/null +++ b/.github/workflows/PR-review.yaml @@ -0,0 +1,55 @@ +name: AI Code Reviewer + +on: + issue_comment: + types: [created] + +permissions: + contents: read + pull-requests: write + issues: write + +jobs: + gemini-code-review: + runs-on: ubuntu-latest + if: | + github.event.issue.pull_request && + contains(github.event.comment.body, '/gemini-review') + steps: + - name: PR Info + env: + #Assign untrusted inputs to environment variables first + COMMENT_BODY: ${{ github.event.comment.body }} + ISSUE_NUM: ${{ github.event.issue.number }} + REPO: ${{ github.repository }} + #Use shell variables ("$VAR") instead of template tags + run: | + echo "Comment: $COMMENT_BODY" + echo "Issue Number: $ISSUE_NUM" + echo "Repository: $REPO" + + - name: Checkout Repo + uses: actions/checkout@v3 + with: + fetch-depth: 0 + ref: refs/pull/${{ github.event.issue.number }}/head + + - name: Get PR Details + id: pr + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + REPO: ${{ github.repository }} + ISSUE_NUM: ${{ github.event.issue.number }} + #Use env vars for the API call to prevent injection + #Use quotes around variables to prevent word splitting + run: | + PR_JSON=$(gh api "repos/$REPO/pulls/$ISSUE_NUM") + echo "head_sha=$(echo "$PR_JSON" | jq -r .head.sha)" >> $GITHUB_OUTPUT + echo "base_sha=$(echo "$PR_JSON" | jq -r .base.sha)" >> $GITHUB_OUTPUT + + - uses: truongnh1992/gemini-ai-code-reviewer@main + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} + GEMINI_MODEL: gemini-2.5-flash + EXCLUDE: "*.md,*.txt,package-lock.json" \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..c61a1679 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,97 @@ +name: CI + +on: + push: + branches: [main, dev] + pull_request: + branches: [main, dev] + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + cache: 'pip' + + - name: Install ruff + run: pip install ruff + + - name: Check formatting + run: ruff format --check . + + - name: Lint + run: ruff check . --output-format=github + + test: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ['3.9', '3.10', '3.11', '3.12'] + + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + cache: 'pip' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements-ci.txt + + - name: Run tests + run: | + pytest --tb=short -q \ + --cov=concore_cli --cov=concore_base \ + --cov-report=term-missing \ + --ignore=measurements/ \ + --ignore=0mq/ \ + --ignore=ratc/ \ + --ignore=linktest/ + + java-test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: '17' + + - name: Download jeromq + run: | + curl -fsSL -o jeromq.jar https://repo1.maven.org/maven2/org/zeromq/jeromq/0.6.0/jeromq-0.6.0.jar + + - name: Compile Java tests + run: | + javac -cp jeromq.jar concoredocker.java TestLiteralEval.java TestConcoredockerApi.java + + - name: Run Java tests + run: | + java -cp .:jeromq.jar TestLiteralEval + java -cp .:jeromq.jar TestConcoredockerApi + + docker-build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Check if Dockerfile.py changed + uses: dorny/paths-filter@v3 + id: filter + with: + filters: | + dockerfile: + - 'Dockerfile.py' + - 'requirements.txt' + + - name: Validate Dockerfile build + if: steps.filter.outputs.dockerfile == 'true' + run: docker build -f Dockerfile.py -t concore-py-test . diff --git a/.github/workflows/greetings.yml b/.github/workflows/greetings.yml new file mode 100644 index 00000000..e9d7e175 --- /dev/null +++ b/.github/workflows/greetings.yml @@ -0,0 +1,41 @@ +name: Greetings + +on: + pull_request_target: + types: [opened] + issues: + types: [opened] + +jobs: + greeting: + runs-on: ubuntu-latest + permissions: + issues: write + pull-requests: write + steps: + - uses: actions/first-interaction@v3 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + + issue_message: | + 👋 Welcome to the CONTROL-CORE Project, @${{ github.actor }}! Thank you for opening your first issue in concore. + We appreciate your contribution to the organization and will review it as soon as possible. + + Before we get started, please check out these resources: + - 📚 [Project Documentation](https://control-core.readthedocs.io/) + - 📘 [Contribution Guidelines](https://github.com/ControlCore-Project/concore/blob/main/CONTRIBUTING.md) + - 📜 [Code of Conduct](https://github.com/ControlCore-Project/concore/blob/main/CODE_OF_CONDUCT.md) + + pr_message: | + 🎉 Welcome aboard, @${{ github.actor }}! Thank you for your first pull request in concore. + + Please ensure that you are contributing to the **dev** branch. + + Your contribution means a lot to us. We'll review it shortly. + + Please ensure you have reviewed our: + - 📚 [Project Documentation](https://control-core.readthedocs.io/) + - 📘 [Contribution Guidelines](https://github.com/ControlCore-Project/concore/blob/main/CONTRIBUTING.md) + - 📜 [Code of Conduct](https://github.com/ControlCore-Project/concore/blob/main/CODE_OF_CONDUCT.md) + + If you have any questions, feel free to ask. Happy coding! diff --git a/.gitignore b/.gitignore index 79b5594d..b323b3eb 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,38 @@ -**/.DS_Store +# Python bytecode/cache +__pycache__/ +*.pyc +*.pyo +*.pyd +*.py[cod] +*.class +*.so +.Python + +# Virtual environments +venv/ +env/ +ENV/ +.env/ +.venv/ + +# IDE +.vscode/ +.idea/ + +# Build/packaging +*.egg-info/ +dist/ +build/ + +# Testing +.pytest_cache/ +htmlcov/ +.coverage + +# Concore specific +concorekill.bat + +.claude +.codex +.cursor +_bmad diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..461fd44e --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,7 @@ +repos: + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.11.12 + hooks: + - id: ruff + args: [--output-format=full] + - id: ruff-format diff --git a/0mq/comm_node.py b/0mq/comm_node.py index edba31db..2602cec7 100644 --- a/0mq/comm_node.py +++ b/0mq/comm_node.py @@ -1,25 +1,21 @@ import concore -import concore2 concore.delay = 0.07 -concore2.delay = 0.07 -concore2.inpath = concore.inpath -concore2.outpath = concore.outpath -concore2.simtime = 0 +concore.simtime = 0 concore.default_maxtime(100) init_simtime_u = "[0.0, 0.0, 0.0]" init_simtime_ym = "[0.0, 0.0, 0.0]" u = concore.initval(init_simtime_u) -ym = concore2.initval(init_simtime_ym) -while(concore2.simtime 0): @@ -50,17 +46,17 @@ if 'U2' in concore.oport: concore.write(concore.oport['U2'], "u", u_data_values) - old_concore2_simtime = concore2.simtime - while concore2.unchanged() or concore2.simtime <= old_concore2_simtime: + old_concore_simtime = float(concore.simtime) + while concore.unchanged() or concore.simtime <= old_concore_simtime: # Assuming concore.iport['Y2'] is a file port (e.g., from pmpymax.py) - ym_data_values = concore2.read(concore.iport['Y2'], "ym", init_simtime_ym_str) - # time.sleep(concore2.delay) # Optional delay + ym_data_values = concore.read(concore.iport['Y2'], "ym", init_simtime_ym_str) + # time.sleep(concore.delay) # Optional delay - ym_full_to_send = [concore2.simtime] + ym_data_values + ym_full_to_send = [concore.simtime] + ym_data_values concore.write(PORT_NAME_F2_OUT, "ym_signal", ym_full_to_send) - print(f"funbody u={u_data_values} ym={ym_data_values} time={concore2.simtime}") + print(f"funbody u={u_data_values} ym={ym_data_values} time={concore.simtime}") print("funbody retry=" + str(concore.retrycount)) diff --git a/0mq/funbody_zmq.py b/0mq/funbody_zmq.py index 6a6b353f..103e4e66 100644 --- a/0mq/funbody_zmq.py +++ b/0mq/funbody_zmq.py @@ -1,7 +1,6 @@ # funbody2_zmq.py import time import concore -import concore2 print("funbody using ZMQ via concore") @@ -15,21 +14,18 @@ # Standard concore initializations concore.delay = 0.07 -concore2.delay = 0.07 -concore2.inpath = concore.inpath -concore2.outpath = concore.outpath -concore2.simtime = 0 +concore.simtime = 0 concore.default_maxtime(100) init_simtime_u_str = "[0.0, 0.0, 0.0]" init_simtime_ym_str = "[0.0, 0.0, 0.0]" u_data_values = concore.initval(init_simtime_u_str) -ym_data_values = concore2.initval(init_simtime_ym_str) +ym_data_values = concore.initval(init_simtime_ym_str) print(f"Initial u_data_values: {u_data_values}, ym_data_values: {ym_data_values}") print(f"Max time: {concore.maxtime}") -while concore2.simtime < concore.maxtime: +while concore.simtime < concore.maxtime: received_u_data = concore.read(PORT_NAME_F2_F1, "u_signal", init_simtime_u_str) if not (isinstance(received_u_data, list) and len(received_u_data) > 0): @@ -49,17 +45,19 @@ if 'U2' in concore.oport: concore.write(concore.oport['U2'], "u", u_data_values) - old_concore2_simtime = concore2.simtime - while concore2.unchanged() or concore2.simtime <= old_concore2_simtime: + # Take a numeric snapshot of the current simulation time to avoid + # inadvertently sharing a reference with concore.simtime. + old_concore_simtime = float(concore.simtime) + while concore.unchanged() or concore.simtime <= old_concore_simtime: # Assuming concore.iport['Y2'] is a file port (e.g., from pmpymax.py) - ym_data_values = concore2.read(concore.iport['Y2'], "ym", init_simtime_ym_str) - # time.sleep(concore2.delay) # Optional delay + ym_data_values = concore.read(concore.iport['Y2'], "ym", init_simtime_ym_str) + # time.sleep(concore.delay) # Optional delay - ym_full_to_send = [concore2.simtime] + ym_data_values + ym_full_to_send = [concore.simtime] + ym_data_values concore.write(PORT_NAME_F2_F1, "ym_signal", ym_full_to_send) - print(f"funbody u={u_data_values} ym={ym_data_values} time={concore2.simtime}") + print(f"funbody u={u_data_values} ym={ym_data_values} time={concore.simtime}") print("funbody retry=" + str(concore.retrycount)) diff --git a/0mq/funbody_zmq2.py b/0mq/funbody_zmq2.py index 04d94873..956af549 100644 --- a/0mq/funbody_zmq2.py +++ b/0mq/funbody_zmq2.py @@ -1,7 +1,6 @@ # funbody2_zmq.py import time import concore -import concore2 print("funbody using ZMQ via concore") @@ -15,21 +14,18 @@ # Standard concore initializations concore.delay = 0.07 -concore2.delay = 0.07 -concore2.inpath = concore.inpath -concore2.outpath = concore.outpath -concore2.simtime = 0 +concore.simtime = 0 concore.default_maxtime(100) init_simtime_u_str = "[0.0, 0.0, 0.0]" init_simtime_ym_str = "[0.0, 0.0, 0.0]" u_data_values = concore.initval(init_simtime_u_str) -ym_data_values = concore2.initval(init_simtime_ym_str) +ym_data_values = concore.initval(init_simtime_ym_str) print(f"Initial u_data_values: {u_data_values}, ym_data_values: {ym_data_values}") print(f"Max time: {concore.maxtime}") -while concore2.simtime < concore.maxtime: +while concore.simtime < concore.maxtime: received_u_data = concore.read(PORT_NAME_F2_OUT, "u_signal", init_simtime_u_str) if not (isinstance(received_u_data, list) and len(received_u_data) > 0): @@ -49,17 +45,17 @@ if 'U2' in concore.oport: concore.write(concore.oport['U2'], "u", u_data_values) - old_concore2_simtime = concore2.simtime - while concore2.unchanged() or concore2.simtime <= old_concore2_simtime: + old_concore_simtime = float(concore.simtime) + while concore.unchanged() or concore.simtime <= old_concore_simtime: # Assuming concore.iport['Y2'] is a file port (e.g., from pmpymax.py) - ym_data_values = concore2.read(concore.iport['Y2'], "ym", init_simtime_ym_str) - # time.sleep(concore2.delay) # Optional delay + ym_data_values = concore.read(concore.iport['Y2'], "ym", init_simtime_ym_str) + # time.sleep(concore.delay) # Optional delay - ym_full_to_send = [concore2.simtime] + ym_data_values + ym_full_to_send = [concore.simtime] + ym_data_values concore.write(PORT_NAME_F2_OUT, "ym_signal", ym_full_to_send) - print(f"funbody u={u_data_values} ym={ym_data_values} time={concore2.simtime}") + print(f"funbody u={u_data_values} ym={ym_data_values} time={concore.simtime}") print("funbody retry=" + str(concore.retrycount)) diff --git a/0mq/funcall.py b/0mq/funcall.py index b316ff1b..c5c63093 100644 --- a/0mq/funcall.py +++ b/0mq/funcall.py @@ -1,37 +1,35 @@ import concore -import concore2 from osparc_control import PairedTransmitter print("funcall 0mq") concore.delay = 0.07 -concore2.delay = 0.07 -concore2.inpath = concore.inpath -concore2.outpath = concore.outpath -concore2.simtime = 0 +concore.simtime = 0 concore.default_maxtime(100) init_simtime_u = "[0.0, 0.0, 0.0]" init_simtime_ym = "[0.0, 0.0, 0.0]" u = concore.initval(init_simtime_u) -ym = concore2.initval(init_simtime_ym) -while(concore2.simtime 0: response_time = received_ym_data[0] if isinstance(response_time, (int, float)): - concore2.simtime = response_time + concore.simtime = response_time ym = received_ym_data[1:] else: print(f"Warning: Received ZMQ data's first element is not time: {received_ym_data}. Using as is.") ym = received_ym_data else: print(f"Warning: Received unexpected ZMQ data format: {received_ym_data}. Using default ym.") - ym = concore2.initval(init_simtime_ym_str) + ym = concore.initval(init_simtime_ym_str) # Assuming concore.oport['Y'] is a file port (e.g., to cpymax.py) - concore2.write(concore.oport['Y'], "ym", ym) + concore.write(concore.oport['Y'], "ym", ym) - print(f"funcall ZMQ u={u} ym={ym} time={concore2.simtime}") + print(f"funcall ZMQ u={u} ym={ym} time={concore.simtime}") print("funcall retry=" + str(concore.retrycount)) diff --git a/0mq/funcall_zmq.py b/0mq/funcall_zmq.py index b2f991d2..ca12a784 100644 --- a/0mq/funcall_zmq.py +++ b/0mq/funcall_zmq.py @@ -1,7 +1,6 @@ # funcall2_zmq.py import time import concore -import concore2 print("funcall using ZMQ via concore") @@ -15,21 +14,18 @@ # Standard concore initializations concore.delay = 0.07 -concore2.delay = 0.07 -concore2.inpath = concore.inpath -concore2.outpath = concore.outpath -concore2.simtime = 0 +concore.simtime = 0 concore.default_maxtime(100) init_simtime_u_str = "[0.0, 0.0, 0.0]" init_simtime_ym_str = "[0.0, 0.0, 0.0]" u = concore.initval(init_simtime_u_str) -ym = concore2.initval(init_simtime_ym_str) +ym = concore.initval(init_simtime_ym_str) -print(f"Initial u: {u}, ym: {ym}, concore.simtime: {concore.simtime}, concore2.simtime: {concore2.simtime}") +print(f"Initial u: {u}, ym: {ym}, concore.simtime: {concore.simtime}, concore.simtime: {concore.simtime}") print(f"Max time: {concore.maxtime}") -while concore2.simtime < concore.maxtime: +while concore.simtime < concore.maxtime: while concore.unchanged(): # Assuming concore.iport['U'] is a file port (e.g., from cpymax.py) u = concore.read(concore.iport['U'], "u", init_simtime_u_str) @@ -44,19 +40,19 @@ if isinstance(received_ym_data, list) and len(received_ym_data) > 0: response_time = received_ym_data[0] if isinstance(response_time, (int, float)): - concore2.simtime = response_time + concore.simtime = response_time ym = received_ym_data[1:] else: print(f"Warning: Received ZMQ data's first element is not time: {received_ym_data}. Using as is.") ym = received_ym_data else: print(f"Warning: Received unexpected ZMQ data format: {received_ym_data}. Using default ym.") - ym = concore2.initval(init_simtime_ym_str) + ym = concore.initval(init_simtime_ym_str) # Assuming concore.oport['Y'] is a file port (e.g., to cpymax.py) - concore2.write(concore.oport['Y'], "ym", ym) + concore.write(concore.oport['Y'], "ym", ym) - print(f"funcall ZMQ u={u} ym={ym} time={concore2.simtime}") + print(f"funcall ZMQ u={u} ym={ym} time={concore.simtime}") print("funcall retry=" + str(concore.retrycount)) diff --git a/0mq/funcall_zmq2.py b/0mq/funcall_zmq2.py index 13c65842..168fc929 100644 --- a/0mq/funcall_zmq2.py +++ b/0mq/funcall_zmq2.py @@ -1,7 +1,6 @@ # funcall2_zmq.py import time import concore -import concore2 print("funcall using ZMQ via concore") @@ -15,21 +14,18 @@ # Standard concore initializations concore.delay = 0.07 -concore2.delay = 0.07 -concore2.inpath = concore.inpath -concore2.outpath = concore.outpath -concore2.simtime = 0 +concore.simtime = 0 concore.default_maxtime(100) init_simtime_u_str = "[0.0, 0.0, 0.0]" init_simtime_ym_str = "[0.0, 0.0, 0.0]" u = concore.initval(init_simtime_u_str) -ym = concore2.initval(init_simtime_ym_str) +ym = concore.initval(init_simtime_ym_str) -print(f"Initial u: {u}, ym: {ym}, concore.simtime: {concore.simtime}, concore2.simtime: {concore2.simtime}") +print(f"Initial u: {u}, ym: {ym}, concore.simtime: {concore.simtime}, concore.simtime: {concore.simtime}") print(f"Max time: {concore.maxtime}") -while concore2.simtime < concore.maxtime: +while concore.simtime < concore.maxtime: while concore.unchanged(): # Assuming concore.iport['U'] is a file port (e.g., from cpymax.py) u = concore.read(concore.iport['U'], "u", init_simtime_u_str) @@ -44,19 +40,19 @@ if isinstance(received_ym_data, list) and len(received_ym_data) > 0: response_time = received_ym_data[0] if isinstance(response_time, (int, float)): - concore2.simtime = response_time + concore.simtime = response_time ym = received_ym_data[1:] else: print(f"Warning: Received ZMQ data's first element is not time: {received_ym_data}. Using as is.") ym = received_ym_data else: print(f"Warning: Received unexpected ZMQ data format: {received_ym_data}. Using default ym.") - ym = concore2.initval(init_simtime_ym_str) + ym = concore.initval(init_simtime_ym_str) # Assuming concore.oport['Y'] is a file port (e.g., to cpymax.py) - concore2.write(concore.oport['Y'], "ym", ym) + concore.write(concore.oport['Y'], "ym", ym) - print(f"funcall ZMQ u={u} ym={ym} time={concore2.simtime}") + print(f"funcall ZMQ u={u} ym={ym} time={concore.simtime}") print("funcall retry=" + str(concore.retrycount)) diff --git a/DEV-GUIDE.md b/DEV-GUIDE.md new file mode 100644 index 00000000..6c0eb772 --- /dev/null +++ b/DEV-GUIDE.md @@ -0,0 +1,120 @@ +# Concore Developer Guide + +## `concore init --interactive` + +The interactive init wizard lets users scaffold a new multi-language concore project without writing any boilerplate by hand. + +### Usage + +```bash +concore init --interactive +# or shorthand +concore init -i +``` + +The wizard prompts for each supported language with a `y/n` question (default: yes): + +``` +Select the node types to include (Enter = yes) + + Include Python node? [Y/n] + Include C++ node? [Y/n] + Include Octave/MATLAB node? [Y/n] + Include Verilog node? [Y/n] + Include Java node? [Y/n] +``` + +### What gets generated + +For each selected language, the wizard: + +1. Creates a **source stub** in `src/` with the correct concore API calls for that language. +2. Adds an **unconnected node** to `workflow.graphml`, colour-coded and vertically positioned for easy viewing in yEd. + +Example output for Python + Java selected: + +``` +my_project/ +├── workflow.graphml ← 2 unconnected nodes +├── src/ +│ ├── script.py ← Python stub +│ └── Script.java ← Java stub +├── README.md +└── STUDY.json +``` + +### Architecture + +The feature lives entirely in `concore_cli/commands/init.py`: + +| Symbol | Role | +|---|---| +| `LANGUAGE_NODES` | Dict mapping language key → `label`, `filename`, node `color`, source `stub` | +| `GRAPHML_HEADER` | XML template for the GraphML wrapper; `project_name` is escaped via `xml.sax.saxutils.quoteattr` | +| `GRAPHML_NODE` | XML template for a single node block | +| `run_wizard()` | Prompts y/n for each language; returns list of selected keys | +| `_build_graphml()` | Assembles the full GraphML string from selected languages | +| `init_project_interactive()` | Orchestrates directory creation, file writes, and success output | + +--- + +## Adding a New Language + +Adding Julia (or any other language) to the interactive wizard is a **one-file change** — just add an entry to the `LANGUAGE_NODES` dictionary in `concore_cli/commands/init.py`. + +### Step-by-step: adding Julia + +**1. Add the entry to `LANGUAGE_NODES`** in `concore_cli/commands/init.py`: + +```python +"julia": { + "label": "Julia", + "filename": "script.jl", + "color": "#9558b2", # Julia purple + "stub": ( + "using Concore\n\n" + "Concore.state.delay = 0.02\n" + "Concore.state.inpath = \"./in\"\n" + "Concore.state.outpath = \"./out\"\n\n" + "maxtime = default_maxtime(100.0)\n" + 'init_val = "[0.0, 0.0]"\n\n' + "while Concore.state.simtime < maxtime\n" + " while unchanged()\n" + ' val = Float64.(concore_read(1, "data", init_val))\n' + " end\n" + " # TODO: process val\n" + " result = val .* 2\n" + ' concore_write(1, "result", result, 0.0)\n' + "end\n" + "println(\"retry=$(Concore.state.retrycount)\")\n" + ), +}, +``` + +Key Julia API points (based on real concore Julia scripts): + +| Element | Julia equivalent | +|---|---| +| Import | `using Concore` | +| Setup | `Concore.state.delay`, `.inpath`, `.outpath` | +| Max time | `default_maxtime(100.0)` — returns the value | +| Sim time | `Concore.state.simtime` | +| Unchanged check | `unchanged()` — no module prefix | +| Read | `concore_read(port, name, initstr)` — snake\_case, no prefix | +| Write | `concore_write(port, name, val, delta)` — snake\_case, no prefix | +| Type cast | `Float64.(concore_read(...))` to ensure numeric type | + +**2. That's it.** No other files need to change — the wizard, GraphML builder, and file writer all iterate over `LANGUAGE_NODES` dynamically. + +### Node colours used + +| Language | Hex colour | +|---|---| +| Python | `#ffcc00` (yellow) | +| C++ | `#ae85ca` (purple) | +| Octave/MATLAB | `#6db3f2` (blue) | +| Verilog | `#f28c8c` (red) | +| Java | `#a8d8a8` (green) | +| Julia *(proposed)* | `#9558b2` (Julia purple) | + +--- diff --git a/Dockerfile.java b/Dockerfile.java index 178c4512..b78f1e6b 100644 --- a/Dockerfile.java +++ b/Dockerfile.java @@ -1,14 +1,14 @@ -FROM openjdk:17-jdk-alpine +#build stage +FROM eclipse-temurin:17-jdk-alpine AS builder +WORKDIR /build +RUN apk add --no-cache wget +RUN wget -q "https://search.maven.org/remotecontent?filepath=org/zeromq/jeromq/0.6.0/jeromq-0.6.0.jar" -O /opt/jeromq.jar +COPY *.java . +RUN javac -cp /opt/jeromq.jar *.java -WORKDIR /app - -# Only copy the JAR if it exists -COPY ./target/concore-0.0.1-SNAPSHOT.jar /app/concore.jar || true - -# Ensure the JAR file is executable if present -RUN [ -f /app/concore.jar ] && chmod +x /app/concore.jar || true +#runtime stage +FROM eclipse-temurin:17-jre-alpine -EXPOSE 3000 - -# Run Java app only if the JAR exists, otherwise do nothing -CMD ["/bin/sh", "-c", "if [ -f /app/concore.jar ]; then java -jar /app/concore.jar; else echo 'No Java application found, exiting'; fi"] +WORKDIR /app +COPY --from=builder /build/*.class /app/ +COPY --from=builder /opt/jeromq.jar /app/jeromq.jar \ No newline at end of file diff --git a/Dockerfile.m b/Dockerfile.m index 54881fcb..02eaeff4 100644 --- a/Dockerfile.m +++ b/Dockerfile.m @@ -1,3 +1,8 @@ FROM mtmiller/octave + +USER root +RUN apt-get update && apt-get install -y --no-install-recommends octave-control octave-signal && rm -rf /var/lib/apt/lists/* +RUN echo "pkg load signal;" >> /etc/octave.conf && echo "pkg load control;" >> /etc/octave.conf + COPY . /src WORKDIR /src diff --git a/Dockerfile.py b/Dockerfile.py index fb13fb56..f769c959 100644 --- a/Dockerfile.py +++ b/Dockerfile.py @@ -1,10 +1,8 @@ -FROM jupyter/base-notebook +FROM python:3.10-slim -USER root -RUN apt-get update -RUN apt-get install -y build-essential g++ libgl1-mesa-glx libx11-6 -RUN conda install matplotlib scipy -RUN pip install cvxopt -COPY . /src WORKDIR /src +RUN apt-get update && apt-get install -y build-essential g++ libgl1 libx11-6 && rm -rf /var/lib/apt/lists/* +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt +COPY . . diff --git a/Dockerfile.sh b/Dockerfile.sh index 0e17693b..ccddea30 100644 --- a/Dockerfile.sh +++ b/Dockerfile.sh @@ -9,9 +9,12 @@ RUN mkdir /mcr-install \ WORKDIR /mcr-install -RUN wget https://ssd.mathworks.com/supportfiles/downloads/R2021a/Release/1/deployment_files/installer/complete/glnxa64/MATLAB_Runtime_R2021a_Update_1_glnxa64.zip +ARG MATLAB_RUNTIME_SHA256="b821022690804e498d2e5ad814dccb64aab17c5e4bc10a1e2a12498ef5364e0d" +ENV MATLAB_RUNTIME_SHA256=${MATLAB_RUNTIME_SHA256} -RUN unzip MATLAB_Runtime_R2021a_Update_1_glnxa64.zip \ +RUN wget https://ssd.mathworks.com/supportfiles/downloads/R2021a/Release/1/deployment_files/installer/complete/glnxa64/MATLAB_Runtime_R2021a_Update_1_glnxa64.zip \ + && echo "${MATLAB_RUNTIME_SHA256} MATLAB_Runtime_R2021a_Update_1_glnxa64.zip" | sha256sum -c - \ + && unzip MATLAB_Runtime_R2021a_Update_1_glnxa64.zip \ && ./install -destinationFolder /opt/mcr -agreeToLicense yes -mode silent \ && cd / \ && rm -rf mcr-install diff --git a/Dockerfile.v b/Dockerfile.v index e00aae73..ebe581a9 100644 --- a/Dockerfile.v +++ b/Dockerfile.v @@ -1,7 +1,7 @@ FROM ubuntu:20.04 RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ - iverilog + iverilog && rm -rf /var/lib/apt/lists/* COPY . /src WORKDIR /src diff --git a/README.md b/README.md index 815627c7..c5c570d4 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,16 @@ The CONTROL-CORE framework consists of the below projects. _concore_ enables composing studies from programs developed in different languages. Currently supported languages are, Python, Matlab/Octave, Verilog, and C++. The studies are designed through the visual _concore_ Editor (DHGWorkflow) and interpreted into _concore_ through its parser. Neural control systems consist of loops (dicycles). Therefore, they cannot be represented by classic workflow standards (such as CWL or WDL). Therefore, _concore_ addresses a significant research gap to model closed-loop neuromodulation control systems. The _concore_ protocol shares data between the programs through file sharing, with no centralized entity (a broker or an orchestrator) to arbitrate communications between the programs. (In the distributed executions, the CONTROL-CORE Mediator enables connecting the disjoint pieces of the study through REST APIs). +## Wire Format + +Concore payloads follow Python literal syntax compatible with `ast.literal_eval()`. The Python, C++, and Java implementations parse this shared format; the MATLAB and Verilog implementations currently support only flat numeric arrays derived from it. Supported value types include: + +* **Numbers** — integers and floats, including scientific notation (e.g., `1e3`, `-2.5`) +* **Booleans** — `True` / `False` (converted to `1.0` / `0.0` in numeric contexts) +* **Strings** — single- or double-quoted (e.g., `"start"`, `'label'`) +* **Nested arrays** — `[1, [2, 3]]` +* **Tuples** — `(1.0, 2.0)` (treated identically to arrays) + # Installation and Getting Started Guide @@ -28,8 +38,118 @@ Please follow the [ReadTheDocs](https://control-core.readthedocs.io/en/latest/in Installation instructions for concore can be found [here](https://control-core.readthedocs.io/en/latest/installation.html). Usage instructions can be found [here](https://control-core.readthedocs.io/en/latest/usage.html). +## Command-Line Interface (CLI) + +_concore_ now includes a command-line interface for easier workflow management. Install it with: + +```bash +pip install -e . +``` + +Quick start with the CLI: + +```bash +# Create a new project +concore init my-project + +# Validate your workflow +concore validate workflow.graphml + +# Compile your workflow +concore build workflow.graphml --auto-build + +# Monitor running processes +concore status + +# Stop all processes +concore stop +``` + +For detailed CLI documentation, see [concore_cli/README.md](concore_cli/README.md). + +## Configuration + +_concore_ supports customization through configuration files in the `CONCOREPATH` directory (defaults to the _concore_ installation directory): + +- **concore.tools** - Override tool paths (one per line, `KEY=value` format): + ``` + CPPEXE=/usr/local/bin/g++-12 + PYTHONEXE=/usr/bin/python3.11 + VEXE=/opt/iverilog/bin/iverilog + OCTAVEEXE=/snap/bin/octave + ``` + Supported keys: `CPPWIN`, `CPPEXE`, `VWIN`, `VEXE`, `PYTHONEXE`, `PYTHONWIN`, `MATLABEXE`, `MATLABWIN`, `OCTAVEEXE`, `OCTAVEWIN` + +- **concore.octave** - Treat `.m` files as Octave instead of MATLAB (presence = enabled) +- **concore.mcr** - MATLAB Compiler Runtime path (single line) +- **concore.sudo** - Docker command override (e.g., `docker` instead of `sudo docker`) +- **concore.repo** - Docker repository override + +Tool paths can also be set via environment variables (e.g., `CONCORE_CPPEXE=/usr/bin/g++`). Priority: config file > env var > defaults. + +### Docker Executable Configuration + +The Docker executable used by generated scripts (`build`, `run`, `stop`, `maxtime`, `params`, `unlock`) is controlled by the `DOCKEREXE` variable. It defaults to `docker` and can be overridden in three ways (highest priority first): + +1. **Config file** — Write the desired command into `concore.sudo` in your `CONCOREPATH` directory: + ``` + docker + ``` + This remains the highest-priority override, preserving backward compatibility. + +2. **Environment variable** — Set `DOCKEREXE` before running `mkconcore.py`: + ```bash + # Rootless Docker / Docker Desktop (macOS, Windows) + export DOCKEREXE="docker" + + # Podman + export DOCKEREXE="podman" + + # Traditional Linux with sudo + export DOCKEREXE="sudo docker" + ``` + +3. **Default** — If neither the config file nor the environment variable is set, `docker` is used. + +> **Note:** Previous versions defaulted to `sudo docker`, which failed on Docker Desktop (macOS/Windows), rootless Docker, and Podman. The new default (`docker`) works out of the box on those platforms. If you still need `sudo`, set `DOCKEREXE="sudo docker"` via the environment variable or `concore.sudo` config file. + +### Security Configuration + +Set a secure secret key for the Flask server before running in production: + +```bash +export FLASK_SECRET_KEY=$(python -c "import secrets; print(secrets.token_hex(32))") +``` + +Do **NOT** commit your secret key to version control. If `FLASK_SECRET_KEY` is not set, a temporary random key will be generated automatically (suitable for local development only). + For a detailed and more scientific documentation, please read our extensive [open-access research paper on CONTROL-CORE](https://doi.org/10.1109/ACCESS.2022.3161471). This paper has a complete discussion on the CONTROL-CORE architecture and deployment, together with the commands to execute the studies in different programming languages and programming environments (Ubuntu, Windows, MacOS, Docker, and distributed execution). +## C++ ZMQ Transport + +`concore.hpp` supports ZMQ-based communication as an opt-in transport alongside the default file-based I/O. + +To enable it, compile with `-DCONCORE_USE_ZMQ` and link against cppzmq: + +```bash +g++ -DCONCORE_USE_ZMQ my_node.cpp -lzmq -o my_node +``` + +In your C++ node, register a ZMQ port before reading or writing: + +```cpp +#include "concore.hpp" + +Concore c; +c.init_zmq_port("in1", "bind", "tcp://*:5555", "REP"); +c.init_zmq_port("out1", "connect", "tcp://localhost:5556", "REQ"); + +vector val = c.read("in1", "", "0.0"); +c.write("out1", "", val, 0); +``` + +Builds without `-DCONCORE_USE_ZMQ` are unaffected. + # The _concore_ Repository @@ -50,5 +170,6 @@ Please make sure to send your _concore_ pull requests to the [dev branch](https: If you use _concore_ in your research, please cite the below papers: -* Kathiravelu, P., Arnold, M., Vijay, S., Jagwani, R., Goyal, P., Goel, A.K., Li, N., Horn, C., Pan, T., Kothare, M. V., and Mahmoudi, B. **Distributed Executions with CONTROL-CORE Integrated Development Environment (IDE) for Closed-loop Neuromodulation Control Systems.** In Cluster Computing – The Journal of Networks Software Tools and Applications (CLUSTER). May 2025. Accepted. Springer. +* Kathiravelu, P., Arnold, M., Vijay, S., Jagwani, R., Goyal, P., Goel, A.K., Li, N., Horn, C., Pan, T., Kothare, M. V., and Mahmoudi, B. **Distributed Executions with CONTROL-CORE Integrated Development Environment (IDE) for Closed-loop Neuromodulation Control Systems.** In Cluster Computing – The Journal of Networks Software Tools and Applications (CLUSTER). 28, 697. September 2025. Springer. https://doi.org/10.1007/s10586-025-05476-w + * Kathiravelu, P., Arnold, M., Fleischer, J., Yao, Y., Awasthi, S., Goel, A. K., Branen, A., Sarikhani, P., Kumar, G., Kothare, M. V., and Mahmoudi, B. **CONTROL-CORE: A Framework for Simulation and Design of Closed-Loop Peripheral Neuromodulation Control Systems**. In IEEE Access. March 2022. https://doi.org/10.1109/ACCESS.2022.3161471 diff --git a/TestConcoreHpp.cpp b/TestConcoreHpp.cpp new file mode 100644 index 00000000..64c77224 --- /dev/null +++ b/TestConcoreHpp.cpp @@ -0,0 +1,345 @@ +/** + * TestConcoreHpp.cpp + * + * Test suite for the Concore class API in concore.hpp. + * Covers: read_FM/write_FM round-trip, unchanged(), initval(), + * mapParser(), default_maxtime(), and tryparam(). + * + * Addresses Issue #484: adds coverage for the Concore class API. + * + * Compile: g++ -std=c++11 -o TestConcoreHpp TestConcoreHpp.cpp + * Run: ./TestConcoreHpp (Linux/macOS) + * TestConcoreHpp.exe (Windows) + */ + +#include +#include +#include +#include +#include +#include +#include + +#ifdef _WIN32 + #include + #define MAKE_DIR(p) _mkdir(p) +#else + #include + #define MAKE_DIR(p) mkdir(p, 0755) +#endif + +#include "concore.hpp" + +static int passed = 0; +static int failed = 0; + +// ------------- helpers --------------------------------------------------- + +static void check(const std::string& testName, bool condition) { + if (condition) { + std::cout << "PASS: " << testName << std::endl; + ++passed; + } else { + std::cout << "FAIL: " << testName << std::endl; + ++failed; + } +} + +static bool approx(double a, double b, double eps = 1e-9) { + return std::fabs(a - b) < eps; +} + +static void write_file(const std::string& path, const std::string& data) { + std::ofstream f(path); + f << data; +} + +static void rm(const std::string& path) { + std::remove(path.c_str()); +} + +static void setup_dirs() { + MAKE_DIR("in"); + MAKE_DIR("in/1"); + MAKE_DIR("in1"); + MAKE_DIR("out1"); +} + +// ------------- read_FM --------------------------------------------------- + +static void test_read_FM_file() { + setup_dirs(); + write_file("in1/v", "[3.0,0.5,1.5]"); + + Concore c; + c.delay = 0; + c.simtime = 0.0; + + std::vector v = c.read_FM(1, "v", "[0.0]"); + check("read_FM size==2", v.size() == 2); + check("read_FM[0]==0.5", approx(v[0], 0.5)); + check("read_FM[1]==1.5", approx(v[1], 1.5)); + check("read_FM simtime updated", approx(c.simtime, 3.0)); + + rm("in1/v"); +} + +static void test_read_FM_missing_file_uses_initstr() { + setup_dirs(); + rm("in1/no_port"); + + Concore c; + c.delay = 0; + c.simtime = 0.0; + + std::vector v = c.read_FM(1, "no_port", "[9.0,3.0]"); + check("missing_file fallback size==1", v.size() == 1); + check("missing_file fallback val==3.0", approx(v[0], 3.0)); +} + +static void test_read_result_missing_file_status() { + setup_dirs(); + rm("in1/no_port_status"); + + Concore c; + c.delay = 0; + c.simtime = 0.0; + + Concore::ReadResult r = c.read_result(1, "no_port_status", "[9.0,3.0]"); + check("read_result missing status FILE_NOT_FOUND", + r.status == Concore::ReadStatus::FILE_NOT_FOUND); + check("read_result missing uses initstr", r.data.size() == 1 && approx(r.data[0], 3.0)); +} + +// ------------- write_FM -------------------------------------------------- + +static void test_write_FM_creates_file() { + setup_dirs(); + + Concore c; + c.delay = 0; + c.simtime = 2.0; + + c.write_FM(1, "w_out", {10.0, 20.0}); + + std::ifstream f("out1/w_out"); + std::ostringstream ss; + ss << f.rdbuf(); + std::string content = ss.str(); + + check("write_FM file not empty", !content.empty()); + check("write_FM contains value 10", content.find("10") != std::string::npos); + check("write_FM contains simtime 2", content.find("2") != std::string::npos); + + rm("out1/w_out"); +} + +// ------------- unchanged() ----------------------------------------------- + +static void test_unchanged_after_read_is_false() { + setup_dirs(); + write_file("in1/uc", "[1.0,0.1]"); + + Concore c; + c.delay = 0; + c.simtime = 0.0; + + c.read_FM(1, "uc", "[0.0]"); + check("unchanged after read is false", !c.unchanged()); + + rm("in1/uc"); +} + +static void test_unchanged_fresh_object_is_true() { + Concore c; + c.delay = 0; + check("unchanged fresh object is true", c.unchanged()); +} + +static void test_unchanged_second_call_after_false_is_true() { + setup_dirs(); + write_file("in1/uc2", "[5.0,7.0]"); + + Concore c; + c.delay = 0; + c.simtime = 0.0; + + c.read_FM(1, "uc2", "[0.0]"); + c.unchanged(); + bool u = c.unchanged(); + check("unchanged second call is true", u); + + rm("in1/uc2"); +} + +// ------------- retry exhaustion on empty file ---------------------------- + +static void test_retry_empty_file_falls_back_to_initstr() { + setup_dirs(); + write_file("in1/empty_port", ""); + + Concore c; + c.delay = 0; + c.simtime = 0.0; + + std::vector v = c.read_FM(1, "empty_port", "[7.0,5.0]"); + check("retry exhaustion: initstr used", v.size() == 1 && approx(v[0], 5.0)); + check("retry exhaustion: retrycount>0", c.retrycount > 0); + + rm("in1/empty_port"); +} + +// ------------- initval() ------------------------------------------------- + +static void test_initval_parses_simtime_and_data() { + Concore c; + c.delay = 0; + + std::vector v = c.initval("[4.0,1.0,2.0]"); + check("initval size==2", v.size() == 2); + check("initval[0]==1.0", approx(v[0], 1.0)); + check("initval[1]==2.0", approx(v[1], 2.0)); + check("initval simtime==4.0", approx(c.simtime, 4.0)); +} + +static void test_initval_empty_input_returns_empty() { + Concore c; + c.delay = 0; + + std::vector v = c.initval("[]"); + check("initval empty returns empty", v.empty()); +} + +// ------------- mapParser() ----------------------------------------------- + +static void test_mapParser_reads_file() { + setup_dirs(); + write_file("tmp_iport_test.txt", "{'u': 1}"); + + Concore c; + c.delay = 0; + + auto m = c.mapParser("tmp_iport_test.txt"); + check("mapParser has key u", m.count("u") == 1); + check("mapParser value==1", m["u"] == 1); + + rm("tmp_iport_test.txt"); +} + +static void test_mapParser_missing_file_is_empty() { + rm("concore_noport_tmp.txt"); + + Concore c; + c.delay = 0; + + auto m = c.mapParser("concore_noport_tmp.txt"); + check("mapParser missing file is empty", m.empty()); +} + +// ------------- default_maxtime() ----------------------------------------- + +static void test_default_maxtime_reads_file() { + setup_dirs(); + write_file("in/1/concore.maxtime", "200"); + + Concore c; + c.delay = 0; + c.default_maxtime(100); + + check("default_maxtime from file==200", c.maxtime == 200); + + rm("in/1/concore.maxtime"); +} + +static void test_default_maxtime_fallback() { + rm("in/1/concore.maxtime"); + + Concore c; + c.delay = 0; + c.default_maxtime(42); + + check("default_maxtime fallback==42", c.maxtime == 42); +} + +// ------------- tryparam() ------------------------------------------------ + +static void test_tryparam_found() { + setup_dirs(); + write_file("in/1/concore.params", "{'alpha': '0.5', 'beta': '1.0'}"); + + Concore c; + c.delay = 0; + c.load_params(); + + check("tryparam found alpha", c.tryparam("alpha", "0.0") == "0.5"); + check("tryparam found beta", c.tryparam("beta", "0.0") == "1.0"); + + rm("in/1/concore.params"); +} + +static void test_tryparam_missing_key_uses_default() { + rm("in/1/concore.params"); + + Concore c; + c.delay = 0; + c.load_params(); + + check("tryparam missing key uses default", + c.tryparam("no_key", "def_val") == "def_val"); +} + +static void test_load_params_semicolon_format() { + setup_dirs(); + write_file("in/1/concore.params", "a=1;b=2"); + + Concore c; + c.delay = 0; + c.load_params(); + + check("load_params semicolon parses a", c.tryparam("a", "") == "1"); + check("load_params semicolon parses b", c.tryparam("b", "") == "2"); + + rm("in/1/concore.params"); +} + +// ------------- main ------------------------------------------------------- + +int main() { + std::cout << "===== Concore API Tests (Issue #484) =====\n\n"; + + // read_FM / write_FM + test_read_FM_file(); + test_read_FM_missing_file_uses_initstr(); + test_read_result_missing_file_status(); + test_write_FM_creates_file(); + + // unchanged() + test_unchanged_after_read_is_false(); + test_unchanged_fresh_object_is_true(); + test_unchanged_second_call_after_false_is_true(); + + // retry exhaustion on empty file + test_retry_empty_file_falls_back_to_initstr(); + + // initval() + test_initval_parses_simtime_and_data(); + test_initval_empty_input_returns_empty(); + + // mapParser() + test_mapParser_reads_file(); + test_mapParser_missing_file_is_empty(); + + // default_maxtime() + test_default_maxtime_reads_file(); + test_default_maxtime_fallback(); + + // tryparam() + test_tryparam_found(); + test_tryparam_missing_key_uses_default(); + test_load_params_semicolon_format(); + + std::cout << "\n=== Results: " << passed << " passed, " << failed + << " failed out of " << (passed + failed) << " tests ===\n"; + + return (failed > 0) ? 1 : 0; +} diff --git a/TestConcoredockerApi.java b/TestConcoredockerApi.java new file mode 100644 index 00000000..90472430 --- /dev/null +++ b/TestConcoredockerApi.java @@ -0,0 +1,233 @@ +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.*; + +/** + * Tests for concoredocker read(), write(), unchanged(), initVal() + * using temp directories for file-based IPC. + */ +public class TestConcoredockerApi { + static int passed = 0; + static int failed = 0; + + public static void main(String[] args) { + // zero delay so tests don't sleep for 1s per read() + concoredocker.setDelay(0); + + testWriteProducesCorrectFormat(); + testReadParsesFileAndStripsSimtime(); + testReadWriteRoundtrip(); + testSimtimeAdvancesWithDelta(); + testUnchangedReturnsFalseAfterRead(); + testUnchangedReturnsTrueOnSameData(); + testInitValExtractsSimtime(); + testInitValReturnsRemainingValues(); + testOutputFileMatchesPythonWireFormat(); + testReadFileNotFound(); + testReadRetriesExceeded(); + testReadParseError(); + + System.out.println("\n=== Results: " + passed + " passed, " + failed + " failed out of " + (passed + failed) + " tests ==="); + if (failed > 0) { + System.exit(1); + } + } + + static void check(String testName, Object expected, Object actual) { + if (Objects.equals(expected, actual)) { + System.out.println("PASS: " + testName); + passed++; + } else { + System.out.println("FAIL: " + testName + " | expected: " + expected + " | actual: " + actual); + failed++; + } + } + + static Path makeTempDir() { + try { + return Files.createTempDirectory("concore_test_"); + } catch (IOException e) { + throw new RuntimeException("Failed to create temp dir", e); + } + } + + /** Creates temp dir with port subdirectory ready for write(). */ + static Path makeTempDir(int port) { + Path tmp = makeTempDir(); + try { + Files.createDirectories(tmp.resolve(String.valueOf(port))); + } catch (IOException e) { + throw new RuntimeException(e); + } + return tmp; + } + + static void writeFile(Path base, int port, String name, String content) { + try { + Path dir = base.resolve(String.valueOf(port)); + Files.createDirectories(dir); + Files.write(dir.resolve(name), content.getBytes()); + } catch (IOException e) { + throw new RuntimeException("Failed to write test file", e); + } + } + + static String readFile(Path base, int port, String name) { + try { + return new String(Files.readAllBytes(base.resolve(String.valueOf(port)).resolve(name))); + } catch (IOException e) { + throw new RuntimeException("Failed to read test file", e); + } + } + + static void testWriteProducesCorrectFormat() { + Path tmp = makeTempDir(1); + concoredocker.resetState(); + concoredocker.setOutPath(tmp.toString()); + + List vals = new ArrayList<>(); + vals.add(10.0); + vals.add(20.0); + concoredocker.write(1, "signal", vals, 1); + + String content = readFile(tmp, 1, "signal"); + @SuppressWarnings("unchecked") + List parsed = (List) concoredocker.literalEval(content); + check("write: simtime+delta as first element", 1.0, parsed.get(0)); + check("write: val1 correct", 10.0, parsed.get(1)); + check("write: val2 correct", 20.0, parsed.get(2)); + } + + static void testReadParsesFileAndStripsSimtime() { + Path tmp = makeTempDir(); + concoredocker.resetState(); + concoredocker.setInPath(tmp.toString()); + writeFile(tmp, 1, "sensor", "[0.0, 42.0, 99.0]"); + + concoredocker.ReadResult result = concoredocker.read(1, "sensor", "[0.0, 0.0, 0.0]"); + check("read: status SUCCESS", concoredocker.ReadStatus.SUCCESS, result.status); + check("read: strips simtime, size=2", 2, result.data.size()); + check("read: val1 correct", 42.0, result.data.get(0)); + check("read: val2 correct", 99.0, result.data.get(1)); + } + + static void testReadWriteRoundtrip() { + Path tmp = makeTempDir(1); + concoredocker.resetState(); + concoredocker.setInPath(tmp.toString()); + concoredocker.setOutPath(tmp.toString()); + + List outVals = new ArrayList<>(); + outVals.add(7.0); + outVals.add(8.0); + concoredocker.write(1, "data", outVals, 1); + + concoredocker.ReadResult inVals = concoredocker.read(1, "data", "[0.0, 0.0, 0.0]"); + check("roundtrip: status", concoredocker.ReadStatus.SUCCESS, inVals.status); + check("roundtrip: size", 2, inVals.data.size()); + check("roundtrip: val1", 7.0, inVals.data.get(0)); + check("roundtrip: val2", 8.0, inVals.data.get(1)); + } + + static void testSimtimeAdvancesWithDelta() { + Path tmp = makeTempDir(1); + concoredocker.resetState(); + concoredocker.setInPath(tmp.toString()); + concoredocker.setOutPath(tmp.toString()); + + List v = Collections.singletonList((Object) 1.0); + + // iteration 1: simtime=0, delta=1 -> file has [1.0, 1.0], read -> simtime becomes 1.0 + concoredocker.write(1, "tick", v, 1); + concoredocker.read(1, "tick", "[0.0, 0.0]"); + check("simtime after iter 1", 1.0, concoredocker.getSimtime()); + + // iteration 2: write again with delta=1 -> file has [2.0, 1.0], read -> simtime becomes 2.0 + concoredocker.write(1, "tick", v, 1); + concoredocker.read(1, "tick", "[0.0, 0.0]"); + check("simtime after iter 2", 2.0, concoredocker.getSimtime()); + } + + static void testUnchangedReturnsFalseAfterRead() { + Path tmp = makeTempDir(); + concoredocker.resetState(); + concoredocker.setInPath(tmp.toString()); + writeFile(tmp, 1, "sig", "[0.0, 5.0]"); + + concoredocker.read(1, "sig", "[0.0, 0.0]"); + check("unchanged: false right after read", false, concoredocker.unchanged()); + } + + static void testUnchangedReturnsTrueOnSameData() { + Path tmp = makeTempDir(); + concoredocker.resetState(); + concoredocker.setInPath(tmp.toString()); + writeFile(tmp, 1, "sig", "[0.0, 5.0]"); + + concoredocker.read(1, "sig", "[0.0, 0.0]"); + concoredocker.unchanged(); // first call: false, locks olds = s + check("unchanged: true on second call with same data", true, concoredocker.unchanged()); + } + + static void testInitValExtractsSimtime() { + concoredocker.resetState(); + concoredocker.initVal("[2.0, 10, 20]"); + check("initVal: simtime extracted", 2.0, concoredocker.getSimtime()); + } + + static void testInitValReturnsRemainingValues() { + concoredocker.resetState(); + List result = concoredocker.initVal("[3.5, 100, 200]"); + check("initVal: size of returned list", 2, result.size()); + check("initVal: first remaining val", 100, result.get(0)); + check("initVal: second remaining val", 200, result.get(1)); + } + + static void testOutputFileMatchesPythonWireFormat() { + Path tmp = makeTempDir(1); + concoredocker.resetState(); + concoredocker.setOutPath(tmp.toString()); + + List vals = new ArrayList<>(); + vals.add(1.0); + vals.add(2.0); + concoredocker.write(1, "out", vals, 0); + + String raw = readFile(tmp, 1, "out"); + check("wire format: starts with '['", true, raw.startsWith("[")); + check("wire format: ends with ']'", true, raw.endsWith("]")); + Object reparsed = concoredocker.literalEval(raw); + check("wire format: re-parseable as list", true, reparsed instanceof List); + } + + static void testReadFileNotFound() { + Path tmp = makeTempDir(); + concoredocker.resetState(); + concoredocker.setInPath(tmp.toString()); + // no file written, port 1/missing does not exist + concoredocker.ReadResult result = concoredocker.read(1, "missing", "[0.0, 0.0]"); + check("read file not found: status", concoredocker.ReadStatus.FILE_NOT_FOUND, result.status); + check("read file not found: data is default", 1, result.data.size()); + } + + static void testReadRetriesExceeded() { + Path tmp = makeTempDir(); + concoredocker.resetState(); + concoredocker.setInPath(tmp.toString()); + writeFile(tmp, 1, "empty", ""); // always empty, exhausts retries + concoredocker.ReadResult result = concoredocker.read(1, "empty", "[0.0, 0.0]"); + check("read retries exceeded: status", concoredocker.ReadStatus.RETRIES_EXCEEDED, result.status); + check("read retries exceeded: data is default", 1, result.data.size()); + } + + static void testReadParseError() { + Path tmp = makeTempDir(); + concoredocker.resetState(); + concoredocker.setInPath(tmp.toString()); + writeFile(tmp, 1, "bad", "not_a_valid_list"); + concoredocker.ReadResult result = concoredocker.read(1, "bad", "[0.0, 0.0]"); + check("read parse error: status", concoredocker.ReadStatus.PARSE_ERROR, result.status); + check("read parse error: data is default", 1, result.data.size()); + } +} diff --git a/TestLiteralEval.java b/TestLiteralEval.java new file mode 100644 index 00000000..5e36d325 --- /dev/null +++ b/TestLiteralEval.java @@ -0,0 +1,313 @@ +import java.util.*; + +/** + * Test suite for concoredocker.literalEval() recursive descent parser. + * Covers: dicts, lists, tuples, numbers, strings, booleans, None, + * nested structures, escape sequences, scientific notation, + * fractional simtime, and related round-trip parsing behavior. + */ +public class TestLiteralEval { + static int passed = 0; + static int failed = 0; + + public static void main(String[] args) { + testEmptyDict(); + testSimpleDict(); + testDictWithIntValues(); + testEmptyList(); + testSimpleList(); + testListOfDoubles(); + testNestedDictWithList(); + testNestedListsDeep(); + testBooleansAndNone(); + testStringsWithCommas(); + testStringsWithColons(); + testStringEscapeSequences(); + testScientificNotation(); + testNegativeNumbers(); + testTuple(); + testTrailingComma(); + testToPythonLiteralBooleans(); + testToPythonLiteralNone(); + testToPythonLiteralString(); + testFractionalSimtime(); + testRoundTripSerialization(); + testStringEscapingSerialization(); + testUnterminatedList(); + testUnterminatedDict(); + testUnterminatedTuple(); + testNonStringDictKey(); + testJsonKeywordTrue(); + testJsonKeywordFalse(); + testJsonKeywordNull(); + testJsonMixedList(); + testJsonRoundTrip(); + + System.out.println("\n=== Results: " + passed + " passed, " + failed + " failed out of " + (passed + failed) + " tests ==="); + if (failed > 0) { + System.exit(1); + } + } + + static void check(String testName, Object expected, Object actual) { + if (Objects.equals(expected, actual)) { + System.out.println("PASS: " + testName); + passed++; + } else { + System.out.println("FAIL: " + testName + " | expected: " + expected + " | actual: " + actual); + failed++; + } + } + + static void testEmptyDict() { + Object result = concoredocker.literalEval("{}"); + check("empty dict", new HashMap<>(), result); + } + + static void testSimpleDict() { + @SuppressWarnings("unchecked") + Map result = (Map) concoredocker.literalEval("{'PYM': 1}"); + check("simple dict key", true, result.containsKey("PYM")); + check("simple dict value", 1, result.get("PYM")); + } + + static void testDictWithIntValues() { + @SuppressWarnings("unchecked") + Map result = (Map) concoredocker.literalEval("{'a': 10, 'b': 20}"); + check("dict int value a", 10, result.get("a")); + check("dict int value b", 20, result.get("b")); + } + + static void testEmptyList() { + Object result = concoredocker.literalEval("[]"); + check("empty list", new ArrayList<>(), result); + } + + static void testSimpleList() { + @SuppressWarnings("unchecked") + List result = (List) concoredocker.literalEval("[1, 2, 3]"); + check("simple list size", 3, result.size()); + check("simple list[0]", 1, result.get(0)); + check("simple list[2]", 3, result.get(2)); + } + + static void testListOfDoubles() { + @SuppressWarnings("unchecked") + List result = (List) concoredocker.literalEval("[0.0, 1.5, 2.7]"); + check("list doubles[0]", 0.0, result.get(0)); + check("list doubles[1]", 1.5, result.get(1)); + } + + static void testNestedDictWithList() { + @SuppressWarnings("unchecked") + Map result = (Map) concoredocker.literalEval("{'key': [1, 2, 3]}"); + check("nested dict has key", true, result.containsKey("key")); + @SuppressWarnings("unchecked") + List inner = (List) result.get("key"); + check("nested list size", 3, inner.size()); + check("nested list[0]", 1, inner.get(0)); + } + + static void testNestedListsDeep() { + @SuppressWarnings("unchecked") + List result = (List) concoredocker.literalEval("[[1, 2], [3, 4]]"); + check("nested lists size", 2, result.size()); + @SuppressWarnings("unchecked") + List inner = (List) result.get(0); + check("inner list[0]", 1, inner.get(0)); + check("inner list[1]", 2, inner.get(1)); + } + + static void testBooleansAndNone() { + @SuppressWarnings("unchecked") + List result = (List) concoredocker.literalEval("[True, False, None]"); + check("boolean True", Boolean.TRUE, result.get(0)); + check("boolean False", Boolean.FALSE, result.get(1)); + check("None", null, result.get(2)); + } + + static void testStringsWithCommas() { + @SuppressWarnings("unchecked") + Map result = (Map) concoredocker.literalEval("{'key': 'hello, world'}"); + check("string with comma", "hello, world", result.get("key")); + } + + static void testStringsWithColons() { + @SuppressWarnings("unchecked") + Map result = (Map) concoredocker.literalEval("{'url': 'http://example.com'}"); + check("string with colon", "http://example.com", result.get("url")); + } + + static void testStringEscapeSequences() { + Object result = concoredocker.literalEval("'hello\\nworld'"); + check("escaped newline", "hello\nworld", result); + } + + static void testScientificNotation() { + Object result = concoredocker.literalEval("1.5e3"); + check("scientific notation", 1500.0, result); + } + + static void testNegativeNumbers() { + @SuppressWarnings("unchecked") + List result = (List) concoredocker.literalEval("[-1, -2.5, 3]"); + check("negative int", -1, result.get(0)); + check("negative double", -2.5, result.get(1)); + check("positive int", 3, result.get(2)); + } + + static void testTuple() { + @SuppressWarnings("unchecked") + List result = (List) concoredocker.literalEval("(1, 2, 3)"); + check("tuple size", 3, result.size()); + check("tuple[0]", 1, result.get(0)); + } + + static void testTrailingComma() { + @SuppressWarnings("unchecked") + List result = (List) concoredocker.literalEval("[1, 2, 3,]"); + check("trailing comma size", 3, result.size()); + } + + // --- Serialization tests (toPythonLiteral via write format) --- + + static void testToPythonLiteralBooleans() { + // Test that booleans serialize to Python format (True/False, not true/false) + @SuppressWarnings("unchecked") + List input = (List) concoredocker.literalEval("[True, False]"); + // Re-parse and check the values are correct Java booleans + check("parsed True is Boolean.TRUE", Boolean.TRUE, input.get(0)); + check("parsed False is Boolean.FALSE", Boolean.FALSE, input.get(1)); + } + + static void testToPythonLiteralNone() { + @SuppressWarnings("unchecked") + List input = (List) concoredocker.literalEval("[None, 1]"); + check("parsed None is null", null, input.get(0)); + check("parsed 1 is Integer 1", 1, input.get(1)); + } + + static void testToPythonLiteralString() { + Object result = concoredocker.literalEval("'hello'"); + check("parsed string", "hello", result); + } + + static void testFractionalSimtime() { + // Simtime values like [0.5, 1.0, 2.0] should preserve fractional part + @SuppressWarnings("unchecked") + List result = (List) concoredocker.literalEval("[0.5, 1.0, 2.0]"); + check("fractional simtime[0]", 0.5, result.get(0)); + check("fractional simtime[1]", 1.0, result.get(1)); + check("fractional simtime[2]", 2.0, result.get(2)); + } + + // --- Round-trip serialization tests --- + + static void testRoundTripSerialization() { + // Conceptually: a list with mixed types [1, 2.5, true, false, null, "hello"] + // Use reflection-free approach: build the Python literal manually + // and verify round-trip through literalEval + String serialized = "[1, 2.5, True, False, None, 'hello']"; + @SuppressWarnings("unchecked") + List roundTripped = (List) concoredocker.literalEval(serialized); + check("round-trip int", 1, roundTripped.get(0)); + check("round-trip double", 2.5, roundTripped.get(1)); + check("round-trip True", Boolean.TRUE, roundTripped.get(2)); + check("round-trip False", Boolean.FALSE, roundTripped.get(3)); + check("round-trip None", null, roundTripped.get(4)); + check("round-trip string", "hello", roundTripped.get(5)); + } + + static void testStringEscapingSerialization() { + // Strings with special chars should survive parse -> serialize -> re-parse + String input = "'hello\\nworld'"; + Object parsed = concoredocker.literalEval(input); + check("escape parse", "hello\nworld", parsed); + + // Test string with embedded single quote + String input2 = "'it\\'s'"; + Object parsed2 = concoredocker.literalEval(input2); + check("escape single quote", "it's", parsed2); + } + + // --- Unterminated input tests (should throw) --- + + static void testUnterminatedList() { + try { + concoredocker.literalEval("[1, 2"); + System.out.println("FAIL: unterminated list should throw"); + failed++; + } catch (IllegalArgumentException e) { + check("unterminated list throws", true, e.getMessage().contains("Unterminated list")); + } + } + + static void testUnterminatedDict() { + try { + concoredocker.literalEval("{'a': 1"); + System.out.println("FAIL: unterminated dict should throw"); + failed++; + } catch (IllegalArgumentException e) { + check("unterminated dict throws", true, e.getMessage().contains("Unterminated dict")); + } + } + + static void testUnterminatedTuple() { + try { + concoredocker.literalEval("(1, 2"); + System.out.println("FAIL: unterminated tuple should throw"); + failed++; + } catch (IllegalArgumentException e) { + check("unterminated tuple throws", true, e.getMessage().contains("Unterminated tuple")); + } + } + + static void testNonStringDictKey() { + // Dict keys must be strings - numeric keys should throw + try { + concoredocker.literalEval("{123: 'value'}"); + System.out.println("FAIL: numeric dict key should throw"); + failed++; + } catch (IllegalArgumentException e) { + check("numeric dict key throws", true, e.getMessage().contains("Dict keys must be non-null strings")); + } + } + + // --- JSON keyword interop tests --- + + static void testJsonKeywordTrue() { + Object result = concoredocker.literalEval("true"); + check("json true -> Boolean.TRUE", Boolean.TRUE, result); + } + + static void testJsonKeywordFalse() { + Object result = concoredocker.literalEval("false"); + check("json false -> Boolean.FALSE", Boolean.FALSE, result); + } + + static void testJsonKeywordNull() { + Object result = concoredocker.literalEval("null"); + check("json null -> null", null, result); + } + + static void testJsonMixedList() { + // Python sends [0.0, true, null] via json.dumps — Java must parse it + @SuppressWarnings("unchecked") + List result = (List) concoredocker.literalEval("[0.0, true, null]"); + check("json list[0] simtime", 0.0, result.get(0)); + check("json list[1] true", Boolean.TRUE, result.get(1)); + check("json list[2] null", null, result.get(2)); + } + + static void testJsonRoundTrip() { + // JSON-style payload: true/false/null and double-quoted strings + String jsonPayload = "[0.0, 1.5, true, false, null, \"hello\"]"; + @SuppressWarnings("unchecked") + List result = (List) concoredocker.literalEval(jsonPayload); + check("json round-trip double", 1.5, result.get(1)); + check("json round-trip true", Boolean.TRUE, result.get(2)); + check("json round-trip false", Boolean.FALSE, result.get(3)); + check("json round-trip null", null, result.get(4)); + check("json round-trip string", "hello", result.get(5)); + } +} diff --git a/TestLiteralEvalCpp.cpp b/TestLiteralEvalCpp.cpp new file mode 100644 index 00000000..5746837a --- /dev/null +++ b/TestLiteralEvalCpp.cpp @@ -0,0 +1,307 @@ +/** + * TestLiteralEvalCpp.cpp + * + * Test suite for the C++ Python-literal-compatible parser in concore_base.hpp. + * Validates Issue #389 fix: C++ parser must accept all valid concore payloads + * that Python's ast.literal_eval() accepts. + * + * Compile: g++ -std=c++11 -o TestLiteralEvalCpp TestLiteralEvalCpp.cpp + * Run: ./TestLiteralEvalCpp (Linux/macOS) + * TestLiteralEvalCpp.exe (Windows) + */ + +#include +#include +#include +#include +#include +#include + +#include "concore_base.hpp" + +using namespace concore_base; + +static int passed = 0; +static int failed = 0; + +// ------------- helpers ------------------------------------------------- + +static void check(const std::string& testName, bool condition) { + if (condition) { + std::cout << "PASS: " << testName << std::endl; + ++passed; + } else { + std::cout << "FAIL: " << testName << std::endl; + ++failed; + } +} + +static bool approx(double a, double b, double eps = 1e-9) { + return std::fabs(a - b) < eps; +} + +// ------------- backward-compatibility tests ---------------------------- + +static void test_flat_numeric_list() { + std::vector v = parselist_double("[10.0, 0.5, 2.3]"); + check("flat_numeric size==3", v.size() == 3); + check("flat_numeric[0]==10.0", approx(v[0], 10.0)); + check("flat_numeric[1]==0.5", approx(v[1], 0.5)); + check("flat_numeric[2]==2.3", approx(v[2], 2.3)); +} + +static void test_empty_list() { + std::vector v = parselist_double("[]"); + check("empty_list size==0", v.size() == 0); +} + +static void test_single_element() { + std::vector v = parselist_double("[42.0]"); + check("single_element size==1", v.size() == 1); + check("single_element[0]==42", approx(v[0], 42.0)); +} + +static void test_negative_numbers() { + std::vector v = parselist_double("[-1.5, -3.0, 2.0]"); + check("negative size==3", v.size() == 3); + check("negative[0]==-1.5", approx(v[0], -1.5)); + check("negative[1]==-3.0", approx(v[1], -3.0)); +} + +static void test_scientific_notation() { + std::vector v = parselist_double("[1e3, 2.5E-2, -1.0e+1]"); + check("sci size==3", v.size() == 3); + check("sci[0]==1000", approx(v[0], 1000.0)); + check("sci[1]==0.025", approx(v[1], 0.025)); + check("sci[2]==-10", approx(v[2], -10.0)); +} + +static void test_integer_values() { + std::vector v = parselist_double("[1, 2, 3]"); + check("int size==3", v.size() == 3); + check("int[0]==1", approx(v[0], 1.0)); + check("int[2]==3", approx(v[2], 3.0)); +} + +// ------------- mixed-type payload tests (Issue #389 core) -------------- + +static void test_string_element() { + // [10.0, "start", 0.5] – string should be skipped in numeric flatten + std::vector v = parselist_double("[10.0, \"start\", 0.5]"); + check("string_elem size==2", v.size() == 2); + check("string_elem[0]==10.0", approx(v[0], 10.0)); + check("string_elem[1]==0.5", approx(v[1], 0.5)); +} + +static void test_boolean_element() { + // [10.0, True, 0.5] + std::vector v = parselist_double("[10.0, True, 0.5]"); + check("bool_elem size==3", v.size() == 3); + check("bool_elem[0]==10.0", approx(v[0], 10.0)); + check("bool_elem[1]==1.0 (True)", approx(v[1], 1.0)); + check("bool_elem[2]==0.5", approx(v[2], 0.5)); +} + +static void test_bool_false() { + std::vector v = parselist_double("[False, 5.0]"); + check("bool_false size==2", v.size() == 2); + check("bool_false[0]==0.0", approx(v[0], 0.0)); +} + +static void test_nested_list() { + // [10.0, [0.5, 0.3], 0.1] – nested list flattened to [10.0, 0.5, 0.3, 0.1] + std::vector v = parselist_double("[10.0, [0.5, 0.3], 0.1]"); + check("nested size==4", v.size() == 4); + check("nested[0]==10.0", approx(v[0], 10.0)); + check("nested[1]==0.5", approx(v[1], 0.5)); + check("nested[2]==0.3", approx(v[2], 0.3)); + check("nested[3]==0.1", approx(v[3], 0.1)); +} + +static void test_tuple_payload() { + // (10.0, 0.3) – tuple treated as array + std::vector v = parselist_double("(10.0, 0.3)"); + check("tuple size==2", v.size() == 2); + check("tuple[0]==10.0", approx(v[0], 10.0)); + check("tuple[1]==0.3", approx(v[1], 0.3)); +} + +static void test_nested_tuple() { + // [10.0, (0.5, 0.3)] + std::vector v = parselist_double("[10.0, (0.5, 0.3)]"); + check("nested_tuple size==3", v.size() == 3); + check("nested_tuple[0]==10.0", approx(v[0], 10.0)); + check("nested_tuple[1]==0.5", approx(v[1], 0.5)); + check("nested_tuple[2]==0.3", approx(v[2], 0.3)); +} + +static void test_mixed_types() { + // [10.0, "label", True, [1, 2], (3,), False, "end"] + std::vector v = parselist_double("[10.0, \"label\", True, [1, 2], (3,), False, \"end\"]"); + // numeric values: 10.0, 1.0(True), 1, 2, 3, 0.0(False) = 6 values + check("mixed size==6", v.size() == 6); + check("mixed[0]==10.0", approx(v[0], 10.0)); + check("mixed[1]==1.0", approx(v[1], 1.0)); // True + check("mixed[2]==1.0", approx(v[2], 1.0)); // nested [1,...] + check("mixed[3]==2.0", approx(v[3], 2.0)); // nested [...,2] + check("mixed[4]==3.0", approx(v[4], 3.0)); // tuple (3,) + check("mixed[5]==0.0", approx(v[5], 0.0)); // False +} + +// ------------- full ConcoreValue parse tests --------------------------- + +static void test_parse_literal_string() { + ConcoreValue v = parse_literal("[10.0, \"start\", 0.5]"); + check("literal_string is ARRAY", v.type == ConcoreValueType::ARRAY); + check("literal_string len==3", v.array.size() == 3); + check("literal_string[0] NUMBER", v.array[0].type == ConcoreValueType::NUMBER); + check("literal_string[1] STRING", v.array[1].type == ConcoreValueType::STRING); + check("literal_string[1]==\"start\"", v.array[1].str == "start"); + check("literal_string[2] NUMBER", v.array[2].type == ConcoreValueType::NUMBER); +} + +static void test_parse_literal_bool() { + ConcoreValue v = parse_literal("[True, False]"); + check("literal_bool is ARRAY", v.type == ConcoreValueType::ARRAY); + check("literal_bool[0] BOOL", v.array[0].type == ConcoreValueType::BOOL); + check("literal_bool[0]==true", v.array[0].boolean == true); + check("literal_bool[1]==false", v.array[1].boolean == false); +} + +static void test_parse_literal_nested() { + ConcoreValue v = parse_literal("[1, [2, [3]]]"); + check("literal_nested outer ARRAY", v.type == ConcoreValueType::ARRAY); + check("literal_nested[1] ARRAY", v.array[1].type == ConcoreValueType::ARRAY); + check("literal_nested[1][1] ARRAY", v.array[1].array[1].type == ConcoreValueType::ARRAY); + check("literal_nested[1][1][0]==3", approx(v.array[1].array[1].array[0].number, 3.0)); +} + +static void test_parse_single_quoted_string() { + ConcoreValue v = parse_literal("['hello']"); + check("single_quote ARRAY", v.type == ConcoreValueType::ARRAY); + check("single_quote[0] STRING", v.array[0].type == ConcoreValueType::STRING); + check("single_quote[0]=='hello'", v.array[0].str == "hello"); +} + +static void test_parse_escape_sequences() { + ConcoreValue v = parse_literal("[\"line\\none\"]"); + check("escape STRING", v.array[0].type == ConcoreValueType::STRING); + check("escape has newline", v.array[0].str == "line\none"); +} + +static void test_parse_none() { + ConcoreValue v = parse_literal("[None, 1]"); + check("none[0] STRING", v.array[0].type == ConcoreValueType::STRING); + check("none[0]==\"None\"", v.array[0].str == "None"); +} + +static void test_trailing_comma() { + // Python allows trailing comma: [1, 2,] + std::vector v = parselist_double("[1, 2,]"); + check("trailing_comma size==2", v.size() == 2); + check("trailing_comma[1]==2", approx(v[1], 2.0)); +} + +// ------------- error / failure case tests ------------------------------ + +static void test_malformed_bracket() { + bool caught = false; + try { + parse_literal("[1, 2"); + } catch (const std::runtime_error&) { + caught = true; + } + check("malformed_bracket throws", caught); +} + +static void test_malformed_string() { + bool caught = false; + try { + parse_literal("[\"unterminated]"); + } catch (const std::runtime_error&) { + caught = true; + } + check("malformed_string throws", caught); +} + +static void test_unsupported_object() { + bool caught = false; + try { + parse_literal("{1: 2}"); + } catch (const std::runtime_error&) { + caught = true; + } + check("unsupported_object throws", caught); +} + +static void test_empty_string_input() { + std::vector v = parselist_double(""); + check("empty_input size==0", v.size() == 0); +} + +// ------------- cross-language round-trip tests ------------------------- + +static void test_python_write_cpp_read_flat() { + // Simulate Python write: "[5.0, 1.0, 2.0]" + std::vector v = parselist_double("[5.0, 1.0, 2.0]"); + check("py2cpp_flat size==3", v.size() == 3); + check("py2cpp_flat[0]==5.0", approx(v[0], 5.0)); +} + +static void test_python_write_cpp_read_mixed() { + // Simulate Python write: "[5.0, 'sensor_a', True, [0.1, 0.2]]" + std::vector v = parselist_double("[5.0, 'sensor_a', True, [0.1, 0.2]]"); + // numeric: 5.0, 1.0(True), 0.1, 0.2 = 4 + check("py2cpp_mixed size==4", v.size() == 4); + check("py2cpp_mixed[0]==5.0", approx(v[0], 5.0)); + check("py2cpp_mixed[1]==1.0", approx(v[1], 1.0)); + check("py2cpp_mixed[2]==0.1", approx(v[2], 0.1)); + check("py2cpp_mixed[3]==0.2", approx(v[3], 0.2)); +} + +// ------------- main ---------------------------------------------------- + +int main() { + std::cout << "===== C++ Literal Parser Tests (Issue #389) =====\n\n"; + + // Backward compatibility + test_flat_numeric_list(); + test_empty_list(); + test_single_element(); + test_negative_numbers(); + test_scientific_notation(); + test_integer_values(); + + // Mixed-type payloads (core of Issue #389) + test_string_element(); + test_boolean_element(); + test_bool_false(); + test_nested_list(); + test_tuple_payload(); + test_nested_tuple(); + test_mixed_types(); + + // Full ConcoreValue structure tests + test_parse_literal_string(); + test_parse_literal_bool(); + test_parse_literal_nested(); + test_parse_single_quoted_string(); + test_parse_escape_sequences(); + test_parse_none(); + test_trailing_comma(); + + // Error / failure cases + test_malformed_bracket(); + test_malformed_string(); + test_unsupported_object(); + test_empty_string_input(); + + // Cross-language round-trip + test_python_write_cpp_read_flat(); + test_python_write_cpp_read_mixed(); + + std::cout << "\n=== Results: " << passed << " passed, " << failed + << " failed out of " << (passed + failed) << " tests ===\n"; + + return (failed > 0) ? 1 : 0; +} diff --git a/concore.hpp b/concore.hpp index b74ddd7e..e3fb5e1a 100644 --- a/concore.hpp +++ b/concore.hpp @@ -1,4 +1,7 @@ // concore.hpp -- this C++ include file will be the equivalent of concore.py +#ifndef CONCORE_HPP +#define CONCORE_HPP + #include #include #include //for setprecision @@ -12,11 +15,16 @@ //libraries for platform independent delay. Supports C++11 upwards #include #include +#ifdef __linux__ #include #include #include +#endif #include #include +#include + +#include "concore_base.hpp" using namespace std; @@ -32,21 +40,43 @@ class Concore{ string inpath = "./in"; string outpath = "./out"; - int shmId_create; - int shmId_get; + static constexpr size_t SHM_SIZE = 4096; + + int shmId_create = -1; + int shmId_get = -1; - char* sharedData_create; - char* sharedData_get; + char* sharedData_create = nullptr; + char* sharedData_get = nullptr; // File sharing:- 0, Shared Memory:- 1 int communication_iport = 0; // iport refers to input port int communication_oport = 0; // oport refers to input port +#ifdef CONCORE_USE_ZMQ + map zmq_ports; +#endif + public: + enum class ReadStatus { + SUCCESS, + TIMEOUT, + PARSE_ERROR, + FILE_NOT_FOUND, + RETRIES_EXCEEDED + }; + + struct ReadResult { + ReadStatus status; + vector data; + }; + double delay = 1; int retrycount = 0; double simtime; + int maxtime = 100; map iport; map oport; + map params; + ReadStatus last_read_status = ReadStatus::SUCCESS; /** * @brief Constructor for Concore class. @@ -55,15 +85,26 @@ class Concore{ */ Concore(){ iport = mapParser("concore.iport"); - oport = mapParser("concore.oport"); - std::map::iterator it_iport = iport.begin(); - std::map::iterator it_oport = oport.begin(); - int iport_number = ExtractNumeric(it_iport->first); - int oport_number = ExtractNumeric(it_oport->first); + oport = mapParser("concore.oport"); + default_maxtime(100); + load_params(); + + int iport_number = -1; + int oport_number = -1; + + if (!iport.empty()) { + std::map::iterator it_iport = iport.begin(); + iport_number = ExtractNumeric(it_iport->first); + } + if (!oport.empty()) { + std::map::iterator it_oport = oport.begin(); + oport_number = ExtractNumeric(it_oport->first); + } // if iport_number and oport_number is equal to -1 then it refers to File Method, // otherwise it refers to Shared Memory and the number represent the unique key. +#ifdef __linux__ if(oport_number != -1) { // oport_number is not equal to -1 so refers to SM and value is key. @@ -76,7 +117,8 @@ class Concore{ // iport_number is not equal to -1 so refers to SM and value is key. communication_iport = 1; this->getSharedMemory(iport_number); - } + } +#endif } /** @@ -85,12 +127,97 @@ class Concore{ */ ~Concore() { +#ifdef CONCORE_USE_ZMQ + for (auto& kv : zmq_ports) + delete kv.second; + zmq_ports.clear(); +#endif +#ifdef __linux__ // Detach the shared memory segment from the process - shmdt(sharedData_create); - shmdt(sharedData_get); + if (communication_oport == 1 && sharedData_create != nullptr) { + shmdt(sharedData_create); + } + if (communication_iport == 1 && sharedData_get != nullptr) { + shmdt(sharedData_get); + } // Remove the shared memory segment - shmctl(shmId_create, IPC_RMID, nullptr); + if (shmId_create != -1) { + shmctl(shmId_create, IPC_RMID, nullptr); + } +#endif + } + + /** + * @brief Concore is not copyable as it owns shared memory handles. + */ + Concore(const Concore&) = delete; + Concore& operator=(const Concore&) = delete; + + /** + * @brief Move constructor. Transfers SHM handle ownership to the new instance. + */ + Concore(Concore&& other) noexcept + : s(std::move(other.s)), olds(std::move(other.olds)), + inpath(std::move(other.inpath)), outpath(std::move(other.outpath)), + shmId_create(other.shmId_create), shmId_get(other.shmId_get), + sharedData_create(other.sharedData_create), sharedData_get(other.sharedData_get), + communication_iport(other.communication_iport), communication_oport(other.communication_oport), + delay(other.delay), retrycount(other.retrycount), simtime(other.simtime), + maxtime(other.maxtime), iport(std::move(other.iport)), oport(std::move(other.oport)), + params(std::move(other.params)) + { + other.shmId_create = -1; + other.shmId_get = -1; + other.sharedData_create = nullptr; + other.sharedData_get = nullptr; + other.communication_iport = 0; + other.communication_oport = 0; + } + + /** + * @brief Move assignment. Cleans up current SHM resources, then takes ownership from other. + */ + Concore& operator=(Concore&& other) noexcept + { + if (this == &other) + return *this; + +#ifdef __linux__ + if (communication_oport == 1 && sharedData_create != nullptr) + shmdt(sharedData_create); + if (communication_iport == 1 && sharedData_get != nullptr) + shmdt(sharedData_get); + if (shmId_create != -1) + shmctl(shmId_create, IPC_RMID, nullptr); +#endif + + s = std::move(other.s); + olds = std::move(other.olds); + inpath = std::move(other.inpath); + outpath = std::move(other.outpath); + shmId_create = other.shmId_create; + shmId_get = other.shmId_get; + sharedData_create = other.sharedData_create; + sharedData_get = other.sharedData_get; + communication_iport = other.communication_iport; + communication_oport = other.communication_oport; + delay = other.delay; + retrycount = other.retrycount; + simtime = other.simtime; + maxtime = other.maxtime; + iport = std::move(other.iport); + oport = std::move(other.oport); + params = std::move(other.params); + + other.shmId_create = -1; + other.shmId_get = -1; + other.sharedData_create = nullptr; + other.sharedData_get = nullptr; + other.communication_iport = 0; + other.communication_oport = 0; + + return *this; } /** @@ -127,22 +254,38 @@ class Concore{ return std::stoi(numberString); } +#ifdef __linux__ /** * @brief Creates a shared memory segment with the given key. * @param key The key for the shared memory segment. */ void createSharedMemory(key_t key) { - shmId_create = shmget(key, 256, IPC_CREAT | 0666); + shmId_create = shmget(key, SHM_SIZE, IPC_CREAT | 0666); if (shmId_create == -1) { std::cerr << "Failed to create shared memory segment." << std::endl; + return; + } + + // Verify the segment is large enough (shmget won't resize an existing segment) + struct shmid_ds shm_info; + if (shmctl(shmId_create, IPC_STAT, &shm_info) == 0 && shm_info.shm_segsz < SHM_SIZE) { + std::cerr << "Shared memory segment too small (" << shm_info.shm_segsz + << " bytes, need " << SHM_SIZE << "). Removing and recreating." << std::endl; + shmctl(shmId_create, IPC_RMID, nullptr); + shmId_create = shmget(key, SHM_SIZE, IPC_CREAT | 0666); + if (shmId_create == -1) { + std::cerr << "Failed to recreate shared memory segment." << std::endl; + return; + } } // Attach the shared memory segment to the process's address space sharedData_create = static_cast(shmat(shmId_create, NULL, 0)); if (sharedData_create == reinterpret_cast(-1)) { std::cerr << "Failed to attach shared memory segment." << std::endl; + sharedData_create = nullptr; } } @@ -153,9 +296,11 @@ class Concore{ */ void getSharedMemory(key_t key) { - while (true) { + int retry = 0; + const int MAX_RETRY = 100; + while (retry < MAX_RETRY) { // Get the shared memory segment created by Writer - shmId_get = shmget(key, 256, 0666); + shmId_get = shmget(key, SHM_SIZE, 0666); // Check if shared memory exists if (shmId_get != -1) { break; // Break the loop if shared memory exists @@ -163,11 +308,22 @@ class Concore{ std::cout << "Shared memory does not exist. Make sure the writer process is running." << std::endl; sleep(1); // Sleep for 1 second before checking again + retry++; + } + + if (shmId_get == -1) { + std::cerr << "Failed to get shared memory segment after max retries." << std::endl; + return; } // Attach the shared memory segment to the process's address space sharedData_get = static_cast(shmat(shmId_get, NULL, 0)); + if (sharedData_get == reinterpret_cast(-1)) { + std::cerr << "Failed to attach shared memory segment." << std::endl; + sharedData_get = nullptr; + } } +#endif /** * @brief Parses a file containing port and number mappings and returns a map of the values. @@ -176,44 +332,13 @@ class Concore{ */ map mapParser(string filename){ map ans; - - ifstream portfile; - string portstr; - portfile.open(filename); - if(portfile){ - ostringstream ss; - ss << portfile.rdbuf(); - portstr = ss.str(); - portfile.close(); - } - - portstr[portstr.size()-1]=','; - portstr+='}'; - int i=0; - string portname=""; - string portnum=""; - - while(portstr[i]!='}'){ - if(portstr[i]=='\''){ - i++; - while(portstr[i]!='\''){ - portname+=portstr[i]; - i++; - } - ans.insert({portname,0}); + auto str_map = concore_base::safe_literal_eval_dict(filename, {}); + for (const auto& kv : str_map) { + try { + ans[kv.first] = std::stoi(kv.second); + } catch (...) { + ans[kv.first] = 0; } - - if(portstr[i]==':'){ - i++; - while(portstr[i]!=','){ - portnum+=portstr[i]; - i++; - } - ans[portname]=stoi(portnum); - portnum=""; - portname=""; - } - i++; } return ans; } @@ -239,24 +364,25 @@ class Concore{ * @return A vector of double values extracted from the input string. */ vector parser(string f){ - vector temp; - string value = ""; - - //Changing last bracket to comma to use comma as a delimiter - f[f.length()-1]=','; + return concore_base::parselist_double(f); + } - for(int i=1;i flatten_numeric(const concore_base::ConcoreValue& v){ + return concore_base::flatten_numeric(v); } /** @@ -276,6 +402,14 @@ class Concore{ return read_FM(port, name, initstr); } + ReadResult read_result(int port, string name, string initstr) + { + ReadResult result; + result.data = read(port, name, initstr); + result.status = last_read_status; + return result; + } + /** * @brief Reads data from a specified port and name using the FM (File Method) communication protocol. * @param port The port number. @@ -287,6 +421,7 @@ class Concore{ chrono::milliseconds timespan((int)(1000*delay)); this_thread::sleep_for(timespan); string ins; + ReadStatus status = ReadStatus::SUCCESS; try { ifstream infile; infile.open(inpath+to_string(port)+"/"+name, ios::in); @@ -297,13 +432,18 @@ class Concore{ infile.close(); } else { + status = ReadStatus::FILE_NOT_FOUND; throw 505;} } catch (...) { ins = initstr; + if (status == ReadStatus::SUCCESS) + status = ReadStatus::FILE_NOT_FOUND; } - while ((int)ins.length()==0){ + int retry = 0; + const int MAX_RETRY = 100; + while ((int)ins.length()==0 && retry < MAX_RETRY){ this_thread::sleep_for(timespan); try{ ifstream infile; @@ -324,13 +464,26 @@ class Concore{ catch(...){ cout<<"Read error"; } - - + retry++; } + if ((int)ins.length()==0) + status = ReadStatus::RETRIES_EXCEEDED; s += ins; vector inval = parser(ins); + if(inval.empty()) { + if (status == ReadStatus::SUCCESS) + status = ReadStatus::PARSE_ERROR; + inval = parser(initstr); + } + if(inval.empty()) { + if (status == ReadStatus::SUCCESS) + status = ReadStatus::PARSE_ERROR; + last_read_status = status; + return inval; + } simtime = simtime > inval[0] ? simtime : inval[0]; + last_read_status = status; //returning a string with data excluding simtime inval.erase(inval.begin()); @@ -349,10 +502,11 @@ class Concore{ chrono::milliseconds timespan((int)(1000*delay)); this_thread::sleep_for(timespan); string ins = ""; + ReadStatus status = ReadStatus::SUCCESS; try { if (shmId_get != -1) { if (sharedData_get && sharedData_get[0] != '\0') { - std::string message(sharedData_get, strnlen(sharedData_get, 256)); + std::string message(sharedData_get, strnlen(sharedData_get, SHM_SIZE)); ins = message; } else @@ -362,17 +516,22 @@ class Concore{ } else { + status = ReadStatus::FILE_NOT_FOUND; throw 505; } } catch (...) { ins = initstr; + if (status == ReadStatus::SUCCESS) + status = ReadStatus::FILE_NOT_FOUND; } - while ((int)ins.length()==0){ + int retry = 0; + const int MAX_RETRY = 100; + while ((int)ins.length()==0 && retry < MAX_RETRY){ this_thread::sleep_for(timespan); try{ if(shmId_get != -1) { - std::string message(sharedData_get, strnlen(sharedData_get, 256)); + std::string message(sharedData_get, strnlen(sharedData_get, SHM_SIZE)); ins = message; retrycount++; } @@ -385,11 +544,26 @@ class Concore{ catch(...){ std::cout << "Read error" << std::endl; } + retry++; } + if ((int)ins.length()==0) + status = ReadStatus::RETRIES_EXCEEDED; s += ins; vector inval = parser(ins); + if(inval.empty()) { + if (status == ReadStatus::SUCCESS) + status = ReadStatus::PARSE_ERROR; + inval = parser(initstr); + } + if(inval.empty()) { + if (status == ReadStatus::SUCCESS) + status = ReadStatus::PARSE_ERROR; + last_read_status = status; + return inval; + } simtime = simtime > inval[0] ? simtime : inval[0]; + last_read_status = status; //returning a string with data excluding simtime inval.erase(inval.begin()); @@ -451,6 +625,7 @@ class Concore{ outfile<= SHM_SIZE) { + std::cerr << "ERROR: write_SM payload (" << result.size() + << " bytes) exceeds " << SHM_SIZE - 1 + << "-byte shared memory limit. Data truncated!" << std::endl; + } + std::strncpy(sharedData_create, result.c_str(), SHM_SIZE - 1); + sharedData_create[SHM_SIZE - 1] = '\0'; + // simtime must not be mutated here (issue #385). } else{ throw 505; @@ -529,7 +713,15 @@ class Concore{ this_thread::sleep_for(timespan); try { if(shmId_create != -1){ - std::strncpy(sharedData_create, val.c_str(), 256 - 1); + if (sharedData_create == nullptr) + throw 506; + if (val.size() >= SHM_SIZE) { + std::cerr << "ERROR: write_SM payload (" << val.size() + << " bytes) exceeds " << SHM_SIZE - 1 + << "-byte shared memory limit. Data truncated!" << std::endl; + } + std::strncpy(sharedData_create, val.c_str(), SHM_SIZE - 1); + sharedData_create[SHM_SIZE - 1] = '\0'; } else throw 505; } @@ -538,6 +730,188 @@ class Concore{ } } +#ifdef CONCORE_USE_ZMQ + /** + * @brief Registers a ZMQ port for use with read()/write(). + * @param port_name The ZMQ port name. + * @param port_type "bind" or "connect". + * @param address The ZMQ address. + * @param socket_type_str The socket type string. + */ + void init_zmq_port(string port_name, string port_type, string address, string socket_type_str) { + if (zmq_ports.count(port_name)) return; + int sock_type = concore_base::zmq_socket_type_from_string(socket_type_str); + if (sock_type == -1) { + cerr << "init_zmq_port: unknown socket type '" << socket_type_str << "'" << endl; + return; + } + zmq_ports[port_name] = new concore_base::ZeroMQPort(port_type, address, sock_type); + } + + /** + * @brief Reads data from a ZMQ port. Strips simtime prefix, updates simtime. + * @param port_name The ZMQ port name. + * @param name The name of the file. + * @param initstr The initial string. + * @return a vector of double values + */ + vector read_ZMQ(string port_name, string name, string initstr) { + ReadStatus status = ReadStatus::SUCCESS; + auto it = zmq_ports.find(port_name); + if (it == zmq_ports.end()) { + cerr << "read_ZMQ: port '" << port_name << "' not initialized" << endl; + status = ReadStatus::FILE_NOT_FOUND; + last_read_status = status; + return parser(initstr); + } + vector inval = it->second->recv_with_retry(); + if (inval.empty()) { + status = ReadStatus::TIMEOUT; + inval = parser(initstr); + } + if (inval.empty()) { + if (status == ReadStatus::SUCCESS) + status = ReadStatus::PARSE_ERROR; + last_read_status = status; + return inval; + } + last_read_status = status; + simtime = simtime > inval[0] ? simtime : inval[0]; + s += port_name; + inval.erase(inval.begin()); + return inval; + } + + /** + * @brief Writes a vector of double values to a ZMQ port. Prepends simtime+delta. + * @param port_name The ZMQ port name. + * @param name The name of the file. + * @param val The vector of double values to write. + * @param delta The delta value (default: 0). + */ + void write_ZMQ(string port_name, string name, vector val, int delta=0) { + auto it = zmq_ports.find(port_name); + if (it == zmq_ports.end()) { + cerr << "write_ZMQ: port '" << port_name << "' not initialized" << endl; + return; + } + val.insert(val.begin(), simtime + delta); + it->second->send_with_retry(val); + // simtime must not be mutated here (issue #385). + } + + /** + * @brief Writes a string to a ZMQ port. + * @param port_name The ZMQ port name. + * @param name The name of the file. + * @param val The string to write. + * @param delta The delta value (default: 0). + */ + void write_ZMQ(string port_name, string name, string val, int delta=0) { + auto it = zmq_ports.find(port_name); + if (it == zmq_ports.end()) { + cerr << "write_ZMQ: port '" << port_name << "' not initialized" << endl; + return; + } + chrono::milliseconds timespan((int)(2000*delay)); + this_thread::sleep_for(timespan); + it->second->send_string_with_retry(val); + } + + /** + * @brief deviate the read to ZMQ communication protocol when port identifier is a string key. + * @param port_name The ZMQ port name. + * @param name The name of the file. + * @param initstr The initial string. + * @return + */ + vector read(string port_name, string name, string initstr) { + return read_ZMQ(port_name, name, initstr); + } + + ReadResult read_result(string port_name, string name, string initstr) { + ReadResult result; + result.data = read(port_name, name, initstr); + result.status = last_read_status; + return result; + } + + /** + * @brief deviate the write to ZMQ communication protocol when port identifier is a string key. + * @param port_name The ZMQ port name. + * @param name The name of the file. + * @param val The vector of double values to write. + * @param delta The delta value (default: 0). + */ + void write(string port_name, string name, vector val, int delta=0) { + return write_ZMQ(port_name, name, val, delta); + } + + /** + * @brief deviate the write to ZMQ communication protocol when port identifier is a string key. + * @param port_name The ZMQ port name. + * @param name The name of the file. + * @param val The string to write. + * @param delta The delta value (default: 0). + */ + void write(string port_name, string name, string val, int delta=0) { + return write_ZMQ(port_name, name, val, delta); + } +#endif // CONCORE_USE_ZMQ + + /** + * @brief Strips leading and trailing whitespace from a string. + * @param str The input string. + * @return The stripped string. + */ + string stripstr(string str){ + return concore_base::stripstr(str); + } + + /** + * @brief Strips surrounding single or double quotes from a string. + * @param str The input string. + * @return The unquoted string. + */ + string stripquotes(string str){ + return concore_base::stripquotes(str); + } + + /** + * @brief Parses a dict-formatted string into a string-to-string map. + * @param str The input string in {key: val, ...} format. + * @return A map of key-value string pairs. + */ + map parsedict(string str){ + return concore_base::parsedict(str); + } + + /** + * @brief Sets maxtime from the concore.maxtime file, falling back to defaultValue. + * @param defaultValue The fallback value if the file is missing. + */ + void default_maxtime(int defaultValue){ + maxtime = (int)concore_base::load_maxtime( + inpath + "/1/concore.maxtime", (double)defaultValue); + } + + /** + * @brief Loads simulation parameters from concore.params into the params map. + */ + void load_params(){ + params = concore_base::load_params(inpath + "/1/concore.params"); + } + + /** + * @brief Returns the value of a param by name, or a default if not found. + * @param n The parameter name. + * @param i The default value. + * @return The parameter value or the default. + */ + string tryparam(string n, string i){ + return concore_base::tryparam(params, n, i); + } + /** * @brief Initializes the system with the given input values. * @param f The input string containing the values. @@ -547,6 +921,8 @@ class Concore{ //parsing vector val = parser(f); + if (val.empty()) return val; + //determining simtime simtime = val[0]; @@ -555,3 +931,5 @@ class Concore{ return val; } }; + +#endif // CONCORE_HPP diff --git a/concore.java b/concore.java new file mode 100644 index 00000000..3cb7d021 --- /dev/null +++ b/concore.java @@ -0,0 +1,923 @@ +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.channels.FileLock; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; +import java.util.HashMap; +import java.util.Map; +import java.util.ArrayList; +import java.util.List; +import org.zeromq.ZMQ; + +/** + * Java implementation of concore local communication. + * + * This class provides file-based inter-process communication for control systems, + * mirroring the functionality of concore.py. + */ +public class concore { + private static Map iport = new HashMap<>(); + private static Map oport = new HashMap<>(); + private static String s = ""; + private static String olds = ""; + // delay in milliseconds (Python uses time.sleep(1) = 1 second) + private static int delay = 1000; + private static int retrycount = 0; + private static int maxRetries = 5; + private static String inpath = "./in"; + private static String outpath = "./out"; + private static Map params = new HashMap<>(); + private static Map zmqPorts = new HashMap<>(); + private static ZMQ.Context zmqContext = null; + // simtime as double to preserve fractional values (e.g. "[0.0, ...]") + private static double simtime = 0; + private static double maxtime; + + private static final Path BASE_DIR = Paths.get("").toAbsolutePath().normalize(); + private static final Path PID_REGISTRY_FILE = BASE_DIR.resolve("concorekill_pids.txt"); + private static final Path KILL_SCRIPT_FILE = BASE_DIR.resolve("concorekill.bat"); + + // initialize on class load, same as Python module-level init + static { + if (isWindows()) { + registerPid(); + writeKillScript(); + Runtime.getRuntime().addShutdownHook(new Thread(concore::cleanupPid)); + } + try { + iport = parseFile("concore.iport"); + } catch (IOException e) { + } + try { + oport = parseFile("concore.oport"); + } catch (IOException e) { + } + try { + String paramsFile = Paths.get(portPath(inpath, 1), "concore.params").toString(); + String sparams = new String(Files.readAllBytes(Paths.get(paramsFile)), java.nio.charset.StandardCharsets.UTF_8); + if (sparams.length() > 0 && sparams.charAt(0) == '"') { // windows keeps "" need to remove + sparams = sparams.substring(1); + sparams = sparams.substring(0, sparams.indexOf('"')); + } + params = parseParams(sparams); + } catch (IOException e) { + params = new HashMap<>(); + } + defaultMaxTime(100); + Runtime.getRuntime().addShutdownHook(new Thread(concore::terminateZmq)); + } + + private static boolean isWindows() { + String os = System.getProperty("os.name"); + return os != null && os.toLowerCase().contains("win"); + } + + private static void registerPid() { + try { + String pid = String.valueOf(ProcessHandle.current().pid()); + try (FileChannel channel = FileChannel.open(PID_REGISTRY_FILE, + StandardOpenOption.CREATE, StandardOpenOption.WRITE, StandardOpenOption.APPEND)) { + try (FileLock lock = channel.lock()) { + channel.write(ByteBuffer.wrap((pid + System.lineSeparator()).getBytes(java.nio.charset.StandardCharsets.UTF_8))); + } + } + } catch (IOException e) { + } + } + + private static void cleanupPid() { + String pid = String.valueOf(ProcessHandle.current().pid()); + if (!Files.exists(PID_REGISTRY_FILE)) return; + List remaining = new ArrayList<>(); + try (FileChannel channel = FileChannel.open(PID_REGISTRY_FILE, + StandardOpenOption.READ, StandardOpenOption.WRITE)) { + try (FileLock lock = channel.lock()) { + ByteBuffer buf = ByteBuffer.allocate((int) channel.size()); + channel.read(buf); + buf.flip(); + String content = java.nio.charset.StandardCharsets.UTF_8.decode(buf).toString(); + for (String line : content.split("\\R")) { + String trimmed = line.trim(); + if (!trimmed.isEmpty() && !trimmed.equals(pid)) { + remaining.add(trimmed); + } + } + channel.truncate(0); + channel.position(0); + if (!remaining.isEmpty()) { + StringBuilder sb = new StringBuilder(); + for (String r : remaining) sb.append(r).append(System.lineSeparator()); + channel.write(ByteBuffer.wrap(sb.toString().getBytes(java.nio.charset.StandardCharsets.UTF_8))); + } + } + } catch (IOException e) { + } + if (remaining.isEmpty()) { + try { Files.deleteIfExists(PID_REGISTRY_FILE); } catch (IOException e) {} + try { Files.deleteIfExists(KILL_SCRIPT_FILE); } catch (IOException e) {} + } + } + + private static void writeKillScript() { + try { + String regName = PID_REGISTRY_FILE.getFileName().toString(); + String batName = KILL_SCRIPT_FILE.getFileName().toString(); + String script = "@echo off\r\n"; + script += "if not exist \"%~dp0" + regName + "\" (\r\n"; + script += " echo No PID registry found. Nothing to kill.\r\n"; + script += " exit /b 0\r\n"; + script += ")\r\n"; + script += "for /f \"usebackq tokens=*\" %%p in (\"%~dp0" + regName + "\") do (\r\n"; + script += " wmic process where \"ProcessId=%%p\" get CommandLine /value 2>nul | find /i \"concore\" >nul\r\n"; + script += " if not errorlevel 1 (\r\n"; + script += " echo Killing concore process %%p\r\n"; + script += " taskkill /F /PID %%p >nul 2>&1\r\n"; + script += " ) else (\r\n"; + script += " echo Skipping PID %%p - not a concore process or not running\r\n"; + script += " )\r\n"; + script += ")\r\n"; + script += "del /q \"%~dp0" + regName + "\" 2>nul\r\n"; + script += "del /q \"%~dp0" + batName + "\" 2>nul\r\n"; + Files.write(KILL_SCRIPT_FILE, script.getBytes(java.nio.charset.StandardCharsets.UTF_8)); + } catch (IOException e) { + } + } + + /** + * Parses a param string into a map, matching concore_base.parse_params. + * Tries dict literal first, then falls back to semicolon-separated key=value pairs. + */ + private static Map parseParams(String sparams) { + Map result = new HashMap<>(); + if (sparams == null || sparams.isEmpty()) return result; + String trimmed = sparams.trim(); + if (trimmed.startsWith("{") && trimmed.endsWith("}")) { + try { + Object val = literalEval(trimmed); + if (val instanceof Map) { + @SuppressWarnings("unchecked") + Map map = (Map) val; + return map; + } + } catch (Exception e) { + } + } + for (String item : trimmed.split(";")) { + if (item.contains("=")) { + String[] parts = item.split("=", 2); // split on first '=' only + String key = parts[0].trim(); + String value = parts[1].trim(); + try { + result.put(key, literalEval(value)); + } catch (Exception e) { + result.put(key, value); + } + } + } + return result; + } + + /** + * Parses a file containing a Python-style dictionary literal. + * Returns empty map if file is empty or malformed (matches Python safe_literal_eval). + */ + private static Map parseFile(String filename) throws IOException { + String content = new String(Files.readAllBytes(Paths.get(filename)), java.nio.charset.StandardCharsets.UTF_8); + content = content.trim(); + if (content.isEmpty()) { + return new HashMap<>(); + } + try { + Object result = literalEval(content); + if (result instanceof Map) { + @SuppressWarnings("unchecked") + Map map = (Map) result; + return map; + } + } catch (IllegalArgumentException e) { + System.err.println("Failed to parse file as map: " + filename + " (" + e.getMessage() + ")"); + } + return new HashMap<>(); + } + + /** + * Sets maxtime from concore.maxtime file, or uses defaultValue if file not found. + * Catches both IOException and RuntimeException to match Python safe_literal_eval. + */ + public static void defaultMaxTime(double defaultValue) { + try { + String maxtimeFile = Paths.get(portPath(inpath, 1), "concore.maxtime").toString(); + String content = new String(Files.readAllBytes(Paths.get(maxtimeFile))); + Object parsed = literalEval(content.trim()); + if (parsed instanceof Number) { + maxtime = ((Number) parsed).doubleValue(); + } else { + maxtime = defaultValue; + } + } catch (IOException | RuntimeException e) { + maxtime = defaultValue; + } + } + + private static String portPath(String base, int portNum) { + return base + portNum; + } + + // package-level helpers for testing with temp directories + static void setInPath(String path) { inpath = path; } + static void setOutPath(String path) { outpath = path; } + static void setDelay(int ms) { delay = ms; } + static double getSimtime() { return simtime; } + static void resetState() { s = ""; olds = ""; simtime = 0; } + + public static boolean unchanged() { + if (olds.equals(s)) { + s = ""; + return true; + } + olds = s; + return false; + } + + public static Object tryParam(String n, Object i) { + if (params.containsKey(n)) { + return params.get(n); + } else { + return i; + } + } + + /** + * Reads data from a port file. Returns the values after extracting simtime. + * Input format: [simtime, val1, val2, ...] + * Returns: list of values after simtime + * Includes max retry limit to avoid infinite blocking (matches Python behavior). + */ + public static ReadResult read(int port, String name, String initstr) { + // Parse default value upfront for consistent return type + List defaultVal = new ArrayList<>(); + try { + List parsed = (List) literalEval(initstr); + if (parsed.size() > 1) { + defaultVal = new ArrayList<>(parsed.subList(1, parsed.size())); + } + } catch (Exception e) { + // initstr not parseable as list; defaultVal stays empty + } + + String filePath = Paths.get(portPath(inpath, port), name).toString(); + try { + Thread.sleep(delay); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + s += initstr; + return new ReadResult(ReadStatus.TIMEOUT, defaultVal); + } + + String ins; + try { + ins = new String(Files.readAllBytes(Paths.get(filePath))); + } catch (IOException e) { + System.out.println("File " + filePath + " not found, using default value."); + s += initstr; + return new ReadResult(ReadStatus.FILE_NOT_FOUND, defaultVal); + } + + int attempts = 0; + while (ins.length() == 0 && attempts < maxRetries) { + try { + Thread.sleep(delay); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + s += initstr; + return new ReadResult(ReadStatus.TIMEOUT, defaultVal); + } + try { + ins = new String(Files.readAllBytes(Paths.get(filePath))); + } catch (IOException e) { + System.out.println("Retry " + (attempts + 1) + ": Error reading " + filePath); + } + attempts++; + retrycount++; + } + + if (ins.length() == 0) { + System.out.println("Max retries reached for " + filePath + ", using default value."); + return new ReadResult(ReadStatus.RETRIES_EXCEEDED, defaultVal); + } + + s += ins; + try { + List inval = (List) literalEval(ins); + if (!inval.isEmpty()) { + double firstSimtime = ((Number) inval.get(0)).doubleValue(); + simtime = Math.max(simtime, firstSimtime); + return new ReadResult(ReadStatus.SUCCESS, new ArrayList<>(inval.subList(1, inval.size()))); + } + } catch (Exception e) { + System.out.println("Error parsing " + ins + ": " + e.getMessage()); + } + return new ReadResult(ReadStatus.PARSE_ERROR, defaultVal); + } + + /** + * Escapes a Java string so it can be safely used as a single-quoted Python string literal. + * At minimum, escapes backslash, single quote, newline, carriage return, and tab. + */ + private static String escapePythonString(String s) { + StringBuilder sb = new StringBuilder(s.length()); + for (int i = 0; i < s.length(); i++) { + char c = s.charAt(i); + switch (c) { + case '\\': sb.append("\\\\"); break; + case '\'': sb.append("\\'"); break; + case '\n': sb.append("\\n"); break; + case '\r': sb.append("\\r"); break; + case '\t': sb.append("\\t"); break; + default: sb.append(c); break; + } + } + return sb.toString(); + } + + /** + * Converts a Java object to its Python-literal string representation. + * True/False/None instead of true/false/null; strings single-quoted. + */ + private static String toPythonLiteral(Object obj) { + if (obj == null) return "None"; + if (obj instanceof Boolean) return ((Boolean) obj) ? "True" : "False"; + if (obj instanceof String) return "'" + escapePythonString((String) obj) + "'"; + if (obj instanceof Number) return obj.toString(); + if (obj instanceof List) { + List list = (List) obj; + StringBuilder sb = new StringBuilder("["); + for (int i = 0; i < list.size(); i++) { + if (i > 0) sb.append(", "); + sb.append(toPythonLiteral(list.get(i))); + } + sb.append("]"); + return sb.toString(); + } + if (obj instanceof Map) { + Map map = (Map) obj; + StringBuilder sb = new StringBuilder("{"); + boolean first = true; + for (Map.Entry entry : map.entrySet()) { + if (!first) sb.append(", "); + sb.append(toPythonLiteral(entry.getKey())).append(": ").append(toPythonLiteral(entry.getValue())); + first = false; + } + sb.append("}"); + return sb.toString(); + } + return obj.toString(); + } + + /** + * Escapes a Java string so it can be safely embedded in a JSON double-quoted string. + * Escapes backslash, double quote, newline, carriage return, and tab. + */ + private static String escapeJsonString(String s) { + StringBuilder sb = new StringBuilder(s.length()); + for (int i = 0; i < s.length(); i++) { + char c = s.charAt(i); + switch (c) { + case '\\': sb.append("\\\\"); break; + case '"': sb.append("\\\""); break; + case '\n': sb.append("\\n"); break; + case '\r': sb.append("\\r"); break; + case '\t': sb.append("\\t"); break; + default: sb.append(c); break; + } + } + return sb.toString(); + } + + /** + * Converts a Java object to its JSON string representation. + * true/false/null instead of True/False/None; strings double-quoted. + */ + private static String toJsonLiteral(Object obj) { + if (obj == null) return "null"; + if (obj instanceof Boolean) return ((Boolean) obj) ? "true" : "false"; + if (obj instanceof String) return "\"" + escapeJsonString((String) obj) + "\""; + if (obj instanceof Number) return obj.toString(); + if (obj instanceof List) { + List list = (List) obj; + StringBuilder sb = new StringBuilder("["); + for (int i = 0; i < list.size(); i++) { + if (i > 0) sb.append(", "); + sb.append(toJsonLiteral(list.get(i))); + } + sb.append("]"); + return sb.toString(); + } + if (obj instanceof Map) { + Map map = (Map) obj; + StringBuilder sb = new StringBuilder("{"); + boolean first = true; + for (Map.Entry entry : map.entrySet()) { + if (!first) sb.append(", "); + sb.append(toJsonLiteral(entry.getKey())).append(": ").append(toJsonLiteral(entry.getValue())); + first = false; + } + sb.append("}"); + return sb.toString(); + } + return obj.toString(); + } + + /** + * Writes data to a port file. + * Prepends simtime+delta to the value list, then serializes to Python-literal format. + * Accepts List or String values (matching Python implementation). + */ + public static void write(int port, String name, Object val, int delta) { + try { + String path = Paths.get(portPath(outpath, port), name).toString(); + StringBuilder content = new StringBuilder(); + if (val instanceof String) { + Thread.sleep(2 * delay); + content.append(val); + } else if (val instanceof List) { + List listVal = (List) val; + content.append("["); + content.append(toPythonLiteral(simtime + delta)); + for (int i = 0; i < listVal.size(); i++) { + content.append(", "); + content.append(toPythonLiteral(listVal.get(i))); + } + content.append("]"); + // simtime must not be mutated here. + // Mutation breaks cross-language determinism. + } else if (val instanceof Object[]) { + // Legacy support for Object[] arguments + Object[] arrayVal = (Object[]) val; + content.append("["); + content.append(toPythonLiteral(simtime + delta)); + for (Object o : arrayVal) { + content.append(", "); + content.append(toPythonLiteral(o)); + } + content.append("]"); + // simtime must not be mutated here. + // Mutation breaks cross-language determinism. + } else { + System.out.println("write must have list or str"); + return; + } + Files.write(Paths.get(path), content.toString().getBytes()); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + System.out.println("skipping " + outpath + "/" + port + "/" + name); + } catch (IOException e) { + System.out.println("skipping " + outpath + "/" + port + "/" + name); + } + } + + /** + * Parses an initial value string like "[0.0, 1.0, 2.0]". + * Extracts simtime from position 0 and returns the remaining values as a List. + */ + public static List initVal(String simtimeVal) { + List val = new ArrayList<>(); + try { + List inval = (List) literalEval(simtimeVal); + if (!inval.isEmpty()) { + simtime = ((Number) inval.get(0)).doubleValue(); + val = new ArrayList<>(inval.subList(1, inval.size())); + } + } catch (Exception e) { + System.out.println("Error parsing initVal: " + e.getMessage()); + } + return val; + } + + private static ZMQ.Context getZmqContext() { + if (zmqContext == null) { + zmqContext = ZMQ.context(1); + } + return zmqContext; + } + + public static void initZmqPort(String portName, String portType, String address, String socketTypeStr) { + if (zmqPorts.containsKey(portName)) return; + int sockType = zmqSocketTypeFromString(socketTypeStr); + if (sockType == -1) { + System.err.println("initZmqPort: unknown socket type '" + socketTypeStr + "'"); + return; + } + zmqPorts.put(portName, new ZeroMQPort(portType, address, sockType)); + } + + public static void terminateZmq() { + for (Map.Entry entry : zmqPorts.entrySet()) { + entry.getValue().socket.close(); + } + zmqPorts.clear(); + if (zmqContext != null) { + zmqContext.term(); + zmqContext = null; + } + } + + private static int zmqSocketTypeFromString(String s) { + switch (s.toUpperCase()) { + case "REQ": return ZMQ.REQ; + case "REP": return ZMQ.REP; + case "PUB": return ZMQ.PUB; + case "SUB": return ZMQ.SUB; + case "PUSH": return ZMQ.PUSH; + case "PULL": return ZMQ.PULL; + case "PAIR": return ZMQ.PAIR; + default: return -1; + } + } + + /** + * Reads data from a ZMQ port. Same wire format as file-based read: + * expects [simtime, val1, val2, ...], strips simtime, returns the rest. + */ + public static ReadResult read(String portName, String name, String initstr) { + List defaultVal = new ArrayList<>(); + try { + List parsed = (List) literalEval(initstr); + if (parsed.size() > 1) { + defaultVal = new ArrayList<>(parsed.subList(1, parsed.size())); + } + } catch (Exception e) { + } + ZeroMQPort port = zmqPorts.get(portName); + if (port == null) { + System.err.println("read: ZMQ port '" + portName + "' not initialized"); + return new ReadResult(ReadStatus.FILE_NOT_FOUND, defaultVal); + } + String msg = port.recvWithRetry(); + if (msg == null) { + System.err.println("read: ZMQ recv timeout on port '" + portName + "'"); + return new ReadResult(ReadStatus.TIMEOUT, defaultVal); + } + s += msg; + try { + List inval = (List) literalEval(msg); + if (!inval.isEmpty()) { + simtime = Math.max(simtime, ((Number) inval.get(0)).doubleValue()); + return new ReadResult(ReadStatus.SUCCESS, new ArrayList<>(inval.subList(1, inval.size()))); + } + } catch (Exception e) { + System.out.println("Error parsing ZMQ message '" + msg + "': " + e.getMessage()); + } + return new ReadResult(ReadStatus.PARSE_ERROR, defaultVal); + } + + /** + * Writes data to a ZMQ port. Prepends [simtime+delta] to match file-based write behavior. + */ + public static void write(String portName, String name, Object val, int delta) { + ZeroMQPort port = zmqPorts.get(portName); + if (port == null) { + System.err.println("write: ZMQ port '" + portName + "' not initialized"); + return; + } + String payload; + if (val instanceof List) { + List listVal = (List) val; + StringBuilder sb = new StringBuilder("["); + sb.append(toJsonLiteral(simtime + delta)); + for (Object o : listVal) { + sb.append(", "); + sb.append(toJsonLiteral(o)); + } + sb.append("]"); + payload = sb.toString(); + // simtime must not be mutated here + } else if (val instanceof String) { + payload = (String) val; + } else { + System.out.println("write must have list or str"); + return; + } + port.sendWithRetry(payload); + } + + /** + * Parses a Python-literal string into Java objects using a recursive descent parser. + * Supports: dict, list, int, float, string (single/double quoted), bool, None, nested structures. + * This replaces the broken split-based parser that could not handle quoted commas or nesting. + */ + static Object literalEval(String s) { + if (s == null) throw new IllegalArgumentException("Input cannot be null"); + s = s.trim(); + if (s.isEmpty()) throw new IllegalArgumentException("Input cannot be empty"); + Parser parser = new Parser(s); + Object result = parser.parseExpression(); + parser.skipWhitespace(); + if (parser.pos < parser.input.length()) { + throw new IllegalArgumentException("Unexpected trailing content at position " + parser.pos); + } + return result; + } + + public enum ReadStatus { + SUCCESS, FILE_NOT_FOUND, TIMEOUT, PARSE_ERROR, RETRIES_EXCEEDED + } + + public static class ReadResult { + public final ReadStatus status; + public final List data; + ReadResult(ReadStatus status, List data) { + this.status = status; + this.data = data; + } + } + + /** + * ZMQ socket wrapper with bind/connect, timeouts, and retry. + */ + private static class ZeroMQPort { + final ZMQ.Socket socket; + final String address; + + ZeroMQPort(String portType, String address, int socketType) { + this.address = address; + ZMQ.Context ctx = getZmqContext(); + this.socket = ctx.socket(socketType); + this.socket.setReceiveTimeOut(2000); + this.socket.setSendTimeOut(2000); + this.socket.setLinger(0); + if (portType.equals("bind")) { + this.socket.bind(address); + } else { + this.socket.connect(address); + } + } + + String recvWithRetry() { + for (int attempt = 0; attempt < 5; attempt++) { + String msg = socket.recvStr(); + if (msg != null) return msg; + try { Thread.sleep(500); } catch (InterruptedException e) { Thread.currentThread().interrupt(); break; } + } + return null; + } + + void sendWithRetry(String message) { + for (int attempt = 0; attempt < 5; attempt++) { + if (socket.send(message)) return; + try { Thread.sleep(500); } catch (InterruptedException e) { Thread.currentThread().interrupt(); break; } + } + } + } + + /** + * Recursive descent parser for Python literal expressions. + * Handles: dicts, lists, tuples, strings, numbers, booleans, None. + */ + private static class Parser { + final String input; + int pos; + + Parser(String input) { + this.input = input; + this.pos = 0; + } + + void skipWhitespace() { + while (pos < input.length() && Character.isWhitespace(input.charAt(pos))) { + pos++; + } + } + + char peek() { + skipWhitespace(); + if (pos >= input.length()) throw new IllegalArgumentException("Unexpected end of input"); + return input.charAt(pos); + } + + char advance() { + char c = input.charAt(pos); + pos++; + return c; + } + + boolean hasMore() { + skipWhitespace(); + return pos < input.length(); + } + + Object parseExpression() { + skipWhitespace(); + if (pos >= input.length()) throw new IllegalArgumentException("Unexpected end of input"); + char c = input.charAt(pos); + + if (c == '{') return parseDict(); + if (c == '[') return parseList(); + if (c == '(') return parseTuple(); + if (c == '\'' || c == '"') return parseString(); + if (c == '-' || c == '+' || Character.isDigit(c)) return parseNumber(); + return parseKeyword(); + } + + Map parseDict() { + Map map = new HashMap<>(); + pos++; // skip '{' + skipWhitespace(); + if (hasMore() && input.charAt(pos) == '}') { + pos++; + return map; + } + while (true) { + skipWhitespace(); + Object key = parseExpression(); + skipWhitespace(); + if (pos >= input.length() || input.charAt(pos) != ':') { + throw new IllegalArgumentException("Expected ':' in dict at position " + pos); + } + pos++; // skip ':' + skipWhitespace(); + Object value = parseExpression(); + if (!(key instanceof String)) { + throw new IllegalArgumentException( + "Dict keys must be non-null strings, but got: " + + (key == null ? "null" : key.getClass().getSimpleName())); + } + map.put((String) key, value); + skipWhitespace(); + if (pos >= input.length()) { + throw new IllegalArgumentException("Unterminated dict: missing '}'"); + } + if (input.charAt(pos) == '}') { + pos++; + break; + } + if (input.charAt(pos) == ',') { + pos++; + skipWhitespace(); + // trailing comma before close + if (hasMore() && input.charAt(pos) == '}') { + pos++; + break; + } + } else { + throw new IllegalArgumentException("Expected ',' or '}' in dict at position " + pos); + } + } + return map; + } + + List parseList() { + List list = new ArrayList<>(); + pos++; // skip '[' + skipWhitespace(); + if (hasMore() && input.charAt(pos) == ']') { + pos++; + return list; + } + while (true) { + skipWhitespace(); + list.add(parseExpression()); + skipWhitespace(); + if (pos >= input.length()) { + throw new IllegalArgumentException("Unterminated list: missing ']'"); + } + if (input.charAt(pos) == ']') { + pos++; + break; + } + if (input.charAt(pos) == ',') { + pos++; + skipWhitespace(); + // trailing comma before close + if (hasMore() && input.charAt(pos) == ']') { + pos++; + break; + } + } else { + throw new IllegalArgumentException("Expected ',' or ']' in list at position " + pos); + } + } + return list; + } + + List parseTuple() { + List list = new ArrayList<>(); + pos++; // skip '(' + skipWhitespace(); + if (hasMore() && input.charAt(pos) == ')') { + pos++; + return list; + } + while (true) { + skipWhitespace(); + list.add(parseExpression()); + skipWhitespace(); + if (pos >= input.length()) { + throw new IllegalArgumentException("Unterminated tuple: missing ')'"); + } + if (input.charAt(pos) == ')') { + pos++; + break; + } + if (input.charAt(pos) == ',') { + pos++; + skipWhitespace(); + // trailing comma before close + if (hasMore() && input.charAt(pos) == ')') { + pos++; + break; + } + } else { + throw new IllegalArgumentException("Expected ',' or ')' in tuple at position " + pos); + } + } + return list; + } + + String parseString() { + char quote = advance(); // opening quote + StringBuilder sb = new StringBuilder(); + while (pos < input.length()) { + char c = input.charAt(pos); + if (c == '\\' && pos + 1 < input.length()) { + pos++; + char escaped = input.charAt(pos); + switch (escaped) { + case 'n': sb.append('\n'); break; + case 't': sb.append('\t'); break; + case 'r': sb.append('\r'); break; + case '\\': sb.append('\\'); break; + case '\'': sb.append('\''); break; + case '"': sb.append('"'); break; + default: sb.append('\\').append(escaped); break; + } + pos++; + } else if (c == quote) { + pos++; + return sb.toString(); + } else { + sb.append(c); + pos++; + } + } + throw new IllegalArgumentException("Unterminated string starting at position " + (pos - sb.length() - 1)); + } + + Number parseNumber() { + int start = pos; + if (pos < input.length() && (input.charAt(pos) == '-' || input.charAt(pos) == '+')) { + pos++; + } + boolean hasDecimal = false; + boolean hasExponent = false; + while (pos < input.length()) { + char c = input.charAt(pos); + if (Character.isDigit(c)) { + pos++; + } else if (c == '.' && !hasDecimal && !hasExponent) { + hasDecimal = true; + pos++; + } else if ((c == 'e' || c == 'E') && !hasExponent) { + hasExponent = true; + pos++; + if (pos < input.length() && (input.charAt(pos) == '+' || input.charAt(pos) == '-')) { + pos++; + } + } else { + break; + } + } + String numStr = input.substring(start, pos); + try { + if (hasDecimal || hasExponent) { + return Double.parseDouble(numStr); + } else { + try { + return Integer.parseInt(numStr); + } catch (NumberFormatException e) { + return Long.parseLong(numStr); + } + } + } catch (NumberFormatException e) { + throw new IllegalArgumentException("Invalid number: '" + numStr + "' at position " + start); + } + } + + Object parseKeyword() { + int start = pos; + while (pos < input.length() && Character.isLetterOrDigit(input.charAt(pos)) || (pos < input.length() && input.charAt(pos) == '_')) { + pos++; + } + String word = input.substring(start, pos); + switch (word) { + case "True": case "true": return Boolean.TRUE; + case "False": case "false": return Boolean.FALSE; + case "None": case "null": return null; + default: throw new IllegalArgumentException("Unknown keyword: '" + word + "' at position " + start); + } + } + } +} diff --git a/concore.py b/concore.py index 6d71f0fc..d2f5b6ff 100644 --- a/concore.py +++ b/concore.py @@ -1,123 +1,154 @@ import time +import logging import os +import atexit from ast import literal_eval import sys import re -import zmq # Added for ZeroMQ +import zmq +import numpy as np +import signal -# if windows, create script to kill this process -# because batch files don't provide easy way to know pid of last command -# ignored for posix != windows, because "concorepid" is handled by script -# ignored for docker (linux != windows), because handled by docker stop -if hasattr(sys, 'getwindowsversion'): - with open("concorekill.bat","w") as fpid: - fpid.write("taskkill /F /PID "+str(os.getpid())+"\n") +import concore_base -# =================================================================== -# ZeroMQ Communication Wrapper -# =================================================================== -class ZeroMQPort: - def __init__(self, port_type, address, zmq_socket_type): - """ - port_type: "bind" or "connect" - address: ZeroMQ address (e.g., "tcp://*:5555") - zmq_socket_type: zmq.REQ, zmq.REP, zmq.PUB, zmq.SUB etc. - """ - self.context = zmq.Context() - self.socket = self.context.socket(zmq_socket_type) - self.port_type = port_type # "bind" or "connect" - self.address = address - - # Configure timeouts & immediate close on failure - self.socket.setsockopt(zmq.RCVTIMEO, 2000) # 2 sec receive timeout - self.socket.setsockopt(zmq.SNDTIMEO, 2000) # 2 sec send timeout - self.socket.setsockopt(zmq.LINGER, 0) # Drop pending messages on close - - # Bind or connect - if self.port_type == "bind": - self.socket.bind(address) - print(f"ZMQ Port bound to {address}") - else: - self.socket.connect(address) - print(f"ZMQ Port connected to {address}") - - def send_json_with_retry(self, message): - """Send JSON message with retries if timeout occurs.""" - for attempt in range(5): - try: - self.socket.send_json(message) - return - except zmq.Again: - print(f"Send timeout (attempt {attempt + 1}/5)") - time.sleep(0.5) - print("Failed to send after retries.") - return - - def recv_json_with_retry(self): - """Receive JSON message with retries if timeout occurs.""" - for attempt in range(5): - try: - return self.socket.recv_json() - except zmq.Again: - print(f"Receive timeout (attempt {attempt + 1}/5)") - time.sleep(0.5) - print("Failed to receive after retries.") - return None - -# Global ZeroMQ ports registry -zmq_ports = {} +logger = logging.getLogger('concore') +logger.addHandler(logging.NullHandler()) -def init_zmq_port(port_name, port_type, address, socket_type_str): - """ - Initializes and registers a ZeroMQ port. - port_name (str): A unique name for this ZMQ port. - port_type (str): "bind" or "connect". - address (str): The ZMQ address (e.g., "tcp://*:5555", "tcp://localhost:5555"). - socket_type_str (str): String representation of ZMQ socket type (e.g., "REQ", "REP", "PUB", "SUB"). - """ - if port_name in zmq_ports: - print(f"ZMQ Port {port_name} already initialized.") - return # Avoid reinitialization - - try: - # Map socket type string to actual ZMQ constant (e.g., zmq.REQ, zmq.REP) - zmq_socket_type = getattr(zmq, socket_type_str.upper()) - zmq_ports[port_name] = ZeroMQPort(port_type, address, zmq_socket_type) - print(f"Initialized ZMQ port: {port_name} ({socket_type_str}) on {address}") - except AttributeError: - print(f"Error: Invalid ZMQ socket type string '{socket_type_str}'.") - except zmq.error.ZMQError as e: - print(f"Error initializing ZMQ port {port_name} on {address}: {e}") - except Exception as e: - print(f"An unexpected error occurred during ZMQ port initialization for {port_name}: {e}") +#these lines mute the noisy library +logging.getLogger('matplotlib').setLevel(logging.WARNING) +logging.getLogger('PIL').setLevel(logging.WARNING) +logging.getLogger('urllib3').setLevel(logging.WARNING) +logging.getLogger('requests').setLevel(logging.WARNING) -def terminate_zmq(): - for port in zmq_ports.values(): - try: - port.socket.close() - port.context.term() - except Exception as e: - print(f"Error while terminating ZMQ port {port.address}: {e}") -# --- ZeroMQ Integration End --- -# =================================================================== -# File & Parameter Handling -# =================================================================== -def safe_literal_eval(filename, defaultValue): +# if windows, register this process PID for safe termination +# Previous approach: single "concorekill.bat" overwritten by each node (race condition). +# New approach: append PID to shared registry; generate validated kill script. +# See: https://github.com/ControlCore-Project/concore/issues/391 + +_LOCK_LEN = 0x7FFFFFFF # lock range large enough to cover entire file +_BASE_DIR = os.path.abspath(".") # capture CWD before atexit can shift it +_PID_REGISTRY_FILE = os.path.join(_BASE_DIR, "concorekill_pids.txt") +_KILL_SCRIPT_FILE = os.path.join(_BASE_DIR, "concorekill.bat") + +def _register_pid(): + """Append current PID to the shared registry file. Uses file locking on Windows.""" try: - with open(filename, "r") as file: - return literal_eval(file.read()) - except (FileNotFoundError, SyntaxError, ValueError, Exception) as e: - # Keep print for debugging, but can be made quieter - # print(f"Info: Error reading {filename} or file not found, using default: {e}") - return defaultValue + with open(_PID_REGISTRY_FILE, "a") as f: + if hasattr(sys, 'getwindowsversion'): + import msvcrt + try: + msvcrt.locking(f.fileno(), msvcrt.LK_LOCK, _LOCK_LEN) + f.write(str(os.getpid()) + "\n") + finally: + try: + f.seek(0) + msvcrt.locking(f.fileno(), msvcrt.LK_UNLCK, _LOCK_LEN) + except OSError: + pass + else: + f.write(str(os.getpid()) + "\n") + except OSError: + pass +def _cleanup_pid(): + """Remove current PID from registry on exit. Uses file locking on Windows.""" + pid = str(os.getpid()) + try: + if not os.path.exists(_PID_REGISTRY_FILE): + return + with open(_PID_REGISTRY_FILE, "r+") as f: + if hasattr(sys, 'getwindowsversion'): + import msvcrt + try: + msvcrt.locking(f.fileno(), msvcrt.LK_LOCK, _LOCK_LEN) + pids = [line.strip() for line in f if line.strip()] + remaining = [p for p in pids if p != pid] + if remaining: + f.seek(0) + f.truncate() + for p in remaining: + f.write(p + "\n") + else: + f.close() + try: + os.remove(_PID_REGISTRY_FILE) + except OSError: + pass + try: + os.remove(_KILL_SCRIPT_FILE) + except OSError: + pass + return + finally: + try: + f.seek(0) + msvcrt.locking(f.fileno(), msvcrt.LK_UNLCK, _LOCK_LEN) + except (OSError, ValueError): + pass + else: + pids = [line.strip() for line in f if line.strip()] + remaining = [p for p in pids if p != pid] + if remaining: + f.seek(0) + f.truncate() + for p in remaining: + f.write(p + "\n") + else: + f.close() + try: + os.remove(_PID_REGISTRY_FILE) + except OSError: + pass + try: + os.remove(_KILL_SCRIPT_FILE) + except OSError: + pass + except OSError: + pass + +def _write_kill_script(): + """Generate concorekill.bat that validates each PID before killing.""" + try: + reg_name = os.path.basename(_PID_REGISTRY_FILE) + bat_name = os.path.basename(_KILL_SCRIPT_FILE) + script = "@echo off\r\n" + script += 'if not exist "%~dp0' + reg_name + '" (\r\n' + script += " echo No PID registry found. Nothing to kill.\r\n" + script += " exit /b 0\r\n" + script += ")\r\n" + script += 'for /f "usebackq tokens=*" %%p in ("%~dp0' + reg_name + '") do (\r\n' + script += ' wmic process where "ProcessId=%%p" get CommandLine /value 2>nul | find /i "concore" >nul\r\n' + script += " if not errorlevel 1 (\r\n" + script += " echo Killing concore process %%p\r\n" + script += " taskkill /F /PID %%p >nul 2>&1\r\n" + script += " ) else (\r\n" + script += " echo Skipping PID %%p - not a concore process or not running\r\n" + script += " )\r\n" + script += ")\r\n" + script += 'del /q "%~dp0' + reg_name + '" 2>nul\r\n' + script += 'del /q "%~dp0' + bat_name + '" 2>nul\r\n' + with open(_KILL_SCRIPT_FILE, "w", newline="") as f: + f.write(script) + except OSError: + pass -# Load input/output ports if present -iport = safe_literal_eval("concore.iport", {}) -oport = safe_literal_eval("concore.oport", {}) +if hasattr(sys, 'getwindowsversion'): + _register_pid() + _write_kill_script() + atexit.register(_cleanup_pid) + +ZeroMQPort = concore_base.ZeroMQPort +convert_numpy_to_python = concore_base.convert_numpy_to_python +safe_literal_eval = concore_base.safe_literal_eval +parse_params = concore_base.parse_params # Global variables +zmq_ports = {} +_cleanup_in_progress = False + +last_read_status = "SUCCESS" + s = '' olds = '' delay = 1 @@ -126,44 +157,51 @@ def safe_literal_eval(filename, defaultValue): outpath = "./out" simtime = 0 -#9/21/22 +def _port_path(base, port_num): + return base + str(port_num) + +concore_params_file = os.path.join(_port_path(inpath, 1), "concore.params") +concore_maxtime_file = os.path.join(_port_path(inpath, 1), "concore.maxtime") + +# Load input/output ports if present +iport = safe_literal_eval("concore.iport", {}) +oport = safe_literal_eval("concore.oport", {}) + +_mod = sys.modules[__name__] + # =================================================================== -# Parameter Parsing +# ZeroMQ Communication Wrapper # =================================================================== -try: - sparams_path = os.path.join(inpath + "1", "concore.params") - if os.path.exists(sparams_path): - with open(sparams_path, "r") as f: - sparams = f.read() - if sparams: # Ensure sparams is not empty - # Windows sometimes keeps quotes - if sparams[0] == '"' and sparams[-1] == '"': #windows keeps "" need to remove - sparams = sparams[1:-1] - - # Convert key=value;key2=value2 to Python dict format - if sparams != '{' and not (sparams.startswith('{') and sparams.endswith('}')): # Check if it needs conversion - print("converting sparams: "+sparams) - sparams = "{'"+re.sub(';',",'",re.sub('=',"':",re.sub(' ','',sparams)))+"}" - print("converted sparams: " + sparams) - try: - params = literal_eval(sparams) - except Exception as e: - print(f"bad params content: {sparams}, error: {e}") - params = dict() - else: - params = dict() - else: - params = dict() -except Exception as e: - # print(f"Info: concore.params not found or error reading, using empty dict: {e}") - params = dict() +def init_zmq_port(port_name, port_type, address, socket_type_str): + concore_base.init_zmq_port(_mod, port_name, port_type, address, socket_type_str) + +def terminate_zmq(): + """Clean up all ZMQ sockets and contexts before exit.""" + concore_base.terminate_zmq(_mod) + +def signal_handler(sig, frame): + """Handle interrupt signals gracefully.""" + print(f"\nReceived signal {sig}, shutting down gracefully...") + try: + atexit.unregister(terminate_zmq) + except Exception: + pass + concore_base.terminate_zmq(_mod) + sys.exit(0) + +# Register cleanup handlers +atexit.register(terminate_zmq) +signal.signal(signal.SIGINT, signal_handler) # Handle Ctrl+C +if not hasattr(sys, 'getwindowsversion'): + signal.signal(signal.SIGTERM, signal_handler) # Handle termination (Unix only) + +params = concore_base.load_params(concore_params_file) #9/30/22 def tryparam(n, i): """Return parameter `n` from params dict, else default `i`.""" return params.get(n, i) - #9/12/21 # =================================================================== # Simulation Time Handling @@ -171,168 +209,48 @@ def tryparam(n, i): def default_maxtime(default): """Read maximum simulation time from file or use default.""" global maxtime - maxtime_path = os.path.join(inpath + "1", "concore.maxtime") - maxtime = safe_literal_eval(maxtime_path, default) + maxtime = safe_literal_eval(concore_maxtime_file, default) default_maxtime(100) def unchanged(): """Check if global string `s` is unchanged since last call.""" - global olds, s - if olds == s: - s = '' - return True - olds = s - return False + return concore_base.unchanged(_mod) # =================================================================== # I/O Handling (File + ZMQ) # =================================================================== def read(port_identifier, name, initstr_val): - global s, simtime, retrycount - - # Default return - default_return_val = initstr_val - if isinstance(initstr_val, str): - try: - default_return_val = literal_eval(initstr_val) - except (SyntaxError, ValueError): - pass - - # Case 1: ZMQ port - if isinstance(port_identifier, str) and port_identifier in zmq_ports: - zmq_p = zmq_ports[port_identifier] - try: - message = zmq_p.recv_json_with_retry() - return message - except zmq.error.ZMQError as e: - print(f"ZMQ read error on port {port_identifier} (name: {name}): {e}. Returning default.") - return default_return_val - except Exception as e: - print(f"Unexpected error during ZMQ read on port {port_identifier} (name: {name}): {e}. Returning default.") - return default_return_val - - # Case 2: File-based port - try: - file_port_num = int(port_identifier) - except ValueError: - print(f"Error: Invalid port identifier '{port_identifier}' for file operation. Must be integer or ZMQ name.") - return default_return_val - - time.sleep(delay) - file_path = os.path.join(inpath+str(file_port_num), name) - ins = "" + """Read data from a ZMQ port or file-based port. + + Returns: + tuple: (data, success_flag) where success_flag is True if real + data was received, False if a fallback/default was used. + Also sets ``concore.last_read_status`` to one of: + SUCCESS, FILE_NOT_FOUND, TIMEOUT, PARSE_ERROR, + EMPTY_DATA, RETRIES_EXCEEDED. + + Backward compatibility: + Legacy callers that do ``value = concore.read(...)`` will + receive a tuple. They can adapt with:: + + result = concore.read(...) + if isinstance(result, tuple): + value, ok = result + else: + value, ok = result, True - try: - with open(file_path, "r") as infile: - ins = infile.read() - except FileNotFoundError: - ins = str(initstr_val) - except Exception as e: - print(f"Error reading {file_path}: {e}. Using default value.") - return default_return_val - - # Retry logic if file is empty - attempts = 0 - max_retries = 5 - while len(ins) == 0 and attempts < max_retries: - time.sleep(delay) - try: - with open(file_path, "r") as infile: - ins = infile.read() - except Exception as e: - print(f"Retry {attempts + 1}: Error reading {file_path} - {e}") - attempts += 1 - retrycount += 1 - - if len(ins) == 0: - print(f"Max retries reached for {file_path}, using default value.") - return default_return_val - - s += ins - - # Try parsing - try: - inval = literal_eval(ins) - if isinstance(inval, list) and len(inval) > 0: - current_simtime_from_file = inval[0] - if isinstance(current_simtime_from_file, (int, float)): - simtime = max(simtime, current_simtime_from_file) - return inval[1:] - else: - print(f"Warning: Unexpected data format in {file_path}: {ins}. Returning raw content or default.") - return inval - except Exception as e: - print(f"Error parsing content from {file_path} ('{ins}'): {e}. Returning default.") - return default_return_val + Alternatively, check ``concore.last_read_status`` after the + call. + """ + global last_read_status + result = concore_base.read(_mod, port_identifier, name, initstr_val) + last_read_status = concore_base.last_read_status + return result def write(port_identifier, name, val, delta=0): - """ - Write data either to ZMQ port or file. - `val` must be list (with simtime prefix) or string. - """ - global simtime - - # Case 1: ZMQ port - if isinstance(port_identifier, str) and port_identifier in zmq_ports: - zmq_p = zmq_ports[port_identifier] - try: - zmq_p.send_json_with_retry(val) - except zmq.error.ZMQError as e: - print(f"ZMQ write error on port {port_identifier} (name: {name}): {e}") - except Exception as e: - print(f"Unexpected error during ZMQ write on port {port_identifier} (name: {name}): {e}") - - # Case 2: File-based port - try: - if isinstance(port_identifier, str) and port_identifier in zmq_ports: - file_path = os.path.join("../"+port_identifier, name) - else: - file_port_num = int(port_identifier) - file_path = os.path.join(outpath+str(file_port_num), name) - except ValueError: - print(f"Error: Invalid port identifier '{port_identifier}' for file operation. Must be integer or ZMQ name.") - return - - # File writing rules - if isinstance(val, str): - time.sleep(2 * delay) # string writes wait longer - elif not isinstance(val, list): - print(f"File write to {file_path} must have list or str value, got {type(val)}") - return - - try: - with open(file_path, "w") as outfile: - if isinstance(val, list): - data_to_write = [simtime + delta] + val - outfile.write(str(data_to_write)) - simtime += delta - else: - outfile.write(val) - except Exception as e: - print(f"Error writing to {file_path}: {e}") + concore_base.write(_mod, port_identifier, name, val, delta) def initval(simtime_val_str): - """ - Initialize simtime from string containing a list. - Example: "[10, 'foo', 'bar']" → simtime=10, returns ['foo','bar'] - """ - global simtime - try: - val = literal_eval(simtime_val_str) - if isinstance(val, list) and len(val) > 0: - first_element = val[0] - if isinstance(first_element, (int, float)): - simtime = first_element - return val[1:] - else: - print(f"Error: First element in initval string '{simtime_val_str}' is not a number. Using data part as is or empty.") - return val[1:] if len(val) > 1 else [] - else: - print(f"Error: initval string '{simtime_val_str}' is not a list or is empty. Returning empty list.") - return [] - - except Exception as e: - print(f"Error parsing simtime_val_str '{simtime_val_str}': {e}. Returning empty list.") - return [] \ No newline at end of file + return concore_base.initval(_mod, simtime_val_str) diff --git a/concore.v b/concore.v index 1bf95257..8d77d933 100644 --- a/concore.v +++ b/concore.v @@ -296,6 +296,7 @@ module concore; end $fdisplay(fout,"]"); $fclose(fout); + // simtime must not be mutated here (issue #385). end endtask diff --git a/concore_base.hpp b/concore_base.hpp new file mode 100644 index 00000000..f38b6edb --- /dev/null +++ b/concore_base.hpp @@ -0,0 +1,497 @@ +// concore_base.hpp -- shared utilities for concore.hpp and concoredocker.hpp +// Extracted to eliminate drift between local and Docker C++ implementations. +#ifndef CONCORE_BASE_HPP +#define CONCORE_BASE_HPP + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace concore_base { + +// =================================================================== +// String Helpers +// =================================================================== +inline std::string stripstr(const std::string& str) { + size_t start = str.find_first_not_of(" \t\n\r"); + if (start == std::string::npos) return ""; + size_t end = str.find_last_not_of(" \t\n\r"); + return str.substr(start, end - start + 1); +} + +inline std::string stripquotes(const std::string& str) { + if (str.size() >= 2 && + ((str.front() == '\'' && str.back() == '\'') || + (str.front() == '"' && str.back() == '"'))) + return str.substr(1, str.size() - 2); + return str; +} + +// =================================================================== +// Parsing Utilities +// =================================================================== + +/** + * Parses a Python-style dict string into a string→string map. + * Format: {'key1': val1, 'key2': val2} + * Handles both quoted and unquoted keys/values. + */ +inline std::map parsedict(const std::string& str) { + std::map result; + std::string trimmed = stripstr(str); + if (trimmed.size() < 2 || trimmed.front() != '{' || trimmed.back() != '}') + return result; + std::string inner = trimmed.substr(1, trimmed.size() - 2); + std::stringstream ss(inner); + std::string token; + while (std::getline(ss, token, ',')) { + size_t colon = token.find(':'); + if (colon == std::string::npos) continue; + std::string key = stripquotes(stripstr(token.substr(0, colon))); + std::string val = stripquotes(stripstr(token.substr(colon + 1))); + if (!key.empty()) result[key] = val; + } + return result; +} + +/** + * Parses a Python-style list string into a vector of strings. + * Format: [val1, val2, val3] + */ +inline std::vector parselist(const std::string& str) { + std::vector result; + std::string trimmed = stripstr(str); + if (trimmed.size() < 2 || trimmed.front() != '[' || trimmed.back() != ']') + return result; + std::string inner = trimmed.substr(1, trimmed.size() - 2); + std::stringstream ss(inner); + std::string token; + while (std::getline(ss, token, ',')) { + std::string val = stripstr(token); + if (!val.empty()) result.push_back(val); + } + return result; +} + +/** + * Parses a double-valued list like "[0.0, 1.5, 2.3]" into a vector. + * Used by concore.hpp's read/write which work with numeric data. + */ +inline std::vector parselist_double(const std::string& str); + +enum class ConcoreValueType { NUMBER, BOOL, STRING, ARRAY }; + +struct ConcoreValue { + ConcoreValueType type; + double number; + bool boolean; + std::string str; + std::vector array; + + ConcoreValue() : type(ConcoreValueType::NUMBER), number(0.0), boolean(false) {} + + static ConcoreValue make_number(double v) { + ConcoreValue cv; + cv.type = ConcoreValueType::NUMBER; + cv.number = v; + return cv; + } + static ConcoreValue make_bool(bool v) { + ConcoreValue cv; + cv.type = ConcoreValueType::BOOL; + cv.boolean = v; + cv.number = v ? 1.0 : 0.0; + return cv; + } + static ConcoreValue make_string(const std::string& v) { + ConcoreValue cv; + cv.type = ConcoreValueType::STRING; + cv.str = v; + return cv; + } + static ConcoreValue make_array(const std::vector& v) { + ConcoreValue cv; + cv.type = ConcoreValueType::ARRAY; + cv.array = v; + return cv; + } +}; + +inline void skip_ws(const std::string& s, size_t& pos) { + while (pos < s.size() && std::isspace(static_cast(s[pos]))) + ++pos; +} + +inline ConcoreValue parse_literal_value(const std::string& s, size_t& pos); + +inline ConcoreValue parse_literal_string(const std::string& s, size_t& pos) { + char quote = s[pos]; + ++pos; + std::string result; + while (pos < s.size() && s[pos] != quote) { + if (s[pos] == '\\' && pos + 1 < s.size()) { + ++pos; + switch (s[pos]) { + case 'n': result += '\n'; break; + case 't': result += '\t'; break; + case '\\': result += '\\'; break; + case '\'': result += '\''; break; + case '"': result += '"'; break; + default: result += '\\'; result += s[pos]; break; + } + } else { + result += s[pos]; + } + ++pos; + } + if (pos >= s.size()) + throw std::runtime_error("Invalid concore payload: unterminated string"); + ++pos; + return ConcoreValue::make_string(result); +} + +inline ConcoreValue parse_literal_array(const std::string& s, size_t& pos) { + char open = s[pos]; + char close = (open == '[') ? ']' : ')'; + ++pos; + std::vector elements; + skip_ws(s, pos); + if (pos < s.size() && s[pos] == close) { ++pos; return ConcoreValue::make_array(elements); } + while (pos < s.size()) { + elements.push_back(parse_literal_value(s, pos)); + skip_ws(s, pos); + if (pos < s.size() && s[pos] == ',') { ++pos; skip_ws(s, pos); } + if (pos < s.size() && s[pos] == close) { ++pos; return ConcoreValue::make_array(elements); } + } + throw std::runtime_error("Invalid concore payload: unterminated array/tuple"); +} + +inline ConcoreValue parse_literal_value(const std::string& s, size_t& pos) { + skip_ws(s, pos); + if (pos >= s.size()) + throw std::runtime_error("Invalid concore payload: unexpected end of input"); + + char c = s[pos]; + + if (c == '[' || c == '(') + return parse_literal_array(s, pos); + + if (c == '\'' || c == '"') + return parse_literal_string(s, pos); + + if (s.compare(pos, 4, "True") == 0 && + (pos + 4 >= s.size() || + (!std::isalnum(static_cast(s[pos + 4])) && s[pos + 4] != '_'))) { + pos += 4; + return ConcoreValue::make_bool(true); + } + if (s.compare(pos, 5, "False") == 0 && + (pos + 5 >= s.size() || + (!std::isalnum(static_cast(s[pos + 5])) && s[pos + 5] != '_'))) { + pos += 5; + return ConcoreValue::make_bool(false); + } + if (s.compare(pos, 4, "None") == 0 && + (pos + 4 >= s.size() || + (!std::isalnum(static_cast(s[pos + 4])) && s[pos + 4] != '_'))) { + pos += 4; + return ConcoreValue::make_string("None"); + } + + { + size_t start = pos; + if (pos < s.size() && (s[pos] == '+' || s[pos] == '-')) ++pos; + bool has_digits = false; + while (pos < s.size() && std::isdigit(static_cast(s[pos]))) { + ++pos; has_digits = true; + } + if (pos < s.size() && s[pos] == '.') { + ++pos; + while (pos < s.size() && std::isdigit(static_cast(s[pos]))) { + ++pos; has_digits = true; + } + } + if (has_digits && pos < s.size() && (s[pos] == 'e' || s[pos] == 'E')) { + ++pos; + if (pos < s.size() && (s[pos] == '+' || s[pos] == '-')) ++pos; + while (pos < s.size() && std::isdigit(static_cast(s[pos]))) ++pos; + } + if (has_digits && pos > start) { + std::string numstr = s.substr(start, pos - start); + try { + double val = std::stod(numstr); + return ConcoreValue::make_number(val); + } catch (...) { + throw std::runtime_error( + "Invalid concore payload: bad number '" + numstr + "'"); + } + } + pos = start; + } + + throw std::runtime_error( + std::string("Invalid concore payload: unsupported literal at position ") + + std::to_string(pos)); +} + +inline ConcoreValue parse_literal(const std::string& s) { + size_t pos = 0; + ConcoreValue v = parse_literal_value(s, pos); + skip_ws(s, pos); + if (pos != s.size()) + throw std::runtime_error( + "Invalid concore payload: unexpected trailing content"); + return v; +} + +inline void flatten_numeric_impl(const ConcoreValue& v, std::vector& out) { + switch (v.type) { + case ConcoreValueType::NUMBER: + out.push_back(v.number); + break; + case ConcoreValueType::BOOL: + out.push_back(v.boolean ? 1.0 : 0.0); + break; + case ConcoreValueType::STRING: + break; + case ConcoreValueType::ARRAY: + for (const auto& elem : v.array) + flatten_numeric_impl(elem, out); + break; + } +} + +inline std::vector flatten_numeric(const ConcoreValue& v) { + std::vector out; + flatten_numeric_impl(v, out); + return out; +} + +inline std::vector parselist_double(const std::string& str) { + std::string trimmed = stripstr(str); + if (trimmed.empty()) return {}; + try { + ConcoreValue v = parse_literal(trimmed); + return flatten_numeric(v); + } catch (...) { + std::vector result; + if (trimmed.size() < 2) return result; + if (trimmed.front() == '[' || trimmed.front() == '(') { + std::vector tokens = parselist(trimmed); + for (const auto& tok : tokens) { + try { result.push_back(std::stod(tok)); } catch (...) {} + } + } + return result; + } +} + +/** + * Reads a file and parses its content as a dict. + * Returns defaultValue on any failure (matches Python safe_literal_eval). + */ +inline std::map safe_literal_eval_dict( + const std::string& filename, + const std::map& defaultValue) +{ + std::ifstream file(filename); + if (!file) return defaultValue; + std::stringstream buf; + buf << file.rdbuf(); + std::string content = buf.str(); + try { + return parsedict(content); + } catch (...) { + return defaultValue; + } +} + +/** + * Loads simulation parameters from a concore.params file. + * Handles Windows quote wrapping, semicolon-separated key=value, + * and dict-literal format. + */ +inline std::map load_params(const std::string& params_file) { + std::ifstream file(params_file); + if (!file) return {}; + std::stringstream buffer; + buffer << file.rdbuf(); + std::string sparams = buffer.str(); + + // Windows sometimes keeps surrounding quotes + if (!sparams.empty() && sparams[0] == '"') { + size_t closing = sparams.find('"', 1); + if (closing != std::string::npos) + sparams = sparams.substr(1, closing - 1); + } + + sparams = stripstr(sparams); + if (sparams.empty()) return {}; + + // If already a dict literal, parse directly + if (sparams.front() == '{') { + try { return parsedict(sparams); } catch (...) {} + } + + // Otherwise convert semicolon-separated key=value to dict format + // e.g. "a=1;b=2" -> {"a":"1","b":"2"} + std::string normalized = std::regex_replace(sparams, std::regex(";"), ","); + std::string converted = "{\"" + + std::regex_replace( + std::regex_replace( + std::regex_replace(normalized, std::regex(","), ",\""), + std::regex("="), "\":"), + std::regex(" "), "") + + "}"; + try { return parsedict(converted); } catch (...) {} + + return {}; +} + +/** + * Reads maxtime from concore.maxtime file, falls back to defaultValue. + */ +inline double load_maxtime(const std::string& maxtime_file, double defaultValue) { + std::ifstream file(maxtime_file); + if (!file) return defaultValue; + double val; + if (file >> val) return val; + return defaultValue; +} + +/** + * Returns param value by name, or default if not found. + */ +inline std::string tryparam( + const std::map& params, + const std::string& name, + const std::string& defaultValue) +{ + auto it = params.find(name); + return (it != params.end()) ? it->second : defaultValue; +} + + +// =================================================================== +// ZeroMQ Transport (opt-in: compile with -DCONCORE_USE_ZMQ) +// =================================================================== +#ifdef CONCORE_USE_ZMQ +#include + +/** + * ZMQ socket wrapper with bind/connect, timeouts, and retry. + */ +class ZeroMQPort { +public: + zmq::context_t context; + zmq::socket_t socket; + std::string port_type; + std::string address; + + ZeroMQPort(const std::string& port_type_, const std::string& address_, int socket_type) + : context(1), socket(context, socket_type), + port_type(port_type_), address(address_) + { + socket.setsockopt(ZMQ_RCVTIMEO, 2000); + socket.setsockopt(ZMQ_SNDTIMEO, 2000); + socket.setsockopt(ZMQ_LINGER, 0); + + if (port_type == "bind") + socket.bind(address); + else + socket.connect(address); + } + + ZeroMQPort(const ZeroMQPort&) = delete; + ZeroMQPort& operator=(const ZeroMQPort&) = delete; + + /** + * Sends a vector as "[v0, v1, ...]" with retry on timeout. + */ + void send_with_retry(const std::vector& payload) { + std::ostringstream ss; + ss << "["; + for (size_t i = 0; i < payload.size(); ++i) { + if (i) ss << ", "; + ss << payload[i]; + } + ss << "]"; + std::string msg = ss.str(); + for (int attempt = 0; attempt < 5; ++attempt) { + try { + zmq::message_t zmsg(msg.begin(), msg.end()); + socket.send(zmsg, zmq::send_flags::none); + return; + } catch (const zmq::error_t&) { + std::this_thread::sleep_for(std::chrono::milliseconds(500)); + } + } + std::cerr << "ZMQ send failed after retries." << std::endl; + } + + /** + * Sends a raw string with retry on timeout. + */ + void send_string_with_retry(const std::string& msg) { + for (int attempt = 0; attempt < 5; ++attempt) { + try { + zmq::message_t zmsg(msg.begin(), msg.end()); + socket.send(zmsg, zmq::send_flags::none); + return; + } catch (const zmq::error_t&) { + std::this_thread::sleep_for(std::chrono::milliseconds(500)); + } + } + std::cerr << "ZMQ send failed after retries." << std::endl; + } + + /** + * Receives and parses "[v0, v1, ...]" back to vector. + */ + std::vector recv_with_retry() { + for (int attempt = 0; attempt < 5; ++attempt) { + try { + zmq::message_t zmsg; + auto res = socket.recv(zmsg, zmq::recv_flags::none); + if (res) { + std::string data(static_cast(zmsg.data()), zmsg.size()); + return parselist_double(data); + } + } catch (const zmq::error_t&) { + std::this_thread::sleep_for(std::chrono::milliseconds(500)); + } + } + std::cerr << "ZMQ recv failed after retries." << std::endl; + return {}; + } +}; + +/** + * Maps socket type string ("REQ", "REP", etc.) to ZMQ constant. + * Returns -1 on unknown type. + */ +inline int zmq_socket_type_from_string(const std::string& s) { + if (s == "REQ") return ZMQ_REQ; + if (s == "REP") return ZMQ_REP; + if (s == "PUB") return ZMQ_PUB; + if (s == "SUB") return ZMQ_SUB; + if (s == "PUSH") return ZMQ_PUSH; + if (s == "PULL") return ZMQ_PULL; + if (s == "PAIR") return ZMQ_PAIR; + return -1; +} +#endif // CONCORE_USE_ZMQ + +} // namespace concore_base + +#endif // CONCORE_BASE_HPP diff --git a/concore_base.py b/concore_base.py new file mode 100644 index 00000000..9173289b --- /dev/null +++ b/concore_base.py @@ -0,0 +1,445 @@ +import time +import logging +import os +from ast import literal_eval +import zmq +import numpy as np + +logger = logging.getLogger('concore') +logger.addHandler(logging.NullHandler()) + +# =================================================================== +# ZeroMQ Communication Wrapper +# =================================================================== +# lazy-initialized shared ZMQ context for the entire process. +# using None until first ZMQ port is created, so file-only workflows +# never spawn ZMQ I/O threads at import time. +_zmq_context = None + +def _get_zmq_context(): + """Return the process-level shared ZMQ context, creating it on first call.""" + global _zmq_context + if _zmq_context is None or _zmq_context.closed: + _zmq_context = zmq.Context() + return _zmq_context + +class ZeroMQPort: + def __init__(self, port_type, address, zmq_socket_type, context=None): + """ + port_type: "bind" or "connect" + address: ZeroMQ address (e.g., "tcp://*:5555") + zmq_socket_type: zmq.REQ, zmq.REP, zmq.PUB, zmq.SUB etc. + context: optional zmq.Context() for the process; defaults to the shared _zmq_context. + """ + if context is None: + context = _get_zmq_context() + self.socket = context.socket(zmq_socket_type) + self.port_type = port_type # "bind" or "connect" + self.address = address + + # Configure timeouts & immediate close on failure + self.socket.setsockopt(zmq.RCVTIMEO, 2000) # 2 sec receive timeout + self.socket.setsockopt(zmq.SNDTIMEO, 2000) # 2 sec send timeout + self.socket.setsockopt(zmq.LINGER, 0) # Drop pending messages on close + + # Bind or connect + if self.port_type == "bind": + self.socket.bind(address) + logger.info(f"ZMQ Port bound to {address}") + else: + self.socket.connect(address) + logger.info(f"ZMQ Port connected to {address}") + + def send_json_with_retry(self, message): + """Send JSON message with retries if timeout occurs.""" + for attempt in range(5): + try: + self.socket.send_json(message) + return + except zmq.Again: + logger.warning(f"Send timeout (attempt {attempt + 1}/5)") + time.sleep(0.5) + raise TimeoutError(f"ZMQ send failed after 5 retries on {self.address}") + + def recv_json_with_retry(self): + """Receive JSON message with retries if timeout occurs.""" + for attempt in range(5): + try: + return self.socket.recv_json() + except zmq.Again: + logger.warning(f"Receive timeout (attempt {attempt + 1}/5)") + time.sleep(0.5) + raise TimeoutError(f"ZMQ recv failed after 5 retries on {self.address}") + + +def init_zmq_port(mod, port_name, port_type, address, socket_type_str): + """ + Initializes and registers a ZeroMQ port. + mod: calling module (has zmq_ports dict) + port_name (str): A unique name for this ZMQ port. + port_type (str): "bind" or "connect". + address (str): The ZMQ address (e.g., "tcp://*:5555", "tcp://localhost:5555"). + socket_type_str (str): String representation of ZMQ socket type (e.g., "REQ", "REP", "PUB", "SUB"). + """ + if port_name in mod.zmq_ports: + logger.info(f"ZMQ Port {port_name} already initialized.") + return # Avoid reinitialization + + try: + # Map socket type string to actual ZMQ constant (e.g., zmq.REQ, zmq.REP) + zmq_socket_type = getattr(zmq, socket_type_str.upper()) + mod.zmq_ports[port_name] = ZeroMQPort(port_type, address, zmq_socket_type, _get_zmq_context()) + logger.info(f"Initialized ZMQ port: {port_name} ({socket_type_str}) on {address}") + except AttributeError: + logger.error(f"Error: Invalid ZMQ socket type string '{socket_type_str}'.") + except zmq.error.ZMQError as e: + logger.error(f"Error initializing ZMQ port {port_name} on {address}: {e}") + except Exception as e: + logger.error(f"An unexpected error occurred during ZMQ port initialization for {port_name}: {e}") + +def terminate_zmq(mod): + """Clean up all ZMQ sockets, then terminate the shared context once.""" + global _zmq_context # declared first — used both in the early-return guard and reset below + if mod._cleanup_in_progress: + return # Already cleaning up, prevent reentrant calls + + if not mod.zmq_ports and (_zmq_context is None or _zmq_context.closed): + return # Nothing to clean up: no ports and no active context + + mod._cleanup_in_progress = True + print("\nCleaning up ZMQ resources...") + + # all sockets must be closed before context.term() is called. + for port_name, port in mod.zmq_ports.items(): + try: + port.socket.close() + print(f"Closed ZMQ port: {port_name}") + except Exception as e: + logger.error(f"Error while terminating ZMQ port {port.address}: {e}") + mod.zmq_ports.clear() + + # terminate the single shared context exactly once, then reset so it + # can be safely recreated if init_zmq_port is called again later. + if _zmq_context is not None and not _zmq_context.closed: + try: + _zmq_context.term() + except Exception as e: + logger.error(f"Error while terminating shared ZMQ context: {e}") + _zmq_context = None + + mod._cleanup_in_progress = False + +# --- ZeroMQ Integration End --- + + +# NumPy Type Conversion Helper +def convert_numpy_to_python(obj): + #Recursively convert numpy types to native Python types. + #This is necessary because literal_eval cannot parse numpy representations + #like np.float64(1.0), but can parse native Python types like 1.0. + if isinstance(obj, np.generic): + # Convert numpy scalar types to Python native types + return obj.item() + elif isinstance(obj, list): + return [convert_numpy_to_python(item) for item in obj] + elif isinstance(obj, tuple): + return tuple(convert_numpy_to_python(item) for item in obj) + elif isinstance(obj, dict): + return {key: convert_numpy_to_python(value) for key, value in obj.items()} + else: + return obj + +# =================================================================== +# File & Parameter Handling +# =================================================================== +def safe_literal_eval(filename, defaultValue): + try: + with open(filename, "r") as file: + return literal_eval(file.read()) + except (FileNotFoundError, SyntaxError, ValueError, Exception) as e: + # print(f"Info: Error reading {filename} or file not found, using default: {e}") + return defaultValue + +#9/21/22 +# =================================================================== +# Parameter Parsing +# =================================================================== +def parse_params(sparams): + params = {} + if not sparams: + return params + + s = sparams.strip() + + #full dict literal + if s.startswith("{") and s.endswith("}"): + try: + val = literal_eval(s) + if isinstance(val, dict): + return val + except (ValueError, SyntaxError): + pass + + for item in s.split(";"): + if "=" in item: + key, value = item.split("=", 1) # split only once + key=key.strip() + value=value.strip() + #try to convert to python type (int, float, list, etc.) + # Use literal_eval to preserve backward compatibility (integers/lists) + # Fallback to string for unquoted values (paths, URLs) + try: + params[key] = literal_eval(value) + except (ValueError, SyntaxError): + params[key] = value + return params + +def load_params(params_file): + try: + if os.path.exists(params_file): + with open(params_file, "r") as f: + sparams = f.read().strip() + if sparams: + # Windows sometimes keeps quotes + if sparams[0] == '"' and sparams[-1] == '"': #windows keeps "" need to remove + sparams = sparams[1:-1] + logger.debug("parsing sparams: "+sparams) + p = parse_params(sparams) + logger.debug("parsed params: " + str(p)) + return p + return dict() + except Exception: + return dict() + +# =================================================================== +# Read Status Tracking +# =================================================================== +last_read_status = "SUCCESS" + +# =================================================================== +# I/O Handling (File + ZMQ) +# =================================================================== + +def unchanged(mod): + """Check if global string `s` is unchanged since last call.""" + if mod.olds == mod.s: + mod.s = '' + return True + mod.olds = mod.s + return False + + +def read(mod, port_identifier, name, initstr_val): + """Read data from a ZMQ port or file-based port. + + Returns: + tuple: (data, success_flag) where success_flag is True if real + data was received, False if a fallback/default was used. + Also sets ``concore.last_read_status`` (and + ``concore_base.last_read_status``) to one of: + SUCCESS, FILE_NOT_FOUND, TIMEOUT, PARSE_ERROR, + EMPTY_DATA, RETRIES_EXCEEDED. + + Backward compatibility: + Legacy callers that do ``value = concore.read(...)`` will + receive a tuple. They can adapt with:: + + result = concore.read(...) + if isinstance(result, tuple): + value, ok = result + else: + value, ok = result, True + + Alternatively, check ``concore.last_read_status`` after the + call. + """ + global last_read_status + + # Default return + default_return_val = initstr_val + if isinstance(initstr_val, str): + try: + default_return_val = literal_eval(initstr_val) + except (SyntaxError, ValueError): + pass + + # Case 1: ZMQ port + if isinstance(port_identifier, str) and port_identifier in mod.zmq_ports: + zmq_p = mod.zmq_ports[port_identifier] + try: + message = zmq_p.recv_json_with_retry() + # Strip simtime prefix if present (mirroring file-based read behavior) + if isinstance(message, list) and len(message) > 0: + first_element = message[0] + if isinstance(first_element, (int, float)): + mod.simtime = max(mod.simtime, first_element) + last_read_status = "SUCCESS" + return message[1:], True + last_read_status = "SUCCESS" + return message, True + except TimeoutError as e: + logger.error(f"ZMQ read timeout on port {port_identifier} (name: {name}): {e}. Returning default.") + last_read_status = "TIMEOUT" + return default_return_val, False + except zmq.error.ZMQError as e: + logger.error(f"ZMQ read error on port {port_identifier} (name: {name}): {e}. Returning default.") + last_read_status = "TIMEOUT" + return default_return_val, False + except Exception as e: + logger.error(f"Unexpected error during ZMQ read on port {port_identifier} (name: {name}): {e}. Returning default.") + last_read_status = "PARSE_ERROR" + return default_return_val, False + + # Case 2: File-based port + try: + file_port_num = int(port_identifier) + except ValueError: + logger.error(f"Error: Invalid port identifier '{port_identifier}' for file operation. Must be integer or ZMQ name.") + last_read_status = "PARSE_ERROR" + return default_return_val, False + + time.sleep(mod.delay) + port_dir = mod._port_path(mod.inpath, file_port_num) + file_path = os.path.join(port_dir, name) + ins = "" + + file_not_found = False + try: + with open(file_path, "r") as infile: + ins = infile.read() + except FileNotFoundError: + file_not_found = True + ins = str(initstr_val) + mod.s += ins # Update s to break unchanged() loop + except Exception as e: + logger.error(f"Error reading {file_path}: {e}. Using default value.") + last_read_status = "FILE_NOT_FOUND" + return default_return_val, False + + # Retry logic if file is empty + attempts = 0 + max_retries = 5 + while len(ins) == 0 and attempts < max_retries: + time.sleep(mod.delay) + try: + with open(file_path, "r") as infile: + ins = infile.read() + except Exception as e: + logger.warning(f"Retry {attempts + 1}: Error reading {file_path} - {e}") + attempts += 1 + mod.retrycount += 1 + + if len(ins) == 0: + logger.error(f"Max retries reached for {file_path}, using default value.") + last_read_status = "RETRIES_EXCEEDED" + return default_return_val, False + + mod.s += ins + + # Try parsing + try: + inval = literal_eval(ins) + if isinstance(inval, list) and len(inval) > 0: + current_simtime_from_file = inval[0] + if isinstance(current_simtime_from_file, (int, float)): + mod.simtime = max(mod.simtime, current_simtime_from_file) + if file_not_found: + last_read_status = "FILE_NOT_FOUND" + return inval[1:], False + last_read_status = "SUCCESS" + return inval[1:], True + else: + logger.warning(f"Warning: Unexpected data format in {file_path}: {ins}. Returning raw content or default.") + if file_not_found: + last_read_status = "FILE_NOT_FOUND" + return inval, False + last_read_status = "SUCCESS" + return inval, True + except Exception as e: + logger.error(f"Error parsing content from {file_path} ('{ins}'): {e}. Returning default.") + if file_not_found: + last_read_status = "FILE_NOT_FOUND" + else: + last_read_status = "PARSE_ERROR" + return default_return_val, False + + +def write(mod, port_identifier, name, val, delta=0): + """ + Write data either to ZMQ port or file. + `val` is the data payload (list or string); write() prepends [simtime + delta] internally. + """ + # Case 1: ZMQ port + if isinstance(port_identifier, str) and port_identifier in mod.zmq_ports: + zmq_p = mod.zmq_ports[port_identifier] + try: + # Keep ZMQ payloads JSON-serializable by normalizing numpy types. + zmq_val = convert_numpy_to_python(val) + if isinstance(zmq_val, list): + # Prepend simtime to match file-based write behavior + payload = [mod.simtime + delta] + zmq_val + zmq_p.send_json_with_retry(payload) + # simtime must not be mutated here. + # Mutation breaks cross-language determinism (see issue #385). + else: + zmq_p.send_json_with_retry(zmq_val) + except TimeoutError as e: + logger.error(f"ZMQ write timeout on port {port_identifier} (name: {name}): {e}") + except zmq.error.ZMQError as e: + logger.error(f"ZMQ write error on port {port_identifier} (name: {name}): {e}") + except Exception as e: + logger.error(f"Unexpected error during ZMQ write on port {port_identifier} (name: {name}): {e}") + return + + # Case 2: File-based port + try: + file_port_num = int(port_identifier) + port_dir = mod._port_path(mod.outpath, file_port_num) + file_path = os.path.join(port_dir, name) + except ValueError: + logger.error(f"Error: Invalid port identifier '{port_identifier}' for file operation. Must be integer or ZMQ name.") + return + + # File writing rules + if isinstance(val, str): + time.sleep(2 * mod.delay) # string writes wait longer + elif not isinstance(val, list): + logger.error(f"File write to {file_path} must have list or str value, got {type(val)}") + return + + try: + with open(file_path, "w") as outfile: + if isinstance(val, list): + # Convert numpy types to native Python types + val_converted = convert_numpy_to_python(val) + data_to_write = [mod.simtime + delta] + val_converted + outfile.write(str(data_to_write)) + # simtime must not be mutated here. + # Mutation breaks cross-language determinism (see issue #385). + else: + outfile.write(val) + except Exception as e: + logger.error(f"Error writing to {file_path}: {e}") + +def initval(mod, simtime_val_str): + """ + Initialize simtime from string containing a list. + Example: "[10, 'foo', 'bar']" -> simtime=10, returns ['foo','bar'] + """ + try: + val = literal_eval(simtime_val_str) + if isinstance(val, list) and len(val) > 0: + first_element = val[0] + if isinstance(first_element, (int, float)): + mod.simtime = first_element + return val[1:] + else: + logger.error(f"Error: First element in initval string '{simtime_val_str}' is not a number. Using data part as is or empty.") + return val[1:] if len(val) > 1 else [] + else: + logger.error(f"Error: initval string '{simtime_val_str}' is not a list or is empty. Returning empty list.") + return [] + + except Exception as e: + logger.error(f"Error parsing simtime_val_str '{simtime_val_str}': {e}. Returning empty list.") + return [] diff --git a/concore_cli/README.md b/concore_cli/README.md new file mode 100644 index 00000000..78f98b1a --- /dev/null +++ b/concore_cli/README.md @@ -0,0 +1,199 @@ +# Concore CLI + +A command-line interface for managing concore neuromodulation workflows. + +## Installation + +```bash +pip install -e . +``` + +## Quick Start + +```bash +# Create a new project +concore init my-project + +# Navigate to your project +cd my-project + +# Validate your workflow +concore validate workflow.graphml + +# Build your workflow +concore build workflow.graphml + +# Check running processes +concore status + +# Stop all processes +concore stop +``` + +## Commands + +### `concore init ` + +Creates a new concore project with a basic structure. + +**Options:** +- `--template` - Template type to use (default: basic) + +**Example:** +```bash +concore init my-workflow +``` + +Creates: +``` +my-workflow/ +├── workflow.graphml # Sample workflow definition +├── src/ +│ └── script.py # Sample processing script +└── README.md # Project documentation +``` + +### `concore build ` + +Compiles a concore workflow GraphML file into executable scripts (POSIX, Windows, or Docker). + +**Options:** +- `-s, --source ` - Source directory (default: src) +- `-o, --output ` - Output directory (default: out) +- `-t, --type ` - Execution type: windows, posix, or docker (default: windows) +- `--auto-build` - Automatically run build script after generation +- `--compose` - Generate `docker-compose.yml` (only valid with `--type docker`) + +**Example:** +```bash +concore build workflow.graphml --source ./src --output ./build --auto-build +``` + +Docker compose example: + +```bash +concore build workflow.graphml --source ./src --output ./out --type docker --compose +cd out +docker compose up +``` + +### `concore validate ` + +Validates a GraphML workflow file before running. + +**Options:** +- `-s, --source ` - Source directory to verify file references exist + +Checks: +- Valid XML structure +- GraphML format compliance +- Node and edge definitions +- File references and naming conventions +- Source file existence (when --source provided) +- ZMQ port conflicts and reserved ports +- Circular dependencies (warns for control loops) +- Edge connectivity + +**Options:** +- `-s, --source ` - Source directory (default: src) + +**Example:** +```bash +concore validate workflow.graphml +concore validate workflow.graphml --source ./src +``` + +### `concore status` + +Shows all currently running concore processes with details: +- Process ID (PID) +- Process name +- Uptime +- Memory usage +- Command + +**Example:** +```bash +concore status +``` + +### `concore stop` + +Stops all running concore processes. Prompts for confirmation before proceeding. + +**Example:** +```bash +concore stop +``` + +## Development Workflow + +1. **Create a new project** + ```bash + concore init my-neuro-study + cd my-neuro-study + ``` + +2. **Edit your workflow** + - Open `workflow.graphml` in yEd or similar GraphML editor + - Add nodes for your processing steps + - Connect nodes with edges to define data flow + +3. **Add processing scripts** + - Place your Python/C++/MATLAB/Verilog files in the `src/` directory + - Reference them in your workflow nodes + +4. **Validate before running** + ```bash + concore validate workflow.graphml + ``` + +5. **Generate and run** + ```bash + concore build workflow.graphml --auto-build + cd out + ./run.bat # or ./run on Linux/Mac + ``` + +6. **Monitor execution** + ```bash + concore status + ``` + +7. **Stop when done** + ```bash + concore stop + ``` + +## Workflow File Format + +Nodes should follow the format: `ID:filename.ext` + +Example: +``` +N1:controller.py +N2:processor.cpp +M1:analyzer.m +``` + +Supported file types: +- `.py` - Python +- `.cpp` - C++ +- `.m` - MATLAB/Octave +- `.v` - Verilog +- `.java` - Java + +## Troubleshooting + +**Issue: "Output directory already exists"** +- Remove the existing output directory or choose a different name +- Use `concore stop` to terminate any running processes first + +**Issue: Validation fails** +- Check that your GraphML file is properly formatted +- Ensure all nodes have labels in the format `ID:filename.ext` +- Verify that edge connections reference valid nodes + +**Issue: Processes won't stop** +- Try running `concore stop` with administrator/sudo privileges +- Manually kill processes using Task Manager (Windows) or `kill` command (Linux/Mac) diff --git a/concore_cli/__init__.py b/concore_cli/__init__.py new file mode 100644 index 00000000..a5ccc5d1 --- /dev/null +++ b/concore_cli/__init__.py @@ -0,0 +1,5 @@ +__version__ = "1.0.0" + +from .cli import cli + +__all__ = ["cli"] diff --git a/concore_cli/cli.py b/concore_cli/cli.py new file mode 100644 index 00000000..a2736755 --- /dev/null +++ b/concore_cli/cli.py @@ -0,0 +1,207 @@ +import click +from rich.console import Console +import os +import sys + +from .commands.init import init_project, init_project_interactive, run_wizard +from .commands.build import build_workflow +from .commands.validate import validate_workflow +from .commands.status import show_status +from .commands.stop import stop_all +from .commands.inspect import inspect_workflow +from .commands.watch import watch_study +from .commands.doctor import doctor_check +from .commands.setup import setup_concore +from . import __version__ + +console = Console() +DEFAULT_EXEC_TYPE = "windows" if os.name == "nt" else "posix" + + +@click.group() +@click.version_option(version=__version__, prog_name="concore") +def cli(): + pass + + +@cli.command() +@click.argument("name", required=False, default=None) +@click.option("--template", default="basic", help="Template type to use") +@click.option( + "--interactive", + "-i", + is_flag=True, + help="Launch guided wizard to select node types", +) +def init(name, template, interactive): + """Create a new concore project""" + try: + if interactive: + if not name: + name = console.input("[cyan]Project name:[/cyan] ").strip() + if not name: + console.print("[red]Error:[/red] Project name is required.") + sys.exit(1) + selected = run_wizard(console) + init_project_interactive(name, selected, console) + else: + if not name: + console.print( + "[red]Error:[/red] Provide a project name or use --interactive." + ) + sys.exit(1) + init_project(name, template, console) + except Exception as e: + console.print(f"[red]Error:[/red] {str(e)}") + sys.exit(1) + + +@cli.command() +@click.argument("workflow_file", type=click.Path(exists=True)) +@click.option("--source", "-s", default="src", help="Source directory") +@click.option("--output", "-o", default="out", help="Output directory") +@click.option( + "--type", + "-t", + default=DEFAULT_EXEC_TYPE, + type=click.Choice(["windows", "posix", "docker"]), + help="Execution type", +) +@click.option( + "--auto-build", is_flag=True, help="Automatically run build script after generation" +) +@click.option( + "--compose", + is_flag=True, + help="Generate docker-compose.yml in output directory (docker type only)", +) +def build(workflow_file, source, output, type, auto_build, compose): + """Compile a concore workflow into executable scripts""" + try: + build_workflow( + workflow_file, + source, + output, + type, + auto_build, + console, + compose=compose, + ) + except Exception as e: + console.print(f"[red]Error:[/red] {str(e)}") + sys.exit(1) + + +@cli.command() +@click.argument("workflow_file", type=click.Path(exists=True)) +@click.option("--source", "-s", default="src", help="Source directory") +@click.option( + "--format", + "output_format", + default="text", + type=click.Choice(["text", "json"]), + help="Validation output format", +) +def validate(workflow_file, source, output_format): + """Validate a workflow file""" + try: + ok = validate_workflow( + workflow_file, + source, + console, + output_format=output_format, + ) + if not ok: + sys.exit(1) + except Exception as e: + console.print(f"[red]Error:[/red] {str(e)}") + sys.exit(1) + + +@cli.command() +@click.argument("workflow_file", type=click.Path(exists=True)) +@click.option("--source", "-s", default="src", help="Source directory") +@click.option("--json", "output_json", is_flag=True, help="Output in JSON format") +def inspect(workflow_file, source, output_json): + """Inspect a workflow file and show its structure""" + try: + inspect_workflow(workflow_file, source, output_json, console) + except Exception as e: + console.print(f"[red]Error:[/red] {str(e)}") + sys.exit(1) + + +@cli.command() +def status(): + """Show running concore processes""" + try: + show_status(console) + except Exception as e: + console.print(f"[red]Error:[/red] {str(e)}") + sys.exit(1) + + +@cli.command() +@click.confirmation_option(prompt="Stop all running concore processes?") +def stop(): + """Stop all running concore processes""" + try: + stop_all(console) + except Exception as e: + console.print(f"[red]Error:[/red] {str(e)}") + sys.exit(1) + + +@cli.command() +@click.argument("study_dir", type=click.Path(exists=True)) +@click.option("--interval", "-n", default=2.0, help="Refresh interval in seconds") +@click.option("--once", is_flag=True, help="Print a single snapshot and exit") +def watch(study_dir, interval, once): + """Watch a running simulation study for live monitoring""" + try: + watch_study(study_dir, interval, once, console) + except Exception as e: + console.print(f"[red]Error:[/red] {str(e)}") + sys.exit(1) + + +@cli.command() +def doctor(): + """Check system readiness for running concore studies""" + try: + ok = doctor_check(console) + if not ok: + sys.exit(1) + except Exception as e: + console.print(f"[red]Error:[/red] {str(e)}") + sys.exit(1) + + +@cli.command() +@click.option("--dry-run", is_flag=True, help="Preview detected config without writing") +@click.option("--force", is_flag=True, help="Overwrite existing config files") +def setup(dry_run, force): + """Auto-detect tools and write concore config files""" + try: + ok = setup_concore(console, dry_run=dry_run, force=force) + if not ok: + sys.exit(1) + except Exception as e: + console.print(f"[red]Error:[/red] {str(e)}") + sys.exit(1) + + +@cli.command() +def editor(): + """Launch concore-editor""" + try: + from .commands.editor import launch_editor + + launch_editor() + except Exception as e: + console.print(f"[red]Error:[/red] {str(e)}") + sys.exit(1) + + +if __name__ == "__main__": + cli() diff --git a/concore_cli/commands/__init__.py b/concore_cli/commands/__init__.py new file mode 100644 index 00000000..fafdf6ec --- /dev/null +++ b/concore_cli/commands/__init__.py @@ -0,0 +1,17 @@ +from .init import init_project +from .build import build_workflow +from .validate import validate_workflow +from .status import show_status +from .stop import stop_all +from .watch import watch_study +from .doctor import doctor_check + +__all__ = [ + "init_project", + "build_workflow", + "validate_workflow", + "show_status", + "stop_all", + "watch_study", + "doctor_check", +] diff --git a/concore_cli/commands/build.py b/concore_cli/commands/build.py new file mode 100644 index 00000000..5865500b --- /dev/null +++ b/concore_cli/commands/build.py @@ -0,0 +1,271 @@ +import re +import shlex +import subprocess +import sys +from pathlib import Path +from rich.panel import Panel +from rich.progress import Progress, SpinnerColumn, TextColumn + +from .metadata import write_study_metadata + + +def _find_mkconcore_path(): + for parent in Path(__file__).resolve().parents: + candidate = parent / "mkconcore.py" + if candidate.exists(): + return candidate + return None + + +def _yaml_quote(value): + return "'" + value.replace("'", "''") + "'" + + +def _parse_docker_run_line(line): + text = line.strip() + if not text or text.startswith("#"): + return None + + if text.endswith("&"): + text = text[:-1].strip() + + try: + tokens = shlex.split(text) + except ValueError: + return None + + if "run" not in tokens: + return None + + run_index = tokens.index("run") + args = tokens[run_index + 1 :] + + container_name = None + volumes = [] + image = None + + i = 0 + while i < len(args): + token = args[i] + if token.startswith("--name="): + container_name = token.split("=", 1)[1] + elif token == "--name" and i + 1 < len(args): + container_name = args[i + 1] + i += 1 + elif token in ("-v", "--volume") and i + 1 < len(args): + volumes.append(args[i + 1]) + i += 1 + elif token.startswith("--volume="): + volumes.append(token.split("=", 1)[1]) + elif token.startswith("-"): + pass + else: + image = token + break + i += 1 + + if not container_name or not image: + return None + + return { + "container_name": container_name, + "volumes": volumes, + "image": image, + } + + +def _write_docker_compose(output_path): + run_script = output_path / "run" + if not run_script.exists(): + return None + + services = [] + for line in run_script.read_text(encoding="utf-8").splitlines(): + parsed = _parse_docker_run_line(line) + if parsed is not None: + services.append(parsed) + + if not services: + return None + + compose_lines = ["services:"] + + for index, service in enumerate(services, start=1): + service_name = re.sub(r"[^A-Za-z0-9_.-]", "-", service["container_name"]).strip( + "-." + ) + if not service_name: + service_name = f"service-{index}" + elif not service_name[0].isalnum(): + service_name = f"service-{service_name}" + + compose_lines.append(f" {service_name}:") + compose_lines.append(f" image: {_yaml_quote(service['image'])}") + compose_lines.append( + f" container_name: {_yaml_quote(service['container_name'])}" + ) + if service["volumes"]: + compose_lines.append(" volumes:") + for volume_spec in service["volumes"]: + compose_lines.append(f" - {_yaml_quote(volume_spec)}") + + compose_lines.append("") + compose_path = output_path / "docker-compose.yml" + compose_path.write_text("\n".join(compose_lines), encoding="utf-8") + return compose_path + + +def build_workflow( + workflow_file, + source, + output, + exec_type, + auto_build, + console, + compose=False, +): + workflow_path = Path(workflow_file).resolve() + source_path = Path(source).resolve() + output_path = Path(output).resolve() + + if not source_path.exists(): + raise FileNotFoundError(f"Source directory '{source}' not found") + + if output_path.exists(): + console.print( + f"[yellow]Warning:[/yellow] Output directory '{output}' already exists" + ) + console.print("Remove it first or choose a different output directory") + return + + console.print(f"[cyan]Workflow:[/cyan] {workflow_path.name}") + console.print(f"[cyan]Source:[/cyan] {source_path}") + console.print(f"[cyan]Output:[/cyan] {output_path}") + console.print(f"[cyan]Type:[/cyan] {exec_type}") + if compose: + console.print("[cyan]Compose:[/cyan] enabled") + console.print() + + if compose and exec_type != "docker": + raise ValueError("--compose can only be used with --type docker") + + mkconcore_path = _find_mkconcore_path() + if mkconcore_path is None: + raise FileNotFoundError( + "mkconcore.py not found. Please install concore from source." + ) + + with Progress( + SpinnerColumn(), + TextColumn("[progress.description]{task.description}"), + console=console, + ) as progress: + task = progress.add_task("Generating workflow...", total=None) + + try: + result = subprocess.run( + [ + sys.executable, + str(mkconcore_path), + str(workflow_path), + str(source_path), + str(output_path), + exec_type, + ], + cwd=mkconcore_path.parent, + capture_output=True, + text=True, + check=True, + ) + + progress.update(task, completed=True) + + if result.stdout: + console.print(result.stdout) + + console.print( + f"[green]✓[/green] Workflow generated in [cyan]{output_path}[/cyan]" + ) + + if compose: + compose_path = _write_docker_compose(output_path) + if compose_path is not None: + console.print( + f"[green]✓[/green] Compose file written to [cyan]{compose_path}[/cyan]" + ) + else: + console.print( + "[yellow]Warning:[/yellow] Could not generate docker-compose.yml from run script" + ) + + try: + metadata_path = write_study_metadata( + output_path, + generated_by="concore build", + workflow_file=workflow_path, + ) + console.print( + f"[green]✓[/green] Metadata written to [cyan]{metadata_path}[/cyan]" + ) + except Exception as exc: + # Metadata is additive, so workflow generation should still succeed on failure. + console.print( + f"[yellow]Warning:[/yellow] Failed to write study metadata for [cyan]{output_path}[/cyan]: {exc}" + ) + + except subprocess.CalledProcessError as e: + progress.stop() + console.print("[red]Generation failed:[/red]") + if e.stdout: + console.print(e.stdout) + if e.stderr: + console.print(e.stderr) + raise + + if auto_build: + console.print() + build_script = output_path / ( + "build.bat" if exec_type == "windows" else "build" + ) + + if build_script.exists(): + with Progress( + SpinnerColumn(), + TextColumn("[progress.description]{task.description}"), + console=console, + ) as progress: + task = progress.add_task("Building workflow...", total=None) + + try: + result = subprocess.run( + [str(build_script)], + cwd=output_path, + capture_output=True, + text=True, + shell=True, + check=True, + ) + progress.update(task, completed=True) + console.print("[green]✓[/green] Build completed") + except subprocess.CalledProcessError as e: + progress.stop() + console.print("[yellow]Build failed[/yellow]") + if e.stderr: + console.print(e.stderr) + + run_command = "docker compose up" if compose else "./run" + if exec_type == "windows": + run_command = "run.bat" + + console.print() + console.print( + Panel.fit( + f"[green]✓[/green] Workflow ready!\n\n" + f"To run your workflow:\n" + f" cd {output_path}\n" + f" {'build.bat' if exec_type == 'windows' else './build'}\n" + f" {run_command}", + title="Next Steps", + border_style="green", + ) + ) diff --git a/concore_cli/commands/doctor.py b/concore_cli/commands/doctor.py new file mode 100644 index 00000000..254edfd0 --- /dev/null +++ b/concore_cli/commands/doctor.py @@ -0,0 +1,411 @@ +import importlib.metadata +import shutil +import subprocess +import sys +import os +import platform +from pathlib import Path +from rich.panel import Panel + +# Map of tool keys to their lookup names per platform +TOOL_DEFINITIONS = { + "C++ compiler": { + "names": { + "posix": ["g++", "clang++"], + "windows": ["g++", "cl"], + }, + "version_flag": "--version", + "config_keys": ["CPPEXE", "CPPWIN"], + "install_hints": { + "Linux": "sudo apt install g++", + "Darwin": "brew install gcc", + "Windows": "winget install -e --id GnuWin32.Gcc", + }, + }, + "Python": { + "names": { + "posix": ["python3", "python"], + "windows": ["python", "python3"], + }, + "version_flag": "--version", + "config_keys": ["PYTHONEXE", "PYTHONWIN"], + "install_hints": { + "Linux": "sudo apt install python3", + "Darwin": "brew install python3", + "Windows": "winget install -e --id Python.Python.3.11", + }, + }, + "Verilog (iverilog)": { + "names": { + "posix": ["iverilog"], + "windows": ["iverilog"], + }, + "version_flag": "-V", + "config_keys": ["VEXE", "VWIN"], + "install_hints": { + "Linux": "sudo apt install iverilog", + "Darwin": "brew install icarus-verilog", + "Windows": "Download from http://bleyer.org/icarus/", + }, + }, + "Octave": { + "names": { + "posix": ["octave", "octave-cli"], + "windows": ["octave", "octave-cli"], + }, + "version_flag": "--version", + "config_keys": ["OCTAVEEXE", "OCTAVEWIN"], + "install_hints": { + "Linux": "sudo apt install octave", + "Darwin": "brew install octave", + "Windows": "winget install -e --id JohnWHiggins.Octave", + }, + }, + "MATLAB": { + "names": { + "posix": ["matlab"], + "windows": ["matlab"], + }, + "version_flag": ["-batch", "disp('ok')"], + "config_keys": ["MATLABEXE", "MATLABWIN"], + "install_hints": { + "Linux": "Install from https://mathworks.com/downloads/", + "Darwin": "Install from https://mathworks.com/downloads/", + "Windows": "Install from https://mathworks.com/downloads/", + }, + }, + "Docker": { + "names": { + "posix": ["docker", "podman"], + "windows": ["docker", "podman"], + }, + "version_flag": "--version", + "config_keys": [], + "install_hints": { + "Linux": "sudo apt install docker.io", + "Darwin": "brew install --cask docker", + "Windows": "winget install -e --id Docker.DockerDesktop", + }, + }, +} + +REQUIRED_PACKAGES = [ + "click", + "rich", + "beautifulsoup4", + "lxml", + "psutil", + "numpy", + "pyzmq", +] + +OPTIONAL_PACKAGES = { + "scipy": "pip install concore[demo]", + "matplotlib": "pip install concore[demo]", +} + + +def _get_platform_key(): + """Return 'posix' or 'windows' based on OS.""" + return "windows" if os.name == "nt" else "posix" + + +def _get_platform_name(): + """Return platform name for install hint lookup.""" + return platform.system() + + +def _resolve_concore_path(): + """Resolve CONCOREPATH the same way mkconcore.py does.""" + script_dir = Path(__file__).resolve().parent.parent.parent + if (script_dir / "concore.py").exists(): + return script_dir + cwd = Path.cwd() + if (cwd / "concore.py").exists(): + return cwd + return script_dir + + +def _detect_tool(names): + """Try to find a tool by checking a list of candidate names. + + Returns (path, name) of the first match, or (None, None). + """ + for name in names: + path = shutil.which(name) + if path: + return path, name + return None, None + + +def _get_version(path, version_flag): + """Run tool with version flag and return first line of output.""" + try: + if isinstance(version_flag, list): + cmd = [path] + version_flag + else: + cmd = [path, version_flag] + result = subprocess.run( + cmd, + capture_output=True, + text=True, + timeout=10, + ) + output = result.stdout.strip() or result.stderr.strip() + if output: + return output.splitlines()[0] + except Exception: + pass + return None + + +def _check_docker_daemon(docker_path): + """Check if Docker daemon is running.""" + try: + result = subprocess.run( + [docker_path, "info"], + capture_output=True, + text=True, + timeout=15, + ) + return result.returncode == 0 + except Exception: + return False + + +def _check_package(package_name): + """Check if a Python package is importable and get its version.""" + try: + version = importlib.metadata.version(package_name) + return True, version + except importlib.metadata.PackageNotFoundError: + return False, None + + +def doctor_check(console): + """Run system readiness checks and display results.""" + passed = 0 + warnings = 0 + errors = 0 + + console.print() + console.print( + Panel.fit( + "[bold]concore Doctor - System Readiness Report[/bold]", + border_style="cyan", + ) + ) + console.print() + + # === Core Checks === + console.print("[bold cyan]Core Checks[/bold cyan]") + + # Python version + py_version = platform.python_version() + if sys.version_info >= (3, 9): + console.print(f" [green]+[/green] Python {py_version} (>= 3.9 required)") + passed += 1 + else: + console.print( + f" [red]x[/red] Python {py_version} - concore requires Python >= 3.9" + ) + errors += 1 + + # concore installation + try: + from concore_cli import __version__ + + console.print(f" [green]+[/green] concore {__version__} installed") + passed += 1 + except ImportError: + console.print(" [red]x[/red] concore package not found") + errors += 1 + + # CONCOREPATH + concore_path = _resolve_concore_path() + if concore_path.exists(): + writable = os.access(str(concore_path), os.W_OK) + status = "writable" if writable else "read-only" + if writable: + console.print(f" [green]+[/green] CONCOREPATH: {concore_path} ({status})") + passed += 1 + else: + console.print( + f" [yellow]![/yellow] CONCOREPATH: {concore_path} ({status})" + ) + warnings += 1 + else: + console.print(f" [red]x[/red] CONCOREPATH: {concore_path} (not found)") + errors += 1 + + console.print() + + # === Tool Detection === + console.print("[bold cyan]Tools[/bold cyan]") + + plat_key = _get_platform_key() + plat_name = _get_platform_name() + + for tool_label, tool_def in TOOL_DEFINITIONS.items(): + candidates = tool_def["names"].get(plat_key, []) + path, found_name = _detect_tool(candidates) + + if path: + version = _get_version(path, tool_def["version_flag"]) + version_str = f" ({version})" if version else "" + exe_info = f" [{found_name}]" if found_name else "" + extra = "" + if tool_label == "Docker": + daemon_ok = _check_docker_daemon(path) + extra = ( + " [green](daemon running)[/green]" + if daemon_ok + else " [yellow](daemon not running)[/yellow]" + ) + if not daemon_ok: + warnings += 1 + console.print( + f" [yellow]![/yellow] {tool_label}{exe_info}" + f"{version_str} -> {path}{extra}" + ) + continue + console.print( + f" [green]+[/green] {tool_label}{exe_info}" + f"{version_str} -> {path}{extra}" + ) + passed += 1 + else: + hint = tool_def["install_hints"].get(plat_name, "") + hint_str = f" (install: {hint})" if hint else "" + # Docker, MATLAB, Verilog are optional - show as warning + if tool_label in ("MATLAB", "Verilog (iverilog)", "Docker"): + console.print( + f" [yellow]![/yellow] {tool_label} -> Not found{hint_str}" + ) + warnings += 1 + else: + console.print(f" [red]x[/red] {tool_label} -> Not found{hint_str}") + errors += 1 + + console.print() + + # === Configuration Checks === + console.print("[bold cyan]Configuration[/bold cyan]") + + config_files = { + "concore.tools": "Tool path overrides", + "concore.octave": "Treat .m files as Octave", + "concore.mcr": "MATLAB Compiler Runtime path", + "concore.repo": "Docker repository path", + "concore.sudo": "Docker executable override", + } + + for filename, description in config_files.items(): + filepath = concore_path / filename + if filepath.exists(): + try: + content = filepath.read_text().strip() + if filename == "concore.tools": + line_count = len( + [ + ln + for ln in content.splitlines() + if ln.strip() and not ln.strip().startswith("#") + ] + ) + console.print( + f" [green]+[/green] {filename} -> " + f"{line_count} tool path(s) configured" + ) + elif filename == "concore.mcr": + if os.path.exists(os.path.expanduser(content)): + console.print(f" [green]+[/green] {filename} -> {content}") + passed += 1 + else: + console.print( + f" [yellow]![/yellow] {filename} -> " + f"path does not exist: {content}" + ) + warnings += 1 + continue + elif filename == "concore.sudo": + console.print(f" [green]+[/green] {filename} -> {content}") + elif filename == "concore.repo": + console.print(f" [green]+[/green] {filename} -> {content}") + else: + console.print(f" [green]+[/green] {filename} -> Enabled") + passed += 1 + except Exception: + console.print(f" [yellow]![/yellow] {filename} -> Could not read") + warnings += 1 + else: + console.print(f" [dim]-[/dim] {filename} -> Not set ({description})") + + # Build environment variable list from TOOL_DEFINITIONS config_keys + env_vars = [] + for tool_def in TOOL_DEFINITIONS.values(): + for key in tool_def.get("config_keys", []): + env_vars.append(f"CONCORE_{key}") + env_vars.append("DOCKEREXE") + env_set = [v for v in env_vars if os.environ.get(v)] + if env_set: + console.print(f" [green]+[/green] Environment variables: {', '.join(env_set)}") + passed += 1 + else: + console.print(" [dim]-[/dim] No concore environment variables set") + + console.print() + + # === Dependency Checks === + console.print("[bold cyan]Dependencies[/bold cyan]") + + for pkg in REQUIRED_PACKAGES: + found, version = _check_package(pkg) + if found: + console.print(f" [green]+[/green] {pkg} {version}") + passed += 1 + else: + console.print(f" [red]x[/red] {pkg} -> Not installed (pip install {pkg})") + errors += 1 + + for pkg, install_hint in OPTIONAL_PACKAGES.items(): + found, version = _check_package(pkg) + if found: + console.print(f" [green]+[/green] {pkg} {version}") + passed += 1 + else: + console.print( + f" [yellow]![/yellow] {pkg} -> Not installed ({install_hint})" + ) + warnings += 1 + + console.print() + + # === Summary === + summary_parts = [] + if passed: + summary_parts.append(f"[green]{passed} passed[/green]") + if warnings: + summary_parts.append(f"[yellow]{warnings} warning(s)[/yellow]") + if errors: + summary_parts.append(f"[red]{errors} error(s)[/red]") + + if summary_parts: + summary_text = ", ".join(summary_parts) + else: + summary_text = "[yellow]No checks were run.[/yellow]" + + console.print(f"[bold]Summary:[/bold] {summary_text}") + + if errors == 0: + console.print() + console.print( + Panel.fit( + "[green]System is ready to run concore studies![/green]", + border_style="green", + ) + ) + + console.print() + + return errors == 0 diff --git a/concore_cli/commands/editor.py b/concore_cli/commands/editor.py new file mode 100644 index 00000000..b40962da --- /dev/null +++ b/concore_cli/commands/editor.py @@ -0,0 +1,32 @@ +import os +import sys +import shutil +import subprocess +from rich.console import Console + +console = Console() +EDITOR_URL = "https://controlcore-project.github.io/concore-editor/" + + +def open_editor_url(url): + try: + if sys.platform == "win32": + os.system(f'start "" chrome "{url}"') + else: + if shutil.which("open"): + if sys.platform == "darwin": + subprocess.run(["open", "-a", "Google Chrome", url]) + elif sys.platform.startswith("linux"): + subprocess.run(["xdg-open", url]) + else: + if shutil.which("xdg-open"): + subprocess.run(["xdg-open", url]) + else: + console.print("unable to open browser for the concore editor.") + except Exception as e: + console.print(f"unable to open browser for the concore editor. ({e})") + + +def launch_editor(): + console.print("[cyan]Opening concore-editor...[/cyan]") + open_editor_url(EDITOR_URL) diff --git a/concore_cli/commands/init.py b/concore_cli/commands/init.py new file mode 100644 index 00000000..53fd53f7 --- /dev/null +++ b/concore_cli/commands/init.py @@ -0,0 +1,378 @@ +from pathlib import Path +from xml.sax.saxutils import quoteattr +from rich.panel import Panel + +from .metadata import write_study_metadata + +# --------------------------------------------------------------------------- +# GraphML templates +# --------------------------------------------------------------------------- + +GRAPHML_HEADER = """ + + + + +{nodes} + + +""" + +GRAPHML_NODE = """ + + + + + + N{idx}:{filename} + + + + """ + +# Single-node fallback used by non-interactive init +SAMPLE_GRAPHML = """ + + + + + + + + + + + N1:script.py + + + + + + +""" + +# --------------------------------------------------------------------------- +# Per-language metadata: label, filename, node colour, source stub +# --------------------------------------------------------------------------- + +LANGUAGE_NODES = { + "python": { + "label": "Python", + "filename": "script.py", + "color": "#ffcc00", + "stub": ( + "import concore\n\n" + "concore.default_maxtime(100)\n" + "concore.delay = 0.02\n\n" + 'init_val = "[0.0, 0.0]"\n' + "val = concore.initval(init_val)\n\n" + "while concore.simtime < concore.maxtime:\n" + " while concore.unchanged():\n" + ' val = concore.read(1, "data", init_val)\n' + " result = [v * 2 for v in val]\n" + ' concore.write(1, "result", result, delta=0)\n' + ), + }, + "cpp": { + "label": "C++", + "filename": "script.cpp", + "color": "#ae85ca", + "stub": ( + '#include "concore.hpp"\n' + "#include \n\n" + "int main() {\n" + " Concore concore;\n" + " concore.default_maxtime(100);\n" + " concore.delay = 0.02;\n\n" + ' std::string init_val = "[0.0, 0.0]";\n' + " std::vector val = concore.initval(init_val);\n\n" + " while (concore.simtime < concore.maxtime) {\n" + " while (concore.unchanged()) {\n" + ' val = concore.read(1, "data", init_val);\n' + " }\n" + " // TODO: process val (e.g. multiply by 2)\n" + ' concore.write(1, "result", val, 0);\n' + " }\n" + " return 0;\n" + "}\n" + ), + }, + "octave": { + "label": "Octave/MATLAB", + "filename": "script.m", + "color": "#6db3f2", + "stub": ( + "global concore;\n" + "import_concore;\n\n" + "concore.delay = 0.02;\n" + "concore_default_maxtime(100);\n\n" + "init_val = '[0.0, 0.0]';\n" + "val = concore_initval(init_val);\n\n" + "while concore.simtime < concore.maxtime\n" + " while concore_unchanged()\n" + " val = concore_read(1, 'data', init_val);\n" + " end\n" + " result = val * 2;\n" + " concore_write(1, 'result', result, 0);\n" + "end\n" + ), + }, + "verilog": { + "label": "Verilog", + "filename": "script.v", + "color": "#f28c8c", + "stub": ( + '`include "concore.v"\n\n' + "module script;\n" + " // concore module provides: simtime, maxtime, readdata, writedata, unchanged\n" + " // data[] and datasize are global arrays filled by readdata\n\n" + " real init_val[1:0]; // [simtime, value]\n" + " integer i;\n\n" + " initial begin\n" + " concore.simtime = 0;\n" + " // set your maxtime (or let concore.maxtime file override)\n\n" + " while (concore.simtime < 100) begin\n" + " while (concore.unchanged(0)) begin\n" + " // readdata fills concore.data[] and updates concore.simtime\n" + ' concore.readdata(1, "data", "[0.0,0.0]");\n' + " end\n" + " // TODO: process concore.data[0..datasize-1]\n" + " concore.data[0] = concore.data[0] * 2;\n" + " concore.datasize = 1;\n" + ' concore.writedata(1, "result", 0); // delta=0\n' + " end\n" + " $finish;\n" + " end\n" + "endmodule\n" + ), + }, + "java": { + "label": "Java", + "filename": "Script.java", + "color": "#a8d8a8", + "stub": ( + "import java.util.List;\n\n" + "public class Script {\n" + " public static void main(String[] args) throws Exception {\n" + " double maxtime = 100;\n" + ' String init_val = "[0.0, 0.0]";\n\n' + " // All concore methods are static\n" + " List val = concore.initVal(init_val);\n" + " while (concore.getSimtime() < maxtime) {\n" + " while (concore.unchanged()) {\n" + ' concore.ReadResult r = concore.read(1, "data", init_val);\n' + " val = r.data;\n" + " }\n" + " // TODO: process val (List)\n" + ' concore.write(1, "result", val, 0);\n' + " }\n" + " }\n" + "}\n" + ), + }, +} + +README_TEMPLATE = """# {project_name} + +A concore workflow project. + +## Getting Started + +1. Edit your workflow in `workflow.graphml` using yEd or similar GraphML editor +2. Add your processing scripts to the `src/` directory +3. Build your workflow: + ``` + concore build workflow.graphml + ``` + +## Project Structure + +- `workflow.graphml` - Your workflow definition +- `src/` - Source files for your nodes +- `README.md` - This file + +## Next Steps + +- Open `workflow.graphml` in yEd and connect the nodes with edges +- Use `concore validate workflow.graphml` to check your workflow +- Use `concore status` to monitor running processes +""" + + +# --------------------------------------------------------------------------- +# Interactive wizard +# --------------------------------------------------------------------------- + + +def run_wizard(console): + """Ask y/n for each supported language. Returns list of selected lang keys.""" + console.print() + console.print( + "[bold cyan]Select the node types to include[/bold cyan] " + "[dim](Enter = yes)[/dim]" + ) + console.print() + + selected = [] + for key, info in LANGUAGE_NODES.items(): + raw = ( + console.input(f" Include [bold]{info['label']}[/bold] node? [Y/n] ") + .strip() + .lower() + ) + if raw in ("", "y", "yes"): + selected.append(key) + + return selected + + +# --------------------------------------------------------------------------- +# GraphML builder +# --------------------------------------------------------------------------- + + +def _build_graphml(project_name, selected_langs): + """Return a GraphML string with one unconnected node per selected language.""" + node_blocks = [] + for idx, lang_key in enumerate(selected_langs, start=1): + info = LANGUAGE_NODES[lang_key] + node_blocks.append( + GRAPHML_NODE.format( + idx=idx, + y=100 + (idx - 1) * 100, # stack vertically, 100 px apart + color=info["color"], + filename=info["filename"], + ) + ) + return GRAPHML_HEADER.format( + project_name=quoteattr(project_name), + nodes="\n".join(node_blocks), + ) + + +# --------------------------------------------------------------------------- +# Public entry points +# --------------------------------------------------------------------------- + + +def init_project_interactive(name, selected_langs, console): + """Create a project with one node per selected language (no edges).""" + project_path = Path(name) + + if project_path.exists(): + raise FileExistsError(f"Directory '{name}' already exists") + + if not selected_langs: + console.print("[yellow]No languages selected — nothing to create.[/yellow]") + return + + console.print() + console.print(f"[cyan]Creating project:[/cyan] {name}") + + project_path.mkdir() + src_path = project_path / "src" + src_path.mkdir() + + # workflow.graphml + workflow_file = project_path / "workflow.graphml" + workflow_file.write_text(_build_graphml(name, selected_langs), encoding="utf-8") + + # one source stub per selected language + for lang_key in selected_langs: + info = LANGUAGE_NODES[lang_key] + (src_path / info["filename"]).write_text(info["stub"], encoding="utf-8") + + # README + (project_path / "README.md").write_text( + README_TEMPLATE.format(project_name=name), encoding="utf-8" + ) + + # Metadata + metadata_info = "" + try: + metadata_path = write_study_metadata( + project_path, + generated_by="concore init --interactive", + workflow_file=workflow_file, + ) + metadata_info = f"Metadata:\n {metadata_path.name}\n\n" + except Exception as exc: + console.print( + f"[yellow]Warning:[/yellow] Failed to write study metadata: {exc}" + ) + + node_lines = "\n".join( + f" N{i}: {LANGUAGE_NODES[k]['filename']}" + for i, k in enumerate(selected_langs, 1) + ) + + console.print() + console.print( + Panel.fit( + f"[green]✓[/green] Project created with {len(selected_langs)} node(s)!\n\n" + f"{metadata_info}" + f"Nodes (unconnected — connect them in yEd):\n{node_lines}\n\n" + f"Next steps:\n" + f" cd {name}\n" + f" concore validate workflow.graphml\n" + f" concore build workflow.graphml", + title="Success", + border_style="green", + ) + ) + + +def init_project(name, template, console): + """Non-interactive init — single Python node skeleton.""" + project_path = Path(name) + + if project_path.exists(): + raise FileExistsError(f"Directory '{name}' already exists") + + console.print(f"[cyan]Creating project:[/cyan] {name}") + + project_path.mkdir() + (project_path / "src").mkdir() + + workflow_file = project_path / "workflow.graphml" + with open(workflow_file, "w", encoding="utf-8") as f: + f.write(SAMPLE_GRAPHML) + + (project_path / "src" / "script.py").write_text( + LANGUAGE_NODES["python"]["stub"], encoding="utf-8" + ) + + (project_path / "README.md").write_text( + README_TEMPLATE.format(project_name=name), encoding="utf-8" + ) + + metadata_info = "" + try: + metadata_path = write_study_metadata( + project_path, + generated_by="concore init", + workflow_file=workflow_file, + ) + metadata_info = f"Metadata:\n {metadata_path.name}\n\n" + except Exception as exc: + # Metadata is additive, so project creation should still succeed on failure. + console.print( + f"[yellow]Warning:[/yellow] Failed to write study metadata: {exc}" + ) + + console.print() + console.print( + Panel.fit( + f"[green]✓[/green] Project created successfully!\n\n" + f"{metadata_info}" + f"Next steps:\n" + f" cd {name}\n" + f" concore validate workflow.graphml\n" + f" concore build workflow.graphml", + title="Success", + border_style="green", + ) + ) diff --git a/concore_cli/commands/inspect.py b/concore_cli/commands/inspect.py new file mode 100644 index 00000000..0dce24a8 --- /dev/null +++ b/concore_cli/commands/inspect.py @@ -0,0 +1,271 @@ +from pathlib import Path +from bs4 import BeautifulSoup +from rich.table import Table +from rich.tree import Tree +from collections import defaultdict +import re + + +def inspect_workflow(workflow_file, source_dir, output_json, console): + workflow_path = Path(workflow_file) + + if output_json: + return _inspect_json(workflow_path, source_dir) + + _inspect_rich(workflow_path, source_dir, console) + + +def _inspect_rich(workflow_path, source_dir, console): + console.print() + console.print(f"[bold cyan]Workflow:[/bold cyan] {workflow_path.name}") + console.print() + + try: + with open(workflow_path, "r") as f: + content = f.read() + + soup = BeautifulSoup(content, "xml") + + if not soup.find("graphml"): + console.print("[red]Not a valid GraphML file[/red]") + return + + nodes = soup.find_all("node") + edges = soup.find_all("edge") + + tree = Tree("📊 [bold]Workflow Overview[/bold]") + + lang_counts = defaultdict(int) + node_files = [] + missing_files = [] + + for node in nodes: + label_tag = node.find("y:NodeLabel") + if label_tag and label_tag.text: + label = label_tag.text.strip() + if ":" in label: + _, filename = label.split(":", 1) + node_files.append(filename) + + ext = Path(filename).suffix + if ext == ".py": + lang_counts["Python"] += 1 + elif ext == ".m": + lang_counts["MATLAB"] += 1 + elif ext == ".java": + lang_counts["Java"] += 1 + elif ext == ".cpp" or ext == ".hpp": + lang_counts["C++"] += 1 + elif ext == ".v": + lang_counts["Verilog"] += 1 + else: + lang_counts["Other"] += 1 + + src_dir = workflow_path.parent / source_dir + if not (src_dir / filename).exists(): + missing_files.append(filename) + + nodes_branch = tree.add(f"Nodes: [bold]{len(nodes)}[/bold]") + if lang_counts: + for lang, count in sorted(lang_counts.items(), key=lambda x: -x[1]): + nodes_branch.add(f"{lang}: {count}") + + edges_branch = tree.add(f"Edges: [bold]{len(edges)}[/bold]") + + edge_label_regex = re.compile(r"0x([a-fA-F0-9]+)_(\S+)") + zmq_count = 0 + file_count = 0 + + for edge in edges: + label_tag = edge.find("y:EdgeLabel") + label_text = label_tag.text.strip() if label_tag and label_tag.text else "" + if label_text and edge_label_regex.match(label_text): + zmq_count += 1 + else: + file_count += 1 + + if zmq_count > 0: + edges_branch.add(f"ZMQ: {zmq_count}") + if file_count > 0: + edges_branch.add(f"File-based: {file_count}") + + comm_type = ( + "ZMQ (0mq)" if zmq_count > 0 else "File-based" if file_count > 0 else "None" + ) + tree.add(f"Communication: [bold]{comm_type}[/bold]") + + if missing_files: + missing_branch = tree.add( + f"[yellow]Missing files: {len(missing_files)}[/yellow]" + ) + for f in missing_files[:5]: + missing_branch.add(f"[yellow]{f}[/yellow]") + if len(missing_files) > 5: + missing_branch.add(f"[dim]...and {len(missing_files) - 5} more[/dim]") + + console.print(tree) + console.print() + + if nodes: + table = Table( + title="Node Details", show_header=True, header_style="bold magenta" + ) + table.add_column("ID", style="cyan", width=12) + table.add_column("File", style="white") + table.add_column("Language", style="green") + table.add_column("Status", style="yellow") + + for node in nodes[:10]: + label_tag = node.find("y:NodeLabel") + if label_tag and label_tag.text: + label = label_tag.text.strip() + if ":" in label: + node_id, filename = label.split(":", 1) + + ext = Path(filename).suffix + lang_map = { + ".py": "Python", + ".m": "MATLAB", + ".java": "Java", + ".cpp": "C++", + ".hpp": "C++", + ".v": "Verilog", + } + lang = lang_map.get(ext, "Other") + + src_dir = workflow_path.parent / source_dir + status = "✓" if (src_dir / filename).exists() else "✗" + + table.add_row(node_id, filename, lang, status) + + if len(nodes) > 10: + table.caption = f"Showing 10 of {len(nodes)} nodes" + + console.print(table) + console.print() + + if edges: + edge_table = Table( + title="Edge Connections", show_header=True, header_style="bold magenta" + ) + edge_table.add_column("From", style="cyan", width=12) + edge_table.add_column("To", style="cyan", width=12) + edge_table.add_column("Type", style="green") + + for edge in edges[:10]: + source = edge.get("source", "unknown") + target = edge.get("target", "unknown") + + label_tag = edge.find("y:EdgeLabel") + edge_type = "File" + if label_tag and label_tag.text: + if edge_label_regex.match(label_tag.text.strip()): + edge_type = "ZMQ" + + edge_table.add_row(source, target, edge_type) + + if len(edges) > 10: + edge_table.caption = f"Showing 10 of {len(edges)} edges" + + console.print(edge_table) + console.print() + + except FileNotFoundError: + console.print(f"[red]File not found:[/red] {workflow_path}") + except Exception as e: + console.print(f"[red]Inspection failed:[/red] {str(e)}") + + +def _inspect_json(workflow_path, source_dir): + import json + + try: + with open(workflow_path, "r") as f: + content = f.read() + + soup = BeautifulSoup(content, "xml") + + if not soup.find("graphml"): + print(json.dumps({"error": "Not a valid GraphML file"}, indent=2)) + return + + nodes = soup.find_all("node") + edges = soup.find_all("edge") + + lang_counts = defaultdict(int) + node_list = [] + edge_list = [] + missing_files = [] + + for node in nodes: + label_tag = node.find("y:NodeLabel") + if label_tag and label_tag.text: + label = label_tag.text.strip() + if ":" in label: + node_id, filename = label.split(":", 1) + + ext = Path(filename).suffix + lang_map = { + ".py": "python", + ".m": "matlab", + ".java": "java", + ".cpp": "cpp", + ".hpp": "cpp", + ".v": "verilog", + } + lang = lang_map.get(ext, "other") + lang_counts[lang] += 1 + + src_dir = workflow_path.parent / source_dir + exists = (src_dir / filename).exists() + if not exists: + missing_files.append(filename) + + node_list.append( + { + "id": node_id, + "file": filename, + "language": lang, + "exists": exists, + } + ) + + edge_label_regex = re.compile(r"0x([a-fA-F0-9]+)_(\S+)") + zmq_count = 0 + file_count = 0 + + for edge in edges: + source = edge.get("source") + target = edge.get("target") + + label_tag = edge.find("y:EdgeLabel") + label_text = label_tag.text.strip() if label_tag and label_tag.text else "" + edge_type = "file" + if label_text and edge_label_regex.match(label_text): + edge_type = "zmq" + zmq_count += 1 + else: + file_count += 1 + + edge_list.append({"source": source, "target": target, "type": edge_type}) + + result = { + "workflow": str(workflow_path.name), + "nodes": { + "total": len(nodes), + "by_language": dict(lang_counts), + "list": node_list, + }, + "edges": { + "total": len(edges), + "zmq": zmq_count, + "file": file_count, + "list": edge_list, + }, + "missing_files": missing_files, + } + + print(json.dumps(result, indent=2)) + + except Exception as e: + print(json.dumps({"error": str(e)}, indent=2)) diff --git a/concore_cli/commands/metadata.py b/concore_cli/commands/metadata.py new file mode 100644 index 00000000..4ccd1082 --- /dev/null +++ b/concore_cli/commands/metadata.py @@ -0,0 +1,77 @@ +import hashlib +import json +import platform +import shutil +from datetime import datetime, timezone +from pathlib import Path + +from concore_cli import __version__ + + +def _checksum_file(path: Path) -> str: + hasher = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(8192), b""): + hasher.update(chunk) + return f"sha256:{hasher.hexdigest()}" + + +def _detect_tools() -> dict: + tool_candidates = { + "python": ["python", "python3"], + "g++": ["g++"], + "docker": ["docker"], + "octave": ["octave"], + "iverilog": ["iverilog"], + } + detected = {} + for tool_name, candidates in tool_candidates.items(): + detected_path = None + for candidate in candidates: + detected_path = shutil.which(candidate) + if detected_path: + break + detected[tool_name] = detected_path or "not found" + return detected + + +def write_study_metadata( + study_path: Path, generated_by: str, workflow_file: Path = None +): + checksums = {} + checksum_candidates = [ + "workflow.graphml", + "docker-compose.yml", + "concore.toml", + "runner.py", + "README.md", + "build", + "run", + "build.bat", + "run.bat", + ] + + if workflow_file is not None and workflow_file.exists(): + checksums[workflow_file.name] = _checksum_file(workflow_file) + + for relative_name in checksum_candidates: + file_path = study_path / relative_name + if file_path.exists() and file_path.is_file(): + checksums[relative_name] = _checksum_file(file_path) + + metadata = { + "generated_by": generated_by, + "concore_version": __version__, + "timestamp": datetime.now(timezone.utc).replace(microsecond=0).isoformat(), + "python_version": platform.python_version(), + "platform": platform.platform(), + "study_name": study_path.name, + "working_directory": str(study_path.resolve()), + "tools_detected": _detect_tools(), + "checksums": checksums, + "schema_version": 1, + } + + metadata_path = study_path / "STUDY.json" + metadata_path.write_text(json.dumps(metadata, indent=2) + "\n", encoding="utf-8") + return metadata_path diff --git a/concore_cli/commands/setup.py b/concore_cli/commands/setup.py new file mode 100644 index 00000000..829f24d4 --- /dev/null +++ b/concore_cli/commands/setup.py @@ -0,0 +1,103 @@ +from pathlib import Path + +from .doctor import ( + TOOL_DEFINITIONS, + _detect_tool, + _get_platform_key, + _resolve_concore_path, +) + + +def _pick_config_key(config_keys, plat_key): + if plat_key == "windows": + for key in config_keys: + if key.endswith("WIN"): + return key + else: + for key in config_keys: + if key.endswith("EXE"): + return key + return config_keys[0] if config_keys else None + + +def _detect_tool_overrides(plat_key): + found = [] + for tool_def in TOOL_DEFINITIONS.values(): + config_keys = tool_def.get("config_keys", []) + if not config_keys: + continue + candidates = tool_def["names"].get(plat_key, []) + path, _ = _detect_tool(candidates) + if not path: + continue + config_key = _pick_config_key(config_keys, plat_key) + if config_key: + found.append((config_key, path)) + return found + + +def _write_text(path, content, dry_run, force, console): + if path.exists() and not force: + console.print( + f"[yellow]![/yellow] Skipping {path.name} (already exists; use --force)" + ) + return True + if dry_run: + preview = content if content else "" + console.print(f"[dim]-[/dim] Would write {path.name}:\n{preview}") + return True + path.write_text(content) + console.print(f"[green]+[/green] Wrote {path.name}") + return True + + +def setup_concore(console, dry_run=False, force=False): + plat_key = _get_platform_key() + concore_path = _resolve_concore_path() + + console.print(f"[cyan]CONCOREPATH:[/cyan] {concore_path}") + + tool_overrides = _detect_tool_overrides(plat_key) + docker_candidates = TOOL_DEFINITIONS["Docker"]["names"].get(plat_key, []) + _, docker_command = _detect_tool(docker_candidates) + octave_candidates = TOOL_DEFINITIONS["Octave"]["names"].get(plat_key, []) + octave_path, _ = _detect_tool(octave_candidates) + octave_found = bool(octave_path) + + wrote_any = False + + tools_file = Path(concore_path) / "concore.tools" + if tool_overrides: + tools_content = "\n".join(f"{k}={v}" for k, v in tool_overrides) + "\n" + wrote_any = ( + _write_text(tools_file, tools_content, dry_run, force, console) or wrote_any + ) + else: + console.print("[yellow]![/yellow] No tool paths detected for concore.tools") + + sudo_file = Path(concore_path) / "concore.sudo" + if docker_command: + sudo_content = f"{docker_command}\n" + wrote_any = ( + _write_text(sudo_file, sudo_content, dry_run, force, console) or wrote_any + ) + else: + console.print( + "[yellow]![/yellow] Docker/Podman not detected; not writing concore.sudo" + ) + + octave_file = Path(concore_path) / "concore.octave" + if octave_found: + wrote_any = _write_text(octave_file, "", dry_run, force, console) or wrote_any + else: + console.print("[dim]-[/dim] Octave not detected; not writing concore.octave") + + if not wrote_any: + console.print("[yellow]No files written.[/yellow]") + return False + + if dry_run: + console.print("[green]Dry run complete.[/green]") + else: + console.print("[green]Setup complete.[/green]") + return True diff --git a/concore_cli/commands/status.py b/concore_cli/commands/status.py new file mode 100644 index 00000000..7ef1fca4 --- /dev/null +++ b/concore_cli/commands/status.py @@ -0,0 +1,106 @@ +import psutil +import os +from rich.table import Table +from rich.panel import Panel +from datetime import datetime + + +def show_status(console): + console.print("[cyan]Scanning for concore processes...[/cyan]\n") + + concore_processes = [] + + try: + current_pid = os.getpid() + + for proc in psutil.process_iter( + ["pid", "name", "cmdline", "create_time", "memory_info", "cpu_percent"] + ): + try: + cmdline = proc.info.get("cmdline") or [] + name = proc.info.get("name", "").lower() + + if proc.info["pid"] == current_pid: + continue + + cmdline_str = " ".join(cmdline) if cmdline else "" + + is_concore = ( + "concore" in cmdline_str.lower() + or "concore.py" in cmdline_str.lower() + or any("concorekill.bat" in str(item) for item in cmdline) + or ( + name in ["python.exe", "python", "python3"] + and "concore" in cmdline_str + ) + ) + + if is_concore: + try: + create_time = datetime.fromtimestamp(proc.info["create_time"]) + uptime = datetime.now() - create_time + hours, remainder = divmod(int(uptime.total_seconds()), 3600) + minutes, seconds = divmod(remainder, 60) + uptime_str = f"{hours}h {minutes}m {seconds}s" + except (OSError, OverflowError, ValueError): + # Failed to calculate uptime + uptime_str = "unknown" + + try: + mem_mb = proc.info["memory_info"].rss / 1024 / 1024 + mem_str = f"{mem_mb:.1f} MB" + except (psutil.NoSuchProcess, psutil.AccessDenied, AttributeError): + # Failed to get memory info + mem_str = "unknown" + + command = ( + " ".join(cmdline[:3]) if len(cmdline) >= 3 else cmdline_str[:50] + ) + + concore_processes.append( + { + "pid": proc.info["pid"], + "name": proc.info.get("name", "unknown"), + "command": command, + "uptime": uptime_str, + "memory": mem_str, + } + ) + except (psutil.NoSuchProcess, psutil.AccessDenied): + # Process may have exited or be inaccessible; safe to ignore + continue + + except Exception as e: + console.print(f"[red]Error scanning processes:[/red] {str(e)}") + return + + if not concore_processes: + console.print( + Panel.fit( + "[yellow]No concore processes currently running[/yellow]", + border_style="yellow", + ) + ) + else: + table = Table( + title=f"Concore Processes ({len(concore_processes)} running)", + show_header=True, + ) + table.add_column("PID", style="cyan", justify="right") + table.add_column("Name", style="green") + table.add_column("Uptime", style="yellow") + table.add_column("Memory", style="magenta") + table.add_column("Command", style="white") + + for proc in concore_processes: + table.add_row( + str(proc["pid"]), + proc["name"], + proc["uptime"], + proc["memory"], + proc["command"], + ) + + console.print(table) + console.print() + console.print("[dim]Use 'concore stop' to terminate all processes[/dim]") diff --git a/concore_cli/commands/stop.py b/concore_cli/commands/stop.py new file mode 100644 index 00000000..27b5796e --- /dev/null +++ b/concore_cli/commands/stop.py @@ -0,0 +1,107 @@ +import psutil +import os +import subprocess +import sys +from rich.panel import Panel + + +def stop_all(console): + console.print("[cyan]Finding concore processes...[/cyan]\n") + + processes_to_kill = [] + current_pid = os.getpid() + + try: + for proc in psutil.process_iter(["pid", "name", "cmdline"]): + try: + if proc.info["pid"] == current_pid: + continue + + cmdline = proc.info.get("cmdline") or [] + name = proc.info.get("name", "").lower() + cmdline_str = " ".join(cmdline) if cmdline else "" + + is_concore = ( + "concore" in cmdline_str.lower() + or "concore.py" in cmdline_str.lower() + or any("concorekill.bat" in str(item) for item in cmdline) + or ( + name in ["python.exe", "python", "python3"] + and "concore" in cmdline_str + ) + ) + + if is_concore: + processes_to_kill.append(proc) + except (psutil.NoSuchProcess, psutil.AccessDenied): + # Process already exited or access denied; continue + continue + + except Exception as e: + console.print(f"[red]Error:[/red] {str(e)}") + return + + if not processes_to_kill: + console.print( + Panel.fit( + "[yellow]No concore processes found[/yellow]", border_style="yellow" + ) + ) + return + + console.print( + f"[yellow]Stopping {len(processes_to_kill)} process(es)...[/yellow]\n" + ) + + killed_count = 0 + failed_count = 0 + + for proc in processes_to_kill: + try: + pid = proc.info["pid"] + name = proc.info.get("name", "unknown") + + if sys.platform == "win32": + result = subprocess.run( + ["taskkill", "/F", "/PID", str(pid)], + capture_output=True, + check=False, + ) + if result.returncode != 0: + raise RuntimeError(f"taskkill failed with code {result.returncode}") + else: + proc.terminate() + proc.wait(timeout=3) + + console.print(f" [green]✓[/green] Stopped {name} (PID: {pid})") + killed_count += 1 + + except psutil.TimeoutExpired: + try: + proc.kill() + console.print(f" [yellow]⚠[/yellow] Force killed {name} (PID: {pid})") + killed_count += 1 + except (psutil.NoSuchProcess, psutil.AccessDenied): + console.print(f" [red]✗[/red] Failed to stop {name} (PID: {pid})") + failed_count += 1 + except Exception as e: + console.print(f" [red]✗[/red] Failed to stop PID {pid}: {str(e)}") + failed_count += 1 + + console.print() + + if failed_count == 0: + console.print( + Panel.fit( + f"[green]✓[/green] Successfully stopped all {killed_count} process(es)", + border_style="green", + ) + ) + else: + console.print( + Panel.fit( + f"[yellow]Stopped {killed_count} process(es)\n" + f"Failed to stop {failed_count} process(es)[/yellow]", + border_style="yellow", + ) + ) diff --git a/concore_cli/commands/validate.py b/concore_cli/commands/validate.py new file mode 100644 index 00000000..a7168b3b --- /dev/null +++ b/concore_cli/commands/validate.py @@ -0,0 +1,474 @@ +import json +from pathlib import Path +from bs4 import BeautifulSoup +from rich.panel import Panel +import re +import xml.etree.ElementTree as ET + + +def _classify_message(message, bucket_name): + if bucket_name == "info": + if message.startswith("Found ") and "node(s)" in message: + return {"info_type": "node_count"} + if message.startswith("Found ") and "edge(s)" in message: + return {"info_type": "edge_count"} + if message.startswith("ZMQ-based edges:"): + return {"info_type": "zmq_edges"} + if message.startswith("File-based edges:"): + return {"info_type": "file_edges"} + return {"info_type": "info"} + + if message == "File is empty": + return {"error_type": "empty_file"} + if message.startswith("Invalid XML:"): + return {"error_type": "invalid_xml"} + if message == "Not a valid GraphML file - missing root element": + return {"error_type": "invalid_graphml"} + if message == "Missing element": + return {"error_type": "missing_graph_element"} + if message == "Graph missing required 'edgedefault' attribute": + return {"error_type": "missing_edgedefault"} + if message.startswith("Invalid edgedefault value"): + return {"error_type": "invalid_edgedefault"} + if message == "No nodes found in workflow": + return {"error_type": "no_nodes"} + if message == "No edges found in workflow": + return {"error_type": "no_edges"} + if message.startswith("Source directory not found:"): + return {"error_type": "missing_source_dir"} + if message == "Node missing required 'id' attribute": + return {"error_type": "missing_node_id"} + if message.startswith("Node '") and message.endswith( + "contains unsafe shell characters" + ): + return {"error_type": "unsafe_node_label"} + if message.startswith("Node '") and "missing format 'ID:filename'" in message: + return {"error_type": "invalid_node_label_format"} + if message.startswith("Node '") and message.endswith("has invalid format"): + return {"error_type": "invalid_node_label_format"} + if message.startswith("Node '") and message.endswith("has no filename"): + return {"error_type": "missing_node_filename"} + if message.startswith("Node '") and message.endswith("has unusual file extension"): + return {"error_type": "unusual_file_extension"} + if message.startswith("Missing source file:"): + return {"error_type": "missing_source_file"} + if message.startswith("Node ") and message.endswith(" has no label"): + return {"error_type": "missing_node_label"} + if message.startswith("Error parsing node:"): + return {"error_type": "node_parse_error"} + if message.startswith("Duplicate node label:"): + return {"error_type": "duplicate_node_label"} + if message == "Edge missing source or target": + return {"error_type": "missing_edge_endpoint"} + if message.startswith("Edge references non-existent source node:"): + return {"error_type": "missing_edge_source"} + if message.startswith("Edge references non-existent target node:"): + return {"error_type": "missing_edge_target"} + if message == "Workflow contains cycles (expected for control loops)": + return {"error_type": "cycle_detected"} + if message.startswith("Invalid port number:"): + return {"error_type": "invalid_port_number"} + if message.startswith("Port conflict:"): + return {"error_type": "port_conflict"} + if message.startswith("Port ") and "is in reserved range" in message: + return {"error_type": "reserved_port"} + if message.startswith("File not found:"): + return {"error_type": "file_not_found"} + if message.startswith("Validation failed:"): + return {"error_type": "validation_exception"} + return {"error_type": "validation_message"} + + +def _build_entries(bucket_name, messages, source_nodes): + entries = [] + for message in messages: + entry = {"message": message} + entry.update(_classify_message(message, bucket_name)) + + if message.startswith("Missing source file:"): + filename = message.split(":", 1)[1].strip() + node_id = source_nodes.get(filename) + if node_id: + entry["node_id"] = node_id + elif message.startswith("Node ") and message.endswith(" has no label"): + entry["node_id"] = message[5:-9] + elif message.startswith("Edge references non-existent source node:"): + entry["node_id"] = message.split(":", 1)[1].strip() + elif message.startswith("Edge references non-existent target node:"): + entry["node_id"] = message.split(":", 1)[1].strip() + + entries.append(entry) + return entries + + +def _build_payload(workflow_path, source_root, errors, warnings, info, source_nodes): + error_entries = _build_entries("errors", errors, source_nodes) + warning_entries = _build_entries("warnings", warnings, source_nodes) + info_entries = _build_entries("info", info, source_nodes) + + nodes_affected = [] + for entry in error_entries + warning_entries: + node_id = entry.get("node_id") + if node_id and node_id not in nodes_affected: + nodes_affected.append(node_id) + + return { + "workflow": workflow_path.name, + "source_dir": str(source_root), + "valid": len(errors) == 0, + "errors": error_entries, + "warnings": warning_entries, + "info": info_entries, + "summary": { + "error_count": len(error_entries), + "warning_count": len(warning_entries), + "info_count": len(info_entries), + "nodes_affected": nodes_affected, + }, + } + + +def validate_workflow(workflow_file, source_dir, console, output_format="text"): + workflow_path = Path(workflow_file) + source_root = workflow_path.parent / source_dir + + if output_format == "text": + console.print(f"[cyan]Validating:[/cyan] {workflow_path.name}") + console.print() + + errors = [] + warnings = [] + info = [] + source_nodes = {} + + def finalize(): + if output_format == "json": + print( + json.dumps( + _build_payload( + workflow_path, + source_root, + errors, + warnings, + info, + source_nodes, + ), + indent=2, + ) + ) + else: + show_results(console, errors, warnings, info) + return len(errors) == 0 + + try: + with open(workflow_path, "r", encoding="utf-8") as f: + content = f.read() + + if not content.strip(): + errors.append("File is empty") + return finalize() + + # strict XML syntax check + try: + ET.fromstring(content) + except ET.ParseError as e: + errors.append(f"Invalid XML: {str(e)}") + return finalize() + + try: + soup = BeautifulSoup(content, "xml") + except Exception as e: + errors.append(f"Invalid XML: {str(e)}") + return finalize() + + root = soup.find("graphml") + if not root: + errors.append("Not a valid GraphML file - missing root element") + return finalize() + + # check the graph attributes + graph = soup.find("graph") + if not graph: + errors.append("Missing element") + else: + edgedefault = graph.get("edgedefault") + if not edgedefault: + errors.append("Graph missing required 'edgedefault' attribute") + elif edgedefault not in ["directed", "undirected"]: + errors.append( + f"Invalid edgedefault value '{edgedefault}' (must be 'directed' or 'undirected')" + ) + + nodes = soup.find_all("node") + edges = soup.find_all("edge") + + if len(nodes) == 0: + warnings.append("No nodes found in workflow") + else: + info.append(f"Found {len(nodes)} node(s)") + + if len(edges) == 0: + warnings.append("No edges found in workflow") + else: + info.append(f"Found {len(edges)} edge(s)") + + if not source_root.exists(): + warnings.append(f"Source directory not found: {source_root}") + + node_labels = [] + for node in nodes: + # check the node id + node_id = node.get("id") + if not node_id: + errors.append("Node missing required 'id' attribute") + # skip further checks for this node to avoid noise + continue + + try: + # robust find: try with namespace prefix first, then without + label_tag = node.find("y:NodeLabel") + if not label_tag: + label_tag = node.find("NodeLabel") + + if label_tag and label_tag.text: + label = label_tag.text.strip() + node_labels.append(label) + + # reject shell metacharacters to prevent command injection (#251) + if re.search(r'[;&|`$\'"()\\]', label): + errors.append( + f"Node '{label}' contains unsafe shell characters" + ) + continue + + if ":" not in label: + warnings.append(f"Node '{label}' missing format 'ID:filename'") + else: + parts = label.split(":") + if len(parts) != 2: + warnings.append(f"Node '{label}' has invalid format") + else: + nodeId_part, filename = parts + source_nodes[filename] = node_id + if not filename: + errors.append(f"Node '{label}' has no filename") + elif not any( + filename.endswith(ext) + for ext in [".py", ".cpp", ".m", ".v", ".java"] + ): + warnings.append( + f"Node '{label}' has unusual file extension" + ) + elif source_root.exists(): + file_path = source_root / filename + if not file_path.exists(): + errors.append(f"Missing source file: {filename}") + else: + warnings.append(f"Node {node_id} has no label") + except Exception as e: + warnings.append(f"Error parsing node: {str(e)}") + + # duplicate labels cause silent corruption in mkconcore.py + seen = set() + for label in node_labels: + if label in seen: + errors.append(f"Duplicate node label: '{label}'") + seen.add(label) + + node_ids = {node.get("id") for node in nodes if node.get("id")} + for edge in edges: + source = edge.get("source") + target = edge.get("target") + + if not source or not target: + errors.append("Edge missing source or target") + continue + + if source not in node_ids: + errors.append(f"Edge references non-existent source node: {source}") + if target not in node_ids: + errors.append(f"Edge references non-existent target node: {target}") + + edge_label_regex = re.compile(r"0x([a-fA-F0-9]+)_(\S+)") + zmq_edges = 0 + file_edges = 0 + + for edge in edges: + try: + label_tag = edge.find("y:EdgeLabel") + if not label_tag: + label_tag = edge.find("EdgeLabel") + + if label_tag and label_tag.text: + if edge_label_regex.match(label_tag.text.strip()): + zmq_edges += 1 + else: + file_edges += 1 + except Exception: + pass + + if zmq_edges > 0: + info.append(f"ZMQ-based edges: {zmq_edges}") + if file_edges > 0: + info.append(f"File-based edges: {file_edges}") + + _check_cycles(soup, errors, warnings) + _check_zmq_ports(soup, errors, warnings) + + return finalize() + + except FileNotFoundError: + if output_format == "json": + print( + json.dumps( + _build_payload( + workflow_path, + source_root, + [f"File not found: {workflow_path}"], + [], + [], + source_nodes, + ), + indent=2, + ) + ) + else: + console.print(f"[red]Error:[/red] File not found: {workflow_path}") + return False + except Exception as e: + if output_format == "json": + print( + json.dumps( + _build_payload( + workflow_path, + source_root, + [f"Validation failed: {str(e)}"], + [], + [], + source_nodes, + ), + indent=2, + ) + ) + else: + console.print(f"[red]Validation failed:[/red] {str(e)}") + return False + + +def _check_cycles(soup, errors, warnings): + nodes = soup.find_all("node") + edges = soup.find_all("edge") + + node_ids = [node.get("id") for node in nodes if node.get("id")] + if not node_ids: + return + + graph = {nid: [] for nid in node_ids} + for edge in edges: + source = edge.get("source") + target = edge.get("target") + if source and target and source in graph: + graph[source].append(target) + + def has_cycle_from(start, visited, rec_stack): + visited.add(start) + rec_stack.add(start) + + for neighbor in graph.get(start, []): + if neighbor not in visited: + if has_cycle_from(neighbor, visited, rec_stack): + return True + elif neighbor in rec_stack: + return True + + rec_stack.remove(start) + return False + + visited = set() + for node_id in node_ids: + if node_id not in visited: + if has_cycle_from(node_id, visited, set()): + warnings.append("Workflow contains cycles (expected for control loops)") + return + + +def _check_zmq_ports(soup, errors, warnings): + edges = soup.find_all("edge") + port_pattern = re.compile(r"0x([a-fA-F0-9]+)_(\S+)") + + ports_used = {} + + for edge in edges: + label_tag = edge.find("y:EdgeLabel") or edge.find("EdgeLabel") + if not label_tag or not label_tag.text: + continue + + match = port_pattern.match(label_tag.text.strip()) + if not match: + continue + + port_hex = match.group(1) + port_name = match.group(2) + port_num = int(port_hex, 16) + + if port_num < 1: + errors.append( + f"Invalid port number: {port_num} (0x{port_hex}) must be at least 1" + ) + continue + elif port_num > 65535: + errors.append( + f"Invalid port number: {port_num} (0x{port_hex}) exceeds maximum (65535)" + ) + continue + + if port_num in ports_used: + existing_name = ports_used[port_num] + if existing_name != port_name: + errors.append( + f"Port conflict: 0x{port_hex} used for both '{existing_name}' and '{port_name}'" + ) + else: + ports_used[port_num] = port_name + + if port_num < 1024: + warnings.append( + f"Port {port_num} (0x{port_hex}) is in reserved range (< 1024)" + ) + + +def show_results(console, errors, warnings, info): + if errors: + console.print("[red]✗ Validation failed[/red]\n") + for error in errors: + console.print(f" [red]✗[/red] {error}") + else: + console.print("[green]✓ Validation passed[/green]\n") + + if warnings: + console.print() + for warning in warnings: + console.print(f" [yellow]⚠[/yellow] {warning}") + + if info: + console.print() + for item in info: + console.print(f" [blue]ℹ[/blue] {item}") + + console.print() + + if not errors: + console.print( + Panel.fit( + "[green]✓[/green] Workflow is valid and ready to run", + border_style="green", + ) + ) + else: + console.print( + Panel.fit( + f"[red]Found {len(errors)} error(s)[/red]\n" + "Fix the errors above before running the workflow", + border_style="red", + ) + ) diff --git a/concore_cli/commands/watch.py b/concore_cli/commands/watch.py new file mode 100644 index 00000000..e82efbab --- /dev/null +++ b/concore_cli/commands/watch.py @@ -0,0 +1,238 @@ +import time +import re +from pathlib import Path +from ast import literal_eval +from datetime import datetime +from rich.table import Table +from rich.live import Live +from rich.panel import Panel +from rich.text import Text +from rich.console import Group + + +def watch_study(study_dir, interval, once, console): + study_path = Path(study_dir).resolve() + if not study_path.is_dir(): + console.print(f"[red]Error:[/red] '{study_dir}' is not a directory") + return + + nodes = _find_nodes(study_path) + edges = _find_edges(study_path, nodes) + + if not nodes and not edges: + console.print( + Panel( + "[yellow]No nodes or edge directories found.[/yellow]\n" + "[dim]Make sure you point to a built study directory (run makestudy/build first).[/dim]", + title="concore watch", + border_style="yellow", + ) + ) + return + + if once: + output = _build_display(study_path, nodes, edges) + console.print(output) + return + + console.print(f"[cyan]Watching:[/cyan] {study_path}") + console.print(f"[dim]Refresh every {interval}s — Ctrl+C to stop[/dim]\n") + + try: + with Live(console=console, refresh_per_second=1, screen=False) as live: + while True: + nodes = _find_nodes(study_path) + edges = _find_edges(study_path, nodes) + live.update(_build_display(study_path, nodes, edges)) + time.sleep(interval) + except KeyboardInterrupt: + console.print("\n[yellow]Watch stopped.[/yellow]") + + +def _build_display(study_path, nodes, edges): + parts = [] + + header = Text() + header.append("Study ", style="bold cyan") + header.append(study_path.name, style="bold white") + header.append(f" | {len(nodes)} node(s), {len(edges)} edge(s)", style="dim") + parts.append(header) + + if edges: + parts.append(Text()) + parts.append(_edge_table(edges)) + + if nodes: + parts.append(Text()) + parts.append(_node_table(nodes)) + + if not edges and not nodes: + parts.append(Panel("[yellow]No data yet[/yellow]", border_style="yellow")) + + return Group(*parts) + + +def _edge_table(edges): + table = Table( + title="Edges (port data)", + show_header=True, + title_style="bold cyan", + expand=True, + ) + table.add_column("Edge", style="green", min_width=10) + table.add_column("Port File", style="cyan") + table.add_column("Simtime", style="yellow", justify="right") + table.add_column("Value", style="white") + table.add_column("Age", style="magenta", justify="right") + + now = time.time() + for edge_name, edge_path in sorted(edges): + files = _read_edge_files(edge_path) + if not files: + table.add_row(edge_name, "[dim]—[/dim]", "", "[dim]empty[/dim]", "") + continue + first = True + for fname, simtime_val, value_str, mtime in files: + age_str = _format_age(now, mtime) + label = edge_name if first else "" + st = str(simtime_val) if simtime_val is not None else "—" + table.add_row(label, fname, st, value_str, age_str) + first = False + + return table + + +def _node_table(nodes): + table = Table(title="Nodes", show_header=True, title_style="bold cyan", expand=True) + table.add_column("Node", style="green", min_width=10) + table.add_column("Ports (in)", style="cyan") + table.add_column("Ports (out)", style="cyan") + table.add_column("Source", style="dim") + + for node_name, node_path in sorted(nodes): + in_dirs = sorted( + d.name + for d in node_path.iterdir() + if d.is_dir() and re.match(r"^in\d+$", d.name) + ) + out_dirs = sorted( + d.name + for d in node_path.iterdir() + if d.is_dir() and re.match(r"^out\d+$", d.name) + ) + src = _detect_source(node_path) + table.add_row( + node_name, + ", ".join(in_dirs) if in_dirs else "—", + ", ".join(out_dirs) if out_dirs else "—", + src, + ) + + return table + + +def _find_nodes(study_path): + nodes = [] + port_re = re.compile(r"^(in|out)\d+$") + skip = {"src", "__pycache__", ".git"} + for entry in study_path.iterdir(): + if not entry.is_dir() or entry.name in skip or entry.name.startswith("."): + continue + try: + has_ports = any( + c.is_dir() and port_re.match(c.name) for c in entry.iterdir() + ) + except PermissionError: + continue + if has_ports: + nodes.append((entry.name, entry)) + return nodes + + +def _find_edges(study_path, nodes=None): + if nodes is None: + nodes = _find_nodes(study_path) + node_names = {name for name, _ in nodes} + skip = {"src", "__pycache__", ".git"} + edges = [] + for entry in study_path.iterdir(): + if not entry.is_dir(): + continue + if entry.name in skip or entry.name in node_names or entry.name.startswith("."): + continue + try: + has_file = any(f.is_file() for f in entry.iterdir()) + except PermissionError: + continue + if has_file: + edges.append((entry.name, entry)) + return edges + + +def _read_edge_files(edge_path): + results = [] + try: + children = sorted(edge_path.iterdir()) + except PermissionError: + return results + for f in children: + if not f.is_file(): + continue + # skip concore internal files + if f.name.startswith("concore."): + continue + simtime_val, value_str = _parse_port_file(f) + try: + mtime = f.stat().st_mtime + except OSError: + mtime = 0 + results.append((f.name, simtime_val, value_str, mtime)) + return results + + +def _detect_source(node_path): + for ext in ("*.py", "*.m", "*.cpp", "*.v", "*.sh", "*.java"): + matches = list(node_path.glob(ext)) + for m in matches: + # skip concore library copies + if m.name.startswith("concore"): + continue + return m.name + return "—" + + +def _parse_port_file(port_file): + try: + content = port_file.read_text().strip() + if not content: + return None, "(empty)" + val = literal_eval(content) + if isinstance(val, list) and len(val) > 0: + simtime = val[0] if isinstance(val[0], (int, float)) else None + data = val[1:] if simtime is not None else val + data_str = str(data) + if len(data_str) > 50: + data_str = data_str[:47] + "..." + return simtime, data_str + raw = str(val) + return None, raw[:50] if len(raw) > 50 else raw + except Exception: + try: + raw = port_file.read_text().strip() + return None, raw[:50] if raw else "(empty)" + except Exception: + return None, "(read error)" + + +def _format_age(now, mtime): + if mtime == 0: + return "—" + age = now - mtime + if age < 3: + return "[bold green]now[/bold green]" + elif age < 60: + return f"{int(age)}s" + elif age < 3600: + return f"{int(age // 60)}m" + else: + return datetime.fromtimestamp(mtime).strftime("%H:%M:%S") diff --git a/concore_default_maxtime.m b/concore_default_maxtime.m index 5627d5f5..489187d0 100644 --- a/concore_default_maxtime.m +++ b/concore_default_maxtime.m @@ -3,7 +3,17 @@ function concore_default_maxtime(default) try maxfile = fopen(strcat(concore.inpath,'1/concore.maxtime')); instr = fscanf(maxfile,'%c'); - concore.maxtime = eval(instr); + % Safe numeric parsing (replaces unsafe eval) + clean_str = strtrim(instr); + clean_str = regexprep(clean_str, '[\[\]]', ''); + % Normalize commas to whitespace so sscanf can parse all tokens + clean_str = strrep(clean_str, ',', ' '); + parsed_values = sscanf(clean_str, '%f'); + if numel(parsed_values) == 1 + concore.maxtime = parsed_values; + else + concore.maxtime = default; + end fclose(maxfile); catch exc concore.maxtime = default; diff --git a/concore_initval.m b/concore_initval.m index 73cc1469..4b92b318 100644 --- a/concore_initval.m +++ b/concore_initval.m @@ -1,6 +1,20 @@ function [result] = concore_initval(simtime_val) global concore; - result = eval(simtime_val); + % Safe numeric parsing (replaces unsafe eval) + clean_str = strtrim(simtime_val); + clean_str = regexprep(clean_str, '[\[\]]', ''); + clean_str = strrep(clean_str, ',', ' '); + result = sscanf(clean_str, '%f').'; + % Guard against empty or invalid numeric input + if isempty(result) + concore.simtime = 0; + result = []; + return; + end concore.simtime = result(1); - result = result(2:length(result)); + if numel(result) >= 2 + result = result(2:end); + else + result = []; + end end diff --git a/concore_iport.m b/concore_iport.m index 128252e2..8a2146f1 100644 --- a/concore_iport.m +++ b/concore_iport.m @@ -7,7 +7,13 @@ if isequal(s(i:i+length(target)-1),target) for j = i+length(target):length(s) if isequal(s(j),',')||isequal(s(j),'}') - result = eval(s(i+length(target):j-1)); + % Safe numeric parsing (replaces unsafe eval) + port_str = strtrim(s(i+length(target):j-1)); + result = sscanf(port_str, '%f'); + if isempty(result) + % Keep the initialized default value (0) if parsing fails + result = 0; + end return end end diff --git a/concore_oport.m b/concore_oport.m index 9cbe3de8..a9ed01b5 100644 --- a/concore_oport.m +++ b/concore_oport.m @@ -7,7 +7,9 @@ if isequal(s(i:i+length(target)-1),target) for j = i+length(target):length(s) if isequal(s(j),',')||isequal(s(j),'}') - result = eval(s(i+length(target):j-1)); + % Safe numeric parsing (replaces unsafe eval) + port_str = strtrim(s(i+length(target):j-1)); + result = sscanf(port_str, '%f'); return end end diff --git a/concore_read.m b/concore_read.m index 2103255d..b4ed1bc0 100644 --- a/concore_read.m +++ b/concore_read.m @@ -8,15 +8,38 @@ catch exc ins = inistr; end - while length(ins) == 0 + maxretries = 5; + attempts = 0; + while length(ins) == 0 && attempts < maxretries pause(concore.delay); - input1 = fopen(strcat(concore.inpath,num2str(port),'/',name)); - ins = fscanf(input1,'%c'); - fclose(input1); + try + input1 = fopen(strcat(concore.inpath,num2str(port),'/',name)); + ins = fscanf(input1,'%c'); + fclose(input1); + catch exc + end concore.retrycount = concore.retrycount + 1; + attempts = attempts + 1; + end + if length(ins) == 0 + ins = inistr; end concore.s = strcat(concore.s, ins); - result = eval(ins); - concore.simtime = max(concore.simtime,result(1)); - result = result(2:length(result)); + % Safe numeric parsing (replaces unsafe eval) + clean_str = strtrim(ins); + clean_str = regexprep(clean_str, '[\[\]]', ''); + % Normalize comma delimiters to whitespace so sscanf parses all values + clean_str = strrep(clean_str, ',', ' '); + result = sscanf(clean_str, '%f').'; + % Guard against empty parse result to avoid indexing errors + if isempty(result) + result = []; + return; + end + concore.simtime = max(concore.simtime, result(1)); + if numel(result) > 1 + result = result(2:end); + else + result = []; + end end diff --git a/concore_write.m b/concore_write.m index 20bce14e..29b2095c 100644 --- a/concore_write.m +++ b/concore_write.m @@ -5,6 +5,7 @@ function concore_write(port, name, val, delta) outstr = cat(2,"[",num2str(concore.simtime+delta),num2str(val,",%e"),"]"); fprintf(output1,'%s',outstr); fclose(output1); + % simtime must not be mutated here (issue #385). catch exc disp(['skipping ' concore.outpath num2str(port) '/' name]); end diff --git a/concoredocker.hpp b/concoredocker.hpp index cbce7184..d9c9a1e0 100644 --- a/concoredocker.hpp +++ b/concoredocker.hpp @@ -1,5 +1,5 @@ -#ifndef CONCORE_HPP -#define CONCORE_HPP +#ifndef CONCOREDOCKER_HPP +#define CONCOREDOCKER_HPP #include #include @@ -12,9 +12,43 @@ #include #include #include +#include +#include +#include + +#ifdef __linux__ +#include +#include +#include +#endif + +#include "concore_base.hpp" class Concore { +private: + static constexpr size_t SHM_SIZE = 4096; + + int shmId_create = -1; + int shmId_get = -1; + char* sharedData_create = nullptr; + char* sharedData_get = nullptr; + int communication_iport = 0; // iport refers to input port + int communication_oport = 0; // oport refers to output port + public: + enum class ReadStatus { + SUCCESS, + TIMEOUT, + PARSE_ERROR, + FILE_NOT_FOUND, + RETRIES_EXCEEDED + }; + + struct ReadResult { + ReadStatus status; + std::vector data; + }; + std::unordered_map iport; std::unordered_map oport; std::string s, olds; @@ -22,53 +56,233 @@ class Concore { int retrycount = 0; std::string inpath = "/in"; std::string outpath = "/out"; - int simtime = 0; - int maxtime = 100; + double simtime = 0; + double maxtime = 100; std::unordered_map params; + ReadStatus last_read_status = ReadStatus::SUCCESS; +#ifdef CONCORE_USE_ZMQ + std::map zmq_ports; +#endif + + std::string stripstr(const std::string& str) { + return concore_base::stripstr(str); + } + + std::string stripquotes(const std::string& str) { + return concore_base::stripquotes(str); + } + + std::unordered_map parsedict(const std::string& str) { + auto ordered = concore_base::parsedict(str); + return std::unordered_map(ordered.begin(), ordered.end()); + } + + std::vector parselist(const std::string& str) { + return concore_base::parselist(str); + } Concore() { iport = safe_literal_eval("concore.iport", {}); oport = safe_literal_eval("concore.oport", {}); default_maxtime(100); load_params(); + +#ifdef __linux__ + int iport_number = -1; + int oport_number = -1; + + if (!iport.empty()) + iport_number = ExtractNumeric(iport.begin()->first); + if (!oport.empty()) + oport_number = ExtractNumeric(oport.begin()->first); + + if (oport_number != -1) { + communication_oport = 1; + createSharedMemory(oport_number); + } + if (iport_number != -1) { + communication_iport = 1; + getSharedMemory(iport_number); + } +#endif + } + + ~Concore() { +#ifdef CONCORE_USE_ZMQ + for (auto& kv : zmq_ports) + delete kv.second; + zmq_ports.clear(); +#endif +#ifdef __linux__ + if (communication_oport == 1 && sharedData_create != nullptr) + shmdt(sharedData_create); + if (communication_iport == 1 && sharedData_get != nullptr) + shmdt(sharedData_get); + if (shmId_create != -1) + shmctl(shmId_create, IPC_RMID, nullptr); +#endif + } + + Concore(const Concore&) = delete; + Concore& operator=(const Concore&) = delete; + + Concore(Concore&& other) noexcept + : iport(std::move(other.iport)), oport(std::move(other.oport)), + s(std::move(other.s)), olds(std::move(other.olds)), + delay(other.delay), retrycount(other.retrycount), + inpath(std::move(other.inpath)), outpath(std::move(other.outpath)), + simtime(other.simtime), maxtime(other.maxtime), + params(std::move(other.params)), + shmId_create(other.shmId_create), shmId_get(other.shmId_get), + sharedData_create(other.sharedData_create), sharedData_get(other.sharedData_get), + communication_iport(other.communication_iport), communication_oport(other.communication_oport) + { +#ifdef CONCORE_USE_ZMQ + zmq_ports = std::move(other.zmq_ports); +#endif + other.shmId_create = -1; + other.shmId_get = -1; + other.sharedData_create = nullptr; + other.sharedData_get = nullptr; + other.communication_iport = 0; + other.communication_oport = 0; + } + + Concore& operator=(Concore&& other) noexcept + { + if (this == &other) + return *this; + +#ifdef CONCORE_USE_ZMQ + for (auto& kv : zmq_ports) + delete kv.second; + zmq_ports = std::move(other.zmq_ports); +#endif +#ifdef __linux__ + if (communication_oport == 1 && sharedData_create != nullptr) + shmdt(sharedData_create); + if (communication_iport == 1 && sharedData_get != nullptr) + shmdt(sharedData_get); + if (shmId_create != -1) + shmctl(shmId_create, IPC_RMID, nullptr); +#endif + + iport = std::move(other.iport); + oport = std::move(other.oport); + s = std::move(other.s); + olds = std::move(other.olds); + delay = other.delay; + retrycount = other.retrycount; + inpath = std::move(other.inpath); + outpath = std::move(other.outpath); + simtime = other.simtime; + maxtime = other.maxtime; + params = std::move(other.params); + shmId_create = other.shmId_create; + shmId_get = other.shmId_get; + sharedData_create = other.sharedData_create; + sharedData_get = other.sharedData_get; + communication_iport = other.communication_iport; + communication_oport = other.communication_oport; + + other.shmId_create = -1; + other.shmId_get = -1; + other.sharedData_create = nullptr; + other.sharedData_get = nullptr; + other.communication_iport = 0; + other.communication_oport = 0; + + return *this; } std::unordered_map safe_literal_eval(const std::string& filename, std::unordered_map defaultValue) { std::ifstream file(filename); - if (!file) { - std::cerr << "Error reading " << filename << "\n"; - return defaultValue; - } - return defaultValue; + if (!file) return defaultValue; + std::stringstream buf; + buf << file.rdbuf(); + auto result = concore_base::parsedict(buf.str()); + if (result.empty()) return defaultValue; + return std::unordered_map(result.begin(), result.end()); } void load_params() { - std::ifstream file(inpath + "/1/concore.params"); - if (!file) return; - std::stringstream buffer; - buffer << file.rdbuf(); - std::string sparams = buffer.str(); + auto ordered = concore_base::load_params(inpath + "/1/concore.params"); + params = std::unordered_map(ordered.begin(), ordered.end()); + } - if (!sparams.empty() && sparams[0] == '"') { - sparams = sparams.substr(1, sparams.find('"') - 1); - } + std::string tryparam(const std::string& n, const std::string& i) { + return params.count(n) ? params[n] : i; + } + + void default_maxtime(double defaultValue) { + maxtime = concore_base::load_maxtime( + inpath + "/1/concore.maxtime", defaultValue); + } - if (!sparams.empty() && sparams[0] != '{') { - sparams = "{'" + std::regex_replace(std::regex_replace(std::regex_replace(sparams, std::regex(","), ", '"), std::regex("="), "':"), std::regex(" "), "") + "}"; + key_t ExtractNumeric(const std::string& str) { + std::string numberString; + size_t numDigits = 0; + while (numDigits < str.length() && std::isdigit(str[numDigits])) { + numberString += str[numDigits]; + ++numDigits; } + if (numDigits == 0) + return -1; + if (numDigits == 1 && std::stoi(numberString) <= 0) + return -1; + return std::stoi(numberString); } - std::string tryparam(const std::string& n, const std::string& i) { - return params.count(n) ? params[n] : i; +#ifdef __linux__ + void createSharedMemory(key_t key) { + shmId_create = shmget(key, SHM_SIZE, IPC_CREAT | 0666); + if (shmId_create == -1) { + std::cerr << "Failed to create shared memory segment.\n"; + return; + } + + // Verify the segment is large enough (shmget won't resize an existing segment) + struct shmid_ds shm_info; + if (shmctl(shmId_create, IPC_STAT, &shm_info) == 0 && shm_info.shm_segsz < SHM_SIZE) { + std::cerr << "Shared memory segment too small (" << shm_info.shm_segsz + << " bytes, need " << SHM_SIZE << "). Removing and recreating.\n"; + shmctl(shmId_create, IPC_RMID, nullptr); + shmId_create = shmget(key, SHM_SIZE, IPC_CREAT | 0666); + if (shmId_create == -1) { + std::cerr << "Failed to recreate shared memory segment.\n"; + return; + } + } + + sharedData_create = static_cast(shmat(shmId_create, NULL, 0)); + if (sharedData_create == reinterpret_cast(-1)) { + std::cerr << "Failed to attach shared memory segment.\n"; + sharedData_create = nullptr; + } } - void default_maxtime(int defaultValue) { - maxtime = defaultValue; - std::ifstream file(inpath + "/1/concore.maxtime"); - if (file) { - file >> maxtime; + void getSharedMemory(key_t key) { + int retry = 0; + const int MAX_RETRY = 100; + while (retry < MAX_RETRY) { + shmId_get = shmget(key, SHM_SIZE, 0666); + if (shmId_get != -1) + break; + std::cout << "Shared memory does not exist. Make sure the writer process is running.\n"; + sleep(1); + retry++; + } + if (shmId_get == -1) { + std::cerr << "Failed to get shared memory segment after max retries.\n"; + return; + } + sharedData_get = static_cast(shmat(shmId_get, NULL, 0)); + if (sharedData_get == reinterpret_cast(-1)) { + std::cerr << "Failed to attach shared memory segment.\n"; + sharedData_get = nullptr; } } +#endif bool unchanged() { if (olds == s) { @@ -79,21 +293,30 @@ class Concore { return false; } - std::vector read(int port, const std::string& name, const std::string& initstr) { + std::vector read(int port, const std::string& name, const std::string& initstr) { +#ifdef __linux__ + if (communication_iport == 1) + return read_SM(port, name, initstr); +#endif + ReadStatus status = ReadStatus::SUCCESS; std::this_thread::sleep_for(std::chrono::seconds(delay)); - std::string file_path = inpath + std::to_string(port) + "/" + name; + std::string file_path = inpath + "/" + std::to_string(port) + "/" + name; std::ifstream infile(file_path); std::string ins; if (!infile) { std::cerr << "File " << file_path << " not found, using default value.\n"; - return {initstr}; + status = ReadStatus::FILE_NOT_FOUND; + std::vector fallback = concore_base::parselist_double(initstr); + last_read_status = status; + return fallback; } std::getline(infile, ins); - + int attempts = 0, max_retries = 5; while (ins.empty() && attempts < max_retries) { std::this_thread::sleep_for(std::chrono::seconds(delay)); + infile.close(); infile.open(file_path); if (infile) std::getline(infile, ins); attempts++; @@ -102,15 +325,96 @@ class Concore { if (ins.empty()) { std::cerr << "Max retries reached for " << file_path << ", using default value.\n"; - return {initstr}; + status = ReadStatus::RETRIES_EXCEEDED; + std::vector fallback = concore_base::parselist_double(initstr); + last_read_status = status; + return fallback; + } + + s += ins; + std::vector inval = concore_base::parselist_double(ins); + if (inval.empty()) { + status = ReadStatus::PARSE_ERROR; + inval = concore_base::parselist_double(initstr); + } + if (inval.empty()) { + last_read_status = status; + return inval; + } + last_read_status = status; + simtime = simtime > inval[0] ? simtime : inval[0]; + inval.erase(inval.begin()); + return inval; + } + + ReadResult read_result(int port, const std::string& name, const std::string& initstr) { + ReadResult result; + result.data = read(port, name, initstr); + result.status = last_read_status; + return result; + } + +#ifdef __linux__ + std::vector read_SM(int port, const std::string& name, const std::string& initstr) { + ReadStatus status = ReadStatus::SUCCESS; + std::this_thread::sleep_for(std::chrono::seconds(delay)); + std::string ins; + try { + if (shmId_get != -1 && sharedData_get && sharedData_get[0] != '\0') + ins = std::string(sharedData_get, strnlen(sharedData_get, SHM_SIZE)); + else + throw 505; + } catch (...) { + status = ReadStatus::FILE_NOT_FOUND; + ins = initstr; + } + + int retry = 0; + const int MAX_RETRY = 100; + while ((int)ins.length() == 0 && retry < MAX_RETRY) { + std::this_thread::sleep_for(std::chrono::seconds(delay)); + try { + if (shmId_get != -1 && sharedData_get) { + ins = std::string(sharedData_get, strnlen(sharedData_get, SHM_SIZE)); + retrycount++; + } else { + retrycount++; + throw 505; + } + } catch (...) { + std::cerr << "Read error\n"; + } + retry++; } - + if ((int)ins.length() == 0) + status = ReadStatus::RETRIES_EXCEEDED; + s += ins; - return {ins}; + std::vector inval = concore_base::parselist_double(ins); + if (inval.empty()) { + if (status == ReadStatus::SUCCESS) + status = ReadStatus::PARSE_ERROR; + inval = concore_base::parselist_double(initstr); + } + if (inval.empty()) { + last_read_status = status; + return inval; + } + last_read_status = status; + simtime = simtime > inval[0] ? simtime : inval[0]; + inval.erase(inval.begin()); + return inval; } +#endif - void write(int port, const std::string& name, const std::vector& val, int delta = 0) { - std::string file_path = outpath + std::to_string(port) + "/" + name; + void write(int port, const std::string& name, const std::vector& val, int delta = 0) { +#ifdef __linux__ + if (communication_oport == 1) { + write_SM(port, name, val, delta); + return; + } +#endif + std::string file_path = outpath + "/" + std::to_string(port) + "/" + name; std::ofstream outfile(file_path); if (!outfile) { std::cerr << "Error writing to " << file_path << "\n"; @@ -122,9 +426,110 @@ class Concore { outfile << val[i] << (i + 1 < val.size() ? ", " : ""); } outfile << "]"; - simtime += delta; + // simtime must not be mutated here (issue #385). } } -}; +#ifdef __linux__ + void write_SM(int port, const std::string& name, std::vector val, int delta = 0) { + try { + if (shmId_create == -1) + throw 505; + if (sharedData_create == nullptr) + throw 506; + val.insert(val.begin(), simtime + delta); + std::ostringstream outfile; + outfile << '['; + for (size_t i = 0; i < val.size() - 1; i++) + outfile << val[i] << ','; + outfile << val[val.size() - 1] << ']'; + std::string result = outfile.str(); + if (result.size() >= SHM_SIZE) { + std::cerr << "ERROR: write_SM payload (" << result.size() + << " bytes) exceeds " << SHM_SIZE - 1 + << "-byte shared memory limit. Data truncated!" << std::endl; + } + std::strncpy(sharedData_create, result.c_str(), SHM_SIZE - 1); + sharedData_create[SHM_SIZE - 1] = '\0'; + } catch (...) { + std::cerr << "skipping +" << outpath << port << "/" << name << "\n"; + } + } #endif + +#ifdef CONCORE_USE_ZMQ + void init_zmq_port(const std::string& port_name, const std::string& port_type, + const std::string& address, const std::string& socket_type_str) { + if (zmq_ports.count(port_name)) return; + int sock_type = concore_base::zmq_socket_type_from_string(socket_type_str); + if (sock_type == -1) { + std::cerr << "init_zmq_port: unknown socket type '" << socket_type_str << "'\n"; + return; + } + zmq_ports[port_name] = new concore_base::ZeroMQPort(port_type, address, sock_type); + } + + std::vector read_ZMQ(const std::string& port_name, const std::string& name, const std::string& initstr) { + ReadStatus status = ReadStatus::SUCCESS; + auto it = zmq_ports.find(port_name); + if (it == zmq_ports.end()) { + std::cerr << "read_ZMQ: port '" << port_name << "' not initialized\n"; + status = ReadStatus::FILE_NOT_FOUND; + last_read_status = status; + return concore_base::parselist_double(initstr); + } + std::vector inval = it->second->recv_with_retry(); + if (inval.empty()) { + status = ReadStatus::TIMEOUT; + inval = concore_base::parselist_double(initstr); + } + if (inval.empty()) { + if (status == ReadStatus::SUCCESS) + status = ReadStatus::PARSE_ERROR; + last_read_status = status; + return inval; + } + last_read_status = status; + simtime = simtime > inval[0] ? simtime : inval[0]; + s += port_name; + inval.erase(inval.begin()); + return inval; + } + + void write_ZMQ(const std::string& port_name, const std::string& name, std::vector val, int delta = 0) { + auto it = zmq_ports.find(port_name); + if (it == zmq_ports.end()) { + std::cerr << "write_ZMQ: port '" << port_name << "' not initialized\n"; + return; + } + val.insert(val.begin(), simtime + delta); + it->second->send_with_retry(val); + // simtime must not be mutated here. + } + + std::vector read(const std::string& port_name, const std::string& name, const std::string& initstr) { + return read_ZMQ(port_name, name, initstr); + } + + ReadResult read_result(const std::string& port_name, const std::string& name, const std::string& initstr) { + ReadResult result; + result.data = read(port_name, name, initstr); + result.status = last_read_status; + return result; + } + + void write(const std::string& port_name, const std::string& name, std::vector val, int delta = 0) { + return write_ZMQ(port_name, name, val, delta); + } +#endif // CONCORE_USE_ZMQ + + std::vector initval(const std::string& simtime_val) { + std::vector val = concore_base::parselist_double(simtime_val); + if (val.empty()) return val; + simtime = val[0]; + val.erase(val.begin()); + return val; + } +}; + +#endif // CONCOREDOCKER_HPP diff --git a/concoredocker.java b/concoredocker.java index dde521c8..d6c7c21f 100644 --- a/concoredocker.java +++ b/concoredocker.java @@ -3,77 +3,150 @@ import java.nio.file.Paths; import java.util.HashMap; import java.util.Map; +import java.util.ArrayList; +import java.util.List; +import org.zeromq.ZMQ; +/** + * Java implementation of concore Docker communication. + * + * This class provides file-based inter-process communication for control systems, + * mirroring the functionality of concoredocker.py. + */ public class concoredocker { private static Map iport = new HashMap<>(); private static Map oport = new HashMap<>(); private static String s = ""; private static String olds = ""; - private static int delay = 1; + // delay in milliseconds (Python uses time.sleep(1) = 1 second) + private static int delay = 1000; private static int retrycount = 0; + private static int maxRetries = 5; private static String inpath = "/in"; private static String outpath = "/out"; private static Map params = new HashMap<>(); - private static int maxtime; + private static Map zmqPorts = new HashMap<>(); + private static ZMQ.Context zmqContext = null; + // simtime as double to preserve fractional values (e.g. "[0.0, ...]") + private static double simtime = 0; + private static double maxtime; - public static void main(String[] args) { + // initialize on class load, same as Python module-level init + static { try { iport = parseFile("concore.iport"); } catch (IOException e) { - e.printStackTrace(); } try { oport = parseFile("concore.oport"); } catch (IOException e) { - e.printStackTrace(); } - try { - String sparams = new String(Files.readAllBytes(Paths.get(inpath + "1/concore.params"))); - if (sparams.charAt(0) == '"') { // windows keeps "" need to remove + String sparams = new String(Files.readAllBytes(Paths.get(inpath + "/1/concore.params")), java.nio.charset.StandardCharsets.UTF_8); + if (sparams.length() > 0 && sparams.charAt(0) == '"') { // windows keeps "" need to remove sparams = sparams.substring(1); sparams = sparams.substring(0, sparams.indexOf('"')); } - if (!sparams.equals("{")) { - System.out.println("converting sparams: " + sparams); - sparams = "{'" + sparams.replaceAll(",", ",'").replaceAll("=", "':").replaceAll(" ", "") + "}"; - System.out.println("converted sparams: " + sparams); - } - try { - params = literalEval(sparams); - } catch (Exception e) { - System.out.println("bad params: " + sparams); - } + params = parseParams(sparams); } catch (IOException e) { params = new HashMap<>(); } - defaultMaxTime(100); + Runtime.getRuntime().addShutdownHook(new Thread(concoredocker::terminateZmq)); } + /** + * Parses a param string into a map, matching concore_base.parse_params. + * Tries dict literal first, then falls back to semicolon-separated key=value pairs. + */ + private static Map parseParams(String sparams) { + Map result = new HashMap<>(); + if (sparams == null || sparams.isEmpty()) return result; + String trimmed = sparams.trim(); + if (trimmed.startsWith("{") && trimmed.endsWith("}")) { + try { + Object val = literalEval(trimmed); + if (val instanceof Map) { + @SuppressWarnings("unchecked") + Map map = (Map) val; + return map; + } + } catch (Exception e) { + } + } + for (String item : trimmed.split(";")) { + if (item.contains("=")) { + String[] parts = item.split("=", 2); // split on first '=' only + String key = parts[0].trim(); + String value = parts[1].trim(); + try { + result.put(key, literalEval(value)); + } catch (Exception e) { + result.put(key, value); + } + } + } + return result; + } + + /** + * Parses a file containing a Python-style dictionary literal. + * Returns empty map if file is empty or malformed (matches Python safe_literal_eval). + */ private static Map parseFile(String filename) throws IOException { - String content = new String(Files.readAllBytes(Paths.get(filename))); - return literalEval(content); + String content = new String(Files.readAllBytes(Paths.get(filename)), java.nio.charset.StandardCharsets.UTF_8); + content = content.trim(); + if (content.isEmpty()) { + return new HashMap<>(); + } + try { + Object result = literalEval(content); + if (result instanceof Map) { + @SuppressWarnings("unchecked") + Map map = (Map) result; + return map; + } + } catch (IllegalArgumentException e) { + System.err.println("Failed to parse file as map: " + filename + " (" + e.getMessage() + ")"); + } + return new HashMap<>(); } - private static void defaultMaxTime(int defaultValue) { + /** + * Sets maxtime from concore.maxtime file, or uses defaultValue if file not found. + * Catches both IOException and RuntimeException to match Python safe_literal_eval. + */ + public static void defaultMaxTime(double defaultValue) { try { - String content = new String(Files.readAllBytes(Paths.get(inpath + "1/concore.maxtime"))); - maxtime = literalEval(content).size(); - } catch (IOException e) { + String content = new String(Files.readAllBytes(Paths.get(inpath + "/1/concore.maxtime"))); + Object parsed = literalEval(content.trim()); + if (parsed instanceof Number) { + maxtime = ((Number) parsed).doubleValue(); + } else { + maxtime = defaultValue; + } + } catch (IOException | RuntimeException e) { maxtime = defaultValue; } } - private static void unchanged() { + // package-level helpers for testing with temp directories + static void setInPath(String path) { inpath = path; } + static void setOutPath(String path) { outpath = path; } + static void setDelay(int ms) { delay = ms; } + static double getSimtime() { return simtime; } + static void resetState() { s = ""; olds = ""; simtime = 0; } + + public static boolean unchanged() { if (olds.equals(s)) { s = ""; - } else { - olds = s; + return true; } + olds = s; + return false; } - private static Object tryParam(String n, Object i) { + public static Object tryParam(String n, Object i) { if (params.containsKey(n)) { return params.get(n); } else { @@ -81,69 +154,674 @@ private static Object tryParam(String n, Object i) { } } - private static Object read(int port, String name, String initstr) { + /** + * Reads data from a port file. Returns the values after extracting simtime. + * Input format: [simtime, val1, val2, ...] + * Returns: list of values after simtime + * Includes max retry limit to avoid infinite blocking (matches Python behavior). + */ + public static ReadResult read(int port, String name, String initstr) { + // Parse default value upfront for consistent return type + List defaultVal = new ArrayList<>(); + try { + List parsed = (List) literalEval(initstr); + if (parsed.size() > 1) { + defaultVal = new ArrayList<>(parsed.subList(1, parsed.size())); + } + } catch (Exception e) { + // initstr not parseable as list; defaultVal stays empty + } + + String filePath = inpath + "/" + port + "/" + name; + try { + Thread.sleep(delay); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + s += initstr; + return new ReadResult(ReadStatus.TIMEOUT, defaultVal); + } + + String ins; try { - String ins = new String(Files.readAllBytes(Paths.get(inpath + port + "/" + name))); - while (ins.length() == 0) { + ins = new String(Files.readAllBytes(Paths.get(filePath))); + } catch (IOException e) { + System.out.println("File " + filePath + " not found, using default value."); + s += initstr; + return new ReadResult(ReadStatus.FILE_NOT_FOUND, defaultVal); + } + + int attempts = 0; + while (ins.length() == 0 && attempts < maxRetries) { + try { Thread.sleep(delay); - ins = new String(Files.readAllBytes(Paths.get(inpath + port + "/" + name))); - retrycount++; + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + s += initstr; + return new ReadResult(ReadStatus.TIMEOUT, defaultVal); + } + try { + ins = new String(Files.readAllBytes(Paths.get(filePath))); + } catch (IOException e) { + System.out.println("Retry " + (attempts + 1) + ": Error reading " + filePath); } - s += ins; - Object[] inval = new Map[] { literalEval(ins) }; - int simtime = Math.max((int) inval[0], 0); // assuming simtime is an integer - return inval[1]; - } catch (IOException | InterruptedException e) { - return initstr; + attempts++; + retrycount++; } + + if (ins.length() == 0) { + System.out.println("Max retries reached for " + filePath + ", using default value."); + return new ReadResult(ReadStatus.RETRIES_EXCEEDED, defaultVal); + } + + s += ins; + try { + List inval = (List) literalEval(ins); + if (!inval.isEmpty()) { + double firstSimtime = ((Number) inval.get(0)).doubleValue(); + simtime = Math.max(simtime, firstSimtime); + return new ReadResult(ReadStatus.SUCCESS, new ArrayList<>(inval.subList(1, inval.size()))); + } + } catch (Exception e) { + System.out.println("Error parsing " + ins + ": " + e.getMessage()); + } + return new ReadResult(ReadStatus.PARSE_ERROR, defaultVal); + } + + /** + * Escapes a Java string so it can be safely used as a single-quoted Python string literal. + * At minimum, escapes backslash, single quote, newline, carriage return, and tab. + */ + private static String escapePythonString(String s) { + StringBuilder sb = new StringBuilder(s.length()); + for (int i = 0; i < s.length(); i++) { + char c = s.charAt(i); + switch (c) { + case '\\': sb.append("\\\\"); break; + case '\'': sb.append("\\'"); break; + case '\n': sb.append("\\n"); break; + case '\r': sb.append("\\r"); break; + case '\t': sb.append("\\t"); break; + default: sb.append(c); break; + } + } + return sb.toString(); + } + + /** + * Converts a Java object to its Python-literal string representation. + * True/False/None instead of true/false/null; strings single-quoted. + */ + private static String toPythonLiteral(Object obj) { + if (obj == null) return "None"; + if (obj instanceof Boolean) return ((Boolean) obj) ? "True" : "False"; + if (obj instanceof String) return "'" + escapePythonString((String) obj) + "'"; + if (obj instanceof Number) return obj.toString(); + if (obj instanceof List) { + List list = (List) obj; + StringBuilder sb = new StringBuilder("["); + for (int i = 0; i < list.size(); i++) { + if (i > 0) sb.append(", "); + sb.append(toPythonLiteral(list.get(i))); + } + sb.append("]"); + return sb.toString(); + } + if (obj instanceof Map) { + Map map = (Map) obj; + StringBuilder sb = new StringBuilder("{"); + boolean first = true; + for (Map.Entry entry : map.entrySet()) { + if (!first) sb.append(", "); + sb.append(toPythonLiteral(entry.getKey())).append(": ").append(toPythonLiteral(entry.getValue())); + first = false; + } + sb.append("}"); + return sb.toString(); + } + return obj.toString(); + } + + /** + * Escapes a Java string so it can be safely embedded in a JSON double-quoted string. + * Escapes backslash, double quote, newline, carriage return, and tab. + */ + private static String escapeJsonString(String s) { + StringBuilder sb = new StringBuilder(s.length()); + for (int i = 0; i < s.length(); i++) { + char c = s.charAt(i); + switch (c) { + case '\\': sb.append("\\\\"); break; + case '"': sb.append("\\\""); break; + case '\n': sb.append("\\n"); break; + case '\r': sb.append("\\r"); break; + case '\t': sb.append("\\t"); break; + default: sb.append(c); break; + } + } + return sb.toString(); + } + + /** + * Converts a Java object to its JSON string representation. + * true/false/null instead of True/False/None; strings double-quoted. + */ + private static String toJsonLiteral(Object obj) { + if (obj == null) return "null"; + if (obj instanceof Boolean) return ((Boolean) obj) ? "true" : "false"; + if (obj instanceof String) return "\"" + escapeJsonString((String) obj) + "\""; + if (obj instanceof Number) return obj.toString(); + if (obj instanceof List) { + List list = (List) obj; + StringBuilder sb = new StringBuilder("["); + for (int i = 0; i < list.size(); i++) { + if (i > 0) sb.append(", "); + sb.append(toJsonLiteral(list.get(i))); + } + sb.append("]"); + return sb.toString(); + } + if (obj instanceof Map) { + Map map = (Map) obj; + StringBuilder sb = new StringBuilder("{"); + boolean first = true; + for (Map.Entry entry : map.entrySet()) { + if (!first) sb.append(", "); + sb.append(toJsonLiteral(entry.getKey())).append(": ").append(toJsonLiteral(entry.getValue())); + first = false; + } + sb.append("}"); + return sb.toString(); + } + return obj.toString(); } - private static void write(int port, String name, Object val, int delta) { + /** + * Writes data to a port file. + * Prepends simtime+delta to the value list, then serializes to Python-literal format. + * Accepts List or String values (matching Python implementation). + */ + public static void write(int port, String name, Object val, int delta) { try { - String path = outpath + port + "/" + name; + String path = outpath + "/" + port + "/" + name; StringBuilder content = new StringBuilder(); if (val instanceof String) { Thread.sleep(2 * delay); - } else if (!(val instanceof Object[])) { - System.out.println("mywrite must have list or str"); - System.exit(1); - } - if (val instanceof Object[]) { + content.append(val); + } else if (val instanceof List) { + List listVal = (List) val; + content.append("["); + content.append(toPythonLiteral(simtime + delta)); + for (int i = 0; i < listVal.size(); i++) { + content.append(", "); + content.append(toPythonLiteral(listVal.get(i))); + } + content.append("]"); + // simtime must not be mutated here. + // Mutation breaks cross-language determinism. + } else if (val instanceof Object[]) { + // Legacy support for Object[] arguments Object[] arrayVal = (Object[]) val; - content.append("[") - .append(maxtime + delta) - .append(",") - .append(arrayVal[0]); - for (int i = 1; i < arrayVal.length; i++) { - content.append(",") - .append(arrayVal[i]); + content.append("["); + content.append(toPythonLiteral(simtime + delta)); + for (Object o : arrayVal) { + content.append(", "); + content.append(toPythonLiteral(o)); } content.append("]"); + // simtime must not be mutated here. + // Mutation breaks cross-language determinism. } else { - content.append(val); + System.out.println("write must have list or str"); + return; } Files.write(Paths.get(path), content.toString().getBytes()); - } catch (IOException | InterruptedException e) { - System.out.println("skipping" + outpath + port + "/" + name); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + System.out.println("skipping " + outpath + "/" + port + "/" + name); + } catch (IOException e) { + System.out.println("skipping " + outpath + "/" + port + "/" + name); } } - private static Object[] initVal(String simtimeVal) { - int simtime = 0; - Object[] val = new Object[] {}; + /** + * Parses an initial value string like "[0.0, 1.0, 2.0]". + * Extracts simtime from position 0 and returns the remaining values as a List. + */ + public static List initVal(String simtimeVal) { + List val = new ArrayList<>(); try { - Object[] arrayVal = new Map[] { literalEval(simtimeVal) }; - simtime = (int) arrayVal[0]; // assuming simtime is an integer - val = new Object[arrayVal.length - 1]; - System.arraycopy(arrayVal, 1, val, 0, val.length); + List inval = (List) literalEval(simtimeVal); + if (!inval.isEmpty()) { + simtime = ((Number) inval.get(0)).doubleValue(); + val = new ArrayList<>(inval.subList(1, inval.size())); + } } catch (Exception e) { - e.printStackTrace(); + System.out.println("Error parsing initVal: " + e.getMessage()); } return val; } - private static Map literalEval(String s) { + private static ZMQ.Context getZmqContext() { + if (zmqContext == null) { + zmqContext = ZMQ.context(1); + } + return zmqContext; + } - return new HashMap<>(); + public static void initZmqPort(String portName, String portType, String address, String socketTypeStr) { + if (zmqPorts.containsKey(portName)) return; + int sockType = zmqSocketTypeFromString(socketTypeStr); + if (sockType == -1) { + System.err.println("initZmqPort: unknown socket type '" + socketTypeStr + "'"); + return; + } + zmqPorts.put(portName, new ZeroMQPort(portType, address, sockType)); + } + + public static void terminateZmq() { + for (Map.Entry entry : zmqPorts.entrySet()) { + entry.getValue().socket.close(); + } + zmqPorts.clear(); + if (zmqContext != null) { + zmqContext.term(); + zmqContext = null; + } + } + + private static int zmqSocketTypeFromString(String s) { + switch (s.toUpperCase()) { + case "REQ": return ZMQ.REQ; + case "REP": return ZMQ.REP; + case "PUB": return ZMQ.PUB; + case "SUB": return ZMQ.SUB; + case "PUSH": return ZMQ.PUSH; + case "PULL": return ZMQ.PULL; + case "PAIR": return ZMQ.PAIR; + default: return -1; + } + } + + /** + * Reads data from a ZMQ port. Same wire format as file-based read: + * expects [simtime, val1, val2, ...], strips simtime, returns the rest. + */ + public static ReadResult read(String portName, String name, String initstr) { + List defaultVal = new ArrayList<>(); + try { + List parsed = (List) literalEval(initstr); + if (parsed.size() > 1) { + defaultVal = new ArrayList<>(parsed.subList(1, parsed.size())); + } + } catch (Exception e) { + } + ZeroMQPort port = zmqPorts.get(portName); + if (port == null) { + System.err.println("read: ZMQ port '" + portName + "' not initialized"); + return new ReadResult(ReadStatus.FILE_NOT_FOUND, defaultVal); + } + String msg = port.recvWithRetry(); + if (msg == null) { + System.err.println("read: ZMQ recv timeout on port '" + portName + "'"); + return new ReadResult(ReadStatus.TIMEOUT, defaultVal); + } + s += msg; + try { + List inval = (List) literalEval(msg); + if (!inval.isEmpty()) { + simtime = Math.max(simtime, ((Number) inval.get(0)).doubleValue()); + return new ReadResult(ReadStatus.SUCCESS, new ArrayList<>(inval.subList(1, inval.size()))); + } + } catch (Exception e) { + System.out.println("Error parsing ZMQ message '" + msg + "': " + e.getMessage()); + } + return new ReadResult(ReadStatus.PARSE_ERROR, defaultVal); + } + + /** + * Writes data to a ZMQ port. Prepends [simtime+delta] to match file-based write behavior. + */ + public static void write(String portName, String name, Object val, int delta) { + ZeroMQPort port = zmqPorts.get(portName); + if (port == null) { + System.err.println("write: ZMQ port '" + portName + "' not initialized"); + return; + } + String payload; + if (val instanceof List) { + List listVal = (List) val; + StringBuilder sb = new StringBuilder("["); + sb.append(toJsonLiteral(simtime + delta)); + for (Object o : listVal) { + sb.append(", "); + sb.append(toJsonLiteral(o)); + } + sb.append("]"); + payload = sb.toString(); + // simtime must not be mutated here + } else if (val instanceof String) { + payload = (String) val; + } else { + System.out.println("write must have list or str"); + return; + } + port.sendWithRetry(payload); + } + + /** + * Parses a Python-literal string into Java objects using a recursive descent parser. + * Supports: dict, list, int, float, string (single/double quoted), bool, None, nested structures. + * This replaces the broken split-based parser that could not handle quoted commas or nesting. + */ + static Object literalEval(String s) { + if (s == null) throw new IllegalArgumentException("Input cannot be null"); + s = s.trim(); + if (s.isEmpty()) throw new IllegalArgumentException("Input cannot be empty"); + Parser parser = new Parser(s); + Object result = parser.parseExpression(); + parser.skipWhitespace(); + if (parser.pos < parser.input.length()) { + throw new IllegalArgumentException("Unexpected trailing content at position " + parser.pos); + } + return result; + } + + public enum ReadStatus { + SUCCESS, FILE_NOT_FOUND, TIMEOUT, PARSE_ERROR, RETRIES_EXCEEDED + } + + public static class ReadResult { + public final ReadStatus status; + public final List data; + ReadResult(ReadStatus status, List data) { + this.status = status; + this.data = data; + } + } + + /** + * ZMQ socket wrapper with bind/connect, timeouts, and retry. + */ + private static class ZeroMQPort { + final ZMQ.Socket socket; + final String address; + + ZeroMQPort(String portType, String address, int socketType) { + this.address = address; + ZMQ.Context ctx = getZmqContext(); + this.socket = ctx.socket(socketType); + this.socket.setReceiveTimeOut(2000); + this.socket.setSendTimeOut(2000); + this.socket.setLinger(0); + if (portType.equals("bind")) { + this.socket.bind(address); + } else { + this.socket.connect(address); + } + } + + String recvWithRetry() { + for (int attempt = 0; attempt < 5; attempt++) { + String msg = socket.recvStr(); + if (msg != null) return msg; + try { Thread.sleep(500); } catch (InterruptedException e) { Thread.currentThread().interrupt(); break; } + } + return null; + } + + void sendWithRetry(String message) { + for (int attempt = 0; attempt < 5; attempt++) { + if (socket.send(message)) return; + try { Thread.sleep(500); } catch (InterruptedException e) { Thread.currentThread().interrupt(); break; } + } + } + } + + /** + * Recursive descent parser for Python literal expressions. + * Handles: dicts, lists, tuples, strings, numbers, booleans, None. + */ + private static class Parser { + final String input; + int pos; + + Parser(String input) { + this.input = input; + this.pos = 0; + } + + void skipWhitespace() { + while (pos < input.length() && Character.isWhitespace(input.charAt(pos))) { + pos++; + } + } + + char peek() { + skipWhitespace(); + if (pos >= input.length()) throw new IllegalArgumentException("Unexpected end of input"); + return input.charAt(pos); + } + + char advance() { + char c = input.charAt(pos); + pos++; + return c; + } + + boolean hasMore() { + skipWhitespace(); + return pos < input.length(); + } + + Object parseExpression() { + skipWhitespace(); + if (pos >= input.length()) throw new IllegalArgumentException("Unexpected end of input"); + char c = input.charAt(pos); + + if (c == '{') return parseDict(); + if (c == '[') return parseList(); + if (c == '(') return parseTuple(); + if (c == '\'' || c == '"') return parseString(); + if (c == '-' || c == '+' || Character.isDigit(c)) return parseNumber(); + return parseKeyword(); + } + + Map parseDict() { + Map map = new HashMap<>(); + pos++; // skip '{' + skipWhitespace(); + if (hasMore() && input.charAt(pos) == '}') { + pos++; + return map; + } + while (true) { + skipWhitespace(); + Object key = parseExpression(); + skipWhitespace(); + if (pos >= input.length() || input.charAt(pos) != ':') { + throw new IllegalArgumentException("Expected ':' in dict at position " + pos); + } + pos++; // skip ':' + skipWhitespace(); + Object value = parseExpression(); + if (!(key instanceof String)) { + throw new IllegalArgumentException( + "Dict keys must be non-null strings, but got: " + + (key == null ? "null" : key.getClass().getSimpleName())); + } + map.put((String) key, value); + skipWhitespace(); + if (pos >= input.length()) { + throw new IllegalArgumentException("Unterminated dict: missing '}'"); + } + if (input.charAt(pos) == '}') { + pos++; + break; + } + if (input.charAt(pos) == ',') { + pos++; + skipWhitespace(); + // trailing comma before close + if (hasMore() && input.charAt(pos) == '}') { + pos++; + break; + } + } else { + throw new IllegalArgumentException("Expected ',' or '}' in dict at position " + pos); + } + } + return map; + } + + List parseList() { + List list = new ArrayList<>(); + pos++; // skip '[' + skipWhitespace(); + if (hasMore() && input.charAt(pos) == ']') { + pos++; + return list; + } + while (true) { + skipWhitespace(); + list.add(parseExpression()); + skipWhitespace(); + if (pos >= input.length()) { + throw new IllegalArgumentException("Unterminated list: missing ']'"); + } + if (input.charAt(pos) == ']') { + pos++; + break; + } + if (input.charAt(pos) == ',') { + pos++; + skipWhitespace(); + // trailing comma before close + if (hasMore() && input.charAt(pos) == ']') { + pos++; + break; + } + } else { + throw new IllegalArgumentException("Expected ',' or ']' in list at position " + pos); + } + } + return list; + } + + List parseTuple() { + List list = new ArrayList<>(); + pos++; // skip '(' + skipWhitespace(); + if (hasMore() && input.charAt(pos) == ')') { + pos++; + return list; + } + while (true) { + skipWhitespace(); + list.add(parseExpression()); + skipWhitespace(); + if (pos >= input.length()) { + throw new IllegalArgumentException("Unterminated tuple: missing ')'"); + } + if (input.charAt(pos) == ')') { + pos++; + break; + } + if (input.charAt(pos) == ',') { + pos++; + skipWhitespace(); + // trailing comma before close + if (hasMore() && input.charAt(pos) == ')') { + pos++; + break; + } + } else { + throw new IllegalArgumentException("Expected ',' or ')' in tuple at position " + pos); + } + } + return list; + } + + String parseString() { + char quote = advance(); // opening quote + StringBuilder sb = new StringBuilder(); + while (pos < input.length()) { + char c = input.charAt(pos); + if (c == '\\' && pos + 1 < input.length()) { + pos++; + char escaped = input.charAt(pos); + switch (escaped) { + case 'n': sb.append('\n'); break; + case 't': sb.append('\t'); break; + case 'r': sb.append('\r'); break; + case '\\': sb.append('\\'); break; + case '\'': sb.append('\''); break; + case '"': sb.append('"'); break; + default: sb.append('\\').append(escaped); break; + } + pos++; + } else if (c == quote) { + pos++; + return sb.toString(); + } else { + sb.append(c); + pos++; + } + } + throw new IllegalArgumentException("Unterminated string starting at position " + (pos - sb.length() - 1)); + } + + Number parseNumber() { + int start = pos; + if (pos < input.length() && (input.charAt(pos) == '-' || input.charAt(pos) == '+')) { + pos++; + } + boolean hasDecimal = false; + boolean hasExponent = false; + while (pos < input.length()) { + char c = input.charAt(pos); + if (Character.isDigit(c)) { + pos++; + } else if (c == '.' && !hasDecimal && !hasExponent) { + hasDecimal = true; + pos++; + } else if ((c == 'e' || c == 'E') && !hasExponent) { + hasExponent = true; + pos++; + if (pos < input.length() && (input.charAt(pos) == '+' || input.charAt(pos) == '-')) { + pos++; + } + } else { + break; + } + } + String numStr = input.substring(start, pos); + try { + if (hasDecimal || hasExponent) { + return Double.parseDouble(numStr); + } else { + try { + return Integer.parseInt(numStr); + } catch (NumberFormatException e) { + return Long.parseLong(numStr); + } + } + } catch (NumberFormatException e) { + throw new IllegalArgumentException("Invalid number: '" + numStr + "' at position " + start); + } + } + + Object parseKeyword() { + int start = pos; + while (pos < input.length() && Character.isLetterOrDigit(input.charAt(pos)) || (pos < input.length() && input.charAt(pos) == '_')) { + pos++; + } + String word = input.substring(start, pos); + switch (word) { + case "True": case "true": return Boolean.TRUE; + case "False": case "false": return Boolean.FALSE; + case "None": case "null": return null; + default: throw new IllegalArgumentException("Unknown keyword: '" + word + "' at position " + start); + } + } } } diff --git a/concoredocker.py b/concoredocker.py index 5c5689d7..51df0eb3 100644 --- a/concoredocker.py +++ b/concoredocker.py @@ -2,17 +2,30 @@ from ast import literal_eval import re import os +import logging +import atexit +import sys +import zmq +import numpy as np +import signal -def safe_literal_eval(filename, defaultValue): - try: - with open(filename, "r") as file: - return literal_eval(file.read()) - except (FileNotFoundError, SyntaxError, ValueError, Exception) as e: - print(f"Error reading {filename}: {e}") - return defaultValue - -iport = safe_literal_eval("concore.iport", {}) -oport = safe_literal_eval("concore.oport", {}) +import concore_base + +logging.basicConfig( + level=logging.INFO, + format='%(levelname)s - %(message)s' +) + +ZeroMQPort = concore_base.ZeroMQPort +convert_numpy_to_python = concore_base.convert_numpy_to_python +safe_literal_eval = concore_base.safe_literal_eval +parse_params = concore_base.parse_params + +# Global variables +zmq_ports = {} +_cleanup_in_progress = False + +last_read_status = "SUCCESS" s = '' olds = '' @@ -22,22 +35,44 @@ def safe_literal_eval(filename, defaultValue): outpath = os.path.abspath("/out") simtime = 0 -#9/21/22 -try: - sparams = open(inpath+"1/concore.params").read() - if sparams[0] == '"': #windows keeps "" need to remove - sparams = sparams[1:] - sparams = sparams[0:sparams.find('"')] - if sparams != '{': - print("converting sparams: "+sparams) - sparams = "{'"+re.sub(',',",'",re.sub('=',"':",re.sub(' ','',sparams)))+"}" - print("converted sparams: " + sparams) +def _port_path(base, port_num): + return os.path.join(base, str(port_num)) + +concore_params_file = os.path.join(_port_path(inpath, 1), "concore.params") +concore_maxtime_file = os.path.join(_port_path(inpath, 1), "concore.maxtime") + +# Load input/output ports if present +iport = safe_literal_eval("concore.iport", {}) +oport = safe_literal_eval("concore.oport", {}) + +_mod = sys.modules[__name__] + +# =================================================================== +# ZeroMQ Communication Wrapper +# =================================================================== +def init_zmq_port(port_name, port_type, address, socket_type_str): + concore_base.init_zmq_port(_mod, port_name, port_type, address, socket_type_str) + +def terminate_zmq(): + """Clean up all ZMQ sockets and contexts before exit.""" + concore_base.terminate_zmq(_mod) + +def signal_handler(sig, frame): + """Handle interrupt signals gracefully.""" + print(f"\nReceived signal {sig}, shutting down gracefully...") try: - params = literal_eval(sparams) - except: - print("bad params: "+sparams) -except: - params = dict() + atexit.unregister(terminate_zmq) + except Exception: + pass + concore_base.terminate_zmq(_mod) + sys.exit(0) + +# Register cleanup handlers (docker containers receive SIGTERM from docker stop) +atexit.register(terminate_zmq) +signal.signal(signal.SIGINT, signal_handler) +signal.signal(signal.SIGTERM, signal_handler) + +params = concore_base.load_params(concore_params_file) #9/30/22 def tryparam(n, i): @@ -46,86 +81,24 @@ def tryparam(n, i): #9/12/21 def default_maxtime(default): global maxtime - maxtime = safe_literal_eval(os.path.join(inpath, "1", "concore.maxtime"), default) + maxtime = safe_literal_eval(concore_maxtime_file, default) default_maxtime(100) def unchanged(): - global olds, s - if olds == s: - s = '' - return True - olds = s - return False - -def read(port, name, initstr): - global s, simtime, retrycount - max_retries=5 - time.sleep(delay) - file_path = os.path.join(inpath+str(port), name) - - try: - with open(file_path, "r") as infile: - ins = infile.read() - except FileNotFoundError: - print(f"File {file_path} not found, using default value.") - ins = initstr - except Exception as e: - print(f"Error reading {file_path}: {e}") - return initstr - - attempts = 0 - while len(ins) == 0 and attempts < max_retries: - time.sleep(delay) - try: - with open(file_path, "r") as infile: - ins = infile.read() - except Exception as e: - print(f"Retry {attempts + 1}: Error reading {file_path} - {e}") - attempts += 1 - retrycount += 1 - - if len(ins) == 0: - print(f"Max retries reached for {file_path}, using default value.") - return initstr - - s += ins - try: - inval = literal_eval(ins) - simtime = max(simtime, inval[0]) - return inval[1:] - except Exception as e: - print(f"Error parsing {ins}: {e}") - return initstr - - -def write(port, name, val, delta=0): - global simtime - file_path = os.path.join(outpath+str(port), name) + return concore_base.unchanged(_mod) - if isinstance(val, str): - time.sleep(2 * delay) - elif not isinstance(val, list): - print("write must have list or str") - return +# =================================================================== +# I/O Handling (File + ZMQ) +# =================================================================== +def read(port_identifier, name, initstr_val): + global last_read_status + result = concore_base.read(_mod, port_identifier, name, initstr_val) + last_read_status = concore_base.last_read_status + return result - try: - with open(file_path, "w") as outfile: - if isinstance(val, list): - outfile.write(str([simtime + delta] + val)) - simtime += delta - else: - outfile.write(val) - except Exception as e: - print(f"Error writing to {file_path}: {e}") +def write(port_identifier, name, val, delta=0): + concore_base.write(_mod, port_identifier, name, val, delta) def initval(simtime_val): - global simtime - try: - val = literal_eval(simtime_val) - simtime = val[0] - return val[1:] - except Exception as e: - print(f"Error parsing simtime_val: {e}") - return [] - + return concore_base.initval(_mod, simtime_val) diff --git a/contribute.py b/contribute.py index 43972f28..59fba5a9 100644 --- a/contribute.py +++ b/contribute.py @@ -1,163 +1,198 @@ -import github -from github import Github -import os,sys,platform,base64,time - -# Intializing the Variables -# Hashed token -BOT_TOKEN = "Z2l0aHViX3BhdF8xMUFYS0pGVFkwU2VhNW9ORjRyN0E5X053WDAwTVBUUU5RVUNTa2lNNlFYZHJET1lZa3B4cTIxS091YVhkeVhUYmRQMzdVUkZaRWpFMjlRRXM5" -BOT_ACCOUNT = 'concore-bot' #bot account name -REPO_NAME = 'concore-studies' #study repo name -UPSTREAM_ACCOUNT = 'ControlCore-Project' #upstream account name -STUDY_NAME = sys.argv[1] -STUDY_NAME_PATH = sys.argv[2] -AUTHOR_NAME = sys.argv[3] -BRANCH_NAME = sys.argv[4] -PR_TITLE = sys.argv[5] -PR_BODY = sys.argv[6] - -# Defining Functions -def checkInputValidity(): - if not AUTHOR_NAME or not STUDY_NAME or not STUDY_NAME_PATH: - print("Please Provide necessary Inputs") - exit(1) - if not os.path.isdir(STUDY_NAME_PATH): - print("Directory doesnot Exists.Invalid Path") - exit(1) - -def printPR(pr): - print(f'Check your example here https://github.com/{UPSTREAM_ACCOUNT}/{REPO_NAME}/pulls/{pr.number}',end="") - -def anyOpenPR(upstream_repo): - try: - prs = upstream_repo.get_pulls(state='open', head=f'{BOT_ACCOUNT}:{BRANCH_NAME}') - return prs[0] if prs.totalCount > 0 else None - except Exception: - print("Unable to fetch PR status. Try again later.") - exit(1) - -def commitAndUpdateRef(repo,tree_content,commit,branch): - try: - new_tree = repo.create_git_tree(tree=tree_content,base_tree=commit.commit.tree) - new_commit = repo.create_git_commit(f"Committing Study Named {STUDY_NAME}",new_tree,[commit.commit]) - if len(repo.compare(base=commit.commit.sha,head=new_commit.sha).files) == 0: - print("Your don't have any new changes.May be your example is already accepted.If this is not the case try with different fields.") - exit(1) - ref = repo.get_git_ref("heads/"+branch.name) - ref.edit(new_commit.sha,True) - except Exception as e: - print("failed to Upload your example.Please try after some time.",end="") - exit(1) - - -def appendBlobInTree(repo,content,file_path,tree_content): - blob = repo.create_git_blob(content,'utf-8') - tree_content.append( github.InputGitTreeElement(path=file_path,mode="100644",type="blob",sha=blob.sha)) - - -def runWorkflow(repo,upstream_repo): - openPR = anyOpenPR(upstream_repo) - if not openPR: - try: - repo.get_workflow("pull_request.yml").create_dispatch( - ref=BRANCH_NAME, - inputs={'title': f"[BOT]: {PR_TITLE}", 'body': PR_BODY, 'upstreamRepo': UPSTREAM_ACCOUNT, 'botRepo': BOT_ACCOUNT, 'repo': REPO_NAME} - ) - printPRStatus(upstream_repo) - except Exception as e: - print(f"Error triggering workflow. Try again later.\n ERROR: {e}") - exit(1) - else: - print(f"Successfully uploaded. Waiting for approval: https://github.com/{UPSTREAM_ACCOUNT}/{REPO_NAME}/pull/{openPR.number}") - -def printPRStatus(upstream_repo): - attempts = 5 - delay = 2 - for i in range(attempts): - print(f"Attempt: {i}") - try: - latest_pr = upstream_repo.get_pulls(state='open', sort='created', direction='desc')[0] - print(f"Check your example here: https://github.com/{UPSTREAM_ACCOUNT}/{REPO_NAME}/pull/{latest_pr.number}") - return - except Exception: - time.sleep(delay) - delay *= 2 - print("Uploaded successfully, but unable to fetch status.") - - -def isImageFile(filename): - image_extensions = ['.jpeg', '.jpg', '.png','.gif'] - return any(filename.endswith(ext) for ext in image_extensions) - -def remove_prefix(text, prefix): - if text.startswith(prefix): - return text[len(prefix):] - return text - - -# Decode Github Token -def decode_token(encoded_token): - decoded_bytes = encoded_token.encode("ascii") - convertedbytes = base64.b64decode(decoded_bytes) - decoded_token = convertedbytes.decode("ascii") - return decoded_token - - -# check if directory path is Valid -checkInputValidity() - - -# Authenticating Github with Access token -try: - BRANCH_NAME = AUTHOR_NAME.replace(" ", "_") + "_" + STUDY_NAME if BRANCH_NAME == "#" else BRANCH_NAME.replace(" ", "_") - PR_TITLE = f"Contributing Study {STUDY_NAME} by {AUTHOR_NAME}" if PR_TITLE == "#" else PR_TITLE - PR_BODY = f"Study Name: {STUDY_NAME}\nAuthor Name: {AUTHOR_NAME}" if PR_BODY == "#" else PR_BODY - DIR_PATH = STUDY_NAME - DIR_PATH = DIR_PATH.replace(" ","_") - g = Github(decode_token(BOT_TOKEN)) - repo = g.get_user(BOT_ACCOUNT).get_repo(REPO_NAME) - upstream_repo = g.get_repo(f'{UPSTREAM_ACCOUNT}/{REPO_NAME}') #controlcore-Project/concore-studies - base_ref = upstream_repo.get_branch(repo.default_branch) - - try: - repo.get_branch(BRANCH_NAME) - is_present = True - except github.GithubException: - print(f"No Branch is available with the name {BRANCH_NAME}") - is_present = False -except Exception as e: - print("Authentication failed", end="") - exit(1) - - -try: - if not is_present: - repo.create_git_ref(f"refs/heads/{BRANCH_NAME}", base_ref.commit.sha) - branch = repo.get_branch(BRANCH_NAME) -except Exception: - print("Unable to create study. Try again later.") - exit(1) - - -tree_content = [] - -try: - for root, dirs, files in os.walk(STUDY_NAME_PATH): - files = [f for f in files if not f[0] == '.'] - for filename in files: - path = f"{root}/{filename}" - if isImageFile(filename): - with open(file=path, mode='rb') as file: - image = file.read() - content = base64.b64encode(image).decode('utf-8') - else: - with open(file=path, mode='r') as file: - content = file.read() - file_path = f'{DIR_PATH+remove_prefix(path,STUDY_NAME_PATH)}' - if(platform.uname()[0]=='Windows'): file_path=file_path.replace("\\","/") - appendBlobInTree(repo,content,file_path,tree_content) - commitAndUpdateRef(repo,tree_content,base_ref.commit,branch) - runWorkflow(repo,upstream_repo) -except Exception as e: - print(e) - print("Some error Occured.Please try again after some time.",end="") +import github +from github import Github +import os,sys,platform,base64,time,re + +# Initializing the Variables +BOT_TOKEN = os.environ.get('CONCORE_BOT_TOKEN', '') + +# Fail fast if token is missing +if not BOT_TOKEN: + print("Error: CONCORE_BOT_TOKEN environment variable is not set.") + sys.exit(1) + +# Token format validation +token_pattern = r"^((ghp_|github_pat_|ghs_)[A-Za-z0-9_]{20,}|[0-9a-fA-F]{40})$" +if not re.match(token_pattern, BOT_TOKEN): + print("Error: Invalid GitHub token format.") + sys.exit(1) +BOT_ACCOUNT = 'concore-bot' #bot account name +REPO_NAME = 'concore-studies' #study repo name +UPSTREAM_ACCOUNT = 'ControlCore-Project' #upstream account name +STUDY_NAME = sys.argv[1] +STUDY_NAME_PATH = sys.argv[2] +AUTHOR_NAME = sys.argv[3] +BRANCH_NAME = sys.argv[4] +PR_TITLE = sys.argv[5] +PR_BODY = sys.argv[6] + +# Defining Functions +def checkInputValidity(): + if not AUTHOR_NAME or not STUDY_NAME or not STUDY_NAME_PATH: + print("Please Provide necessary Inputs") + exit(1) + if not os.path.isdir(STUDY_NAME_PATH): + print("Directory does not Exists.Invalid Path") + exit(1) + +# Retry + backoff wrapper for PyGithub operations +def with_retry(operation, retries=3): + """Retry wrapper for PyGithub operations with exponential backoff.""" + for attempt in range(retries): + try: + return operation() + except github.GithubException as e: + if (e.status == 429 or e.status >= 500) and attempt < retries - 1: + wait_time = 2 ** attempt + time.sleep(wait_time) + continue + raise + print("Error: GitHub API request failed after retries.") + sys.exit(1) + +# Correct PR URL (singular 'pull' not 'pulls') +def printPR(pr): + print(f'Check your example here https://github.com/{UPSTREAM_ACCOUNT}/{REPO_NAME}/pull/{pr.number}',end="") + +def anyOpenPR(upstream_repo): + try: + prs = upstream_repo.get_pulls(state='open', head=f'{BOT_ACCOUNT}:{BRANCH_NAME}') + return prs[0] if prs.totalCount > 0 else None + except github.GithubException as e: + if e.status == 429 or e.status >= 500: + print("GitHub API rate limit or server error while fetching PR status.") + else: + print("Unable to fetch PR status. Try again later.") + exit(1) + except Exception: + print("Unable to fetch PR status. Try again later.") + exit(1) + +def commitAndUpdateRef(repo,tree_content,commit,branch): + try: + new_tree = repo.create_git_tree(tree=tree_content,base_tree=commit.commit.tree) + new_commit = repo.create_git_commit(f"Committing Study Named {STUDY_NAME}",new_tree,[commit.commit]) + if len(repo.compare(base=commit.commit.sha,head=new_commit.sha).files) == 0: + print("Your don't have any new changes.May be your example is already accepted.If this is not the case try with different fields.") + exit(1) + ref = repo.get_git_ref("heads/"+branch.name) + ref.edit(new_commit.sha,True) + except github.GithubException as e: + print(f"GitHub API error: {e.status}") + exit(1) + except Exception: + print("Failed to upload your example. Please try after some time.",end="") + exit(1) + + +def appendBlobInTree(repo,content,file_path,tree_content): + blob = repo.create_git_blob(content,'utf-8') + tree_content.append( github.InputGitTreeElement(path=file_path,mode="100644",type="blob",sha=blob.sha)) + + +def runWorkflow(repo,upstream_repo): + openPR = anyOpenPR(upstream_repo) + if not openPR: + try: + repo.get_workflow("pull_request.yml").create_dispatch( + ref=BRANCH_NAME, + inputs={'title': f"[BOT]: {PR_TITLE}", 'body': PR_BODY, 'upstreamRepo': UPSTREAM_ACCOUNT, 'botRepo': BOT_ACCOUNT, 'repo': REPO_NAME} + ) + printPRStatus(upstream_repo) + except github.GithubException as e: + print(f"GitHub API error while triggering workflow: {e.status}") + exit(1) + except Exception: + print("Error triggering workflow. Try again later.") + exit(1) + else: + print(f"Successfully uploaded. Waiting for approval: https://github.com/{UPSTREAM_ACCOUNT}/{REPO_NAME}/pull/{openPR.number}") + +def printPRStatus(upstream_repo): + attempts = 5 + delay = 2 + for i in range(attempts): + print(f"Attempt: {i}") + try: + latest_pr = upstream_repo.get_pulls(state='open', sort='created', direction='desc')[0] + print(f"Check your example here: https://github.com/{UPSTREAM_ACCOUNT}/{REPO_NAME}/pull/{latest_pr.number}") + return + except Exception: + time.sleep(delay) + delay *= 2 + print("Uploaded successfully, but unable to fetch status.") + + +def isImageFile(filename): + image_extensions = ['.jpeg', '.jpg', '.png','.gif'] + return any(filename.endswith(ext) for ext in image_extensions) + +def remove_prefix(text, prefix): + if text.startswith(prefix): + return text[len(prefix):] + return text + + +# check if directory path is Valid +checkInputValidity() + + +# Authenticating Github with Access token +try: + BRANCH_NAME = AUTHOR_NAME.replace(" ", "_") + "_" + STUDY_NAME if BRANCH_NAME == "#" else BRANCH_NAME.replace(" ", "_") + PR_TITLE = f"Contributing Study {STUDY_NAME} by {AUTHOR_NAME}" if PR_TITLE == "#" else PR_TITLE + PR_BODY = f"Study Name: {STUDY_NAME}\nAuthor Name: {AUTHOR_NAME}" if PR_BODY == "#" else PR_BODY + DIR_PATH = STUDY_NAME + DIR_PATH = DIR_PATH.replace(" ","_") + g = Github(BOT_TOKEN) + repo = g.get_user(BOT_ACCOUNT).get_repo(REPO_NAME) + upstream_repo = g.get_repo(f'{UPSTREAM_ACCOUNT}/{REPO_NAME}') #controlcore-Project/concore-studies + base_ref = upstream_repo.get_branch(repo.default_branch) + + try: + repo.get_branch(BRANCH_NAME) + is_present = True + except github.GithubException: + print(f"No Branch is available with the name {BRANCH_NAME}") + is_present = False +except github.GithubException as e: + print(f"GitHub API error during authentication: {e.status}") + exit(1) +except Exception: + print("Authentication failed", end="") + exit(1) + + +try: + if not is_present: + repo.create_git_ref(f"refs/heads/{BRANCH_NAME}", base_ref.commit.sha) + branch = repo.get_branch(BRANCH_NAME) +except Exception: + print("Unable to create study. Try again later.") + exit(1) + + +tree_content = [] + +try: + for root, dirs, files in os.walk(STUDY_NAME_PATH): + files = [f for f in files if not f[0] == '.'] + for filename in files: + path = f"{root}/{filename}" + if isImageFile(filename): + with open(file=path, mode='rb') as file: + image = file.read() + content = base64.b64encode(image).decode('utf-8') + else: + with open(file=path, mode='r') as file: + content = file.read() + file_path = f'{DIR_PATH+remove_prefix(path,STUDY_NAME_PATH)}' + if(platform.uname()[0]=='Windows'): file_path=file_path.replace("\\","/") + appendBlobInTree(repo,content,file_path,tree_content) + commitAndUpdateRef(repo,tree_content,base_ref.commit,branch) + runWorkflow(repo,upstream_repo) +except github.GithubException as e: + print(f"GitHub API error: {e.status}") + exit(1) +except Exception: + print("Some error occurred. Please try again after some time.",end="") exit(1) \ No newline at end of file diff --git a/copy_with_port_portname.py b/copy_with_port_portname.py index 7307289f..1398092b 100644 --- a/copy_with_port_portname.py +++ b/copy_with_port_portname.py @@ -5,27 +5,40 @@ import logging import json -logging.basicConfig( - level=logging.INFO, - format='%(message)s', - datefmt='%Y-%m-%d %H:%M:%S' -) - -def run_specialization_script(template_script_path, output_dir, edge_params_list, python_exe, copy_script_path): +def _normalize_output_relpath(template_script_path, output_relpath=None): + if output_relpath: + relpath = output_relpath.replace("\\", "/").lstrip("/") + else: + relpath = os.path.basename(template_script_path) + if not relpath: + raise ValueError("Output relative path cannot be empty.") + return relpath + + +def _join_output_path(output_dir, output_relpath): + return os.path.join(output_dir, *output_relpath.split("/")) + + +def run_specialization_script( + template_script_path, + output_dir, + edge_params_list, + python_exe, + copy_script_path, + output_relpath=None +): """ Calls the copy script to generate a specialized version of a node's script. Returns the basename of the generated script on success, None on failure. """ - # The new copy script generates a standardized filename, e.g., "original.py" base_template_name = os.path.basename(template_script_path) - template_root, template_ext = os.path.splitext(base_template_name) - output_filename = f"{template_root}{template_ext}" - expected_output_path = os.path.join(output_dir, output_filename) + output_relpath = _normalize_output_relpath(template_script_path, output_relpath) + expected_output_path = _join_output_path(output_dir, output_relpath) # If the specialized file already exists, we don't need to regenerate it. if os.path.exists(expected_output_path): logging.info(f"Specialized script '{expected_output_path}' already exists. Using existing.") - return output_filename + return output_relpath # Convert the list of parameters to a JSON string for command line argument edge_params_json_str = json.dumps(edge_params_list) @@ -37,13 +50,15 @@ def run_specialization_script(template_script_path, output_dir, edge_params_list output_dir, edge_params_json_str # Pass the JSON string as the last argument ] + if output_relpath: + cmd.append(output_relpath) logging.info(f"Running specialization for '{base_template_name}': {' '.join(cmd)}") try: result = subprocess.run(cmd, capture_output=True, text=True, check=True, encoding='utf-8') - logging.info(f"Successfully generated specialized script '{output_filename}'.") + logging.info(f"Successfully generated specialized script '{output_relpath}'.") if result.stdout: logging.debug(f"copy_with_port_portname.py stdout:\n{result.stdout.strip()}") if result.stderr: logging.warning(f"copy_with_port_portname.py stderr:\n{result.stderr.strip()}") - return output_filename + return output_relpath except subprocess.CalledProcessError as e: logging.error(f"Error calling specialization script for '{template_script_path}':") logging.error(f"Command: {' '.join(e.cmd)}") @@ -56,7 +71,7 @@ def run_specialization_script(template_script_path, output_dir, edge_params_list return None -def create_modified_script(template_script_path, output_dir, edge_params_json_str): +def create_modified_script(template_script_path, output_dir, edge_params_json_str, output_relpath=None): """ Creates a modified Python script by injecting ZMQ port and port name definitions from a JSON object. @@ -127,17 +142,16 @@ def create_modified_script(template_script_path, output_dir, edge_params_json_st modified_lines = lines[:insert_index] + definitions + lines[insert_index:] # --- Determine and create output file --- - base_template_name = os.path.basename(template_script_path) - template_root, template_ext = os.path.splitext(base_template_name) - - # Standardized output filename for a node with one or more specializations - output_filename = f"{template_root}{template_ext}" - output_script_path = os.path.join(output_dir, output_filename) + output_relpath = _normalize_output_relpath(template_script_path, output_relpath) + output_script_path = _join_output_path(output_dir, output_relpath) try: if not os.path.exists(output_dir): os.makedirs(output_dir) print(f"Created output directory: {output_dir}") + output_parent = os.path.dirname(output_script_path) + if output_parent and not os.path.exists(output_parent): + os.makedirs(output_parent, exist_ok=True) with open(output_script_path, 'w') as f: f.writelines(modified_lines) @@ -149,8 +163,14 @@ def create_modified_script(template_script_path, output_dir, edge_params_json_st sys.exit(1) if __name__ == "__main__": - if len(sys.argv) != 4: - print("\nUsage: python3 copy_with_port_portname.py ''\n") + logging.basicConfig( + level=logging.INFO, + format='%(message)s', + datefmt='%Y-%m-%d %H:%M:%S' + ) + + if len(sys.argv) not in [4, 5]: + print("\nUsage: python3 copy_with_port_portname.py '' [OUTPUT_RELATIVE_PATH]\n") print("Example JSON: '[{\"port\": \"2355\", \"port_name\": \"FUNBODY_REP_1\", \"source_node_label\": \"nodeA\", \"target_node_label\": \"nodeB\"}]'") print("Note: The JSON string must be enclosed in single quotes in shell.\n") sys.exit(1) @@ -158,5 +178,6 @@ def create_modified_script(template_script_path, output_dir, edge_params_json_st template_script_path_arg = sys.argv[1] output_directory_arg = sys.argv[2] json_params_arg = sys.argv[3] + output_relpath_arg = sys.argv[4] if len(sys.argv) == 5 else None - create_modified_script(template_script_path_arg, output_directory_arg, json_params_arg) \ No newline at end of file + create_modified_script(template_script_path_arg, output_directory_arg, json_params_arg, output_relpath_arg) diff --git a/demo/controller2knob.py b/demo/controller2knob.py index d6427563..284e22e1 100644 --- a/demo/controller2knob.py +++ b/demo/controller2knob.py @@ -1,6 +1,7 @@ import numpy as np import concore +ysp = 3.0 def controller(ym): if ym[0] < ysp: diff --git a/demo/controller_RT.py b/demo/controller_RT.py new file mode 100644 index 00000000..5516fbcc --- /dev/null +++ b/demo/controller_RT.py @@ -0,0 +1,37 @@ +import numpy as np +from ast import literal_eval +import concore + +try: + ysp = literal_eval(open("ysp.txt").read()) +except: + ysp = 3.0 + +def controller(ym): + if ym[0] < ysp: + return 1.01 * ym + else: + return 0.9 * ym + +#//main// +concore.default_maxtime(150) ##maps to-- for i in range(0,150): +concore.delay = 0.02 + +#//initial values-- transforms to string including the simtime as the 0th entry in the list// +# u = np.array([[0.0]]) +# ym = np.array([[0.0]]) +init_simtime_u = "[0.0, 0.0]" +init_simtime_ym = "[0.0, 0.0]" + +u = np.array([concore.initval(init_simtime_u)]).T +while(concore.simtime 1.1*timeout_max: #timeout_count>100: print("timeout or bad POST request "+str(r.status_code)) - quit() + sys.exit(1) if len(r.text)!=0: try: t=literal_eval(r.text)[0] diff --git a/demo/plotym_RT.py b/demo/plotym_RT.py new file mode 100644 index 00000000..fdeac954 --- /dev/null +++ b/demo/plotym_RT.py @@ -0,0 +1,73 @@ +import concore +import logging +import numpy as np +import matplotlib.pyplot as plt +import time +logging.info("plot ym") + +concore.delay = 0.02 +concore.default_maxtime(150) +init_simtime_u = "[0.0, 0.0]" +init_simtime_ym = "[0.0, 0.0]" +ymt = [] +ym = concore.initval(init_simtime_ym) + +# 1. Fetch 'realtime' parameter passed from terminal (defaults to False) +realtime = concore.tryparam('realtime', False) + +# DEBUG: Check if the parameter was successfully caught +logging.info(f"--- Realtime mode is set to: {realtime} ---") + +# 2. Set up interactive plot before the loop if realtime is True +if realtime: + plt.ion() # Turn on interactive mode + fig, ax1 = plt.subplots(1, 1) + line1, = ax1.plot([], []) + + ax1.set_ylabel('ym') + ax1.legend(['ym'], loc=0) + ax1.set_xlabel('Cycles') + plt.show(block=False) # Ensure it does not block the script + +while(concore.simtime 1.1*timeout_max: #timeout_count>200: print("timeout or bad POST request "+str(r.status_code)) - quit() + sys.exit(1) if len(r.text)!=0: try: t=literal_eval(r.text)[0] diff --git a/demo/sample_RT.graphml b/demo/sample_RT.graphml new file mode 100644 index 00000000..a2b36a84 --- /dev/null +++ b/demo/sample_RT.graphml @@ -0,0 +1,208 @@ + + + + + + + + + + + + pm:pm_RT.py + + + + + + + + + + + controller:controller_RT.py + + + + + + + + + + + plotym:plotym_RT.py + + + + + + + + + + edge1 + + + + + + + + + + + + edge2 + + + + + + + + + + + + edge3 + + + + + + + + 1771794664219 + + DEL_NODE + WyI5N2M2OTRlZS0zNTAzLTRlZDctOWZhZS0xOGQ3YjgwNzA5ZmYiXQ== + + + ADD_NODE + WyJwbTpwbV9SVC5weSIseyJ3aWR0aCI6MTE4LCJoZWlnaHQiOjYwLCJzaGFwZSI6InJlY3RhbmdsZSIsIm9wYWNpdHkiOjEsImJhY2tncm91bmRDb2xvciI6IiNmZmNjMDAiLCJib3JkZXJDb2xvciI6IiMwMDAiLCJib3JkZXJXaWR0aCI6MX0sIm9yZGluIix7IngiOjExMCwieSI6MTEwfSx7fSwiOTdjNjk0ZWUtMzUwMy00ZWQ3LTlmYWUtMThkN2I4MDcwOWZmIl0= + + a95ae0297e5f0e63bfa65a29c8d14d92 + + + 1771794687823 + + DEL_NODE + WyIxYmUwMDJkYi0wZmMyLTRjM2ItOGNhMS01NGU0YmJiM2ZiY2IiXQ== + + + ADD_NODE + WyJjb250cm9sbGVyOmNvbnRyb2xsZXJfUlQucHkiLHsid2lkdGgiOjIwNiwiaGVpZ2h0Ijo2MCwic2hhcGUiOiJyZWN0YW5nbGUiLCJvcGFjaXR5IjoxLCJiYWNrZ3JvdW5kQ29sb3IiOiIjZmZjYzAwIiwiYm9yZGVyQ29sb3IiOiIjMDAwIiwiYm9yZGVyV2lkdGgiOjF9LCJvcmRpbiIseyJ4Ijo2NzAsInkiOjkwfSx7fSwiMWJlMDAyZGItMGZjMi00YzNiLThjYTEtNTRlNGJiYjNmYmNiIl0= + + 81165b0ec333e71a4ee794f887493323 + + + 1771794690698 + + SET_POS + WyIxYmUwMDJkYi0wZmMyLTRjM2ItOGNhMS01NGU0YmJiM2ZiY2IiLHsieCI6MTEwLCJ5IjoxMTB9LHsieCI6NDkwLCJ5IjoxMTB9XQ== + + + SET_POS + WyIxYmUwMDJkYi0wZmMyLTRjM2ItOGNhMS01NGU0YmJiM2ZiY2IiLHsieCI6NDkwLCJ5IjoxMTB9LHsieCI6MTEwLCJ5IjoxMTB9XQ== + + 2137de444092ba020db5d395a39a3b79 + + + 1771794720693 + + DEL_NODE + WyIyN2M1ZDUyNC1iZGM5LTQ4Y2QtOWFhNi00Y2IxODk0Y2I1ZDciXQ== + + + ADD_NODE + WyJwbG90eW06cGxvdHltX1JULnB5Iix7IndpZHRoIjoxOTYsImhlaWdodCI6NjAsInNoYXBlIjoicmVjdGFuZ2xlIiwib3BhY2l0eSI6MSwiYmFja2dyb3VuZENvbG9yIjoiI2ZmY2MwMCIsImJvcmRlckNvbG9yIjoiIzAwMCIsImJvcmRlcldpZHRoIjoxfSwib3JkaW4iLHsieCI6MzcwLCJ5Ijo0MzB9LHt9LCIyN2M1ZDUyNC1iZGM5LTQ4Y2QtOWFhNi00Y2IxODk0Y2I1ZDciXQ== + + 64c4ee3fac24cbd01e9cf06ea57dd8c2 + + + 1771794723359 + + SET_POS + WyIyN2M1ZDUyNC1iZGM5LTQ4Y2QtOWFhNi00Y2IxODk0Y2I1ZDciLHsieCI6MTEwLCJ5IjoxMTB9LHsieCI6MzcwLCJ5IjozNTB9XQ== + + + SET_POS + WyIyN2M1ZDUyNC1iZGM5LTQ4Y2QtOWFhNi00Y2IxODk0Y2I1ZDciLHsieCI6MzcwLCJ5IjozNTB9LHsieCI6MTEwLCJ5IjoxMTB9XQ== + + b92c36146ebe446c0907df865f83794d + + + 1771794740309 + + DEL_EDGE + WyJkNGRkZDA2Mi00NTUyLTRkNTktOWIzYy0wYzA0ZGU2MGQxOTciXQ== + + + ADD_EDGE + W3sic291cmNlSUQiOiIxYmUwMDJkYi0wZmMyLTRjM2ItOGNhMS01NGU0YmJiM2ZiY2IiLCJ0YXJnZXRJRCI6Ijk3YzY5NGVlLTM1MDMtNGVkNy05ZmFlLTE4ZDdiODA3MDlmZiIsImxhYmVsIjoiZWRnZTEiLCJzdHlsZSI6eyJ0aGlja25lc3MiOjIsImJhY2tncm91bmRDb2xvciI6IiM1NTUiLCJzaGFwZSI6InNvbGlkIn0sImlkIjoiZDRkZGQwNjItNDU1Mi00ZDU5LTliM2MtMGMwNGRlNjBkMTk3In1d + + 8b3c683a137808fb40dc2921313f8115 + + + 1771794751288 + + SET_POS + WyIxYmUwMDJkYi0wZmMyLTRjM2ItOGNhMS01NGU0YmJiM2ZiY2IiLHsieCI6NDkwLCJ5IjoxMTB9LHsieCI6NjMwLCJ5Ijo5MH1d + + + SET_POS + WyIxYmUwMDJkYi0wZmMyLTRjM2ItOGNhMS01NGU0YmJiM2ZiY2IiLHsieCI6NjMwLCJ5Ijo5MH0seyJ4Ijo0OTAsInkiOjExMH1d + + 0ce00744b96d9ae529645d9297b56821 + + + 1771794765726 + + DEL_EDGE + WyIwOGU0NTRhNS1mYjAwLTQ2MDEtOWMxNC00NDA4NGIyMzFiNTciXQ== + + + ADD_EDGE + W3sic291cmNlSUQiOiI5N2M2OTRlZS0zNTAzLTRlZDctOWZhZS0xOGQ3YjgwNzA5ZmYiLCJ0YXJnZXRJRCI6IjI3YzVkNTI0LWJkYzktNDhjZC05YWE2LTRjYjE4OTRjYjVkNyIsImxhYmVsIjoiZWRnZTIiLCJzdHlsZSI6eyJ0aGlja25lc3MiOjIsImJhY2tncm91bmRDb2xvciI6IiM1NTUiLCJzaGFwZSI6InNvbGlkIn0sImlkIjoiMDhlNDU0YTUtZmIwMC00NjAxLTljMTQtNDQwODRiMjMxYjU3In1d + + 901af9eaa14d4bf0fc1ba715fc5ac3d2 + + + 1771794782565 + + DEL_EDGE + WyI3ZmM3YjQxNy01NDBkLTRhMDgtYTc4Ni0wNDkwYjgyMDg4YWUiXQ== + + + ADD_EDGE + W3sic291cmNlSUQiOiIyN2M1ZDUyNC1iZGM5LTQ4Y2QtOWFhNi00Y2IxODk0Y2I1ZDciLCJ0YXJnZXRJRCI6IjFiZTAwMmRiLTBmYzItNGMzYi04Y2ExLTU0ZTRiYmIzZmJjYiIsImxhYmVsIjoiZWRnZTMiLCJzdHlsZSI6eyJ0aGlja25lc3MiOjIsImJhY2tncm91bmRDb2xvciI6IiM1NTUiLCJzaGFwZSI6InNvbGlkIn0sImlkIjoiN2ZjN2I0MTctNTQwZC00YTA4LWE3ODYtMDQ5MGI4MjA4OGFlIn1d + + e157ed01dca772c8461787c1053794ce + + + 1771794788695 + + SET_POS + WyIyN2M1ZDUyNC1iZGM5LTQ4Y2QtOWFhNi00Y2IxODk0Y2I1ZDciLHsieCI6MzcwLCJ5IjozNTB9LHsieCI6MzcwLCJ5Ijo0MzB9XQ== + + + SET_POS + WyIyN2M1ZDUyNC1iZGM5LTQ4Y2QtOWFhNi00Y2IxODk0Y2I1ZDciLHsieCI6MzcwLCJ5Ijo0MzB9LHsieCI6MzcwLCJ5IjozNTB9XQ== + + aebd758397fec30aac0b59fe643a0da2 + + + 1771794795323 + + SET_POS + WyIxYmUwMDJkYi0wZmMyLTRjM2ItOGNhMS01NGU0YmJiM2ZiY2IiLHsieCI6NjMwLCJ5Ijo5MH0seyJ4Ijo2NzAsInkiOjkwfV0= + + + SET_POS + WyIxYmUwMDJkYi0wZmMyLTRjM2ItOGNhMS01NGU0YmJiM2ZiY2IiLHsieCI6NjcwLCJ5Ijo5MH0seyJ4Ijo2MzAsInkiOjkwfV0= + + 20778f5ad2b1c7bd577f09da007bc0e0 + + + \ No newline at end of file diff --git a/destroy.bat b/destroy.bat index 561ec023..68b31a18 100644 --- a/destroy.bat +++ b/destroy.bat @@ -1,3 +1,25 @@ -@echo off -if exist "%1\stop.bat" (rmdir /s/q %1) else (echo "%1 is not a concore study") - +@echo off + +if not exist "%1" ( + echo "%1 does not exist" + exit /b 1 +) + +if not exist "%1\stop.bat" ( + echo "%1 is not a concore study" + exit /b 1 +) + +echo Stopping study... +call "%1\stop.bat" + +if exist "%1\clear.bat" ( + echo Clearing study... + call "%1\clear.bat" +) + +echo Removing study directory... +rmdir /s /q "%1" + +echo Done. + diff --git a/docs/CONFIG.md b/docs/CONFIG.md new file mode 100644 index 00000000..f72e6289 --- /dev/null +++ b/docs/CONFIG.md @@ -0,0 +1,226 @@ +# concore Configuration Reference + +This document describes the configuration files used by `mkconcore.py` to locate compilers, runtimes, and Docker settings. All config files are read from `CONCOREPATH` — the resolved concore install directory (usually the directory containing `concore.py`). + +## How CONCOREPATH Is Resolved + +`mkconcore.py` resolves `CONCOREPATH` in the following order: + +1. The directory containing `mkconcore.py` itself (if `concore.py` exists there) +2. The current working directory (if `concore.py` exists there) +3. Falls back to the `mkconcore.py` directory + +--- + +## Configuration Files + +### `concore.tools` + +**Purpose:** Override compiler and runtime paths for all supported languages. + +**Format:** Key-value pairs, one per line. Lines starting with `#` and blank lines are ignored. + +``` +KEY=value +``` + +**Supported Keys:** + +| Key | Platform | Default | Description | +|-----|----------|---------|-------------| +| `CPPWIN` | Windows | `g++` | C++ compiler | +| `CPPEXE` | POSIX | `g++` | C++ compiler | +| `VWIN` | Windows | `iverilog` | Verilog compiler | +| `VEXE` | POSIX | `iverilog` | Verilog compiler | +| `PYTHONWIN` | Windows | `python` | Python interpreter | +| `PYTHONEXE` | POSIX | `python3` | Python interpreter | +| `MATLABWIN` | Windows | `matlab` | MATLAB executable | +| `MATLABEXE` | POSIX | `matlab` | MATLAB executable | +| `OCTAVEWIN` | Windows | `octave` | GNU Octave executable | +| `OCTAVEEXE` | POSIX | `octave` | GNU Octave executable | +| `JAVACWIN` | Windows | `javac` | Java compiler | +| `JAVACEXE` | POSIX | `javac` | Java compiler | +| `JAVAWIN` | Windows | `java` | Java runtime | +| `JAVAEXE` | POSIX | `java` | Java runtime | + +**Example (`concore.tools`):** + +``` +CPPEXE=/usr/bin/g++ +PYTHONEXE=/opt/python3.11/bin/python3 +VEXE=/usr/local/bin/iverilog +OCTAVEEXE=/usr/bin/octave +JAVACEXE=/usr/lib/jvm/java-17/bin/javac +JAVAEXE=/usr/lib/jvm/java-17/bin/java +``` + +**Windows Example:** + +``` +CPPWIN=C:\MinGW\bin\g++.exe +PYTHONWIN=C:\Python313\python.exe +JAVACWIN=C:\Program Files\Java\jdk-17\bin\javac.exe +JAVAWIN=C:\Program Files\Java\jdk-17\bin\java.exe +``` + +--- + +### `concore.octave` + +**Purpose:** When this file is present, `.m` files are treated as GNU Octave instead of MATLAB. The file content is ignored — only its existence matters. + +**Format:** Empty file (presence-based flag). + +**How to enable:** + +```bash +touch $CONCOREPATH/concore.octave +``` + +**How to disable:** Delete the file. + +--- + +### `concore.mcr` + +**Purpose:** Path to the local MATLAB Compiler Runtime (MCR) installation. Used when running compiled MATLAB executables instead of MATLAB directly. + +**Format:** Single line containing the absolute path to the MCR installation. + +**Default:** `~/MATLAB/R2021a` + +**Example (`concore.mcr`):** + +``` +/opt/matlab/mcr/v913 +``` + +**Windows Example:** + +``` +C:\Program Files\MATLAB\R2021a\runtime\win64 +``` + +--- + +### `concore.sudo` + +**Purpose:** Override the Docker executable command. Originally intended to control whether Docker runs with `sudo`, but can specify any container runtime (Docker, Podman, etc.). + +**Format:** Single line containing the Docker executable command. + +**Default:** `docker` (from `DOCKEREXE` environment variable, or the literal string `docker`) + +**Example (`concore.sudo`):** + +``` +sudo docker +``` + +**To use Podman instead of Docker:** + +``` +podman +``` + +--- + +### `concore.repo` + +**Purpose:** Override the Docker Hub repository prefix used when pulling pre-built images. + +**Format:** Single line containing the Docker Hub username or organization. + +**Default:** `markgarnold` + +**Example (`concore.repo`):** + +``` +myorganization +``` + +This means Docker images will be pulled from `myorganization/` instead of `markgarnold/`. + +--- + +## Environment Variables + +Environment variables are read **before** config files and provide initial defaults. Config files override environment variables. + +| Variable | Default | Description | +|----------|---------|-------------| +| `CONCORE_CPPWIN` | `g++` | Windows C++ compiler | +| `CONCORE_CPPEXE` | `g++` | POSIX C++ compiler | +| `CONCORE_VWIN` | `iverilog` | Windows Verilog compiler | +| `CONCORE_VEXE` | `iverilog` | POSIX Verilog compiler | +| `CONCORE_PYTHONEXE` | `python3` | POSIX Python interpreter | +| `CONCORE_PYTHONWIN` | `python` | Windows Python interpreter | +| `CONCORE_MATLABEXE` | `matlab` | POSIX MATLAB executable | +| `CONCORE_MATLABWIN` | `matlab` | Windows MATLAB executable | +| `CONCORE_OCTAVEEXE` | `octave` | POSIX GNU Octave executable | +| `CONCORE_OCTAVEWIN` | `octave` | Windows GNU Octave executable | +| `CONCORE_JAVACEXE` | `javac` | POSIX Java compiler | +| `CONCORE_JAVACWIN` | `javac` | Windows Java compiler | +| `CONCORE_JAVAEXE` | `java` | POSIX Java runtime | +| `CONCORE_JAVAWIN` | `java` | Windows Java runtime | +| `DOCKEREXE` | `docker` | Docker executable | + +--- + +## Precedence (Priority Order) + +When `mkconcore.py` resolves a tool path, the following precedence applies (highest priority first): + +1. **`concore.tools` file** — overrides everything for compiler/runtime paths +2. **`concore.sudo` file** — overrides `DOCKEREXE` for Docker executable +3. **`concore.repo` file** — overrides the hardcoded default Docker repository prefix (for example, `markgarnold`) +4. **Environment variables** (`CONCORE_*`, `DOCKEREXE`) — initial defaults +5. **Hardcoded defaults** — `g++`, `python3`, `iverilog`, `octave`, `docker`, etc. + +For `concore.octave` and `concore.mcr`, there is no environment variable equivalent — they are controlled only by file presence/content. + +--- + +## Quick Start + +**Minimal setup (Python-only study on Linux):** + +No config files needed — defaults work out of the box. + +**Typical setup (Python + C++ on Linux):** + +```bash +# Only needed if g++ is not on PATH +echo "CPPEXE=/usr/local/bin/g++-13" > $CONCOREPATH/concore.tools +``` + +**Windows setup (Python + C++):** + +``` +CPPWIN=C:\MinGW\bin\g++.exe +PYTHONWIN=C:\Python313\python.exe +``` + +**Docker without sudo:** + +```bash +echo "docker" > $CONCOREPATH/concore.sudo +``` + +**Using Octave instead of MATLAB:** + +```bash +touch $CONCOREPATH/concore.octave +``` + +--- + +## Diagnostic + +Run `concore doctor` to see all detected tools, config file status, and environment variables in a single readiness report: + +``` +$ concore doctor +``` + +See the [CLI README](../concore_cli/README.md) for details. diff --git a/example/java_e2e/README.md b/example/java_e2e/README.md new file mode 100644 index 00000000..75c0e812 --- /dev/null +++ b/example/java_e2e/README.md @@ -0,0 +1,36 @@ +# Java end-to-end example + +This example runs a Python controller (`controller.py`) and a Java PM node (`pm_java.java`) over the standard concore file-based exchange. + +## Files + +- `controller.py` - Python controller node +- `pm_java.java` - Java PM node using `concoredocker.java` +- `java_e2e.graphml` - workflow graph for the example +- `smoke_check.py` - lightweight verification script + +## Prerequisites + +- Python environment with project dependencies installed +- JDK (for `javac` and `java`) +- `jeromq-0.6.0.jar` + +Download jar (from repo root): + +```bash +mkdir -p .ci-cache/java +curl -fsSL -o .ci-cache/java/jeromq-0.6.0.jar https://repo1.maven.org/maven2/org/zeromq/jeromq/0.6.0/jeromq-0.6.0.jar +``` + +## Run smoke check + +From repo root: + +```bash +python example/java_e2e/smoke_check.py --jar .ci-cache/java/jeromq-0.6.0.jar +``` + +Expected result: + +- script prints `smoke_check passed` +- final `u` and `ym` payloads are printed in concore wire format diff --git a/example/java_e2e/controller.py b/example/java_e2e/controller.py new file mode 100644 index 00000000..e5faa0b7 --- /dev/null +++ b/example/java_e2e/controller.py @@ -0,0 +1,45 @@ +import ast +import os +from pathlib import Path + +import concore +import numpy as np + + +ysp = 3.0 + + +def controller(ym): + if ym[0] < ysp: + return 1.01 * ym + else: + return 0.9 * ym + + +study_dir = os.environ.get("CONCORE_STUDY_DIR", str(Path(__file__).resolve().parent / "study")) +os.makedirs(os.path.join(study_dir, "1"), exist_ok=True) + +concore.inpath = os.path.join(study_dir, "") +concore.outpath = os.path.join(study_dir, "") +concore.default_maxtime(20) +concore.delay = 0.02 + +init_simtime_u = "[0.0, 0.0]" +init_simtime_ym = "[0.0, 0.0]" + +u = np.array([concore.initval(init_simtime_u)]).T +while(concore.simtime + + + + + + + + + + + CJ:example/java_e2e/controller.py + + + + + + + + + + + PJ:example/java_e2e/pm_java.java + + + + + + + + + + CU + + + + + + + + + PYM + + + + + diff --git a/example/java_e2e/pm_java.java b/example/java_e2e/pm_java.java new file mode 100644 index 00000000..1b11e075 --- /dev/null +++ b/example/java_e2e/pm_java.java @@ -0,0 +1,43 @@ +import java.util.ArrayList; +import java.util.List; + +public class pm_java { + private static final String INIT_SIMTIME_U = "[0.0, 0.0]"; + + private static double toDouble(Object value) { + if (value instanceof Number) { + return ((Number) value).doubleValue(); + } + return 0.0; + } + + public static void main(String[] args) { + String studyDir = System.getenv("CONCORE_STUDY_DIR"); + if (studyDir == null || studyDir.isEmpty()) { + studyDir = "example/java_e2e/study"; + } + double maxTime = 20.0; + + concoredocker.setInPath(studyDir); + concoredocker.setOutPath(studyDir); + concoredocker.setDelay(20); + concoredocker.defaultMaxTime(maxTime); + + while (concoredocker.getSimtime() < maxTime) { + concoredocker.ReadResult readResult = concoredocker.read(1, "u", INIT_SIMTIME_U); + List u = readResult.data; + + double u0 = 0.0; + if (!u.isEmpty()) { + u0 = toDouble(u.get(0)); + } + + double ym0 = u0 + 0.01; + List ym = new ArrayList<>(); + ym.add(ym0); + + System.out.println(concoredocker.getSimtime() + ". u=" + u + " ym=" + ym); + concoredocker.write(1, "ym", ym, 1); + } + } +} diff --git a/example/java_e2e/smoke_check.py b/example/java_e2e/smoke_check.py new file mode 100644 index 00000000..5fc04e92 --- /dev/null +++ b/example/java_e2e/smoke_check.py @@ -0,0 +1,114 @@ +import argparse +import ast +import os +from pathlib import Path +import shutil +import subprocess +import sys + + +JEROMQ_URL = "https://repo1.maven.org/maven2/org/zeromq/jeromq/0.6.0/jeromq-0.6.0.jar" + + +def parse_args(): + parser = argparse.ArgumentParser(description="Run Java e2e example smoke check") + parser.add_argument("--jar", type=Path, help="Path to jeromq jar") + parser.add_argument("--keep-study", action="store_true", help="Keep generated study files") + return parser.parse_args() + +def main(): + args = parse_args() + here = Path(__file__).resolve().parent + repo_root = here.parents[1] + study_dir = here / "study" + jar_path = args.jar or (repo_root / ".ci-cache" / "java" / "jeromq-0.6.0.jar") + + if not jar_path.exists(): + raise FileNotFoundError( + f"Missing jeromq jar at {jar_path}. Download from {JEROMQ_URL} or pass --jar." + ) + + if study_dir.exists(): + shutil.rmtree(study_dir) + (study_dir / "1").mkdir(parents=True, exist_ok=True) + (study_dir / "1" / "concore.maxtime").write_text("20", encoding="utf-8") + + subprocess.run( + [ + "javac", + "-cp", + str(jar_path), + str(repo_root / "concoredocker.java"), + str(here / "pm_java.java"), + ], + check=True, + cwd=repo_root, + ) + + env = os.environ.copy() + env["CONCORE_STUDY_DIR"] = str(study_dir) + classpath = os.pathsep.join([str(repo_root), str(here), str(jar_path)]) + + py_log = open(study_dir / "controller.log", "w", encoding="utf-8") + java_log = open(study_dir / "pm_java.log", "w", encoding="utf-8") + + py_proc = subprocess.Popen( + [sys.executable, str(here / "controller.py")], + cwd=repo_root, + env=env, + stdout=py_log, + stderr=subprocess.STDOUT, + ) + java_proc = subprocess.Popen( + ["java", "-cp", classpath, "pm_java"], + cwd=repo_root, + env=env, + stdout=java_log, + stderr=subprocess.STDOUT, + ) + + try: + py_rc = py_proc.wait(timeout=45) + java_rc = java_proc.wait(timeout=45) + except subprocess.TimeoutExpired: + py_proc.kill() + java_proc.kill() + py_log.close() + java_log.close() + raise RuntimeError("Timed out waiting for node processes") + + py_log.close() + java_log.close() + + if py_rc != 0 or java_rc != 0: + raise RuntimeError( + f"Node process failed (controller={py_rc}, pm_java={java_rc}). " + f"See {study_dir / 'controller.log'} and {study_dir / 'pm_java.log'}." + ) + + u_path = study_dir / "1" / "u" + ym_path = study_dir / "1" / "ym" + if not u_path.exists() or not ym_path.exists(): + raise RuntimeError("Expected output files were not produced") + + u_val = ast.literal_eval(u_path.read_text(encoding="utf-8")) + ym_val = ast.literal_eval(ym_path.read_text(encoding="utf-8")) + if not isinstance(u_val, list) or len(u_val) < 2: + raise RuntimeError("u output did not match expected wire format") + if not isinstance(ym_val, list) or len(ym_val) < 2: + raise RuntimeError("ym output did not match expected wire format") + + print("smoke_check passed") + print(f"u: {u_val}") + print(f"ym: {ym_val}") + + if not args.keep_study and study_dir.exists(): + shutil.rmtree(study_dir) + + class_file = here / "pm_java.class" + if class_file.exists(): + class_file.unlink() + + +if __name__ == "__main__": + main() diff --git a/fri/Dockerfile b/fri/Dockerfile index 450274f2..6a639285 100644 --- a/fri/Dockerfile +++ b/fri/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.6-slim +FROM python:3.10-slim RUN apt-get update && apt-get install -y ca-certificates \ && update-ca-certificates \ diff --git a/fri/requirements.txt b/fri/requirements.txt index 2c7516da..85974b38 100644 --- a/fri/requirements.txt +++ b/fri/requirements.txt @@ -1,5 +1,6 @@ Flask gunicorn==20.1.0 -FLASK_CORS +flask-cors +# Optional dependency for /openJupyter endpoint jupyterlab PyGithub \ No newline at end of file diff --git a/fri/server/main.py b/fri/server/main.py index f94bc663..760db294 100644 --- a/fri/server/main.py +++ b/fri/server/main.py @@ -1,20 +1,125 @@ -from flask import Flask, request, jsonify, send_file, send_from_directory +from flask import Flask, request, jsonify, send_file, send_from_directory, abort from werkzeug.utils import secure_filename import xml.etree.ElementTree as ET import os +import secrets import subprocess from subprocess import call,check_output from pathlib import Path import json +import logging import platform +import re +import secrets +import threading from flask_cors import CORS, cross_origin +logger = logging.getLogger(__name__) + +# Input validation pattern for safe names (alphanumeric, dash, underscore, slash, dot, space) +SAFE_INPUT_PATTERN = re.compile(r'^[a-zA-Z0-9_\-/. ]+$') +# Pattern for filenames - no path separators or .. allowed +SAFE_FILENAME_PATTERN = re.compile(r'^[a-zA-Z0-9_\-. ]+$') + +def validate_input(value, field_name, required=False): + """Validate that input contains only safe characters.""" + if value is None: + if required: + raise ValueError(f"Missing required field: {field_name}") + return True + if not isinstance(value, str): + raise ValueError(f"Invalid {field_name}: must be a string") + if required and len(value) == 0: + raise ValueError(f"Missing required field: {field_name}") + if len(value) > 0 and not SAFE_INPUT_PATTERN.match(value): + raise ValueError(f"Invalid {field_name}: contains unsafe characters") + return True + +def validate_filename(value, field_name, required=False): + """Validate filename - no path separators or .. segments allowed.""" + if value is None: + if required: + raise ValueError(f"Missing required field: {field_name}") + return True + if not isinstance(value, str): + raise ValueError(f"Invalid {field_name}: must be a string") + if required and len(value) == 0: + raise ValueError(f"Missing required field: {field_name}") + # Reject path traversal attempts + if '..' in value: + raise ValueError(f"Invalid {field_name}: path traversal not allowed") + # Use basename to strip any path components + basename = os.path.basename(value) + if basename != value: + raise ValueError(f"Invalid {field_name}: must be a filename, not a path") + if len(value) > 0 and not SAFE_FILENAME_PATTERN.match(value): + raise ValueError(f"Invalid {field_name}: contains unsafe characters") + return True + +def validate_text_field(value, field_name, max_length=None): + """Validate text fields like PR title/body - allow more characters but check type/length.""" + if value is None: + return True + if not isinstance(value, str): + raise ValueError(f"Invalid {field_name}: must be a string") + if max_length and len(value) > max_length: + raise ValueError(f"Invalid {field_name}: too long (max {max_length} characters)") + return True + +def get_error_output(e): + """Extract error output from CalledProcessError, preferring stderr then output.""" + raw_output = None + if hasattr(e, 'stderr') and e.stderr: + raw_output = e.stderr + elif hasattr(e, 'output') and e.output: + raw_output = e.output + + if raw_output is None: + return "Command execution failed" + + if isinstance(raw_output, bytes): + try: + return raw_output.decode('utf-8', errors='replace') + except Exception: + return str(raw_output) + elif isinstance(raw_output, str): + return raw_output + else: + return str(raw_output) + cur_path = os.path.dirname(os.path.abspath(__file__)) concore_path = os.path.abspath(os.path.join(cur_path, '../../')) +# API key authentication for sensitive endpoints +API_KEY = os.environ.get("CONCORE_API_KEY") + +def require_api_key(): + """Require a valid API key via X-API-KEY header.""" + if not API_KEY: + abort(500, description="Server not configured with API key") + provided = request.headers.get("X-API-KEY") or "" + if not secrets.compare_digest(provided, API_KEY): + abort(403, description="Unauthorized") + +# Track single Jupyter process to prevent multiple concurrent launches +jupyter_process = None +jupyter_lock = threading.Lock() + app = Flask(__name__) -app.secret_key = "secret key" +app.secret_key = os.getenv("FLASK_SECRET_KEY") + +if not app.secret_key: + # In production, require an explicit secret key to avoid session issues + flask_env = os.getenv("FLASK_ENV", "").lower() + if flask_env in ("development", "dev") or app.debug: + # Generate temporary key for development environments where a secret key + # has not been explicitly configured. + app.secret_key = secrets.token_hex(32) + else: + raise RuntimeError( + "FLASK_SECRET_KEY environment variable must be set in production." + ) cors = CORS(app) app.config['CORS_HEADERS'] = 'Content-Type' @@ -110,76 +215,32 @@ def build(dir): if not os.path.exists(dir_path): - if(platform.uname()[0]=='Windows'): - if(out_dir == None or out_dir == ""): - if(docker == 'true'): - try: - output_bytes = subprocess.check_output(["makedocker", makestudy_dir], cwd=concore_path, shell=True) - output_str = output_bytes.decode("utf-8") - proc = 0 - except subprocess.CalledProcessError as e: - output_str = f"Docker study creation failed with return code {e.returncode} (check duplicate directory)" - proc = 1 - else: - try: - output_bytes = subprocess.check_output(["makestudy", makestudy_dir], cwd=concore_path, shell=True) - output_str = output_bytes.decode("utf-8") - proc = 0 - except subprocess.CalledProcessError as e: - output_str = f"Study creation failed with return code {e.returncode} (check duplicate directory)" - proc = 1 - else: - if(docker == 'true'): - try: - output_bytes = subprocess.check_output(["makedocker", makestudy_dir, out_dir], cwd=concore_path, shell=True) - output_str = output_bytes.decode("utf-8") - proc = 0 - except subprocess.CalledProcessError as e: - output_str = f"Docker study creation failed with return code {e.returncode} (check duplicate directory)" - proc = 1 - else: - try: - output_bytes = subprocess.check_output(["makestudy", makestudy_dir, out_dir], cwd=concore_path, shell=True) - output_str = output_bytes.decode("utf-8") - proc = 0 - except subprocess.CalledProcessError as e: - output_str = f"Study creation failed with return code {e.returncode} (check duplicate directory)" - proc = 1 + # Determine command name and error label based on docker flag + if docker == 'true': + cmd_name = "makedocker" + error_label = "Docker study" else: - if(out_dir == None or out_dir == ""): - if(docker == 'true'): - try: - output_bytes = subprocess.check_output([r"./makedocker", makestudy_dir], cwd=concore_path) - output_str = output_bytes.decode("utf-8") - proc = 0 - except subprocess.CalledProcessError as e: - output_str = f"Docker study creation failed with return code {e.returncode} (check duplicate directory)" - proc = 1 - else: - try: - output_bytes = subprocess.check_output([r"./makestudy", makestudy_dir], cwd=concore_path) - output_str = output_bytes.decode("utf-8") - proc = 0 - except subprocess.CalledProcessError as e: - output_str = f"Study creation failed with return code {e.returncode} (check duplicate directory)" - proc = 1 - else: - if(docker == 'true'): - try: - output_bytes = subprocess.check_output([r"./makedocker", makestudy_dir, out_dir], cwd=concore_path) - output_str = output_bytes.decode("utf-8") - proc = 0 - except subprocess.CalledProcessError as e: - output_str = f"Docker study creation failed with return code {e.returncode} (check duplicate directory)" - proc = 1 - else: - try: - output_bytes = subprocess.check_output([r"./makestudy", makestudy_dir, out_dir], cwd=concore_path) - output_str = output_bytes.decode("utf-8") - proc = 0 - except subprocess.CalledProcessError as e: - output_str = f"Study creation failed with return code {e.returncode} (check duplicate directory)" - proc = 1 + cmd_name = "makestudy" + error_label = "Study" + + # Build command list + cmd = [cmd_name, makestudy_dir] + if out_dir != None and out_dir != "": + cmd.append(out_dir) + + # OS-specific adjustments + is_windows = platform.uname()[0] == 'Windows' + if not is_windows: + cmd[0] = f"./{cmd[0]}" + + try: + output_bytes = subprocess.check_output(cmd, cwd=concore_path, shell=is_windows) + output_str = output_bytes.decode("utf-8") + proc = 0 + except subprocess.CalledProcessError as e: + output_str = f"{error_label} creation failed with return code {e.returncode} (check duplicate directory)" + proc = 1 + if(proc == 0): resp = jsonify({'message': 'Directory successfully created'}) resp.status_code = 201 @@ -298,20 +359,36 @@ def clear(dir): def contribute(): try: data = request.json - PR_TITLE = data.get('title') - PR_BODY = data.get('desc') - AUTHOR_NAME = data.get('auth') - STUDY_NAME = data.get('study') - STUDY_NAME_PATH = data.get('path') - BRANCH_NAME = data.get('branch') + PR_TITLE = data.get('title') or '' + PR_BODY = data.get('desc') or '' + AUTHOR_NAME = data.get('auth') or '' + STUDY_NAME = data.get('study') or '' + STUDY_NAME_PATH = data.get('path') or '' + BRANCH_NAME = data.get('branch') or '' + + # Validate all user inputs to prevent command injection + # Strict validation for names/paths that go into command arguments + validate_input(STUDY_NAME, 'study', required=True) + validate_input(STUDY_NAME_PATH, 'path', required=True) + validate_input(AUTHOR_NAME, 'auth', required=True) + validate_input(BRANCH_NAME, 'branch', required=False) + + # For PR title/body, allow more characters but enforce type/length + validate_text_field(PR_TITLE, 'title', max_length=512) + validate_text_field(PR_BODY, 'desc', max_length=8192) + + # Build base command depending on platform if(platform.uname()[0]=='Windows'): - proc=check_output(["contribute",STUDY_NAME,STUDY_NAME_PATH,AUTHOR_NAME,BRANCH_NAME,PR_TITLE,PR_BODY],cwd=concore_path,shell=True) + cmd = ["cmd.exe", "/c", "contribute.bat", STUDY_NAME, STUDY_NAME_PATH, AUTHOR_NAME] else: - if len(BRANCH_NAME)==0: - proc = check_output([r"./contribute",STUDY_NAME,STUDY_NAME_PATH,AUTHOR_NAME],cwd=concore_path) - else: - proc = check_output([r"./contribute",STUDY_NAME,STUDY_NAME_PATH,AUTHOR_NAME,BRANCH_NAME,PR_TITLE,PR_BODY],cwd=concore_path) - output_string = proc.decode() + cmd = [r"./contribute", STUDY_NAME, STUDY_NAME_PATH, AUTHOR_NAME] + + # Append optional branch/PR args only when BRANCH_NAME is provided + if len(BRANCH_NAME) > 0: + cmd.extend([BRANCH_NAME, PR_TITLE, PR_BODY]) + + proc = subprocess.run(cmd, cwd=concore_path, check=True, capture_output=True, text=True) + output_string = proc.stdout status=200 if output_string.find("/pulls/")!=-1: status=200 @@ -320,6 +397,11 @@ def contribute(): else: status=400 return jsonify({'message': output_string}),status + except ValueError as e: + return jsonify({'message': str(e)}), 400 + except subprocess.CalledProcessError as e: + output_string = get_error_output(e) + return jsonify({'message': output_string}), 501 except Exception as e: output_string = "Some Error occured.Please try after some time" status=501 @@ -330,15 +412,44 @@ def contribute(): def download(dir): download_file = request.args.get('fetch') sub_folder = request.args.get('fetchDir') + + if not download_file: + abort(400, description="Missing file parameter") + + download_file = secure_filename(download_file) + + if download_file == "": + abort(400, description="Invalid filename") + + # Normalize the requested file path + safe_path = os.path.normpath(download_file) + + # Prevent absolute paths + if os.path.isabs(safe_path): + abort(400, description="Invalid file path") + + # Prevent directory traversal + if ".." in safe_path.split(os.sep): + abort(400, description="Directory traversal attempt detected") + dirname = secure_filename(dir) + "/" + secure_filename(sub_folder) - directory_name = os.path.abspath(os.path.join(concore_path, dirname)) + concore_real = os.path.realpath(concore_path) + directory_name = os.path.realpath(os.path.join(concore_real, dirname)) + if not directory_name.startswith(concore_real + os.sep): + abort(403, description="Access denied") if not os.path.exists(directory_name): resp = jsonify({'message': 'Directory not found'}) resp.status_code = 400 return resp + + # Ensure final resolved path is within the intended directory, resolving symlinks + full_path = os.path.realpath(os.path.join(directory_name, safe_path)) + if not full_path.startswith(directory_name + os.sep): + abort(403, description="Access denied") + try: - return send_from_directory(directory_name, download_file, as_attachment=True) - except: + return send_from_directory(directory_name, safe_path, as_attachment=True) + except Exception: resp = jsonify({'message': 'file not found'}) resp.status_code = 400 return resp @@ -365,19 +476,37 @@ def library(dir): dir_path = os.path.abspath(os.path.join(concore_path, dir_name)) filename = request.args.get('filename') library_path = request.args.get('path') - proc = 0 + + # Validate user inputs to prevent command injection + try: + # Use strict filename validation - no path separators or .. allowed + validate_filename(filename, 'filename', required=True) + validate_input(library_path, 'path', required=False) + except ValueError as e: + resp = jsonify({'message': str(e)}) + resp.status_code = 400 + return resp + if (library_path == None or library_path == ''): library_path = r"../tools" - if(platform.uname()[0]=='Windows'): - proc = subprocess.check_output([r"..\library", library_path, filename],shell=True, cwd=dir_path) - else: - proc = subprocess.check_output([r"../library", library_path, filename], cwd=dir_path) - if(proc != 0): - resp = jsonify({'message': proc.decode("utf-8")}) - resp.status_code = 201 + try: + if(platform.uname()[0]=='Windows'): + # Use cmd.exe /c to invoke library.bat on Windows + result = subprocess.run(["cmd.exe", "/c", r"..\library.bat", library_path, filename], cwd=dir_path, check=True, capture_output=True, text=True) + proc = result.stdout + else: + proc = subprocess.check_output([r"../library", library_path, filename], cwd=dir_path, stderr=subprocess.STDOUT) + proc = proc.decode("utf-8") + resp = jsonify({'message': proc}) + resp.status_code = 200 return resp - else: - resp = jsonify({'message': 'There is an Error'}) + except subprocess.CalledProcessError as e: + error_output = get_error_output(e) + resp = jsonify({'message': f'Command execution failed: {error_output}'}) + resp.status_code = 400 + return resp + except Exception as e: + resp = jsonify({'message': 'Internal server error'}) resp.status_code = 500 return resp @@ -395,16 +524,53 @@ def getFilesList(dir): @app.route('/openJupyter/', methods=['POST']) def openJupyter(): - proc = subprocess.Popen(['jupyter', 'lab'], shell=False, stdout=subprocess.PIPE, cwd=concore_path) - if proc.poll() is None: - resp = jsonify({'message': 'Successfuly opened Jupyter'}) - resp.status_code = 308 - return resp - else: - resp = jsonify({'message': 'There is an Error'}) - resp.status_code = 500 - return resp + global jupyter_process + + require_api_key() + + with jupyter_lock: + if jupyter_process and jupyter_process.poll() is None: + return jsonify({"message": "Jupyter already running"}), 409 + + try: + jupyter_process = subprocess.Popen( + ['jupyter', 'lab', '--no-browser'], + shell=False, + cwd=concore_path, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL + ) + return jsonify({"message": "Jupyter Lab started"}), 200 + except (FileNotFoundError, PermissionError, OSError) as e: + logger.error("Failed to start Jupyter: %s", e) + return jsonify({"error": "Failed to start Jupyter"}), 500 + + +@app.route('/stopJupyter/', methods=['POST']) +def stopJupyter(): + global jupyter_process + + require_api_key() + + with jupyter_lock: + if not jupyter_process or jupyter_process.poll() is not None: + return jsonify({"message": "No running Jupyter instance"}), 404 + + try: + jupyter_process.terminate() + # Wait for Jupyter to terminate gracefully + jupyter_process.wait(timeout=5) + except subprocess.TimeoutExpired: + # Force kill if it did not exit in time + jupyter_process.kill() + jupyter_process.wait(timeout=5) + finally: + jupyter_process = None + + return jsonify({"message": "Jupyter stopped"}), 200 if __name__ == "__main__": - app.run(host="0.0.0.0", port=5000) + # In production, use: + # gunicorn -w 4 -b 0.0.0.0:5000 fri.server.main:app + app.run(host="0.0.0.0", port=5000, debug=False) diff --git a/humanc/bangbang.py b/humanc/bangbang.py index eed72b83..b5020514 100644 --- a/humanc/bangbang.py +++ b/humanc/bangbang.py @@ -8,9 +8,8 @@ def bangbang_controller(ym): amp = 3 elif ym[1]<65: amp = 1 - - - ustar = np.array([amp,30]) + + ustar = np.array([amp,30]) return ustar diff --git a/humanc/cardiac_pm.py b/humanc/cardiac_pm.py index d3edb300..67a55cf7 100644 --- a/humanc/cardiac_pm.py +++ b/humanc/cardiac_pm.py @@ -1,4 +1,10 @@ import numpy as np +import sys +import os + +cardiac_pm_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'cardiac_pm.dir') +if cardiac_pm_dir not in sys.path: + sys.path.insert(0, cardiac_pm_dir) import pulsatile_model_functions as pmf import healthy_params as K import concore diff --git a/import_concore.m b/import_concore.m index 018c10e5..ad95889c 100644 --- a/import_concore.m +++ b/import_concore.m @@ -1,42 +1,63 @@ -function import_concore - global concore; - - try - pid = getpid(); - catch exc - pid = feature('getpid'); - end - outputpid = fopen('concorekill.bat','w'); - fprintf(outputpid,'%s',['taskkill /F /PID ',num2str(pid)]); - fclose(outputpid); - - - try - iportfile = fopen('concore.iport'); - concore.iports = fscanf(iportfile,'%c'); - catch exc - iportfile = ''; - end - - try - oportfile = fopen('concore.oport'); - concore.oports = fscanf(oportfile,'%c'); - catch exc - oportfile = ''; - end - - concore.s = ''; - concore.olds = ''; - concore.delay = 1; - concore.retrycount = 0; - if exist('/in1','dir')==7 % 5/20/21 work for docker or local - concore.inpath = '/in'; - concore.outpath = '/out'; - else - concore.inpath = 'in'; - concore.outpath = 'out'; - end - concore.simtime = 0; - - concore_default_maxtime(100); -end +function import_concore + global concore; + + if ispc + try + pid = getpid(); + catch exc + pid = feature('getpid'); + end + outputpid = fopen('concorekill.bat','w'); + if outputpid ~= -1 + fprintf(outputpid,'%s',['taskkill /F /PID ',num2str(pid)]); + fclose(outputpid); + else + warning('import_concore:ConcoreKillFileOpenFailed', ... + 'Could not create concorekill.bat. Continuing without kill script.'); + end + end + + + try + iportfile = fopen('concore.iport'); + concore.iports = fscanf(iportfile,'%c'); + if isnumeric(iportfile) && iportfile ~= -1 + fclose(iportfile); + end + iportfile = -1; + catch exc + if exist('iportfile', 'var') && isnumeric(iportfile) && iportfile ~= -1 + fclose(iportfile); + end + iportfile = -1; + end + + try + oportfile = fopen('concore.oport'); + concore.oports = fscanf(oportfile,'%c'); + if isnumeric(oportfile) && oportfile ~= -1 + fclose(oportfile); + end + oportfile = -1; + catch exc + if exist('oportfile', 'var') && isnumeric(oportfile) && oportfile ~= -1 + fclose(oportfile); + end + oportfile = -1; + end + + concore.s = ''; + concore.olds = ''; + concore.delay = 1; + concore.retrycount = 0; + if exist('/in1','dir')==7 % 5/20/21 work for docker or local + concore.inpath = '/in'; + concore.outpath = '/out'; + else + concore.inpath = 'in'; + concore.outpath = 'out'; + end + concore.simtime = 0; + + concore_default_maxtime(100); +end diff --git a/import_concoredocker.m b/import_concoredocker.m index 018c10e5..dd385e16 100644 --- a/import_concoredocker.m +++ b/import_concoredocker.m @@ -1,42 +1,45 @@ -function import_concore - global concore; - - try - pid = getpid(); - catch exc - pid = feature('getpid'); - end - outputpid = fopen('concorekill.bat','w'); - fprintf(outputpid,'%s',['taskkill /F /PID ',num2str(pid)]); - fclose(outputpid); - - - try - iportfile = fopen('concore.iport'); - concore.iports = fscanf(iportfile,'%c'); - catch exc - iportfile = ''; - end - - try - oportfile = fopen('concore.oport'); - concore.oports = fscanf(oportfile,'%c'); - catch exc - oportfile = ''; - end - - concore.s = ''; - concore.olds = ''; - concore.delay = 1; - concore.retrycount = 0; - if exist('/in1','dir')==7 % 5/20/21 work for docker or local - concore.inpath = '/in'; - concore.outpath = '/out'; - else - concore.inpath = 'in'; - concore.outpath = 'out'; - end - concore.simtime = 0; - - concore_default_maxtime(100); -end +function import_concore + global concore; + + % Docker/Linux environment — no Windows batch file generation + + + iportfile = fopen('concore.iport'); + if iportfile ~= -1 + try + concore.iports = fscanf(iportfile,'%c'); + catch exc + concore.iports = ''; + end + fclose(iportfile); + else + concore.iports = ''; + end + + oportfile = fopen('concore.oport'); + if oportfile ~= -1 + try + concore.oports = fscanf(oportfile,'%c'); + catch exc + concore.oports = ''; + end + fclose(oportfile); + else + concore.oports = ''; + end + + concore.s = ''; + concore.olds = ''; + concore.delay = 1; + concore.retrycount = 0; + if exist('/in1','dir')==7 % 5/20/21 work for docker or local + concore.inpath = '/in'; + concore.outpath = '/out'; + else + concore.inpath = 'in'; + concore.outpath = 'out'; + end + concore.simtime = 0; + + concore_default_maxtime(100); +end diff --git a/measurements/A.py b/measurements/A.py index b7d0fb6e..0db49368 100644 --- a/measurements/A.py +++ b/measurements/A.py @@ -5,6 +5,23 @@ import psutil import sys +# --- Fallback Definitions for Direct Execution --- +# These values are normally injected by copy_with_port_portname.py during study generation. +# When running this script directly, safe defaults are used instead. +_STANDALONE_MODE = False + +if 'PORT_NAME_F1_F2' not in globals(): + PORT_NAME_F1_F2 = "F1_F2" + _STANDALONE_MODE = True + +if 'PORT_F1_F2' not in globals(): + PORT_F1_F2 = "5555" + _STANDALONE_MODE = True + +if _STANDALONE_MODE: + print("Warning: Port variables not injected. Running in standalone mode with default values.") + print(" For full study behavior, run via study generation (makestudy).") + # --- ZMQ Initialization --- # This REQ socket connects to Node B concore.init_zmq_port( diff --git a/measurements/B.py b/measurements/B.py index 8ab05d09..595f3d8b 100644 --- a/measurements/B.py +++ b/measurements/B.py @@ -2,6 +2,31 @@ import concore import time +# --- Fallback Definitions for Direct Execution --- +# These values are normally injected by copy_with_port_portname.py during study generation. +# When running this script directly, safe defaults are used instead. +_STANDALONE_MODE = False + +if 'PORT_NAME_F1_F2' not in globals(): + PORT_NAME_F1_F2 = "F1_F2" + _STANDALONE_MODE = True + +if 'PORT_F1_F2' not in globals(): + PORT_F1_F2 = "5555" + _STANDALONE_MODE = True + +if 'PORT_NAME_F2_F3' not in globals(): + PORT_NAME_F2_F3 = "F2_F3" + _STANDALONE_MODE = True + +if 'PORT_F2_F3' not in globals(): + PORT_F2_F3 = "5556" + _STANDALONE_MODE = True + +if _STANDALONE_MODE: + print("Warning: Port variables not injected. Running in standalone mode with default values.") + print(" For full study behavior, run via study generation (makestudy).") + # --- ZMQ Initialization --- # This REP socket binds and waits for requests from Node A concore.init_zmq_port( diff --git a/measurements/C.py b/measurements/C.py index 1448069d..03337657 100644 --- a/measurements/C.py +++ b/measurements/C.py @@ -5,6 +5,23 @@ import os import sys +# --- Fallback Definitions for Direct Execution --- +# These values are normally injected by copy_with_port_portname.py during study generation. +# When running this script directly, safe defaults are used instead. +_STANDALONE_MODE = False + +if 'PORT_NAME_F2_F3' not in globals(): + PORT_NAME_F2_F3 = "F2_F3" + _STANDALONE_MODE = True + +if 'PORT_F2_F3' not in globals(): + PORT_F2_F3 = "5556" + _STANDALONE_MODE = True + +if _STANDALONE_MODE: + print("Warning: Port variables not injected. Running in standalone mode with default values.") + print(" For full study behavior, run via study generation (makestudy).") + # --- ZMQ Initialization --- # This REP socket binds and waits for requests from Node B concore.init_zmq_port( diff --git a/measurements/Latency/funbody_distributed.py b/measurements/Latency/funbody_distributed.py index eaae295c..33da80b2 100644 --- a/measurements/Latency/funbody_distributed.py +++ b/measurements/Latency/funbody_distributed.py @@ -1,7 +1,6 @@ # funbody2_zmq.py import time import concore -import concore2 print("funbody using ZMQ via concore") @@ -16,21 +15,18 @@ # Standard concore initializations concore.delay = 0.07 -concore2.delay = 0.07 -concore2.inpath = concore.inpath -concore2.outpath = concore.outpath -concore2.simtime = 0 +concore.simtime = 0 concore.default_maxtime(100) init_simtime_u_str = "[0.0, 0.0, 0.0]" init_simtime_ym_str = "[0.0, 0.0, 0.0]" u_data_values = concore.initval(init_simtime_u_str) -ym_data_values = concore2.initval(init_simtime_ym_str) +ym_data_values = concore.initval(init_simtime_ym_str) print(f"Initial u_data_values: {u_data_values}, ym_data_values: {ym_data_values}") print(f"Max time: {concore.maxtime}") -while concore2.simtime < concore.maxtime: +while concore.simtime < concore.maxtime: received_u_data = concore.read(PORT_NAME_F2_OUT, "u_signal", init_simtime_u_str) if not (isinstance(received_u_data, list) and len(received_u_data) > 0): @@ -50,17 +46,17 @@ if 'U2' in concore.oport: concore.write(concore.oport['U2'], "u", u_data_values) - old_concore2_simtime = concore2.simtime - while concore2.unchanged() or concore2.simtime <= old_concore2_simtime: + old_concore_simtime = float(concore.simtime) + while concore.unchanged() or concore.simtime <= old_concore_simtime: # Assuming concore.iport['Y2'] is a file port (e.g., from pmpymax.py) - ym_data_values = concore2.read(concore.iport['Y2'], "ym", init_simtime_ym_str) - # time.sleep(concore2.delay) # Optional delay + ym_data_values = concore.read(concore.iport['Y2'], "ym", init_simtime_ym_str) + # time.sleep(concore.delay) # Optional delay - ym_full_to_send = [concore2.simtime] + ym_data_values + ym_full_to_send = [concore.simtime] + ym_data_values concore.write(PORT_NAME_F2_OUT, "ym_signal", ym_full_to_send) - print(f"funbody u={u_data_values} ym={ym_data_values} time={concore2.simtime}") + print(f"funbody u={u_data_values} ym={ym_data_values} time={concore.simtime}") print("funbody retry=" + str(concore.retrycount)) diff --git a/measurements/Latency/funcall_distributed.py b/measurements/Latency/funcall_distributed.py index 779324fb..8e8e7590 100644 --- a/measurements/Latency/funcall_distributed.py +++ b/measurements/Latency/funcall_distributed.py @@ -1,7 +1,6 @@ # funcall_distributed.py (MODIFIED FOR LATENCY MEASUREMENT) import time import concore -import concore2 import csv # <--- ADDED: Import CSV library print("funcall using ZMQ via concore") @@ -16,24 +15,21 @@ # Standard concore initializations concore.delay = 0.07 -concore2.delay = 0.07 -concore2.inpath = concore.inpath -concore2.outpath = concore.outpath -concore2.simtime = 0 +concore.simtime = 0 concore.default_maxtime(100) # Recommend increasing this for more data points, e.g., 1000 init_simtime_u_str = "[0.0, 0.0, 0.0]" init_simtime_ym_str = "[0.0, 0.0, 0.0]" u = concore.initval(init_simtime_u_str) -ym = concore2.initval(init_simtime_ym_str) +ym = concore.initval(init_simtime_ym_str) # --- ADDED: Initialize a list to store latency values --- zeromq_latencies = [] -print(f"Initial u: {u}, ym: {ym}, concore.simtime: {concore.simtime}, concore2.simtime: {concore2.simtime}") +print(f"Initial u: {u}, ym: {ym}, concore.simtime: {concore.simtime}, concore.simtime: {concore.simtime}") print(f"Max time: {concore.maxtime}") -while concore2.simtime < concore.maxtime: +while concore.simtime < concore.maxtime: while concore.unchanged(): u = concore.read(concore.iport['U'], "u", init_simtime_u_str) @@ -53,18 +49,18 @@ if isinstance(received_ym_data, list) and len(received_ym_data) > 0: response_time = received_ym_data[0] if isinstance(response_time, (int, float)): - concore2.simtime = response_time + concore.simtime = response_time ym = received_ym_data[1:] else: print(f"Warning: Received ZMQ data's first element is not time: {received_ym_data}. Using as is.") ym = received_ym_data else: print(f"Warning: Received unexpected ZMQ data format: {received_ym_data}. Using default ym.") - ym = concore2.initval(init_simtime_ym_str) + ym = concore.initval(init_simtime_ym_str) - concore2.write(concore.oport['Y'], "ym", ym) + concore.write(concore.oport['Y'], "ym", ym) - print(f"funcall ZMQ u={u} ym={ym} time={concore2.simtime} | ZMQ Latency: {latency_ms:.4f} ms") + print(f"funcall ZMQ u={u} ym={ym} time={concore.simtime} | ZMQ Latency: {latency_ms:.4f} ms") print("funcall retry=" + str(concore.retrycount)) diff --git a/measurements/comm_node_test.py b/measurements/comm_node_test.py index dac015ed..465344d0 100644 --- a/measurements/comm_node_test.py +++ b/measurements/comm_node_test.py @@ -1,14 +1,10 @@ import concore -import concore2 import time import sys # --- Script Configuration --- concore.delay = 0.07 -concore2.delay = 0.07 -concore2.inpath = concore.inpath -concore2.outpath = concore.outpath -concore2.simtime = 0 +concore.simtime = 0 concore.default_maxtime(100) # This will be ignored by the new logic init_simtime_u = "[0.0, 0.0, 0.0]" init_simtime_ym = "[0.0, 0.0, 0.0]" @@ -19,7 +15,7 @@ # --- Main Script Logic --- u = concore.initval(init_simtime_u) -ym = concore2.initval(init_simtime_ym) +ym = concore.initval(init_simtime_ym) curr = 0 max_value = 100 iteration = 0 @@ -39,16 +35,16 @@ # Break if the loop condition is met after the first read if curr >= max_value: # Forward a final message to ensure the next node also terminates - concore2.write(concore.oport['Y'], "ym", [curr]) + concore.write(concore.oport['Y'], "ym", [curr]) break # 3. Wait for a message from the 'Y1' channel - old2 = concore2.simtime - while concore2.unchanged() or concore2.simtime <= old2: - ym = concore2.read(concore.iport['Y1'], "ym", init_simtime_ym) + old2 = float(concore.simtime) + while concore.unchanged() or concore.simtime <= old2: + ym = concore.read(concore.iport['Y1'], "ym", init_simtime_ym) # 4. Forward it to the 'Y' channel - concore2.write(concore.oport['Y'], "ym", ym) + concore.write(concore.oport['Y'], "ym", ym) curr = ym[0] print(f"comm_node: u={u[0]:.2f} | ym={ym[0]:.2f}") diff --git a/measurements/readme.md b/measurements/readme.md index f22f9f46..621a296e 100644 --- a/measurements/readme.md +++ b/measurements/readme.md @@ -32,4 +32,7 @@ This folder contains studies and source code specifically for measuring communic Here you will find studies and source code focused on measuring communication times using ZeroMQ within a single system setup. # Network Communication Measurements (Two Systems Required) -For the Latency, Throughput, and CPU Usage measurements, two different systems are required. These systems should be connected over the same network to ensure efficient and accurate communication measurements between them. This setup is crucial for evaluating network-dependent performance metrics effectively. \ No newline at end of file +For the Latency, Throughput, and CPU Usage measurements, two different systems are required. These systems should be connected to the same network to ensure efficient, accurate communication between them. This setup is crucial for effectively evaluating network-dependent performance metrics. + +# Important Note on Port Variables +The measurement benchmark scripts (A.py, B.py, C.py) expect port variables such as `PORT_NAME_F1_F2`, `PORT_F1_F2`, `PORT_NAME_F2_F3`, and `PORT_F2_F3` to be injected by `copy_with_port_portname.py` during study generation. Running these scripts directly will use safe fallback defaults and may not reflect full study behavior. For accurate results, always run measurements through the study generation workflow (e.g., `makestudy`). diff --git a/mkconcore.py b/mkconcore.py index b243c230..514e384d 100644 --- a/mkconcore.py +++ b/mkconcore.py @@ -63,6 +63,7 @@ # - Sets the executable permission (`stat.S_IRWXU`) for the generated scripts on POSIX systems. from bs4 import BeautifulSoup +import atexit import logging import re import sys @@ -71,25 +72,97 @@ import stat import copy_with_port_portname import numpy as np - -MKCONCORE_VER = "22-09-18" - -GRAPHML_FILE = sys.argv[1] -TRIMMED_LOGS = True -CONCOREPATH = "." -CPPWIN = "g++" #Windows C++ 6/22/21 -CPPEXE = "g++" #Ubuntu/macOS C++ 6/22/21 -VWIN = "iverilog" #Windows verilog 6/25/21 -VEXE = "iverilog" #Ubuntu/macOS verilog 6/25/21 -PYTHONEXE = "python3" #Ubuntu/macOS python3 -PYTHONWIN = "python" #Windows python3 -MATLABEXE = "matlab" #Ubuntu/macOS matlab -MATLABWIN = "matlab" #Windows matlab -OCTAVEEXE = "octave" #Ubuntu/macOS octave -OCTAVEWIN = "octave" #Windows octave +import shlex # Added for POSIX shell escaping + +# input validation helper +def safe_name(value, context, allow_path=False): + """ + Validates that the input string does not contain characters dangerous + for filesystem paths or shell command injection. + """ + if not value: + raise ValueError(f"{context} cannot be empty") + # blocks control characters and shell metacharacters + # allow path separators and drive colons for full paths when needed + if allow_path: + pattern = r'[\x00-\x1F\x7F*?"<>|;&`$\'()]' + else: + # blocks path traversal (/, \, :) in addition to shell metacharacters + pattern = r'[\x00-\x1F\x7F\\/:*?"<>|;&`$\'()]' + if re.search(pattern, value): + raise ValueError(f"Unsafe {context}: '{value}' contains illegal characters.") + return value + +def safe_relpath(value, context): + """ + Allow relative subpaths while blocking traversal and absolute/drive paths. + """ + if not value: + raise ValueError(f"{context} cannot be empty") + normalized = value.replace("\\", "/") + safe_name(normalized, context, allow_path=True) + if normalized.startswith("/") or normalized.startswith("~"): + raise ValueError(f"Unsafe {context}: absolute paths are not allowed.") + if re.match(r"^[A-Za-z]:", normalized): + raise ValueError(f"Unsafe {context}: drive paths are not allowed.") + if ":" in normalized: + raise ValueError(f"Unsafe {context}: ':' is not allowed in relative paths.") + if any(part in ("", "..") for part in normalized.split("/")): + raise ValueError(f"Unsafe {context}: invalid path segment.") + return normalized + +MKCONCORE_VER = "22-09-18" + +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) + +def _load_tool_config(filepath): + tools = {} + with open(filepath, "r") as f: + for line in f: + line = line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + k, v = line.split("=", 1) + k, v = k.strip(), v.strip() + if v: + tools[k] = v + return tools + +def _resolve_concore_path(): + script_concore = os.path.join(SCRIPT_DIR, "concore.py") + if os.path.exists(script_concore): + return SCRIPT_DIR + cwd_concore = os.path.join(os.getcwd(), "concore.py") + if os.path.exists(cwd_concore): + return os.getcwd() + return SCRIPT_DIR + +if len(sys.argv) < 4: + print("usage: py mkconcore.py file.graphml sourcedir outdir [type]") + print(" type must be posix (macos or ubuntu), windows, or docker") + sys.exit(1) + +ORIGINAL_CWD = os.getcwd() +GRAPHML_FILE = os.path.abspath(sys.argv[1]) +TRIMMED_LOGS = True +CONCOREPATH = _resolve_concore_path() +CPPWIN = os.environ.get("CONCORE_CPPWIN", "g++") #Windows C++ 6/22/21 +CPPEXE = os.environ.get("CONCORE_CPPEXE", "g++") #Ubuntu/macOS C++ 6/22/21 +VWIN = os.environ.get("CONCORE_VWIN", "iverilog") #Windows verilog 6/25/21 +VEXE = os.environ.get("CONCORE_VEXE", "iverilog") #Ubuntu/macOS verilog 6/25/21 +PYTHONEXE = os.environ.get("CONCORE_PYTHONEXE", "python3") #Ubuntu/macOS python3 +PYTHONWIN = os.environ.get("CONCORE_PYTHONWIN", "python") #Windows python3 +MATLABEXE = os.environ.get("CONCORE_MATLABEXE", "matlab") #Ubuntu/macOS matlab +MATLABWIN = os.environ.get("CONCORE_MATLABWIN", "matlab") #Windows matlab +OCTAVEEXE = os.environ.get("CONCORE_OCTAVEEXE", "octave") #Ubuntu/macOS octave +OCTAVEWIN = os.environ.get("CONCORE_OCTAVEWIN", "octave") #Windows octave +JAVACEXE = os.environ.get("CONCORE_JAVACEXE", "javac") #Ubuntu/macOS javac +JAVACWIN = os.environ.get("CONCORE_JAVACWIN", "javac") #Windows javac +JAVAEXE = os.environ.get("CONCORE_JAVAEXE", "java") #Ubuntu/macOS java +JAVAWIN = os.environ.get("CONCORE_JAVAWIN", "java") #Windows java M_IS_OCTAVE = False #treat .m as octave MCRPATH = "~/MATLAB/R2021a" #path to local Ubunta Matlab Compiler Runtime -DOCKEREXE = "sudo docker"#assume simple docker install +DOCKEREXE = os.environ.get("DOCKEREXE", "docker")#default to docker, allow env override DOCKEREPO = "markgarnold"#where pulls come from 3/28/21 INDIRNAME = ":/in" OUTDIRNAME = ":/out" @@ -104,30 +177,50 @@ M_IS_OCTAVE = True #treat .m as octave 9/27/21 if os.path.exists(CONCOREPATH+"/concore.mcr"): # 11/12/21 - MCRPATH = open(CONCOREPATH+"/concore.mcr", "r").readline().strip() #path to local Ubunta Matlab Compiler Runtime + with open(CONCOREPATH+"/concore.mcr", "r") as f: + MCRPATH = f.readline().strip() #path to local Ubunta Matlab Compiler Runtime if os.path.exists(CONCOREPATH+"/concore.sudo"): # 12/04/21 - DOCKEREXE = open(CONCOREPATH+"/concore.sudo", "r").readline().strip() #to omit sudo in docker + with open(CONCOREPATH+"/concore.sudo", "r") as f: + DOCKEREXE = f.readline().strip() #to omit sudo in docker if os.path.exists(CONCOREPATH+"/concore.repo"): # 12/04/21 - DOCKEREPO = open(CONCOREPATH+"/concore.repo", "r").readline().strip() #docker id for repo - + with open(CONCOREPATH+"/concore.repo", "r") as f: + DOCKEREPO = f.readline().strip() #docker id for repo + +if os.path.exists(CONCOREPATH+"/concore.tools"): + _tools = _load_tool_config(CONCOREPATH+"/concore.tools") + CPPWIN = _tools.get("CPPWIN", CPPWIN) + CPPEXE = _tools.get("CPPEXE", CPPEXE) + VWIN = _tools.get("VWIN", VWIN) + VEXE = _tools.get("VEXE", VEXE) + PYTHONEXE = _tools.get("PYTHONEXE", PYTHONEXE) + PYTHONWIN = _tools.get("PYTHONWIN", PYTHONWIN) + MATLABEXE = _tools.get("MATLABEXE", MATLABEXE) + MATLABWIN = _tools.get("MATLABWIN", MATLABWIN) + OCTAVEEXE = _tools.get("OCTAVEEXE", OCTAVEEXE) + OCTAVEWIN = _tools.get("OCTAVEWIN", OCTAVEWIN) + JAVACEXE = _tools.get("JAVACEXE", JAVACEXE) + JAVACWIN = _tools.get("JAVACWIN", JAVACWIN) + JAVAEXE = _tools.get("JAVAEXE", JAVAEXE) + JAVAWIN = _tools.get("JAVAWIN", JAVAWIN) + +prefixedgenode = "" +sourcedir = os.path.abspath(sys.argv[2]) +outdir = os.path.abspath(sys.argv[3]) + +# Validate outdir argument (allow full paths) +safe_name(outdir, "Output directory argument", allow_path=True) -prefixedgenode = "" -sourcedir = sys.argv[2] -outdir = sys.argv[3] if not os.path.isdir(sourcedir): logging.error(f"{sourcedir} does not exist") quit() -if len(sys.argv) < 4: - logging.error("usage: py mkconcore.py file.graphml sourcedir outdir [type]") - logging.error(" type must be posix (macos or ubuntu), windows, or docker") - quit() -elif len(sys.argv) == 4: - prefixedgenode = outdir+"_" #nodes and edges prefixed with outdir_ only in case no type specified 3/24/21 - concoretype = "docker" -else: +if len(sys.argv) == 4: + # Use only the output directory name in generated prefixes. + prefixedgenode = os.path.basename(os.path.normpath(outdir)) + "_" + concoretype = "docker" +else: concoretype = sys.argv[4] if not (concoretype in ["posix","windows","docker","macos","ubuntu"]): logging.error(" type must be posix (macos or ubuntu), windows, or docker") @@ -144,8 +237,8 @@ logging.error(f"if intended, Remove/Rename {outdir} first") quit() -os.mkdir(outdir) -os.chdir(outdir) +os.makedirs(outdir) +os.chdir(outdir) if concoretype == "windows": fbuild = open("build.bat","w") frun = open("run.bat", "w") @@ -166,8 +259,14 @@ funlock = open("unlock", "w") # 12/4/21 fparams = open("params", "w") # 9/18/22 -os.mkdir("src") -os.chdir("..") +def cleanup_script_files(): + for fh in [fbuild, frun, fdebug, fstop, fclear, fmaxtime, funlock, fparams]: + if not fh.closed: + fh.close() +atexit.register(cleanup_script_files) + +os.mkdir("src") +os.chdir(ORIGINAL_CWD) logging.info(f"mkconcore {MKCONCORE_VER}") logging.info(f"Concore path: {CONCOREPATH}") @@ -179,8 +278,8 @@ logging.info(f"MCR path: {MCRPATH}") logging.info(f"Docker repository: {DOCKEREPO}") -f = open(GRAPHML_FILE, "r") -text_str = f.read() +with open(GRAPHML_FILE, "r") as f: + text_str = f.read() soup = BeautifulSoup(text_str, 'xml') @@ -200,11 +299,30 @@ node_label_tag = data.find('y:NodeLabel') if node_label_tag: node_label = prefixedgenode + node_label_tag.text - nodes_dict[node['id']] = re.sub(r'(\s+|\n)', ' ', node_label) + node_label = re.sub(r'(\s+|\n)', ' ', node_label) + + #Validate node labels + if ':' in node_label: + container_part, source_part = node_label.split(':', 1) + safe_name(container_part, f"Node container name '{container_part}'") + source_part = safe_relpath(source_part, f"Node source file '{source_part}'") + node_label = f"{container_part}:{source_part}" + else: + safe_name(node_label, f"Node label '{node_label}'") + # Explicitly reject incorrect format to prevent later crashes and ambiguity + raise ValueError(f"Invalid node label '{node_label}': expected format 'container:source' with a ':' separator.") + + nodes_dict[node['id']] = node_label node_id_to_label_map[node['id']] = node_label.split(':')[0] except (IndexError, AttributeError): logging.debug('A node with no valid properties encountered and ignored') +label_values = list(nodes_dict.values()) +duplicates = {label for label in label_values if label_values.count(label) > 1} +if duplicates: + logging.error(f"Duplicate node labels found: {sorted(duplicates)}") + quit() + for edge in edges_text: try: data = edge.find('data', recursive=False) @@ -215,6 +333,10 @@ edge_label = prefixedgenode + raw_label # Filter out ZMQ edges from the file-based edge dictionary by checking the raw label if not edge_label_regex.match(raw_label): + + #Validate edge labels + safe_name(edge_label, f"Edge label '{edge_label}'") + if edge_label not in edges_dict: edges_dict[edge_label] = [nodes_dict[edge['source']], []] edges_dict[edge_label][1].append(nodes_dict[edge['target']]) @@ -312,40 +434,69 @@ logging.warning(f"Error processing edge for parameter aggregation: {e}") # --- Now, run the specialization for each node that has aggregated parameters --- -if node_edge_params: - logging.info("Running script specialization process...") - specialized_scripts_output_dir = os.path.abspath(os.path.join(outdir, "src")) - os.makedirs(specialized_scripts_output_dir, exist_ok=True) - - for node_id, params_list in node_edge_params.items(): - current_node_full_label = nodes_dict[node_id] - try: - container_name, original_script = current_node_full_label.split(':', 1) - except ValueError: - continue # Skip if label format is wrong - - if not original_script or "." not in original_script: - continue # Skip if not a script file - - template_script_full_path = os.path.join(sourcedir, original_script) - if not os.path.exists(template_script_full_path): - logging.error(f"Cannot specialize: Original script '{template_script_full_path}' not found in '{sourcedir}'.") - continue - - new_script_basename = copy_with_port_portname.run_specialization_script( - template_script_full_path, - specialized_scripts_output_dir, - params_list, - python_executable, - copy_script_py_path - ) - - if new_script_basename: - # Update nodes_dict to point to the new comprehensive specialized script - nodes_dict[node_id] = f"{container_name}:{new_script_basename}" - logging.info(f"Node ID '{node_id}' ('{container_name}') updated to use specialized script '{new_script_basename}'.") - else: - logging.error(f"Failed to generate specialized script for node ID '{node_id}'. It will retain its original script.") +if node_edge_params: + logging.info("Running script specialization process...") + specialized_scripts_output_dir = os.path.abspath(os.path.join(outdir, "src")) + os.makedirs(specialized_scripts_output_dir, exist_ok=True) + + # Build one specialization plan per source script. This avoids collisions + # when multiple nodes reference the same script and need different ZMQ params. + script_edge_params = {} + script_nodes = {} + for node_id, params_list in node_edge_params.items(): + current_node_full_label = nodes_dict.get(node_id, "") + try: + container_name, original_script = current_node_full_label.split(':', 1) + except ValueError: + continue + + if not original_script or "." not in original_script: + continue + + script_nodes.setdefault(original_script, []).append((node_id, container_name)) + script_edge_params.setdefault(original_script, []) + seen_keys = { + ( + p.get("port"), + p.get("port_name"), + p.get("source_node_label"), + p.get("target_node_label") + ) + for p in script_edge_params[original_script] + } + for edge_param in params_list: + edge_key = ( + edge_param.get("port"), + edge_param.get("port_name"), + edge_param.get("source_node_label"), + edge_param.get("target_node_label") + ) + if edge_key not in seen_keys: + script_edge_params[original_script].append(edge_param) + seen_keys.add(edge_key) + + for original_script, merged_params in script_edge_params.items(): + template_script_full_path = os.path.join(sourcedir, original_script) + if not os.path.exists(template_script_full_path): + logging.error(f"Cannot specialize: Original script '{template_script_full_path}' not found in '{sourcedir}'.") + continue + + new_script_relpath = copy_with_port_portname.run_specialization_script( + template_script_full_path, + specialized_scripts_output_dir, + merged_params, + python_executable, + copy_script_py_path, + output_relpath=original_script + ) + + if not new_script_relpath: + logging.error(f"Failed to generate specialized script for source '{original_script}'.") + continue + + for node_id, container_name in script_nodes.get(original_script, []): + nodes_dict[node_id] = f"{container_name}:{new_script_relpath}" + logging.info(f"Node ID '{node_id}' ('{container_name}') updated to use specialized script '{new_script_relpath}'.") #not right for PM2_1_1 and PM2_1_2 volswr = len(nodes_dict)*[''] @@ -380,12 +531,15 @@ if not sourcecode: continue - if "." in sourcecode: - dockername, langext = os.path.splitext(sourcecode) - else: - dockername, langext = sourcecode, "" - - script_target_path = os.path.join(outdir, "src", sourcecode) + if "." in sourcecode: + dockername, langext = os.path.splitext(sourcecode) + else: + dockername, langext = sourcecode, "" + + script_target_path = os.path.join(outdir, "src", sourcecode) + script_target_parent = os.path.dirname(script_target_path) + if script_target_parent: + os.makedirs(script_target_parent, exist_ok=True) # If the script was specialized, it's already in outdir/src. If not, copy from sourcedir. if node_id_key not in node_edge_params: @@ -407,124 +561,115 @@ shutil.copytree(os.path.join(sourcedir, dir_for_node), os.path.join(outdir, "src", dir_for_node), dirs_exist_ok=True) -#copy proper concore.py into /src -try: - if concoretype=="docker": - fsource = open(CONCOREPATH+"/concoredocker.py") - else: - fsource = open(CONCOREPATH+"/concore.py") -except: - logging.error(f"{CONCOREPATH} is not correct path to concore") - quit() -with open(outdir+"/src/concore.py","w") as fcopy: - fcopy.write(fsource.read()) -fsource.close() - -#copy proper concore.hpp into /src 6/22/21 -try: - if concoretype=="docker": - fsource = open(CONCOREPATH+"/concoredocker.hpp") - else: - fsource = open(CONCOREPATH+"/concore.hpp") -except: - logging.error(f"{CONCOREPATH} is not correct path to concore") - quit() -with open(outdir+"/src/concore.hpp","w") as fcopy: - fcopy.write(fsource.read()) -fsource.close() - -#copy proper concore.v into /src 6/25/21 -try: - if concoretype=="docker": - fsource = open(CONCOREPATH+"/concoredocker.v") - else: - fsource = open(CONCOREPATH+"/concore.v") -except: - logging.error(f"{CONCOREPATH} is not correct path to concore") - quit() -with open(outdir+"/src/concore.v","w") as fcopy: - fcopy.write(fsource.read()) -fsource.close() - -#copy mkcompile into /src 5/27/21 -try: - fsource = open(CONCOREPATH+"/mkcompile") -except: - logging.error(f"{CONCOREPATH} is not correct path to concore") - quit() -with open(outdir+"/src/mkcompile","w") as fcopy: - fcopy.write(fsource.read()) -fsource.close() -os.chmod(outdir+"/src/mkcompile",stat.S_IRWXU) - -#copy concore*.m into /src 4/2/21 -try: #maxtime in matlab 11/22/21 - fsource = open(CONCOREPATH+"/concore_default_maxtime.m") -except: - logging.error(f"{CONCOREPATH} is not correct path to concore") - quit() -with open(outdir+"/src/concore_default_maxtime.m","w") as fcopy: - fcopy.write(fsource.read()) -fsource.close() -try: - fsource = open(CONCOREPATH+"/concore_unchanged.m") -except: - logging.error(f"{CONCOREPATH} is not correct path to concore") - quit() -with open(outdir+"/src/concore_unchanged.m","w") as fcopy: - fcopy.write(fsource.read()) -fsource.close() -try: - fsource = open(CONCOREPATH+"/concore_read.m") -except: - logging.error(f"{CONCOREPATH} is not correct path to concore") - quit() -with open(outdir+"/src/concore_read.m","w") as fcopy: - fcopy.write(fsource.read()) -fsource.close() -try: - fsource = open(CONCOREPATH+"/concore_write.m") -except: - logging.error(f"{CONCOREPATH} is not correct path to concore") - quit() -with open(outdir+"/src/concore_write.m","w") as fcopy: - fcopy.write(fsource.read()) -fsource.close() -try: #4/9/21 - fsource = open(CONCOREPATH+"/concore_initval.m") -except: - logging.error(f"{CONCOREPATH} is not correct path to concore") - quit() -with open(outdir+"/src/concore_initval.m","w") as fcopy: - fcopy.write(fsource.read()) -fsource.close() -try: #11/19/21 - fsource = open(CONCOREPATH+"/concore_iport.m") -except: - logging.error(f"{CONCOREPATH} is not correct path to concore") - quit() -with open(outdir+"/src/concore_iport.m","w") as fcopy: - fcopy.write(fsource.read()) -fsource.close() -try: #11/19/21 - fsource = open(CONCOREPATH+"/concore_oport.m") -except: - logging.error(f"{CONCOREPATH} is not correct path to concore") - quit() -with open(outdir+"/src/concore_oport.m","w") as fcopy: - fcopy.write(fsource.read()) -fsource.close() -try: # 4/4/21 - if concoretype=="docker": - fsource = open(CONCOREPATH+"/import_concoredocker.m") - else: - fsource = open(CONCOREPATH+"/import_concore.m") -except: - logging.error(f"{CONCOREPATH} is not correct path to concore") - quit() -with open(outdir+"/src/import_concore.m","w") as fcopy: - fcopy.write(fsource.read()) -fsource.close() +#determine languages used in the graphml +required_langs = set() +for node in nodes_dict: + containername, sourcecode = nodes_dict[node].split(':') + if len(sourcecode) != 0 and "." in sourcecode: + langext = sourcecode.split(".")[-1] + required_langs.add(langext) +logging.info(f"Languages detected in graph: {required_langs}") + +if 'py' in required_langs: + try: + if concoretype=="docker": + fsource = open(CONCOREPATH+"/concoredocker.py") + else: + fsource = open(CONCOREPATH+"/concore.py") + except (FileNotFoundError, IOError): + print(CONCOREPATH+" is not correct path to concore (missing python files)") + quit() + with open(outdir+"/src/concore.py","w") as fcopy: + fcopy.write(fsource.read()) + fsource.close() + try: + with open(CONCOREPATH+"/concore_base.py") as fbase: + with open(outdir+"/src/concore_base.py","w") as fcopy: + fcopy.write(fbase.read()) + except (FileNotFoundError, IOError): + print(CONCOREPATH+" is not correct path to concore (missing concore_base.py)") + quit() + +if 'cpp' in required_langs: + try: + if concoretype=="docker": + fsource = open(CONCOREPATH+"/concoredocker.hpp") + else: + fsource = open(CONCOREPATH+"/concore.hpp") + except (FileNotFoundError, IOError): + print(CONCOREPATH+" is not correct path to concore (missing C++ files)") + quit() + with open(outdir+"/src/concore.hpp","w") as fcopy: + fcopy.write(fsource.read()) + fsource.close() + +if 'v' in required_langs: + try: + if concoretype=="docker": + fsource = open(CONCOREPATH+"/concoredocker.v") + else: + fsource = open(CONCOREPATH+"/concore.v") + except (FileNotFoundError, IOError): + print(CONCOREPATH+" is not correct path to concore (missing Verilog files)") + quit() + with open(outdir+"/src/concore.v","w") as fcopy: + fcopy.write(fsource.read()) + fsource.close() + +if 'java' in required_langs and concoretype != "docker": + try: + fsource = open(CONCOREPATH+"/concore.java") + except (FileNotFoundError, IOError): + print(CONCOREPATH+" is not correct path to concore (missing Java files)") + quit() + with open(outdir+"/src/concore.java","w") as fcopy: + fcopy.write(fsource.read()) + fsource.close() + +if 'm' in required_langs: + try: + fsource = open(CONCOREPATH+"/concore_default_maxtime.m") + with open(outdir+"/src/concore_default_maxtime.m","w") as fcopy: fcopy.write(fsource.read()) + fsource.close() + + fsource = open(CONCOREPATH+"/concore_unchanged.m") + with open(outdir+"/src/concore_unchanged.m","w") as fcopy: fcopy.write(fsource.read()) + fsource.close() + + fsource = open(CONCOREPATH+"/concore_read.m") + with open(outdir+"/src/concore_read.m","w") as fcopy: fcopy.write(fsource.read()) + fsource.close() + + fsource = open(CONCOREPATH+"/concore_write.m") + with open(outdir+"/src/concore_write.m","w") as fcopy: fcopy.write(fsource.read()) + fsource.close() + + fsource = open(CONCOREPATH+"/concore_initval.m") + with open(outdir+"/src/concore_initval.m","w") as fcopy: fcopy.write(fsource.read()) + fsource.close() + + fsource = open(CONCOREPATH+"/concore_iport.m") + with open(outdir+"/src/concore_iport.m","w") as fcopy: fcopy.write(fsource.read()) + fsource.close() + + fsource = open(CONCOREPATH+"/concore_oport.m") + with open(outdir+"/src/concore_oport.m","w") as fcopy: fcopy.write(fsource.read()) + fsource.close() + + if concoretype=="docker": + fsource = open(CONCOREPATH+"/import_concoredocker.m") + else: + fsource = open(CONCOREPATH+"/import_concore.m") + with open(outdir+"/src/import_concore.m","w") as fcopy: fcopy.write(fsource.read()) + fsource.close() + + fsource = open(CONCOREPATH+"/mkcompile") + with open(outdir+"/src/mkcompile","w") as fcopy: fcopy.write(fsource.read()) + fsource.close() + os.chmod(outdir+"/src/mkcompile",stat.S_IRWXU) + except Exception as e: + print(CONCOREPATH+" is not correct path to concore (missing MATLAB files):", e) + quit() # --- Generate iport and oport mappings --- logging.info("Generating iport/oport mappings...") @@ -571,47 +716,58 @@ # 4. Write final iport/oport files logging.info("Writing .iport and .oport files...") -for node_label, ports in node_port_mappings.items(): +for node_label, ports in node_port_mappings.items(): try: containername, sourcecode = node_label.split(':', 1) - if not sourcecode or "." not in sourcecode: continue - dockername = os.path.splitext(sourcecode)[0] - with open(os.path.join(outdir, "src", f"{dockername}.iport"), "w") as fport: - fport.write(str(ports['iport']).replace("'" + prefixedgenode, "'")) - with open(os.path.join(outdir, "src", f"{dockername}.oport"), "w") as fport: - fport.write(str(ports['oport']).replace("'" + prefixedgenode, "'")) + if not sourcecode or "." not in sourcecode: continue + dockername = os.path.splitext(sourcecode)[0] + iport_path = os.path.join(outdir, "src", f"{dockername}.iport") + oport_path = os.path.join(outdir, "src", f"{dockername}.oport") + iport_parent = os.path.dirname(iport_path) + if iport_parent: + os.makedirs(iport_parent, exist_ok=True) + with open(iport_path, "w") as fport: + fport.write(str(ports['iport']).replace("'" + prefixedgenode, "'")) + with open(oport_path, "w") as fport: + fport.write(str(ports['oport']).replace("'" + prefixedgenode, "'")) except ValueError: continue #if docker, make docker-dirs, generate build, run, stop, clear scripts and quit -if (concoretype=="docker"): - for node in nodes_dict: - containername,sourcecode = nodes_dict[node].split(':') - if len(sourcecode)!=0 and sourcecode.find(".")!=-1: #3/28/21 - dockername,langext = sourcecode.split(".") - if not os.path.exists(outdir+"/src/Dockerfile."+dockername): # 3/30/21 - try: +if (concoretype=="docker"): + for node in nodes_dict: + containername,sourcecode = nodes_dict[node].split(':') + if len(sourcecode)!=0 and sourcecode.find(".")!=-1: #3/28/21 + dockername,langext = sourcecode.rsplit(".", 1) + dockerfile_path = os.path.join(outdir, "src", f"Dockerfile.{dockername}") + if not os.path.exists(dockerfile_path): # 3/30/21 + try: if langext=="py": - fsource = open(CONCOREPATH+"/Dockerfile.py") + src_path = CONCOREPATH+"/Dockerfile.py" logging.info("assuming .py extension for Dockerfile") elif langext == "cpp": # 6/22/21 - fsource = open(CONCOREPATH+"/Dockerfile.cpp") + src_path = CONCOREPATH+"/Dockerfile.cpp" logging.info("assuming .cpp extension for Dockerfile") elif langext == "v": # 6/26/21 - fsource = open(CONCOREPATH+"/Dockerfile.v") + src_path = CONCOREPATH+"/Dockerfile.v" logging.info("assuming .v extension for Dockerfile") elif langext == "sh": # 5/19/21 - fsource = open(CONCOREPATH+"/Dockerfile.sh") + src_path = CONCOREPATH+"/Dockerfile.sh" logging.info("assuming .sh extension for Dockerfile") else: - fsource = open(CONCOREPATH+"/Dockerfile.m") + src_path = CONCOREPATH+"/Dockerfile.m" logging.info("assuming .m extension for Dockerfile") - except: - logging.error(f"{CONCOREPATH} is not correct path to concore") - quit() - with open(outdir+"/src/Dockerfile."+dockername,"w") as fcopy: - fcopy.write(fsource.read()) + with open(src_path) as fsource: + source_content = fsource.read() + except: + logging.error(f"{CONCOREPATH} is not correct path to concore") + quit() + dockerfile_parent = os.path.dirname(dockerfile_path) + if dockerfile_parent: + os.makedirs(dockerfile_parent, exist_ok=True) + with open(dockerfile_path,"w") as fcopy: + fcopy.write(source_content) if langext=="py": fcopy.write('CMD ["python", "-i", "'+sourcecode+'"]\n') if langext=="m": @@ -622,20 +778,21 @@ if langext=="v": fcopy.write('RUN iverilog ./'+sourcecode+'\n') # 7/02/21 fcopy.write('CMD ["./a.out"]\n') # 7/02/21 - fsource.close() fbuild.write('#!/bin/bash' + "\n") for node in nodes_dict: containername,sourcecode = nodes_dict[node].split(':') if len(sourcecode)!=0 and sourcecode.find(".")!=-1: #3/28/21 - dockername,langext = sourcecode.split(".") - fbuild.write("mkdir docker-"+dockername+"\n") - fbuild.write("cd docker-"+dockername+"\n") + dockername,langext = sourcecode.rsplit(".", 1) + dockerbuilddir = "docker-"+dockername.replace("/", "__").replace("\\", "__") + fbuild.write("mkdir "+dockerbuilddir+"\n") + fbuild.write("cd "+dockerbuilddir+"\n") fbuild.write("cp ../src/Dockerfile."+dockername+" Dockerfile\n") #copy sourcefiles from ./src into corresponding directories fbuild.write("cp ../src/"+sourcecode+" .\n") if langext == "py": #4/29/21 fbuild.write("cp ../src/concore.py .\n") + fbuild.write("cp ../src/concore_base.py .\n") elif langext == "cpp": #6/22/21 fbuild.write("cp ../src/concore.hpp .\n") elif langext == "v": #6/25/21 @@ -660,19 +817,16 @@ for node in nodes_dict: containername,sourcecode = nodes_dict[node].split(':') if len(sourcecode)!=0: + safe_container = shlex.quote(containername) if sourcecode.find(".")==-1: logging.debug(f"Generating Docker run command: {DOCKEREXE} run --name={containername+volswr[i]+volsro[i]} {DOCKEREPO}/docker- {sourcecode}") - frun.write(DOCKEREXE+' run --name='+containername+volswr[i]+volsro[i]+" "+DOCKEREPO+"/docker-"+sourcecode+"&\n") + # Use safe_container + frun.write(DOCKEREXE+' run --name='+safe_container+volswr[i]+volsro[i]+" "+DOCKEREPO+"/docker-"+shlex.quote(sourcecode)+"&\n") else: - dockername,langext = sourcecode.split(".") + dockername,langext = sourcecode.rsplit(".", 1) logging.debug(f"Generating Docker run command for {dockername}: {DOCKEREXE} run --name={containername+volswr[i]+volsro[i]} docker-{dockername}") - frun.write(DOCKEREXE+' run --name='+containername+volswr[i]+volsro[i]+" docker-"+dockername+"&\n") - #if langext != "m": #3/27/21 - # print(DOCKEREXE+' run --name='+containername+volswr[i]+volsro[i]+" docker-"+dockername) - # frun.write(DOCKEREXE+' run --name='+containername+volswr[i]+volsro[i]+" docker-"+dockername+"&\n") - #else: - # print(DOCKEREXE+' run --name='+containername+volswr[i]+volsro[i]+" docker-"+dockername+' octave -qf --eval "run('+"'"+sourcecode+"'"+')"'+"&\n") - # frun.write(DOCKEREXE+' run --name='+containername+volswr[i]+volsro[i]+" docker-"+dockername+' octave -qf --eval "run('+"'"+sourcecode+"'"+')"'+"&\n") + # Use safe_container + frun.write(DOCKEREXE+' run --name='+safe_container+volswr[i]+volsro[i]+" docker-"+shlex.quote(dockername)+"&\n") i=i+1 frun.close() @@ -681,10 +835,11 @@ for node in nodes_dict: containername,sourcecode = nodes_dict[node].split(':') if len(sourcecode)!=0: - #dockername,langext = sourcecode.split(".") - dockername = sourcecode.split(".")[0] # 3/28/21 - fstop.write(DOCKEREXE+' stop '+containername+"\n") - fstop.write(DOCKEREXE+' rm '+containername+"\n") + #dockername,langext = sourcecode.rsplit(".", 1) + dockername = sourcecode.rsplit(".", 1)[0] # 3/28/21 + safe_container = shlex.quote(containername) + fstop.write(DOCKEREXE+' stop '+safe_container+"\n") + fstop.write(DOCKEREXE+' rm '+safe_container+"\n") i=i+1 fstop.close() @@ -693,10 +848,12 @@ for node in nodes_dict: containername,sourcecode = nodes_dict[node].split(':') if len(sourcecode)!=0: - dockername = sourcecode.split(".")[0] #3/28/21 + dockername = sourcecode.rsplit(".", 1)[0] #3/28/21 writeedges = volswr[i] while writeedges.find(":") != -1: - fclear.write(DOCKEREXE+' volume rm ' +writeedges.split(":")[0].split("-v")[1]+"\n") + #scape volume path using shlex.quote for Docker commands (defense-in-depth) + volume_path = writeedges.split(":")[0].split("-v")[1].strip() + fclear.write(DOCKEREXE+' volume rm ' + shlex.quote(volume_path) +"\n") # Added strip() and quote writeedges = writeedges[writeedges.find(":")+1:] i=i+1 fclear.close() @@ -704,19 +861,21 @@ fmaxtime.write('#!/bin/bash' + "\n") fmaxtime.write('echo "$1" >concore.maxtime\n') fmaxtime.write('echo "FROM alpine:3.8" > Dockerfile\n') - fmaxtime.write('sudo docker build -t docker-concore .\n') - fmaxtime.write('sudo docker run --name=concore') + fmaxtime.write(f'{DOCKEREXE} build -t docker-concore .\n') + fmaxtime.write(f'{DOCKEREXE} run --name=concore') # -v VCZ:/VCZ -v VPZ:/VPZ i=0 # 9/12/21 for node in nodes_dict: containername,sourcecode = nodes_dict[node].split(':') if len(sourcecode)!=0: - dockername = sourcecode.split(".")[0] #3/28/21 + dockername = sourcecode.rsplit(".", 1)[0] #3/28/21 writeedges = volswr[i] while writeedges.find(":") != -1: fmaxtime.write(' -v ') - fmaxtime.write(writeedges.split(":")[0].split("-v ")[1]+":/") - fmaxtime.write(writeedges.split(":")[0].split("-v ")[1]) + # escape volume paths in Docker run + vol_path = writeedges.split(":")[0].split("-v ")[1].strip() + fmaxtime.write(shlex.quote(vol_path)+":/") + fmaxtime.write(shlex.quote(vol_path)) writeedges = writeedges[writeedges.find(":")+1:] i=i+1 fmaxtime.write(' docker-concore >/dev/null &\n') @@ -726,16 +885,18 @@ for node in nodes_dict: containername,sourcecode = nodes_dict[node].split(':') if len(sourcecode)!=0: - dockername = sourcecode.split(".")[0] #3/28/21 + dockername = sourcecode.rsplit(".", 1)[0] #3/28/21 writeedges = volswr[i] while writeedges.find(":") != -1: - fmaxtime.write('sudo docker cp concore.maxtime concore:/') - fmaxtime.write(writeedges.split(":")[0].split("-v ")[1]+"/concore.maxtime\n") + fmaxtime.write(f'{DOCKEREXE} cp concore.maxtime concore:/') + # escape destination path in docker cp + vol_path = writeedges.split(":")[0].split("-v ")[1].strip() + fmaxtime.write(shlex.quote(vol_path+"/concore.maxtime")+"\n") writeedges = writeedges[writeedges.find(":")+1:] i=i+1 - fmaxtime.write('sudo docker stop concore \n') - fmaxtime.write('sudo docker rm concore\n') - fmaxtime.write('sudo docker rmi docker-concore\n') + fmaxtime.write(f'{DOCKEREXE} stop concore \n') + fmaxtime.write(f'{DOCKEREXE} rm concore\n') + fmaxtime.write(f'{DOCKEREXE} rmi docker-concore\n') fmaxtime.write('rm Dockerfile\n') fmaxtime.write('rm concore.maxtime\n') fmaxtime.close() @@ -743,19 +904,21 @@ fparams.write('#!/bin/bash' + "\n") fparams.write('echo "$1" >concore.params\n') fparams.write('echo "FROM alpine:3.8" > Dockerfile\n') - fparams.write('sudo docker build -t docker-concore .\n') - fparams.write('sudo docker run --name=concore') + fparams.write(f'{DOCKEREXE} build -t docker-concore .\n') + fparams.write(f'{DOCKEREXE} run --name=concore') # -v VCZ:/VCZ -v VPZ:/VPZ i=0 # 9/12/21 for node in nodes_dict: containername,sourcecode = nodes_dict[node].split(':') if len(sourcecode)!=0: - dockername = sourcecode.split(".")[0] #3/28/21 + dockername = sourcecode.rsplit(".", 1)[0] #3/28/21 writeedges = volswr[i] while writeedges.find(":") != -1: fparams.write(' -v ') - fparams.write(writeedges.split(":")[0].split("-v ")[1]+":/") - fparams.write(writeedges.split(":")[0].split("-v ")[1]) + #escape volume paths + vol_path = writeedges.split(":")[0].split("-v ")[1].strip() + fparams.write(shlex.quote(vol_path)+":/") + fparams.write(shlex.quote(vol_path)) writeedges = writeedges[writeedges.find(":")+1:] i=i+1 fparams.write(' docker-concore >/dev/null &\n') @@ -765,35 +928,39 @@ for node in nodes_dict: containername,sourcecode = nodes_dict[node].split(':') if len(sourcecode)!=0: - dockername = sourcecode.split(".")[0] #3/28/21 + dockername = sourcecode.rsplit(".", 1)[0] #3/28/21 writeedges = volswr[i] while writeedges.find(":") != -1: - fparams.write('sudo docker cp concore.params concore:/') - fparams.write(writeedges.split(":")[0].split("-v ")[1]+"/concore.params\n") + fparams.write(f'{DOCKEREXE} cp concore.params concore:/') + # escape destination path + vol_path = writeedges.split(":")[0].split("-v ")[1].strip() + fparams.write(shlex.quote(vol_path+"/concore.params")+"\n") writeedges = writeedges[writeedges.find(":")+1:] i=i+1 - fparams.write('sudo docker stop concore \n') - fparams.write('sudo docker rm concore\n') - fparams.write('sudo docker rmi docker-concore\n') + fparams.write(f'{DOCKEREXE} stop concore \n') + fparams.write(f'{DOCKEREXE} rm concore\n') + fparams.write(f'{DOCKEREXE} rmi docker-concore\n') fparams.write('rm Dockerfile\n') fparams.write('rm concore.params\n') fparams.close() funlock.write('#!/bin/bash' + "\n") funlock.write('echo "FROM alpine:3.8" > Dockerfile\n') - funlock.write('sudo docker build -t docker-concore .\n') - funlock.write('sudo docker run --name=concore') + funlock.write(f'{DOCKEREXE} build -t docker-concore .\n') + funlock.write(f'{DOCKEREXE} run --name=concore') # -v VCZ:/VCZ -v VPZ:/VPZ i=0 # 9/12/21 for node in nodes_dict: containername,sourcecode = nodes_dict[node].split(':') if len(sourcecode)!=0: - dockername = sourcecode.split(".")[0] #3/28/21 + dockername = sourcecode.rsplit(".", 1)[0] #3/28/21 writeedges = volswr[i] while writeedges.find(":") != -1: funlock.write(' -v ') - funlock.write(writeedges.split(":")[0].split("-v ")[1]+":/") - funlock.write(writeedges.split(":")[0].split("-v ")[1]) + # escape volume paths + vol_path = writeedges.split(":")[0].split("-v ")[1].strip() + funlock.write(shlex.quote(vol_path)+":/") + funlock.write(shlex.quote(vol_path)) writeedges = writeedges[writeedges.find(":")+1:] i=i+1 funlock.write(' docker-concore >/dev/null &\n') @@ -803,16 +970,18 @@ for node in nodes_dict: containername,sourcecode = nodes_dict[node].split(':') if len(sourcecode)!=0: - dockername = sourcecode.split(".")[0] #3/28/21 + dockername = sourcecode.rsplit(".", 1)[0] #3/28/21 writeedges = volswr[i] while writeedges.find(":") != -1: - funlock.write('sudo docker cp ~/concore.apikey concore:/') - funlock.write(writeedges.split(":")[0].split("-v ")[1]+"/concore.apikey\n") + funlock.write(f'{DOCKEREXE} cp ~/concore.apikey concore:/') + # escape destination path + vol_path = writeedges.split(":")[0].split("-v ")[1].strip() + funlock.write(shlex.quote(vol_path+"/concore.apikey")+"\n") writeedges = writeedges[writeedges.find(":")+1:] i=i+1 - funlock.write('sudo docker stop concore \n') - funlock.write('sudo docker rm concore\n') - funlock.write('sudo docker rmi docker-concore\n') + funlock.write(f'{DOCKEREXE} stop concore \n') + funlock.write(f'{DOCKEREXE} rm concore\n') + funlock.write(f'{DOCKEREXE} rmi docker-concore\n') funlock.write('rm Dockerfile\n') funlock.close() @@ -821,8 +990,11 @@ for node in nodes_dict: containername,sourcecode = nodes_dict[node].split(':') if len(sourcecode)!=0 and sourcecode.find(".")!=-1: #3/28/21 - dockername,langext = sourcecode.split(".") - fdebug.write(DOCKEREXE+' run -it --name='+containername+volswr[i]+volsro[i]+" docker-"+dockername+"&\n") + dockername,langext = sourcecode.rsplit(".", 1) + # safe_container added to debug line (POSIX) + safe_container = shlex.quote(containername) + safe_image = shlex.quote("docker-" + dockername) # escape docker image name + fdebug.write(DOCKEREXE+' run -it --name='+safe_container+volswr[i]+volsro[i]+" "+safe_image+"&\n") i=i+1 fdebug.close() os.chmod(outdir+"/build",stat.S_IRWXU) @@ -841,24 +1013,33 @@ if concoretype=="posix": fbuild.write('#!/bin/bash' + "\n") -for node in nodes_dict: - containername,sourcecode = nodes_dict[node].split(':') - if len(sourcecode)!=0: - if sourcecode.find(".")==-1: - logging.error("cannot pull container "+sourcecode+" with control core type "+concoretype) #3/28/21 - quit() - dockername,langext = sourcecode.split(".") - fbuild.write('mkdir '+containername+"\n") - if concoretype == "windows": - fbuild.write("copy .\\src\\"+sourcecode+" .\\"+containername+"\\"+sourcecode+"\n") +for node in nodes_dict: + containername,sourcecode = nodes_dict[node].split(':') + if len(sourcecode)!=0: + if sourcecode.find(".")==-1: + logging.error("cannot pull container "+sourcecode+" with control core type "+concoretype) #3/28/21 + quit() + dockername,langext = sourcecode.rsplit(".", 1) + fbuild.write('mkdir '+containername+"\n") + source_subdir = os.path.dirname(sourcecode).replace("\\", "/") + if source_subdir: + if concoretype == "windows": + fbuild.write("mkdir .\\"+containername+"\\"+source_subdir.replace("/", "\\")+"\n") + else: + fbuild.write("mkdir -p ./"+containername+"/"+source_subdir+"\n") + if concoretype == "windows": + fbuild.write("copy .\\src\\"+sourcecode+" .\\"+containername+"\\"+sourcecode+"\n") if langext == "py": fbuild.write("copy .\\src\\concore.py .\\" + containername + "\\concore.py\n") + fbuild.write("copy .\\src\\concore_base.py .\\" + containername + "\\concore_base.py\n") elif langext == "cpp": # 6/22/21 fbuild.write("copy .\\src\\concore.hpp .\\" + containername + "\\concore.hpp\n") elif langext == "v": # 6/25/21 fbuild.write("copy .\\src\\concore.v .\\" + containername + "\\concore.v\n") + elif langext == "java": + fbuild.write("copy .\\src\\concore.java .\\" + containername + "\\concore.java\n") elif langext == "m": # 4/2/21 fbuild.write("copy .\\src\\concore_*.m .\\" + containername + "\\\n") fbuild.write("copy .\\src\\import_concore.m .\\" + containername + "\\\n") @@ -871,10 +1052,13 @@ fbuild.write("cp ./src/"+sourcecode+" ./"+containername+"/"+sourcecode+"\n") if langext == "py": fbuild.write("cp ./src/concore.py ./"+containername+"/concore.py\n") + fbuild.write("cp ./src/concore_base.py ./"+containername+"/concore_base.py\n") elif langext == "cpp": fbuild.write("cp ./src/concore.hpp ./"+containername+"/concore.hpp\n") elif langext == "v": fbuild.write("cp ./src/concore.v ./"+containername+"/concore.v\n") + elif langext == "java": + fbuild.write("cp ./src/concore.java ./"+containername+"/concore.java\n") elif langext == "m": # 4/2/21 fbuild.write("cp ./src/concore_*.m ./"+containername+"/\n") fbuild.write("cp ./src/import_concore.m ./"+containername+"/\n") @@ -898,7 +1082,7 @@ containername,sourcecode = edges_dict[edges][0].split(':') outcount[nodes_num[edges_dict[edges][0]]] += 1 if len(sourcecode)!=0: - dockername,langext = sourcecode.split(".") + dockername,langext = sourcecode.rsplit(".", 1) fbuild.write('cd '+containername+"\n") if concoretype=="windows": fbuild.write("mklink /J out"+str(outcount[nodes_num[edges_dict[edges][0]]])+" ..\\"+str(edges)+"\n") @@ -911,7 +1095,7 @@ for node in nodes_dict: containername,sourcecode = nodes_dict[node].split(':') if len(sourcecode)!=0: - dockername,langext = sourcecode.split(".") + dockername,langext = sourcecode.rsplit(".", 1) fbuild.write('cd '+containername+"\n") for pair in indir[i]: volname,dirname = pair.split(':/') @@ -932,109 +1116,152 @@ for node in nodes_dict: containername,sourcecode = nodes_dict[node].split(':') if len(sourcecode)!=0: - dockername,langext = sourcecode.split(".") - if not (langext in ["py","m","sh","cpp","v"]): # 6/22/21 + dockername,langext = sourcecode.rsplit(".", 1) + if not (langext in ["py","m","sh","cpp","v","java"]): # 6/22/21 logging.error(f"Extension .{langext} is unsupported") quit() if concoretype=="windows": + # manual double quoting for Windows + Input validation above prevents breakout + q_container = f'"{containername}"' + q_source = f'"{sourcecode}"' + if langext=="py": - frun.write('start /B /D '+containername+" "+PYTHONWIN+" "+sourcecode+" >"+containername+"\\concoreout.txt\n") - fdebug.write('start /D '+containername+" cmd /K "+PYTHONWIN+" "+sourcecode+"\n") + frun.write('start /B /D '+q_container+" "+PYTHONWIN+" "+q_source+" >"+q_container+"\\concoreout.txt\n") + fdebug.write('start /D '+q_container+" cmd /K "+PYTHONWIN+" "+q_source+"\n") elif langext=="cpp": #6/25/21 - frun.write('cd '+containername+'\n') - frun.write(CPPWIN+' '+sourcecode+'\n') + frun.write('cd '+q_container+'\n') + frun.write(CPPWIN+' '+q_source+'\n') frun.write('cd ..\n') - frun.write('start /B /D '+containername+' cmd /c a >'+containername+'\\concoreout.txt\n') + frun.write('start /B /D '+q_container+' cmd /c a >'+q_container+'\\concoreout.txt\n') #frun.write('start /B /D '+containername+' "'+CPPWIN+' '+sourcecode+'|a >'+containername+'\\concoreout.txt"\n') - fdebug.write('cd '+containername+'\n') - fdebug.write(CPPWIN+' '+sourcecode+'\n') + fdebug.write('cd '+q_container+'\n') + fdebug.write(CPPWIN+' '+q_source+'\n') fdebug.write('cd ..\n') - fdebug.write('start /D '+containername+' cmd /K a\n') + fdebug.write('start /D '+q_container+' cmd /K a\n') #fdebug.write('start /D '+containername+' cmd /K "'+CPPWIN+' '+sourcecode+'|a"\n') elif langext=="v": #6/25/21 - frun.write('cd '+containername+'\n') - frun.write(VWIN+' '+sourcecode+'\n') + frun.write('cd '+q_container+'\n') + frun.write(VWIN+' '+q_source+'\n') frun.write('cd ..\n') - frun.write('start /B /D '+containername+' cmd /c vvp a.out >'+containername+'\\concoreout.txt\n') - fdebug.write('cd '+containername+'\n') - fdebug.write(VWIN+' '+sourcecode+'\n') + frun.write('start /B /D '+q_container+' cmd /c vvp a.out >'+q_container+'\\concoreout.txt\n') + fdebug.write('cd '+q_container+'\n') + fdebug.write(VWIN+' '+q_source+'\n') fdebug.write('cd ..\n') - fdebug.write('start /D '+containername+' cmd /K vvp a.out\n') + fdebug.write('start /D '+q_container+' cmd /K vvp a.out\n') #fdebug.write('start /D '+containername+' cmd /K "'+CPPWIN+' '+sourcecode+'|a"\n') + elif langext=="java": + javaclass = os.path.splitext(os.path.basename(sourcecode))[0] + frun.write('cd '+q_container+'\n') + frun.write(JAVACWIN+' '+q_source+'\n') + frun.write('cd ..\n') + frun.write('start /B /D '+q_container+' cmd /c '+JAVAWIN+' -cp .;..\\src\\jeromq.jar '+javaclass+' >'+q_container+'\\concoreout.txt\n') + fdebug.write('cd '+q_container+'\n') + fdebug.write(JAVACWIN+' '+q_source+'\n') + fdebug.write('cd ..\n') + fdebug.write('start /D '+q_container+' cmd /K '+JAVAWIN+' -cp .;..\\src\\jeromq.jar '+javaclass+'\n') elif langext=="m": #3/23/21 + # Use q_source in Windows commands to ensure quoting consistency if M_IS_OCTAVE: - frun.write('start /B /D '+containername+" "+OCTAVEWIN+' -qf --eval "run('+"'"+sourcecode+"'"+')"'+" >"+containername+"\\concoreout.txt\n") - fdebug.write('start /D '+containername+" cmd /K " +OCTAVEWIN+' -qf --eval "run('+"'"+sourcecode+"'"+')"'+"\n") + frun.write('start /B /D '+q_container+" "+OCTAVEWIN+' -qf --eval "run('+q_source+')"'+" >"+q_container+"\\concoreout.txt\n") + fdebug.write('start /D '+q_container+" cmd /K " +OCTAVEWIN+' -qf --eval "run('+q_source+')"'+"\n") else: # 4/2/21 - frun.write('start /B /D '+containername+" "+MATLABWIN+' -batch "run('+"'"+sourcecode+"'"+')"'+" >"+containername+"\\concoreout.txt\n") - fdebug.write('start /D '+containername+" cmd /K " +MATLABWIN+' -batch "run('+"'"+sourcecode+"'"+')"'+"\n") + frun.write('start /B /D '+q_container+" "+MATLABWIN+' -batch "run('+q_source+')"'+" >"+q_container+"\\concoreout.txt\n") + fdebug.write('start /D '+q_container+" cmd /K " +MATLABWIN+' -batch "run('+q_source+')"'+"\n") else: + #use shlex.quote for POSIX systems + safe_container = shlex.quote(containername) + safe_source = shlex.quote(sourcecode) + if langext == "py": - frun.write('(cd "' + containername + '"; ' + PYTHONEXE + ' ' + sourcecode + ' >concoreout.txt & echo $! >concorepid) &\n') + frun.write('(cd ' + safe_container + '; ' + PYTHONEXE + ' ' + safe_source + ' >concoreout.txt & echo $! >concorepid) &\n') if ubuntu: fdebug.write('concorewd="$(pwd)"\n') - fdebug.write('xterm -e bash -c "cd \\"$concorewd/' + containername + '\\"; ' + PYTHONEXE + ' ' + sourcecode + '; bash" &\n') + # quote the directory path inside the inner bash command + fdebug.write('xterm -e bash -c "cd \\"$concorewd/' + safe_container + '\\"; ' + PYTHONEXE + ' ' + safe_source + '; bash" &\n') else: fdebug.write('concorewd="$(pwd)"\n') - fdebug.write('osascript -e "tell application \\"Terminal\\" to do script \\"cd \\\\\\"$concorewd/' + containername + '\\\\\\"; ' + PYTHONEXE + ' ' + sourcecode + '\\"" \n') + fdebug.write('osascript -e "tell application \\"Terminal\\" to do script \\"cd \\\\\\"$concorewd/' + safe_container + '\\\\\\"; ' + PYTHONEXE + ' ' + safe_source + '\\"" \n') elif langext == "cpp": # 6/22/21 - frun.write('(cd "' + containername + '"; ' + CPPEXE + ' ' + sourcecode + '; ./a.out >concoreout.txt & echo $! >concorepid) &\n') + frun.write('(cd ' + safe_container + '; ' + CPPEXE + ' ' + safe_source + '; ./a.out >concoreout.txt & echo $! >concorepid) &\n') if ubuntu: fdebug.write('concorewd="$(pwd)"\n') - fdebug.write('xterm -e bash -c "cd \\"$concorewd/' + containername + '\\"; ' + CPPEXE + ' ' + sourcecode + '; ./a.out; bash" &\n') + fdebug.write('xterm -e bash -c "cd \\"$concorewd/' + safe_container + '\\"; ' + CPPEXE + ' ' + safe_source + '; ./a.out; bash" &\n') else: fdebug.write('concorewd="$(pwd)"\n') - fdebug.write('osascript -e "tell application \\"Terminal\\" to do script \\"cd \\\\\\"$concorewd/' + containername + '\\\\\\"; ' + CPPEXE + ' ' + sourcecode + '; ./a.out\\"" \n') + fdebug.write('osascript -e "tell application \\"Terminal\\" to do script \\"cd \\\\\\"$concorewd/' + safe_container + '\\\\\\"; ' + CPPEXE + ' ' + safe_source + '; ./a.out\\"" \n') elif langext == "v": # 6/25/21 - frun.write('(cd "' + containername + '"; ' + VEXE + ' ' + sourcecode + '; ./a.out >concoreout.txt & echo $! >concorepid) &\n') + frun.write('(cd ' + safe_container + '; ' + VEXE + ' ' + safe_source + '; ./a.out >concoreout.txt & echo $! >concorepid) &\n') + if ubuntu: + fdebug.write('concorewd="$(pwd)"\n') + fdebug.write('xterm -e bash -c "cd \\"$concorewd/' + safe_container + '\\"; ' + VEXE + ' ' + safe_source + '; ./a.out; bash" &\n') + else: + fdebug.write('concorewd="$(pwd)"\n') + fdebug.write('osascript -e "tell application \\"Terminal\\" to do script \\"cd \\\\\\"$concorewd/' + safe_container + '\\\\\\"; ' + VEXE + ' ' + safe_source + '; vvp a.out\\"" \n') + + elif langext == "java": + javaclass = os.path.splitext(os.path.basename(sourcecode))[0] + safe_javaclass = shlex.quote(javaclass) + frun.write('(cd ' + safe_container + '; ' + JAVACEXE + ' ' + safe_source + '; ' + JAVAEXE + ' -cp .:../src/jeromq.jar ' + safe_javaclass + ' >concoreout.txt & echo $! >concorepid) &\n') if ubuntu: fdebug.write('concorewd="$(pwd)"\n') - fdebug.write('xterm -e bash -c "cd \\"$concorewd/' + containername + '\\"; ' + VEXE + ' ' + sourcecode + '; ./a.out; bash" &\n') + fdebug.write('xterm -e bash -c "cd \\"$concorewd/' + safe_container + '\\"; ' + JAVACEXE + ' ' + safe_source + '; ' + JAVAEXE + ' -cp .:../src/jeromq.jar ' + safe_javaclass + '; bash" &\n') else: fdebug.write('concorewd="$(pwd)"\n') - fdebug.write('osascript -e "tell application \\"Terminal\\" to do script \\"cd \\\\\\"$concorewd/' + containername + '\\\\\\"; ' + VEXE + ' ' + sourcecode + '; vvp a.out\\"" \n') + fdebug.write('osascript -e "tell application \\"Terminal\\" to do script \\"cd \\\\\\"$concorewd/' + safe_container + '\\\\\\\"; ' + JAVACEXE + ' ' + safe_source + '; ' + JAVAEXE + ' -cp .:../src/jeromq.jar ' + safe_javaclass + '\\"" \n') elif langext == "sh": # 5/19/21 - frun.write('(cd "' + containername + '"; ./' + sourcecode + ' ' + MCRPATH + ' >concoreout.txt & echo $! >concorepid) &\n') + # FIX: Escape MCRPATH to prevent shell injection + safe_mcr = shlex.quote(MCRPATH) + frun.write('(cd ' + safe_container + '; ./' + safe_source + ' ' + safe_mcr + ' >concoreout.txt & echo $! >concorepid) &\n') if ubuntu: fdebug.write('concorewd="$(pwd)"\n') - fdebug.write('xterm -e bash -c "cd \\"$concorewd/' + containername + '\\"; ./' + sourcecode + ' ' + MCRPATH + '; bash" &\n') + fdebug.write('xterm -e bash -c "cd \\"$concorewd/' + safe_container + '\\"; ./' + safe_source + ' ' + safe_mcr + '; bash" &\n') else: fdebug.write('concorewd="$(pwd)"\n') - fdebug.write('osascript -e "tell application \\"Terminal\\" to do script \\"cd \\\\\\"$concorewd/' + containername + '\\\\\\"; ./' + sourcecode + ' ' + MCRPATH + '\\"" \n') + fdebug.write('osascript -e "tell application \\"Terminal\\" to do script \\"cd \\\\\\"$concorewd/' + safe_container + '\\\\\\"; ./' + safe_source + ' ' + safe_mcr + '\\"" \n') elif langext == "m": #3/23/21 + # FIX: Verify filename safety for MATLAB to prevent injection in run() + # MATLAB/Octave run('filename') is vulnerable if filename contains quotes or metachars. + if not re.match(r'^[A-Za-z0-9_./\-]+$', sourcecode): + raise ValueError(f"Invalid MATLAB/Octave source file name: {sourcecode!r}") + + # construct safe eval command + safe_eval_cmd = shlex.quote(f"run('{sourcecode}')") if M_IS_OCTAVE: - frun.write('(cd "' + containername + '"; ' + OCTAVEEXE + ' -qf --eval run(\\\'' + sourcecode + '\\\') >concoreout.txt & echo $! >concorepid) &\n') + frun.write('(cd ' + safe_container + '; ' + OCTAVEEXE + ' -qf --eval ' + safe_eval_cmd + ' >concoreout.txt & echo $! >concorepid) &\n') if ubuntu: fdebug.write('concorewd="$(pwd)"\n') - fdebug.write('xterm -e bash -c "cd \\"$concorewd/' + containername + '\\"; ' + OCTAVEEXE + ' -qf --eval run(\\\'' + sourcecode + '\\\'); bash" &\n') + fdebug.write('xterm -e bash -c "cd \\"$concorewd/' + safe_container + '\\"; ' + OCTAVEEXE + ' -qf --eval ' + safe_eval_cmd + '; bash" &\n') else: fdebug.write('concorewd="$(pwd)"\n') - fdebug.write('osascript -e "tell application \\"Terminal\\" to do script \\"cd \\\\\\"$concorewd/' + containername + '\\\\\\"; ' + OCTAVEEXE + ' -qf --eval run(\\\\\\\'' + sourcecode + '\\\\\\\')\\"" \n') + #osascript quoting is very complex; minimal safe_container applied + fdebug.write('osascript -e "tell application \\"Terminal\\" to do script \\"cd \\\\\\"$concorewd/' + safe_container + '\\\\\\"; ' + OCTAVEEXE + ' -qf --eval run(\\\\\\\'' + sourcecode + '\\\\\\\')\\"" \n') else: - frun.write('(cd "' + containername + '"; ' + MATLABEXE + ' -batch run(\\\'' + sourcecode + '\\\') >concoreout.txt & echo $! >concorepid) &\n') + frun.write('(cd ' + safe_container + '; ' + MATLABEXE + ' -batch ' + safe_eval_cmd + ' >concoreout.txt & echo $! >concorepid) &\n') if ubuntu: fdebug.write('concorewd="$(pwd)"\n') - fdebug.write('xterm -e bash -c "cd \\"$concorewd/' + containername + '\\"; ' + MATLABEXE + ' -batch run(\\\'' + sourcecode + '\\\'); bash" &\n') + fdebug.write('xterm -e bash -c "cd \\"$concorewd/' + safe_container + '\\"; ' + MATLABEXE + ' -batch ' + safe_eval_cmd + '; bash" &\n') else: fdebug.write('concorewd="$(pwd)"\n') - fdebug.write('osascript -e "tell application \\"Terminal\\" to do script \\"cd \\\\\\"$concorewd/' + containername + '\\\\\\"; ' + MATLABEXE + ' -batch run(\\\\\\\'' + sourcecode + '\\\\\\\')\\"" \n') + fdebug.write('osascript -e "tell application \\"Terminal\\" to do script \\"cd \\\\\\"$concorewd/' + safe_container + '\\\\\\"; ' + MATLABEXE + ' -batch run(\\\\\\\'' + sourcecode + '\\\\\\\')\\"" \n') if concoretype=="posix": fstop.write('#!/bin/bash' + "\n") i=0 # 3/30/21 for node in nodes_dict: containername,sourcecode = nodes_dict[node].split(':') if len(sourcecode)!=0: - dockername = sourcecode.split(".")[0] # 3/28/21 + dockername = sourcecode.rsplit(".", 1)[0] # 3/28/21 if concoretype=="windows": - fstop.write('cmd /C '+containername+"\\concorekill\n") - fstop.write('del '+containername+"\\concorekill.bat\n") + q_container = f'"{containername}"' + fstop.write('cmd /C '+q_container+"\\concorekill\n") + fstop.write('del '+q_container+"\\concorekill.bat\n") else: - fstop.write('kill -9 `cat '+containername+"/concorepid`\n") - fstop.write('rm '+containername+"/concorepid\n") + safe_pidfile = shlex.quote(f"{containername}/concorepid") + fstop.write('kill -9 `cat '+safe_pidfile+'`\n') + fstop.write('rm '+safe_pidfile+'\n') i=i+1 fstop.close() @@ -1044,13 +1271,16 @@ for node in nodes_dict: containername,sourcecode = nodes_dict[node].split(':') if len(sourcecode)!=0: - dockername = sourcecode.split(".")[0] #3/28/21 + dockername = sourcecode.rsplit(".", 1)[0] #3/28/21 writeedges = volswr[i] while writeedges.find(":") != -1: + path_part = writeedges.split(":")[0].split("-v")[1].strip() if concoretype=="windows": - fclear.write('del /Q' + writeedges.split(":")[0].split("-v")[1]+ "\\*\n") + fclear.write('del /Q "' + path_part + '\\*"\n') else: - fclear.write('rm ' + writeedges.split(":")[0].split("-v")[1]+ "/*\n") + # FIX: Safer wildcard removal. + # Avoid quoting the wildcard itself ('path/*'). Instead cd into directory and remove contents. + fclear.write(f'cd {shlex.quote(path_part)} && rm -f *\n') writeedges = writeedges[writeedges.find(":")+1:] i=i+1 fclear.close() @@ -1061,13 +1291,14 @@ for node in nodes_dict: containername,sourcecode = nodes_dict[node].split(':') if len(sourcecode)!=0: - dockername = sourcecode.split(".")[0] #3/28/21 + dockername = sourcecode.rsplit(".", 1)[0] #3/28/21 writeedges = volswr[i] while writeedges.find(":") != -1: + path_part = writeedges.split(":")[0].split("-v")[1].strip() if concoretype=="windows": - fmaxtime.write('echo %1 >' + writeedges.split(":")[0].split("-v")[1]+ "\\concore.maxtime\n") + fmaxtime.write('echo %1 >"' + path_part + '\\concore.maxtime"\n') else: - fmaxtime.write('echo "$1" >' + writeedges.split(":")[0].split("-v")[1]+ "/concore.maxtime\n") + fmaxtime.write('echo "$1" >' + shlex.quote(path_part + "/concore.maxtime") + '\n') writeedges = writeedges[writeedges.find(":")+1:] i=i+1 fmaxtime.close() @@ -1078,13 +1309,14 @@ for node in nodes_dict: containername,sourcecode = nodes_dict[node].split(':') if len(sourcecode)!=0: - dockername = sourcecode.split(".")[0] #3/28/21 + dockername = sourcecode.rsplit(".", 1)[0] #3/28/21 writeedges = volswr[i] while writeedges.find(":") != -1: + path_part = writeedges.split(":")[0].split("-v")[1].strip() if concoretype=="windows": - fparams.write('echo %1 >' + writeedges.split(":")[0].split("-v")[1]+ "\\concore.params\n") + fparams.write('echo %1 >"' + path_part + '\\concore.params"\n') else: - fparams.write('echo "$1" >' + writeedges.split(":")[0].split("-v")[1]+ "/concore.params\n") + fparams.write('echo "$1" >' + shlex.quote(path_part + "/concore.params") + '\n') writeedges = writeedges[writeedges.find(":")+1:] i=i+1 fparams.close() @@ -1095,13 +1327,14 @@ for node in nodes_dict: containername,sourcecode = nodes_dict[node].split(':') if len(sourcecode)!=0: - dockername = sourcecode.split(".")[0] #3/28/21 + dockername = sourcecode.rsplit(".", 1)[0] #3/28/21 writeedges = volswr[i] while writeedges.find(":") != -1: + path_part = writeedges.split(":")[0].split("-v")[1].strip() if concoretype=="windows": - funlock.write('copy %HOMEDRIVE%%HOMEPATH%\\concore.apikey' + writeedges.split(":")[0].split("-v")[1]+ "\\concore.apikey\n") + funlock.write('copy %HOMEDRIVE%%HOMEPATH%\\concore.apikey "' + path_part + '\\concore.apikey"\n') else: - funlock.write('cp ~/concore.apikey ' + writeedges.split(":")[0].split("-v")[1]+ "/concore.apikey\n") + funlock.write('cp ~/concore.apikey ' + shlex.quote(path_part + "/concore.apikey") + '\n') writeedges = writeedges[writeedges.find(":")+1:] i=i+1 funlock.close() @@ -1110,10 +1343,6 @@ frun.close() fbuild.close() fdebug.close() -fstop.close() -fclear.close() -fmaxtime.close() -fparams.close() if concoretype != "windows": os.chmod(outdir+"/build",stat.S_IRWXU) os.chmod(outdir+"/run",stat.S_IRWXU) @@ -1122,5 +1351,4 @@ os.chmod(outdir+"/clear",stat.S_IRWXU) os.chmod(outdir+"/maxtime",stat.S_IRWXU) os.chmod(outdir+"/params",stat.S_IRWXU) - os.chmod(outdir+"/unlock",stat.S_IRWXU) - + os.chmod(outdir+"/unlock",stat.S_IRWXU) diff --git a/nintan/powermetermax.py b/nintan/powermetermax.py index 5d6828ef..5294ceb8 100644 --- a/nintan/powermetermax.py +++ b/nintan/powermetermax.py @@ -1,14 +1,10 @@ #CW import concore -import concore2 import time print("powermeter") concore.delay = 0.07 -concore2.delay = 0.07 -concore2.inpath = concore.inpath -concore2.outpath = concore.outpath -concore2.simtime = 0 +concore.simtime = 0 #Nsim = 100 concore.default_maxtime(100) init_simtime_u = "[0.0, 0.0, 0.0]" @@ -19,9 +15,9 @@ while(concore.simtime=45", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "concore" +dynamic = ["version"] +description = "Concore workflow management CLI" +readme = "README.md" +requires-python = ">=3.9" +license = {text = "MIT"} +dependencies = [ + "click>=8.0.0", + "rich>=10.0.0", + "beautifulsoup4>=4.9.0", + "lxml>=4.6.0", + "psutil>=5.8.0", + "numpy>=1.19.0", + "pyzmq>=22.0.0", +] + +[project.optional-dependencies] +dev = [ + "pytest>=6.0.0", + "pytest-cov>=2.10.0", +] +demo = [ + "scipy>=1.5.0", + "matplotlib>=3.3.0", +] + +[project.scripts] +concore = "concore_cli.cli:cli" + +[tool.setuptools] +packages = ["concore_cli", "concore_cli.commands"] +py-modules = ["concore", "concoredocker", "concore_base", "mkconcore"] + +[tool.setuptools.dynamic] +version = {attr = "concore_cli.__version__"} diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 00000000..5881b607 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,6 @@ +[pytest] +testpaths = tests +python_files = test_*.py +python_classes = Test* +python_functions = test_* +addopts = -v --tb=short --cov=concore_cli --cov=concore_base --cov-report=term-missing \ No newline at end of file diff --git a/ratc/cwrap.py b/ratc/cwrap.py index b8cfb4b6..f9f33e59 100644 --- a/ratc/cwrap.py +++ b/ratc/cwrap.py @@ -58,7 +58,6 @@ except: init_simtime_ym = "[0.0, 0.0, 0.0]" -print(apikey) print(yuyu) print(name1+'='+init_simtime_u) print(name2+'='+init_simtime_ym) @@ -83,7 +82,7 @@ u = concore.read(1,name1,init_simtime_u) f = {'file1': open(concore.inpath+'1/'+name1, 'rb')} print("CW: before post u="+str(u)) - print('http://www.controlcore.org/pm/'+yuyu+apikey+'&fetch='+name2) + print('http://www.controlcore.org/pm/'+yuyu+''+'&fetch='+name2) r = requests.post('http://www.controlcore.org/pm/'+yuyu+apikey+'&fetch='+name2, files=f,timeout=timeout_max) if r.status_code!=200: print("bad POST request "+str(r.status_code)) diff --git a/ratc/learn3.py b/ratc/learn3.py index 496067ac..bbbc5476 100644 --- a/ratc/learn3.py +++ b/ratc/learn3.py @@ -1,14 +1,10 @@ import concore -import concore2 import numpy as np import matplotlib.pyplot as plt import time GENERATE_PLOT = 0 concore.delay = 0.002 -concore2.delay = 0.002 -concore2.inpath = concore.inpath -concore2.outpath = concore.outpath -concore2.simtime = 0 +concore.simtime = 0 fout=open(concore.outpath+'1/history.txt','w') fout2=open('historyfull.txt','a+') @@ -23,9 +19,9 @@ while concore.unchanged(): u = concore.read(concore.iport["VCY"],"u",init_simtime_u) ut[int(concore.simtime)] = np.array(u).T - while concore2.unchanged(): - ym = concore2.read(concore.iport["VPY"],"ym",init_simtime_ym) - ymt[int(concore2.simtime)] = np.array(ym).T + while concore.unchanged(): + ym = concore.read(concore.iport["VPY"],"ym",init_simtime_ym) + ymt[int(concore.simtime)] = np.array(ym).T #fout.write(str(u)+str(ym)+'\n') #fout2.write(str(u)+str(ym)+'\n') print("retry="+str(concore.retrycount)) diff --git a/ratc/pwrap.py b/ratc/pwrap.py index beb90116..283ca0ce 100644 --- a/ratc/pwrap.py +++ b/ratc/pwrap.py @@ -61,7 +61,6 @@ except: init_simtime_ym = "[0.0, 0.0, 0.0]" -print(apikey) print(yuyu) print(name1+'='+init_simtime_u) print(name2+'='+init_simtime_ym) diff --git a/requirements-ci.txt b/requirements-ci.txt new file mode 100644 index 00000000..1f26eb20 --- /dev/null +++ b/requirements-ci.txt @@ -0,0 +1,12 @@ +# Minimal dependencies for CI (linting and testing) +# Does not include heavyweight packages like tensorflow +pytest +pytest-cov +ruff +pyzmq +numpy +click>=8.0.0 +rich>=10.0.0 +psutil>=5.8.0 +beautifulsoup4 +lxml diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 00000000..7571d09b --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,2 @@ +pytest>=7.0.0 +pytest-cov>=4.0.0 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 067a3ec1..f63b0bb3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,9 @@ beautifulsoup4 lxml -tensorflow numpy +pyzmq scipy matplotlib -cvxopt -PyGithub \ No newline at end of file +click>=8.0.0 +rich>=10.0.0 +psutil>=5.8.0 \ No newline at end of file diff --git a/ruff.toml b/ruff.toml new file mode 100644 index 00000000..d487540d --- /dev/null +++ b/ruff.toml @@ -0,0 +1,41 @@ +exclude = [ + # legacy/experimental dirs - fix incrementally + "demo/", + "ratc/", + "ratc2/", + "example/", + "0mq/", + "measurements/", + "tools/", + "nintan/", + "testsou/", + "linktest/", + "fri/", + "gi/", + "humanc/", + # core modules - maintainer-managed, don't touch yet + "mkconcore.py", + "concore.py", + "concoredocker.py", + "concore_base.py", + "contribute.py", + "copy_with_port_portname.py", + # not valid Python (Dockerfiles) + "Dockerfile.py", + "Dockerfile.m", + "Dockerfile.sh", + "Dockerfile.v", + "Dockerfile.java", +] + +[lint] +select = ["E", "F"] +ignore = [ + "E501", # line too long - enforce incrementally + "E712", # comparison to True/False - common in tests + "E721", # type comparison - common in tests +] + +[format] +quote-style = "double" +indent-style = "space" diff --git a/sample/PZ/pm.py b/sample/PZ/pm.py new file mode 100644 index 00000000..8eff2ca3 --- /dev/null +++ b/sample/PZ/pm.py @@ -0,0 +1,34 @@ +import concore +import numpy as np +import ast + + +def pm(u): + return u + 0.01 + + +concore.default_maxtime(150) +concore.delay = 0.02 + +init_simtime_u = "[0.0, 0.0]" +init_simtime_ym = "[0.0, 0.0]" + +ym = np.array([concore.initval(init_simtime_ym)], dtype=np.float64).T + +while concore.simtime < concore.maxtime: + while concore.unchanged(): + u_raw = concore.read(1, "u", init_simtime_u) + if isinstance(u_raw, str): + try: + u_raw = ast.literal_eval(u_raw) + except (ValueError, SyntaxError): + print("Failed to parse fallback u string:", u_raw) + u_raw = [0.0] + u = np.array([u_raw], dtype=np.float64).T + + ym = pm(u) + + print(f"{concore.simtime}. u={u} ym={ym}") + concore.write(1, "ym", [float(x) for x in ym.T[0]], delta=1) + +print("retry=" + str(concore.retrycount)) diff --git a/sample/src/pm.py b/sample/src/pm.py new file mode 100644 index 00000000..8eff2ca3 --- /dev/null +++ b/sample/src/pm.py @@ -0,0 +1,34 @@ +import concore +import numpy as np +import ast + + +def pm(u): + return u + 0.01 + + +concore.default_maxtime(150) +concore.delay = 0.02 + +init_simtime_u = "[0.0, 0.0]" +init_simtime_ym = "[0.0, 0.0]" + +ym = np.array([concore.initval(init_simtime_ym)], dtype=np.float64).T + +while concore.simtime < concore.maxtime: + while concore.unchanged(): + u_raw = concore.read(1, "u", init_simtime_u) + if isinstance(u_raw, str): + try: + u_raw = ast.literal_eval(u_raw) + except (ValueError, SyntaxError): + print("Failed to parse fallback u string:", u_raw) + u_raw = [0.0] + u = np.array([u_raw], dtype=np.float64).T + + ym = pm(u) + + print(f"{concore.simtime}. u={u} ym={ym}") + concore.write(1, "ym", [float(x) for x in ym.T[0]], delta=1) + +print("retry=" + str(concore.retrycount)) diff --git a/setup.py b/setup.py new file mode 100644 index 00000000..ed9acf30 --- /dev/null +++ b/setup.py @@ -0,0 +1,5 @@ +from setuptools import setup + +# All metadata and configuration is in pyproject.toml. +# This file exists only for legacy compatibility. +setup() diff --git a/startserver b/startserver new file mode 100644 index 00000000..73bd6de6 --- /dev/null +++ b/startserver @@ -0,0 +1,14 @@ +#!/bin/bash +cd fri/server +if ! command -v python3 >/dev/null 2>&1; then + python main.py +else + python3 main.py +fi + +if [ $? -ne 0 ]; then + echo "" + echo "Error: Make sure modules are installed from fri/requirements.txt" + echo "Run: pip install -r fri/requirements.txt" + echo "" +fi diff --git a/startserver.bat b/startserver.bat new file mode 100644 index 00000000..856824a4 --- /dev/null +++ b/startserver.bat @@ -0,0 +1,8 @@ +cd fri\server +python main.py +if errorlevel 1 ( + echo. + echo Error: Make sure modules are installed from fri/requirements.txt + echo Run: pip install -r fri/requirements.txt + echo. +) \ No newline at end of file diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 00000000..c303e450 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,15 @@ +import pytest +import os +import sys +import tempfile +import shutil + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + + +@pytest.fixture +def temp_dir(): + dirpath = tempfile.mkdtemp() + yield dirpath + if os.path.exists(dirpath): + shutil.rmtree(dirpath) diff --git a/tests/protocol_fixtures/PROTOCOL_FIXTURES.md b/tests/protocol_fixtures/PROTOCOL_FIXTURES.md new file mode 100644 index 00000000..dd42bc6c --- /dev/null +++ b/tests/protocol_fixtures/PROTOCOL_FIXTURES.md @@ -0,0 +1,29 @@ +# Protocol Conformance Fixtures (Phase 1) + +This directory contains the phase-1 protocol conformance baseline for Python. + +- `schema.phase1.json`: fixture document shape and supported case targets. +- `python_phase1_cases.json`: initial baseline cases (report-only mode metadata). +- `cross_runtime_matrix.phase2.json`: phase-2 cross-runtime mapping matrix in report-only mode. + +Phase-1 scope: + +- No runtime behavior changes. +- Python-only execution through `tests/test_protocol_conformance.py`. +- Fixture format is language-neutral to enable future cross-binding runners. +- Baseline now includes `read_file` runtime-behavior checks in addition to parser/API targets. + +Phase-2 scope (mapping only): + +- No runtime behavior changes. +- Adds a cross-runtime matrix to track per-case audit status and classification. +- Java runtime entries are tracked with observed status from the Java regression suite (`TestLiteralEval.java`, `TestConcoredockerApi.java`). +- Current baseline records Java as `observed_pass` for the listed phase-2 cases. +- Phase-2 matrix includes `read_file` status rows for cross-runtime tracking. +- Keeps CI non-blocking for non-Python runtimes that are not yet audited by marking them as `not_audited`. + +Java conformance execution in CI: + +- The `java-test` job in `.github/workflows/ci.yml` downloads `jeromq` for classpath compatibility. +- It compiles `concoredocker.java`, `TestLiteralEval.java`, and `TestConcoredockerApi.java`. +- It runs both Java test classes and records initial phase-2 matrix status as observed in CI. diff --git a/tests/protocol_fixtures/cross_runtime_matrix.phase2.json b/tests/protocol_fixtures/cross_runtime_matrix.phase2.json new file mode 100644 index 00000000..712ba967 --- /dev/null +++ b/tests/protocol_fixtures/cross_runtime_matrix.phase2.json @@ -0,0 +1,314 @@ +{ + "schema_version": "1.0", + "phase": "2", + "mode": "report_only", + "source_fixture": "python_phase1_cases.json", + "runtimes": [ + "python", + "cpp", + "java", + "matlab", + "octave", + "verilog" + ], + "classifications": [ + "required", + "implementation_defined", + "known_deviation" + ], + "statuses": [ + "observed_pass", + "observed_fail", + "not_audited" + ], + "cases": [ + { + "id": "parse_params/simple_types_and_whitespace", + "target": "parse_params", + "runtime_results": { + "python": { + "status": "observed_pass", + "classification": "required", + "note": "Phase-1 baseline execution." + }, + "cpp": { + "status": "not_audited", + "classification": "required", + "note": "Audit planned in phase 2." + }, + "java": { + "status": "observed_pass", + "classification": "required", + "note": "Validated by TestLiteralEval.java and TestConcoredockerApi.java." + }, + "matlab": { + "status": "not_audited", + "classification": "required", + "note": "Audit planned in phase 2." + }, + "octave": { + "status": "not_audited", + "classification": "required", + "note": "Audit planned in phase 2." + }, + "verilog": { + "status": "not_audited", + "classification": "implementation_defined", + "note": "May require binding-specific interpretation." + } + } + }, + { + "id": "parse_params/embedded_equals_not_split", + "target": "parse_params", + "runtime_results": { + "python": { + "status": "observed_pass", + "classification": "required", + "note": "Phase-1 baseline execution." + }, + "cpp": { + "status": "not_audited", + "classification": "required", + "note": "Audit planned in phase 2." + }, + "java": { + "status": "observed_pass", + "classification": "required", + "note": "Validated by TestLiteralEval.java and TestConcoredockerApi.java." + }, + "matlab": { + "status": "not_audited", + "classification": "required", + "note": "Audit planned in phase 2." + }, + "octave": { + "status": "not_audited", + "classification": "required", + "note": "Audit planned in phase 2." + }, + "verilog": { + "status": "not_audited", + "classification": "implementation_defined", + "note": "May require binding-specific interpretation." + } + } + }, + { + "id": "initval/valid_list_sets_simtime", + "target": "initval", + "runtime_results": { + "python": { + "status": "observed_pass", + "classification": "required", + "note": "Phase-1 baseline execution." + }, + "cpp": { + "status": "not_audited", + "classification": "required", + "note": "Audit planned in phase 2." + }, + "java": { + "status": "observed_pass", + "classification": "required", + "note": "Validated by TestLiteralEval.java and TestConcoredockerApi.java." + }, + "matlab": { + "status": "not_audited", + "classification": "required", + "note": "Audit planned in phase 2." + }, + "octave": { + "status": "not_audited", + "classification": "required", + "note": "Audit planned in phase 2." + }, + "verilog": { + "status": "not_audited", + "classification": "implementation_defined", + "note": "May require binding-specific interpretation." + } + } + }, + { + "id": "initval/invalid_input_returns_empty_and_preserves_simtime", + "target": "initval", + "runtime_results": { + "python": { + "status": "observed_pass", + "classification": "required", + "note": "Phase-1 baseline execution." + }, + "cpp": { + "status": "not_audited", + "classification": "required", + "note": "Audit planned in phase 2." + }, + "java": { + "status": "observed_pass", + "classification": "required", + "note": "Validated by TestLiteralEval.java and TestConcoredockerApi.java." + }, + "matlab": { + "status": "not_audited", + "classification": "required", + "note": "Audit planned in phase 2." + }, + "octave": { + "status": "not_audited", + "classification": "required", + "note": "Audit planned in phase 2." + }, + "verilog": { + "status": "not_audited", + "classification": "implementation_defined", + "note": "May require binding-specific interpretation." + } + } + }, + { + "id": "write_zmq/list_payload_prepends_timestamp_without_mutation", + "target": "write_zmq", + "runtime_results": { + "python": { + "status": "observed_pass", + "classification": "required", + "note": "Phase-1 baseline execution." + }, + "cpp": { + "status": "not_audited", + "classification": "required", + "note": "Audit planned in phase 2." + }, + "java": { + "status": "observed_pass", + "classification": "required", + "note": "Validated by TestLiteralEval.java and TestConcoredockerApi.java." + }, + "matlab": { + "status": "not_audited", + "classification": "required", + "note": "Audit planned in phase 2." + }, + "octave": { + "status": "not_audited", + "classification": "required", + "note": "Audit planned in phase 2." + }, + "verilog": { + "status": "not_audited", + "classification": "implementation_defined", + "note": "May require binding-specific interpretation." + } + } + }, + { + "id": "write_zmq/non_list_payload_forwarded_as_is", + "target": "write_zmq", + "runtime_results": { + "python": { + "status": "observed_pass", + "classification": "required", + "note": "Phase-1 baseline execution." + }, + "cpp": { + "status": "not_audited", + "classification": "required", + "note": "Audit planned in phase 2." + }, + "java": { + "status": "observed_pass", + "classification": "required", + "note": "Validated by TestLiteralEval.java and TestConcoredockerApi.java." + }, + "matlab": { + "status": "not_audited", + "classification": "required", + "note": "Audit planned in phase 2." + }, + "octave": { + "status": "not_audited", + "classification": "required", + "note": "Audit planned in phase 2." + }, + "verilog": { + "status": "not_audited", + "classification": "implementation_defined", + "note": "May require binding-specific interpretation." + } + } + }, + { + "id": "read_file/missing_file_returns_default_and_false", + "target": "read_file", + "runtime_results": { + "python": { + "status": "observed_pass", + "classification": "required", + "note": "Phase-1 baseline execution." + }, + "cpp": { + "status": "not_audited", + "classification": "required", + "note": "Audit planned in phase 2." + }, + "java": { + "status": "observed_pass", + "classification": "required", + "note": "Validated by TestConcoredockerApi.java." + }, + "matlab": { + "status": "not_audited", + "classification": "required", + "note": "Audit planned in phase 2." + }, + "octave": { + "status": "not_audited", + "classification": "required", + "note": "Audit planned in phase 2." + }, + "verilog": { + "status": "not_audited", + "classification": "implementation_defined", + "note": "May require binding-specific interpretation." + } + } + }, + { + "id": "read_file/older_timestamp_does_not_decrease_simtime", + "target": "read_file", + "runtime_results": { + "python": { + "status": "observed_pass", + "classification": "required", + "note": "Phase-1 baseline execution." + }, + "cpp": { + "status": "not_audited", + "classification": "required", + "note": "Audit planned in phase 2." + }, + "java": { + "status": "observed_pass", + "classification": "required", + "note": "Validated by TestConcoredockerApi.java simtime progression checks." + }, + "matlab": { + "status": "not_audited", + "classification": "required", + "note": "Audit planned in phase 2." + }, + "octave": { + "status": "not_audited", + "classification": "required", + "note": "Audit planned in phase 2." + }, + "verilog": { + "status": "not_audited", + "classification": "implementation_defined", + "note": "May require binding-specific interpretation." + } + } + } + ] +} diff --git a/tests/protocol_fixtures/python_phase1_cases.json b/tests/protocol_fixtures/python_phase1_cases.json new file mode 100644 index 00000000..ccdeb02d --- /dev/null +++ b/tests/protocol_fixtures/python_phase1_cases.json @@ -0,0 +1,142 @@ +{ + "schema_version": "1.0", + "runtime": "python", + "mode": "report_only", + "cases": [ + { + "id": "parse_params/simple_types_and_whitespace", + "target": "parse_params", + "description": "Semicolon-delimited params preserve string values and coerce literals.", + "input": { + "sparams": "delay=5; coeffs=[1,2,3]; label = hello world" + }, + "expected": { + "result": { + "delay": 5, + "coeffs": [ + 1, + 2, + 3 + ], + "label": "hello world" + } + } + }, + { + "id": "parse_params/embedded_equals_not_split", + "target": "parse_params", + "description": "Only the first '=' is used as key/value separator.", + "input": { + "sparams": "url=https://example.com?a=1&b=2" + }, + "expected": { + "result": { + "url": "https://example.com?a=1&b=2" + } + } + }, + { + "id": "initval/valid_list_sets_simtime", + "target": "initval", + "description": "initval sets simtime to first numeric entry and returns payload tail.", + "input": { + "initial_simtime": 0, + "simtime_val_str": "[12.5, \"a\", 3]" + }, + "expected": { + "result": [ + "a", + 3 + ], + "simtime_after": 12.5 + } + }, + { + "id": "initval/invalid_input_returns_empty_and_preserves_simtime", + "target": "initval", + "description": "Invalid non-list input returns [] and leaves simtime unchanged.", + "input": { + "initial_simtime": 7, + "simtime_val_str": "not_a_list" + }, + "expected": { + "result": [], + "simtime_after": 7 + } + }, + { + "id": "write_zmq/list_payload_prepends_timestamp_without_mutation", + "target": "write_zmq", + "description": "write() prepends simtime+delta for list payloads but does not mutate global simtime.", + "input": { + "initial_simtime": 10, + "delta": 2, + "name": "data", + "value": [ + 1.5, + 2.5 + ] + }, + "expected": { + "sent_payload": [ + 12, + 1.5, + 2.5 + ], + "simtime_after": 10 + } + }, + { + "id": "write_zmq/non_list_payload_forwarded_as_is", + "target": "write_zmq", + "description": "Non-list payloads are forwarded as-is and simtime remains unchanged.", + "input": { + "initial_simtime": 10, + "delta": 3, + "name": "status", + "value": "ok" + }, + "expected": { + "sent_payload": "ok", + "simtime_after": 10 + } + }, + { + "id": "read_file/missing_file_returns_default_and_false", + "target": "read_file", + "description": "read() returns init default with ok=False when file is missing.", + "input": { + "initial_simtime": 4, + "port": 1, + "name": "missing", + "initstr_val": "[0.0, 5.0]" + }, + "expected": { + "result": [ + 5.0 + ], + "ok": false, + "simtime_after": 4 + } + }, + { + "id": "read_file/older_timestamp_does_not_decrease_simtime", + "target": "read_file", + "description": "read() keeps simtime monotonic when incoming file timestamp is older.", + "input": { + "initial_simtime": 10, + "port": 1, + "name": "ym", + "file_content": "[7.0, 3.14]", + "initstr_val": "[0.0, 0.0]" + }, + "expected": { + "result": [ + 3.14 + ], + "ok": true, + "simtime_after": 10 + } + } + ] +} diff --git a/tests/protocol_fixtures/schema.phase1.json b/tests/protocol_fixtures/schema.phase1.json new file mode 100644 index 00000000..33495b82 --- /dev/null +++ b/tests/protocol_fixtures/schema.phase1.json @@ -0,0 +1,63 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Concore Protocol Conformance Fixtures (Phase 1)", + "description": "Language-neutral fixture format. Phase 1 executes Python-only baseline checks.", + "type": "object", + "required": [ + "schema_version", + "runtime", + "mode", + "cases" + ], + "properties": { + "schema_version": { + "type": "string" + }, + "runtime": { + "type": "string" + }, + "mode": { + "type": "string", + "enum": [ + "report_only" + ] + }, + "cases": { + "type": "array", + "items": { + "type": "object", + "required": [ + "id", + "target", + "input", + "expected" + ], + "properties": { + "id": { + "type": "string" + }, + "target": { + "type": "string", + "enum": [ + "parse_params", + "initval", + "write_zmq", + "read_file" + ] + }, + "description": { + "type": "string" + }, + "input": { + "type": "object" + }, + "expected": { + "type": "object" + } + }, + "additionalProperties": true + } + } + }, + "additionalProperties": false +} diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 00000000..63d3cb4d --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,487 @@ +import unittest +import tempfile +import shutil +import os +import json +from pathlib import Path +from click.testing import CliRunner +from concore_cli.cli import cli + + +class TestConcoreCLI(unittest.TestCase): + def setUp(self): + self.runner = CliRunner() + self.temp_dir = tempfile.mkdtemp() + + def tearDown(self): + if Path(self.temp_dir).exists(): + shutil.rmtree(self.temp_dir) + + def test_version(self): + result = self.runner.invoke(cli, ["--version"]) + self.assertEqual(result.exit_code, 0) + self.assertIn("1.0.0", result.output) + + def test_help(self): + result = self.runner.invoke(cli, ["--help"]) + self.assertEqual(result.exit_code, 0) + self.assertIn("Usage:", result.output) + self.assertIn("Commands:", result.output) + + def test_init_command(self): + with self.runner.isolated_filesystem(temp_dir=self.temp_dir): + result = self.runner.invoke(cli, ["init", "test-project"]) + self.assertEqual(result.exit_code, 0) + + project_path = Path("test-project") + self.assertTrue(project_path.exists()) + self.assertTrue((project_path / "workflow.graphml").exists()) + self.assertTrue((project_path / "src").exists()) + self.assertTrue((project_path / "README.md").exists()) + self.assertTrue((project_path / "src" / "script.py").exists()) + self.assertTrue((project_path / "STUDY.json").exists()) + + metadata = json.loads((project_path / "STUDY.json").read_text()) + self.assertEqual(metadata["generated_by"], "concore init") + self.assertEqual(metadata["study_name"], "test-project") + self.assertEqual(metadata["schema_version"], 1) + self.assertIn("workflow.graphml", metadata["checksums"]) + + def test_init_existing_directory(self): + with self.runner.isolated_filesystem(temp_dir=self.temp_dir): + Path("existing").mkdir() + result = self.runner.invoke(cli, ["init", "existing"]) + self.assertNotEqual(result.exit_code, 0) + self.assertIn("already exists", result.output) + + def test_validate_missing_file(self): + result = self.runner.invoke(cli, ["validate", "nonexistent.graphml"]) + self.assertNotEqual(result.exit_code, 0) + + def test_validate_valid_file(self): + with self.runner.isolated_filesystem(temp_dir=self.temp_dir): + result = self.runner.invoke(cli, ["init", "test-project"]) + self.assertEqual(result.exit_code, 0) + + result = self.runner.invoke( + cli, ["validate", "test-project/workflow.graphml"] + ) + self.assertEqual(result.exit_code, 0) + self.assertIn("Validation passed", result.output) + + def test_validate_missing_node_file(self): + with self.runner.isolated_filesystem(temp_dir=self.temp_dir): + result = self.runner.invoke(cli, ["init", "test-project"]) + self.assertEqual(result.exit_code, 0) + + missing_file = Path("test-project/src/script.py") + if missing_file.exists(): + missing_file.unlink() + + result = self.runner.invoke( + cli, ["validate", "test-project/workflow.graphml"] + ) + self.assertNotEqual(result.exit_code, 0) + self.assertIn("Missing source file", result.output) + + def test_validate_json_output_for_valid_file(self): + with self.runner.isolated_filesystem(temp_dir=self.temp_dir): + result = self.runner.invoke(cli, ["init", "test-project"]) + self.assertEqual(result.exit_code, 0) + + result = self.runner.invoke( + cli, + ["validate", "test-project/workflow.graphml", "--format", "json"], + ) + self.assertEqual(result.exit_code, 0) + + payload = json.loads(result.output) + self.assertTrue(payload["valid"]) + self.assertEqual(payload["summary"]["error_count"], 0) + self.assertEqual(payload["workflow"], "workflow.graphml") + self.assertIn("src", payload["source_dir"]) + + def test_validate_json_output_for_missing_source_file(self): + with self.runner.isolated_filesystem(temp_dir=self.temp_dir): + result = self.runner.invoke(cli, ["init", "test-project"]) + self.assertEqual(result.exit_code, 0) + + missing_file = Path("test-project/src/script.py") + if missing_file.exists(): + missing_file.unlink() + + result = self.runner.invoke( + cli, + ["validate", "test-project/workflow.graphml", "--format", "json"], + ) + self.assertNotEqual(result.exit_code, 0) + + payload = json.loads(result.output) + self.assertFalse(payload["valid"]) + self.assertEqual(payload["summary"]["error_count"], 1) + self.assertEqual(payload["errors"][0]["error_type"], "missing_source_file") + self.assertEqual(payload["errors"][0]["node_id"], "n1") + + def test_status_command(self): + result = self.runner.invoke(cli, ["status"]) + self.assertEqual(result.exit_code, 0) + + def test_build_command_missing_source(self): + with self.runner.isolated_filesystem(temp_dir=self.temp_dir): + result = self.runner.invoke(cli, ["init", "test-project"]) + result = self.runner.invoke( + cli, + ["build", "test-project/workflow.graphml", "--source", "nonexistent"], + ) + self.assertNotEqual(result.exit_code, 0) + + def test_build_command_from_project_dir(self): + with self.runner.isolated_filesystem(temp_dir=self.temp_dir): + result = self.runner.invoke(cli, ["init", "test-project"]) + self.assertEqual(result.exit_code, 0) + + result = self.runner.invoke( + cli, + [ + "build", + "test-project/workflow.graphml", + "--source", + "test-project/src", + "--output", + "out", + "--type", + "posix", + ], + ) + self.assertEqual(result.exit_code, 0) + self.assertTrue(Path("out/src/concore.py").exists()) + self.assertTrue(Path("out/STUDY.json").exists()) + + metadata = json.loads(Path("out/STUDY.json").read_text()) + self.assertEqual(metadata["generated_by"], "concore build") + self.assertEqual(metadata["study_name"], "out") + self.assertEqual(metadata["schema_version"], 1) + self.assertIn("workflow.graphml", metadata["checksums"]) + + def test_build_command_default_type(self): + with self.runner.isolated_filesystem(temp_dir=self.temp_dir): + result = self.runner.invoke(cli, ["init", "test-project"]) + self.assertEqual(result.exit_code, 0) + + result = self.runner.invoke( + cli, + [ + "build", + "test-project/workflow.graphml", + "--source", + "test-project/src", + "--output", + "out", + ], + ) + self.assertEqual(result.exit_code, 0) + if os.name == "nt": + self.assertTrue(Path("out/build.bat").exists()) + else: + self.assertTrue(Path("out/build").exists()) + + def test_build_command_nested_output_path(self): + with self.runner.isolated_filesystem(temp_dir=self.temp_dir): + result = self.runner.invoke(cli, ["init", "test-project"]) + self.assertEqual(result.exit_code, 0) + + result = self.runner.invoke( + cli, + [ + "build", + "test-project/workflow.graphml", + "--source", + "test-project/src", + "--output", + "build/out", + "--type", + "posix", + ], + ) + self.assertEqual(result.exit_code, 0) + self.assertTrue(Path("build/out/src/concore.py").exists()) + + def test_build_command_subdir_source(self): + with self.runner.isolated_filesystem(temp_dir=self.temp_dir): + result = self.runner.invoke(cli, ["init", "test-project"]) + self.assertEqual(result.exit_code, 0) + + subdir = Path("test-project/src/subdir") + subdir.mkdir(parents=True, exist_ok=True) + shutil.move("test-project/src/script.py", subdir / "script.py") + + workflow_path = Path("test-project/workflow.graphml") + content = workflow_path.read_text() + content = content.replace("N1:script.py", "N1:subdir/script.py") + workflow_path.write_text(content) + + result = self.runner.invoke( + cli, + [ + "build", + "test-project/workflow.graphml", + "--source", + "test-project/src", + "--output", + "out", + "--type", + "posix", + ], + ) + self.assertEqual(result.exit_code, 0) + self.assertTrue(Path("out/src/subdir/script.py").exists()) + + def test_build_command_docker_subdir_source_build_paths(self): + with self.runner.isolated_filesystem(temp_dir=self.temp_dir): + result = self.runner.invoke(cli, ["init", "test-project"]) + self.assertEqual(result.exit_code, 0) + + subdir = Path("test-project/src/subdir") + subdir.mkdir(parents=True, exist_ok=True) + shutil.move("test-project/src/script.py", subdir / "script.py") + + workflow_path = Path("test-project/workflow.graphml") + content = workflow_path.read_text() + content = content.replace("N1:script.py", "N1:subdir/script.py") + workflow_path.write_text(content) + + result = self.runner.invoke( + cli, + [ + "build", + "test-project/workflow.graphml", + "--source", + "test-project/src", + "--output", + "out", + "--type", + "docker", + ], + ) + self.assertEqual(result.exit_code, 0) + + build_script = Path("out/build").read_text() + self.assertIn("mkdir docker-subdir__script", build_script) + self.assertIn("cp ../src/Dockerfile.subdir/script Dockerfile", build_script) + self.assertIn("cp ../src/subdir/script.py .", build_script) + self.assertIn("cp ../src/subdir/script.iport concore.iport", build_script) + self.assertIn("cd ..", build_script) + + def test_build_command_compose_requires_docker_type(self): + with self.runner.isolated_filesystem(temp_dir=self.temp_dir): + result = self.runner.invoke(cli, ["init", "test-project"]) + self.assertEqual(result.exit_code, 0) + + result = self.runner.invoke( + cli, + [ + "build", + "test-project/workflow.graphml", + "--source", + "test-project/src", + "--output", + "out", + "--type", + "posix", + "--compose", + ], + ) + self.assertNotEqual(result.exit_code, 0) + self.assertIn( + "--compose can only be used with --type docker", result.output + ) + + def test_build_command_docker_compose_single_node(self): + with self.runner.isolated_filesystem(temp_dir=self.temp_dir): + result = self.runner.invoke(cli, ["init", "test-project"]) + self.assertEqual(result.exit_code, 0) + + result = self.runner.invoke( + cli, + [ + "build", + "test-project/workflow.graphml", + "--source", + "test-project/src", + "--output", + "out", + "--type", + "docker", + "--compose", + ], + ) + self.assertEqual(result.exit_code, 0) + + compose_path = Path("out/docker-compose.yml") + self.assertTrue(compose_path.exists()) + compose_content = compose_path.read_text() + self.assertIn("services:", compose_content) + self.assertIn("container_name: 'N1'", compose_content) + self.assertIn("image: 'docker-script'", compose_content) + + metadata = json.loads(Path("out/STUDY.json").read_text()) + self.assertIn("docker-compose.yml", metadata["checksums"]) + + def test_build_command_docker_compose_multi_node(self): + with self.runner.isolated_filesystem(temp_dir=self.temp_dir): + Path("src").mkdir() + Path("src/common.py").write_text( + "import concore\n\ndef step():\n return None\n" + ) + + workflow = """ + + + + + A:common.py + B:common.py + C:common.py + 0x1000_AB + 0x1001_BC + + +""" + Path("workflow.graphml").write_text(workflow) + + result = self.runner.invoke( + cli, + [ + "build", + "workflow.graphml", + "--source", + "src", + "--output", + "out", + "--type", + "docker", + "--compose", + ], + ) + self.assertEqual(result.exit_code, 0) + + compose_content = Path("out/docker-compose.yml").read_text() + self.assertIn("container_name: 'A'", compose_content) + self.assertIn("container_name: 'B'", compose_content) + self.assertIn("container_name: 'C'", compose_content) + self.assertIn("image: 'docker-common'", compose_content) + + def test_build_command_shared_source_specialization_merges_edge_params(self): + with self.runner.isolated_filesystem(temp_dir=self.temp_dir): + Path("src").mkdir() + Path("src/common.py").write_text( + "import concore\n\ndef step():\n return None\n" + ) + + workflow = """ + + + + + A:common.py + B:common.py + C:common.py + 0x1000_AB + 0x1001_BC + + +""" + Path("workflow.graphml").write_text(workflow) + + result = self.runner.invoke( + cli, + [ + "build", + "workflow.graphml", + "--source", + "src", + "--output", + "out", + "--type", + "posix", + ], + ) + self.assertEqual(result.exit_code, 0) + + specialized_script = Path("out/src/common.py") + self.assertTrue(specialized_script.exists()) + content = specialized_script.read_text() + self.assertIn("PORT_NAME_A_B", content) + self.assertIn("PORT_A_B", content) + self.assertIn("PORT_NAME_B_C", content) + self.assertIn("PORT_B_C", content) + + def test_build_command_existing_output(self): + with self.runner.isolated_filesystem(temp_dir=self.temp_dir): + result = self.runner.invoke(cli, ["init", "test-project"]) + Path("output").mkdir() + + result = self.runner.invoke( + cli, + [ + "build", + "test-project/workflow.graphml", + "--source", + "test-project/src", + "--output", + "output", + ], + ) + self.assertIn("already exists", result.output.lower()) + + def test_inspect_command_basic(self): + with self.runner.isolated_filesystem(temp_dir=self.temp_dir): + result = self.runner.invoke(cli, ["init", "test-project"]) + self.assertEqual(result.exit_code, 0) + + result = self.runner.invoke( + cli, ["inspect", "test-project/workflow.graphml"] + ) + self.assertEqual(result.exit_code, 0) + self.assertIn("Workflow Overview", result.output) + self.assertIn("Nodes:", result.output) + self.assertIn("Edges:", result.output) + + def test_inspect_missing_file(self): + result = self.runner.invoke(cli, ["inspect", "nonexistent.graphml"]) + self.assertNotEqual(result.exit_code, 0) + + def test_inspect_json_output(self): + with self.runner.isolated_filesystem(temp_dir=self.temp_dir): + result = self.runner.invoke(cli, ["init", "test-project"]) + self.assertEqual(result.exit_code, 0) + + result = self.runner.invoke( + cli, ["inspect", "test-project/workflow.graphml", "--json"] + ) + self.assertEqual(result.exit_code, 0) + + import json + + output_data = json.loads(result.output) + self.assertIn("workflow", output_data) + self.assertIn("nodes", output_data) + self.assertIn("edges", output_data) + self.assertEqual(output_data["workflow"], "workflow.graphml") + + def test_inspect_missing_source_file(self): + with self.runner.isolated_filesystem(temp_dir=self.temp_dir): + result = self.runner.invoke(cli, ["init", "test-project"]) + self.assertEqual(result.exit_code, 0) + + Path("test-project/src/script.py").unlink() + + result = self.runner.invoke( + cli, ["inspect", "test-project/workflow.graphml", "--source", "src"] + ) + self.assertEqual(result.exit_code, 0) + self.assertIn("Missing files", result.output) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_concore.py b/tests/test_concore.py new file mode 100644 index 00000000..dc98ced5 --- /dev/null +++ b/tests/test_concore.py @@ -0,0 +1,721 @@ +import pytest +import os +import sys +import numpy as np +from unittest.mock import patch + + +class TestSafeLiteralEval: + def test_reads_dictionary_from_file(self, temp_dir): + test_file = os.path.join(temp_dir, "config.txt") + with open(test_file, "w") as f: + f.write("{'name': 'test', 'value': 123}") + + from concore import safe_literal_eval + + result = safe_literal_eval(test_file, {}) + + assert result == {"name": "test", "value": 123} + + def test_returns_default_when_file_missing(self): + from concore import safe_literal_eval + + result = safe_literal_eval("nonexistent_file.txt", "fallback") + + assert result == "fallback" + + def test_returns_default_for_empty_file(self, temp_dir): + test_file = os.path.join(temp_dir, "empty.txt") + with open(test_file, "w") as _: + pass + + from concore import safe_literal_eval + + result = safe_literal_eval(test_file, "default") + + assert result == "default" + + +class TestTryparam: + @pytest.fixture(autouse=True) + def reset_params(self): + from concore import params + + original_params = params.copy() + yield + params.clear() + params.update(original_params) + + def test_returns_existing_parameter(self): + from concore import tryparam, params + + params["my_setting"] = "custom_value" + + result = tryparam("my_setting", "default_value") + + assert result == "custom_value" + + def test_returns_default_for_missing_parameter(self): + from concore import tryparam + + result = tryparam("missing_param", "fallback") + + assert result == "fallback" + + +class TestZeroMQPort: + def test_class_is_defined(self): + from concore import ZeroMQPort + + assert ZeroMQPort is not None + + +class TestDefaultConfiguration: + def test_default_input_path(self): + from concore import inpath + + assert inpath == "./in" + + def test_default_output_path(self): + from concore import outpath + + assert outpath == "./out" + + +class TestPublicAPI: + def test_module_imports_successfully(self): + from concore import safe_literal_eval + + assert safe_literal_eval is not None + + def test_core_functions_exist(self): + from concore import safe_literal_eval, tryparam, default_maxtime + + assert callable(safe_literal_eval) + assert callable(tryparam) + assert callable(default_maxtime) + + +class TestNumpyConversion: + def test_convert_scalar(self): + from concore import convert_numpy_to_python + + val = np.float64(3.14) + res = convert_numpy_to_python(val) + assert type(res) == float + assert res == 3.14 + + def test_convert_list_and_dict(self): + from concore import convert_numpy_to_python + + data = {"a": np.int32(10), "b": [np.float64(1.1), np.float64(2.2)]} + res = convert_numpy_to_python(data) + assert type(res["a"]) == int + assert type(res["b"][0]) == float + assert res["b"][1] == 2.2 + + +class TestInitVal: + @pytest.fixture(autouse=True) + def reset_simtime(self): + import concore + + old_simtime = concore.simtime + yield + concore.simtime = old_simtime + + def test_initval_updates_simtime(self): + import concore + + concore.simtime = 0 + # initval takes string repr of a list [time, val1, val2...] + result = concore.initval("[100, 'data']") + + assert concore.simtime == 100 + assert result == ["data"] + + def test_initval_handles_bad_input(self): + import concore + + concore.simtime = 0 + # Input that isn't a list + result = concore.initval("not_a_list") + assert concore.simtime == 0 + assert result == [] + + +class TestDefaultMaxTime: + def test_uses_file_value(self, temp_dir, monkeypatch): + import concore + + # Mock the path to maxtime file + maxtime_file = os.path.join(temp_dir, "concore.maxtime") + with open(maxtime_file, "w") as f: + f.write("500") + + monkeypatch.setattr(concore, "concore_maxtime_file", maxtime_file) + concore.default_maxtime(100) + + assert concore.maxtime == 500 + + def test_uses_default_when_missing(self, monkeypatch): + import concore + + monkeypatch.setattr(concore, "concore_maxtime_file", "missing_file") + concore.default_maxtime(999) + assert concore.maxtime == 999 + + +class TestUnchanged: + @pytest.fixture(autouse=True) + def reset_globals(self): + import concore + + old_s = concore.s + old_olds = concore.olds + yield + concore.s = old_s + concore.olds = old_olds + + def test_unchanged_returns_true_if_same(self): + import concore + + concore.s = "same" + concore.olds = "same" + + # Should return True and reset s to empty + assert concore.unchanged() is True + assert concore.s == "" + + def test_unchanged_returns_false_if_diff(self): + import concore + + concore.s = "new" + concore.olds = "old" + + assert concore.unchanged() is False + assert concore.olds == "new" + + +class TestParseParams: + def test_simple_key_value_pairs(self): + from concore import parse_params + + params = parse_params("a=1;b=2") + assert params == {"a": 1, "b": 2} + + def test_preserves_whitespace_in_values(self): + from concore import parse_params + + params = parse_params("label = hello world ; x = 5") + assert params["label"] == "hello world" + assert params["x"] == 5 + + def test_embedded_equals_in_value(self): + from concore import parse_params + + params = parse_params("url=https://example.com?a=1&b=2") + assert params["url"] == "https://example.com?a=1&b=2" + + def test_numeric_and_list_coercion(self): + from concore import parse_params + + params = parse_params("delay=5;coeffs=[1,2,3]") + assert params["delay"] == 5 + assert params["coeffs"] == [1, 2, 3] + + def test_dict_literal_backward_compatibility(self): + from concore import parse_params + + params = parse_params("{'a': 1, 'b': 2}") + assert params == {"a": 1, "b": 2} + + def test_windows_quoted_input(self): + from concore import parse_params + + s = '"a=1;b=2"' + s = s[1:-1] # simulate quote stripping before parse_params + params = parse_params(s) + assert params == {"a": 1, "b": 2} + + +class TestWriteZMQ: + @pytest.fixture(autouse=True) + def reset_zmq_ports(self): + import concore + + original_ports = concore.zmq_ports.copy() + yield + concore.zmq_ports.clear() + concore.zmq_ports.update(original_ports) + + def test_write_converts_numpy_types_for_zmq(self): + import concore + + class DummyPort: + def __init__(self): + self.sent = None + + def send_json_with_retry(self, message): + self.sent = message + + dummy = DummyPort() + concore.zmq_ports["test_zmq"] = dummy + + # Reset simtime for predictable test behavior + concore.simtime = 0 + + payload = [np.int64(7), np.float64(3.5), {"x": np.float32(1.25)}] + concore.write("test_zmq", "data", payload) + + assert dummy.sent is not None + # ZMQ write now prepends simtime (0 in this case) to match file-based write behavior + assert dummy.sent == [0, 7, 3.5, {"x": 1.25}] + # Data values (after simtime) should be converted from numpy types + assert not isinstance(dummy.sent[1], np.generic) + assert not isinstance(dummy.sent[2], np.generic) + assert not isinstance(dummy.sent[3]["x"], np.generic) + + def test_zmq_write_read_roundtrip(self): + """Test that ZMQ write+read returns original data without simtime prefix.""" + import concore + + class DummyZMQPort: + def __init__(self): + self.buffer = None + + def send_json_with_retry(self, message): + self.buffer = message + + def recv_json_with_retry(self): + return self.buffer + + dummy = DummyZMQPort() + concore.zmq_ports["roundtrip_test"] = dummy + + # Reset simtime for predictable test behavior + concore.simtime = 0 + + original_data = [1.5, 2.5, 3.5] + concore.write("roundtrip_test", "data", original_data) + + # Read should return original data (simtime stripped) plus success flag + result, ok = concore.read("roundtrip_test", "data", "[]") + assert result == original_data + assert ok is True + + +class TestSimtimeNotMutatedByWrite: + """Regression tests for issue #385: + write() must NOT mutate global simtime. Simtime advancement happens + only in read() via max(simtime, file_simtime). Mutating simtime in + write() causes cascading timestamps in multi-output-port nodes and + breaks cross-language determinism. + """ + + @pytest.fixture(autouse=True) + def reset_simtime(self): + import concore + + old_simtime = concore.simtime + yield + concore.simtime = old_simtime + + @pytest.fixture(autouse=True) + def reset_outpath(self): + import concore + + old_outpath = concore.outpath + yield + concore.outpath = old_outpath + + @pytest.fixture(autouse=True) + def reset_zmq_ports(self): + import concore + + original_ports = concore.zmq_ports.copy() + yield + concore.zmq_ports.clear() + concore.zmq_ports.update(original_ports) + + # ---- Test Case 1: single-output write keeps simtime unchanged ---- + + def test_single_file_write_does_not_mutate_simtime(self, temp_dir): + """A single file-based write with delta must not change simtime.""" + import concore + + concore.simtime = 10 + out_dir = os.path.join(temp_dir, "out1") + os.makedirs(out_dir, exist_ok=True) + concore.outpath = os.path.join(temp_dir, "out") + + concore.write(1, "v", [5.0], delta=1) + + assert concore.simtime == 10, ( + "simtime must not be mutated by write(); " + "was %s instead of 10" % concore.simtime + ) + + def test_single_zmq_write_does_not_mutate_simtime(self): + """A single ZMQ-based write with delta must not change simtime.""" + import concore + + class DummyPort: + def send_json_with_retry(self, msg): + self.sent = msg + + dummy = DummyPort() + concore.zmq_ports["zmq_test"] = dummy + concore.simtime = 10 + + concore.write("zmq_test", "v", [5.0], delta=1) + + assert concore.simtime == 10, ( + "simtime must not be mutated by ZMQ write(); " + "was %s instead of 10" % concore.simtime + ) + + # ---- Test Case 2: multi-port write → identical timestamps ---- + + def test_multi_port_file_writes_share_same_timestamp(self, temp_dir): + """Two consecutive file writes with delta=1 must produce the + same timestamp (simtime+delta), proving simtime is not incremented + between calls.""" + import concore + + concore.simtime = 10 + concore.outpath = os.path.join(temp_dir, "out") + for p in (1, 2): + os.makedirs(os.path.join(temp_dir, "out" + str(p)), exist_ok=True) + + concore.write(1, "u", [1.0], delta=1) + concore.write(2, "v", [2.0], delta=1) + + # Read back the written files and compare timestamps + from ast import literal_eval + + payloads = [] + for p in (1, 2): + with open( + os.path.join(temp_dir, "out" + str(p), ("u" if p == 1 else "v")) + ) as f: + payloads.append(literal_eval(f.read())) + + ts1, ts2 = payloads[0][0], payloads[1][0] + assert ts1 == ts2 == 11, ( + "Both ports must share timestamp simtime+delta=11; " + "got %s and %s" % (ts1, ts2) + ) + + def test_multi_port_zmq_writes_share_same_timestamp(self): + """Two consecutive ZMQ writes with delta=1 must produce the + same timestamp.""" + import concore + + class DummyPort: + def __init__(self): + self.sent = None + + def send_json_with_retry(self, msg): + self.sent = msg + + d1, d2 = DummyPort(), DummyPort() + concore.zmq_ports["p1"] = d1 + concore.zmq_ports["p2"] = d2 + concore.simtime = 10 + + concore.write("p1", "u", [1.0], delta=1) + concore.write("p2", "v", [2.0], delta=1) + + assert d1.sent[0] == d2.sent[0] == 11, ( + "Both ZMQ ports must share timestamp 11; got %s and %s" + % (d1.sent[0], d2.sent[0]) + ) + + # ---- Test Case 3: cross-language parity check ---- + + def test_write_timestamp_matches_cpp_semantics(self, temp_dir): + """C++ uses `simtime+delta` as a local expression without mutation. + After N writes with delta=1, simtime must still be the original + value — matching C++ behaviour.""" + import concore + + concore.simtime = 0 + concore.outpath = os.path.join(temp_dir, "out") + for p in range(1, 4): + os.makedirs(os.path.join(temp_dir, "out" + str(p)), exist_ok=True) + + for p in range(1, 4): + concore.write(p, "x", [float(p)], delta=1) + + assert concore.simtime == 0, ( + "After 3 writes with delta=1 simtime must remain 0 " + "(matching C++/MATLAB/Verilog); got %s" % concore.simtime + ) + + +class TestZMQRetryExhaustion: + """Issue #393 – recv_json_with_retry / send_json_with_retry must raise + TimeoutError instead of silently returning None when retries are + exhausted, and callers (read / write) must handle it gracefully.""" + + @patch("concore_base.time.sleep") + def test_recv_raises_timeout_error(self, _mock_sleep): + """recv_json_with_retry must raise TimeoutError after 5 failed attempts.""" + import concore_base + + port = concore_base.ZeroMQPort.__new__(concore_base.ZeroMQPort) + port.address = "tcp://127.0.0.1:9999" + + class FakeSocket: + def recv_json(self, flags=0): + import zmq + + raise zmq.Again("Resource temporarily unavailable") + + port.socket = FakeSocket() + with pytest.raises(TimeoutError): + port.recv_json_with_retry() + + @patch("concore_base.time.sleep") + def test_send_raises_timeout_error(self, _mock_sleep): + """send_json_with_retry must raise TimeoutError after 5 failed attempts.""" + import concore_base + + port = concore_base.ZeroMQPort.__new__(concore_base.ZeroMQPort) + port.address = "tcp://127.0.0.1:9999" + + class FakeSocket: + def send_json(self, data, flags=0): + import zmq + + raise zmq.Again("Resource temporarily unavailable") + + port.socket = FakeSocket() + with pytest.raises(TimeoutError): + port.send_json_with_retry([42]) + + def test_read_returns_default_on_timeout(self, temp_dir): + """read() must return (default, False) when ZMQ recv times out.""" + import concore + import concore_base + + # Save original global state to avoid leaking into other tests. + had_inpath = hasattr(concore, "inpath") + orig_inpath = concore.inpath if had_inpath else None + orig_zmq_ports = dict(concore.zmq_ports) + had_simtime = hasattr(concore, "simtime") + orig_simtime = concore.simtime if had_simtime else None + + try: + concore.inpath = os.path.join(temp_dir, "in") + + class TimeoutPort: + address = "tcp://127.0.0.1:0" + socket = None + + def recv_json_with_retry(self): + raise TimeoutError("ZMQ recv failed after 5 retries") + + concore.zmq_ports["t_in"] = TimeoutPort() + concore.simtime = 0 + + result, ok = concore.read("t_in", "x", "[0.0]") + + assert result == [0.0] + assert ok is False + assert concore_base.last_read_status == "TIMEOUT" + finally: + # Restore zmq_ports and other globals to their original state. + concore.zmq_ports.clear() + concore.zmq_ports.update(orig_zmq_ports) + + if had_inpath: + concore.inpath = orig_inpath + elif hasattr(concore, "inpath"): + delattr(concore, "inpath") + + if had_simtime: + concore.simtime = orig_simtime + elif hasattr(concore, "simtime"): + delattr(concore, "simtime") + + def test_write_does_not_crash_on_timeout(self, temp_dir): + """write() must not propagate TimeoutError to the caller.""" + import concore + + # Save original global state to avoid leaking into other tests. + had_outpath = hasattr(concore, "outpath") + orig_outpath = concore.outpath if had_outpath else None + orig_zmq_ports = dict(concore.zmq_ports) + had_simtime = hasattr(concore, "simtime") + orig_simtime = concore.simtime if had_simtime else None + + try: + concore.outpath = os.path.join(temp_dir, "out") + os.makedirs(os.path.join(temp_dir, "out_t_out"), exist_ok=True) + + class TimeoutPort: + address = "tcp://127.0.0.1:0" + socket = None + + def send_json_with_retry(self, message): + raise TimeoutError("ZMQ send failed after 5 retries") + + concore.zmq_ports["t_out"] = TimeoutPort() + concore.simtime = 0 + + concore.write("t_out", "y", [1.0], delta=1) + finally: + # Restore zmq_ports and other globals to their original state. + concore.zmq_ports.clear() + concore.zmq_ports.update(orig_zmq_ports) + + if had_outpath: + concore.outpath = orig_outpath + elif hasattr(concore, "outpath"): + delattr(concore, "outpath") + + if had_simtime: + concore.simtime = orig_simtime + elif hasattr(concore, "simtime"): + delattr(concore, "simtime") + + +class TestPidRegistry: + """Tests for the Windows PID registry mechanism (Issue #391).""" + + @pytest.fixture(autouse=True) + def use_temp_dir(self, temp_dir, monkeypatch): + self.temp_dir = temp_dir + monkeypatch.chdir(temp_dir) + import concore + + monkeypatch.setattr(concore, "_BASE_DIR", temp_dir) + monkeypatch.setattr( + concore, + "_PID_REGISTRY_FILE", + os.path.join(temp_dir, "concorekill_pids.txt"), + ) + monkeypatch.setattr( + concore, + "_KILL_SCRIPT_FILE", + os.path.join(temp_dir, "concorekill.bat"), + ) + + def test_register_pid_creates_registry_file(self): + from concore import _register_pid, _PID_REGISTRY_FILE + + _register_pid() + assert os.path.exists(_PID_REGISTRY_FILE) + with open(_PID_REGISTRY_FILE) as f: + pids = [line.strip() for line in f if line.strip()] + assert str(os.getpid()) in pids + + def test_register_pid_appends_not_overwrites(self): + from concore import _register_pid, _PID_REGISTRY_FILE + + with open(_PID_REGISTRY_FILE, "w") as f: + f.write("11111\n") + f.write("22222\n") + _register_pid() + with open(_PID_REGISTRY_FILE) as f: + pids = [line.strip() for line in f if line.strip()] + assert "11111" in pids + assert "22222" in pids + assert str(os.getpid()) in pids + assert len(pids) == 3 + + def test_cleanup_pid_removes_current_pid(self): + from concore import _cleanup_pid, _PID_REGISTRY_FILE + + current_pid = str(os.getpid()) + with open(_PID_REGISTRY_FILE, "w") as f: + f.write("99999\n") + f.write(current_pid + "\n") + f.write("88888\n") + _cleanup_pid() + with open(_PID_REGISTRY_FILE) as f: + pids = [line.strip() for line in f if line.strip()] + assert current_pid not in pids + assert "99999" in pids + assert "88888" in pids + + def test_cleanup_pid_deletes_files_when_last_pid(self): + from concore import _cleanup_pid, _PID_REGISTRY_FILE, _KILL_SCRIPT_FILE + + current_pid = str(os.getpid()) + with open(_PID_REGISTRY_FILE, "w") as f: + f.write(current_pid + "\n") + with open(_KILL_SCRIPT_FILE, "w") as f: + f.write("@echo off\n") + _cleanup_pid() + assert not os.path.exists(_PID_REGISTRY_FILE) + assert not os.path.exists(_KILL_SCRIPT_FILE) + + def test_cleanup_pid_handles_missing_registry(self): + from concore import _cleanup_pid, _PID_REGISTRY_FILE + + assert not os.path.exists(_PID_REGISTRY_FILE) + _cleanup_pid() # Should not raise + + def test_write_kill_script_generates_bat_file(self): + from concore import _write_kill_script, _KILL_SCRIPT_FILE, _PID_REGISTRY_FILE + + _write_kill_script() + assert os.path.exists(_KILL_SCRIPT_FILE) + with open(_KILL_SCRIPT_FILE) as f: + content = f.read() + assert os.path.basename(_PID_REGISTRY_FILE) in content + assert "wmic" in content + assert "taskkill" in content + assert "concore" in content.lower() + + def test_multi_node_registration(self): + from concore import _register_pid, _PID_REGISTRY_FILE + + fake_pids = ["1204", "1932", "8120"] + with open(_PID_REGISTRY_FILE, "w") as f: + for pid in fake_pids: + f.write(pid + "\n") + _register_pid() + with open(_PID_REGISTRY_FILE) as f: + pids = [line.strip() for line in f if line.strip()] + for pid in fake_pids: + assert pid in pids + assert str(os.getpid()) in pids + assert len(pids) == 4 + + def test_cleanup_preserves_other_pids(self): + from concore import _cleanup_pid, _PID_REGISTRY_FILE + + current_pid = str(os.getpid()) + other_pids = ["1111", "2222", "3333"] + with open(_PID_REGISTRY_FILE, "w") as f: + for pid in other_pids: + f.write(pid + "\n") + f.write(current_pid + "\n") + _cleanup_pid() + with open(_PID_REGISTRY_FILE) as f: + pids = [line.strip() for line in f if line.strip()] + assert len(pids) == 3 + assert current_pid not in pids + for pid in other_pids: + assert pid in pids + + @pytest.mark.skipif( + not hasattr(sys, "getwindowsversion"), reason="Windows-only test" + ) + def test_import_registers_pid_on_windows(self): + """Verify module-level PID registration on Windows.""" + import importlib + + for mod_name in ("concore", "concore_base"): + sys.modules.pop(mod_name, None) + import concore + + assert os.path.exists(concore._PID_REGISTRY_FILE) + with open(concore._PID_REGISTRY_FILE) as f: + pids = [line.strip() for line in f if line.strip()] + assert str(os.getpid()) in pids + importlib.reload(concore) diff --git a/tests/test_concoredocker.py b/tests/test_concoredocker.py new file mode 100644 index 00000000..fe19dd1b --- /dev/null +++ b/tests/test_concoredocker.py @@ -0,0 +1,340 @@ +import os +import pytest + + +class TestSafeLiteralEval: + def test_reads_dictionary_from_file(self, temp_dir): + test_file = os.path.join(temp_dir, "ports.txt") + with open(test_file, "w") as f: + f.write("{'a': 1, 'b': 2}") + + from concoredocker import safe_literal_eval + + result = safe_literal_eval(test_file, {}) + + assert result == {"a": 1, "b": 2} + + def test_reads_list_from_file(self, temp_dir): + test_file = os.path.join(temp_dir, "data.txt") + with open(test_file, "w") as f: + f.write("[1, 2, 3]") + + from concoredocker import safe_literal_eval + + result = safe_literal_eval(test_file, []) + + assert result == [1, 2, 3] + + def test_returns_default_when_file_missing(self): + from concoredocker import safe_literal_eval + + result = safe_literal_eval("/nonexistent.txt", {"default": True}) + + assert result == {"default": True} + + def test_returns_default_for_bad_syntax(self, temp_dir): + test_file = os.path.join(temp_dir, "bad.txt") + with open(test_file, "w") as f: + f.write("not valid {{{") + + from concoredocker import safe_literal_eval + + result = safe_literal_eval(test_file, "fallback") + + assert result == "fallback" + + +class TestUnchanged: + def test_returns_true_when_unchanged(self): + import concoredocker + + concoredocker.s = "abc" + concoredocker.olds = "abc" + + assert concoredocker.unchanged() == True + assert concoredocker.s == "" + + def test_returns_false_when_changed(self): + import concoredocker + + concoredocker.s = "new" + concoredocker.olds = "old" + + assert concoredocker.unchanged() == False + assert concoredocker.olds == "new" + + +class TestInitval: + def test_parses_simtime_and_values(self): + import concoredocker + + concoredocker.simtime = 0 + result = concoredocker.initval("[5.0, 1.0, 2.0]") + + assert result == [1.0, 2.0] + assert concoredocker.simtime == 5.0 + + def test_parses_single_value(self): + import concoredocker + + concoredocker.simtime = 0 + result = concoredocker.initval("[10.0, 99]") + + assert result == [99] + assert concoredocker.simtime == 10.0 + + +class TestWrite: + def test_writes_list_with_simtime(self, temp_dir): + import concoredocker + + old_outpath = concoredocker.outpath + outdir = os.path.join(temp_dir, "1") + os.makedirs(outdir) + concoredocker.outpath = temp_dir + concoredocker.simtime = 5.0 + + concoredocker.write(1, "testfile", [1.0, 2.0], delta=0) + + with open(os.path.join(outdir, "testfile")) as f: + content = f.read() + assert content == "[5.0, 1.0, 2.0]" + concoredocker.outpath = old_outpath + + def test_writes_with_delta(self, temp_dir): + import concoredocker + + old_outpath = concoredocker.outpath + outdir = os.path.join(temp_dir, "1") + os.makedirs(outdir) + concoredocker.outpath = temp_dir + concoredocker.simtime = 10.0 + + concoredocker.write(1, "testfile", [3.0], delta=2) + + with open(os.path.join(outdir, "testfile")) as f: + content = f.read() + assert content == "[12.0, 3.0]" + # simtime must not be mutated by write() + assert concoredocker.simtime == 10.0 + concoredocker.outpath = old_outpath + + +class TestRead: + def test_reads_and_parses_data(self, temp_dir): + import concoredocker + + old_inpath = concoredocker.inpath + old_delay = concoredocker.delay + indir = os.path.join(temp_dir, "1") + os.makedirs(indir) + concoredocker.inpath = temp_dir + concoredocker.delay = 0.001 + + with open(os.path.join(indir, "data"), "w") as f: + f.write("[7.0, 100, 200]") + + concoredocker.s = "" + concoredocker.simtime = 0 + result, ok = concoredocker.read(1, "data", "[0, 0, 0]") + + assert result == [100, 200] + assert ok is True + assert concoredocker.simtime == 7.0 + concoredocker.inpath = old_inpath + concoredocker.delay = old_delay + + def test_returns_default_when_file_missing(self, temp_dir): + import concoredocker + + old_inpath = concoredocker.inpath + old_delay = concoredocker.delay + indir = os.path.join(temp_dir, "1") + os.makedirs(indir) + concoredocker.inpath = temp_dir + concoredocker.delay = 0.001 + + concoredocker.s = "" + concoredocker.simtime = 0 + result, ok = concoredocker.read(1, "nofile", "[0, 5, 5]") + + assert result == [5, 5] + assert ok is False + concoredocker.inpath = old_inpath + concoredocker.delay = old_delay + + +class TestZMQ: + @pytest.fixture(autouse=True) + def reset_zmq_ports(self): + import concoredocker + + original_ports = concoredocker.zmq_ports.copy() + yield + concoredocker.zmq_ports.clear() + concoredocker.zmq_ports.update(original_ports) + + def test_write_prepends_simtime(self): + import concoredocker + + class DummyPort: + def __init__(self): + self.sent = None + + def send_json_with_retry(self, message): + self.sent = message + + dummy = DummyPort() + concoredocker.zmq_ports["test_zmq"] = dummy + concoredocker.simtime = 3.0 + + concoredocker.write("test_zmq", "data", [1.0, 2.0], delta=2) + + assert dummy.sent == [5.0, 1.0, 2.0] + # simtime must not be mutated by write() + assert concoredocker.simtime == 3.0 + + def test_read_strips_simtime(self): + import concoredocker + + class DummyPort: + def recv_json_with_retry(self): + return [10.0, 4.0, 5.0] + + concoredocker.zmq_ports["test_zmq"] = DummyPort() + concoredocker.simtime = 0 + + result, ok = concoredocker.read("test_zmq", "data", "[]") + + assert result == [4.0, 5.0] + assert ok is True + assert concoredocker.simtime == 10.0 + + def test_read_updates_simtime_monotonically(self): + import concoredocker + + class DummyPort: + def recv_json_with_retry(self): + return [2.0, 99.0] + + concoredocker.zmq_ports["test_zmq"] = DummyPort() + concoredocker.simtime = 5.0 + + concoredocker.read("test_zmq", "data", "[]") + + assert concoredocker.simtime == 5.0 + + def test_write_read_roundtrip(self): + import concoredocker + + class DummyPort: + def __init__(self): + self.buffer = None + + def send_json_with_retry(self, message): + self.buffer = message + + def recv_json_with_retry(self): + return self.buffer + + dummy = DummyPort() + concoredocker.zmq_ports["roundtrip"] = dummy + concoredocker.simtime = 0 + + original = [1.5, 2.5, 3.5] + concoredocker.write("roundtrip", "data", original) + result, ok = concoredocker.read("roundtrip", "data", "[]") + + assert result == original + assert ok is True + + +class TestZMQRetryExhaustion: + """Issue #393 – concoredocker read/write must handle TimeoutError + raised by ZMQ retry exhaustion without crashing.""" + + def test_read_returns_default_on_timeout(self, temp_dir): + """read() must return (default, False) when ZMQ recv times out.""" + import concoredocker + import concore_base + + # Save original global state to avoid leaking into other tests. + had_inpath = hasattr(concoredocker, "inpath") + orig_inpath = concoredocker.inpath if had_inpath else None + orig_zmq_ports = dict(concoredocker.zmq_ports) + had_simtime = hasattr(concoredocker, "simtime") + orig_simtime = concoredocker.simtime if had_simtime else None + + try: + concoredocker.inpath = os.path.join(temp_dir, "in") + + class TimeoutPort: + address = "tcp://127.0.0.1:0" + socket = None + + def recv_json_with_retry(self): + raise TimeoutError("ZMQ recv failed after 5 retries") + + concoredocker.zmq_ports["t_in"] = TimeoutPort() + concoredocker.simtime = 0 + + result, ok = concoredocker.read("t_in", "x", "[0.0]") + + assert result == [0.0] + assert ok is False + assert concore_base.last_read_status == "TIMEOUT" + finally: + # Restore zmq_ports and other globals to their original state. + concoredocker.zmq_ports.clear() + concoredocker.zmq_ports.update(orig_zmq_ports) + + if had_inpath: + concoredocker.inpath = orig_inpath + elif hasattr(concoredocker, "inpath"): + delattr(concoredocker, "inpath") + + if had_simtime: + concoredocker.simtime = orig_simtime + elif hasattr(concoredocker, "simtime"): + delattr(concoredocker, "simtime") + + def test_write_does_not_crash_on_timeout(self, temp_dir): + """write() must not propagate TimeoutError to the caller.""" + import concoredocker + + # Save original global state to avoid leaking into other tests. + had_outpath = hasattr(concoredocker, "outpath") + orig_outpath = concoredocker.outpath if had_outpath else None + orig_zmq_ports = dict(concoredocker.zmq_ports) + had_simtime = hasattr(concoredocker, "simtime") + orig_simtime = concoredocker.simtime if had_simtime else None + + try: + concoredocker.outpath = os.path.join(temp_dir, "out") + os.makedirs(os.path.join(temp_dir, "out_t_out"), exist_ok=True) + + class TimeoutPort: + address = "tcp://127.0.0.1:0" + socket = None + + def send_json_with_retry(self, message): + raise TimeoutError("ZMQ send failed after 5 retries") + + concoredocker.zmq_ports["t_out"] = TimeoutPort() + concoredocker.simtime = 0 + + concoredocker.write("t_out", "y", [1.0], delta=1) + finally: + # Restore zmq_ports and other globals to their original state. + concoredocker.zmq_ports.clear() + concoredocker.zmq_ports.update(orig_zmq_ports) + + if had_outpath: + concoredocker.outpath = orig_outpath + elif hasattr(concoredocker, "outpath"): + delattr(concoredocker, "outpath") + + if had_simtime: + concoredocker.simtime = orig_simtime + elif hasattr(concoredocker, "simtime"): + delattr(concoredocker, "simtime") diff --git a/tests/test_doctor.py b/tests/test_doctor.py new file mode 100644 index 00000000..e9ec5846 --- /dev/null +++ b/tests/test_doctor.py @@ -0,0 +1,213 @@ +import unittest +import tempfile +import shutil +from pathlib import Path +from unittest.mock import patch +from click.testing import CliRunner +from concore_cli.cli import cli +from concore_cli.commands.doctor import ( + _detect_tool, + _get_platform_key, + _check_package, + _resolve_concore_path, + doctor_check, +) + + +class TestDoctorCommand(unittest.TestCase): + """Tests for the concore doctor CLI command.""" + + def setUp(self): + self.runner = CliRunner() + + def test_doctor_command_runs(self): + """Doctor command should run and produce output.""" + result = self.runner.invoke(cli, ["doctor"]) + self.assertIn("concore Doctor", result.output) + self.assertIn("Core Checks", result.output) + self.assertIn("Tools", result.output) + self.assertIn("Configuration", result.output) + self.assertIn("Dependencies", result.output) + self.assertIn("Summary", result.output) + + def test_doctor_help(self): + """Doctor command should have help text.""" + result = self.runner.invoke(cli, ["doctor", "--help"]) + self.assertEqual(result.exit_code, 0) + self.assertIn("Check system readiness", result.output) + + def test_doctor_shows_python_version(self): + """Doctor should show the current Python version.""" + result = self.runner.invoke(cli, ["doctor"]) + import platform + + py_version = platform.python_version() + self.assertIn(py_version, result.output) + + def test_doctor_shows_concore_version(self): + """Doctor should detect and show concore version.""" + from concore_cli import __version__ + + result = self.runner.invoke(cli, ["doctor"]) + self.assertIn("concore", result.output) + self.assertIn(__version__, result.output) + + def test_doctor_shows_concorepath(self): + """Doctor should show the CONCOREPATH.""" + result = self.runner.invoke(cli, ["doctor"]) + self.assertIn("CONCOREPATH", result.output) + + def test_doctor_checks_dependencies(self): + """Doctor should check required Python packages.""" + result = self.runner.invoke(cli, ["doctor"]) + # These should be installed since we're running tests + self.assertIn("click", result.output) + self.assertIn("rich", result.output) + + def test_doctor_shows_summary(self): + """Doctor should show a summary with pass/warn/error counts.""" + result = self.runner.invoke(cli, ["doctor"]) + self.assertIn("Summary", result.output) + self.assertIn("passed", result.output) + + +class TestDetectTool(unittest.TestCase): + """Tests for tool detection helpers.""" + + def test_detect_python(self): + """Should detect the currently running Python.""" + # python or python3 should be findable + path, name = _detect_tool(["python3", "python"]) + self.assertIsNotNone(path) + self.assertIn(name, ["python3", "python"]) + + def test_detect_nonexistent_tool(self): + """Should return None for a tool that doesn't exist.""" + path, name = _detect_tool(["nonexistent_tool_abc123"]) + self.assertIsNone(path) + self.assertIsNone(name) + + def test_detect_tool_tries_multiple_names(self): + """Should try all candidate names and return the first match.""" + path, name = _detect_tool(["nonexistent_tool_abc123", "python3", "python"]) + self.assertIsNotNone(path) + + def test_detect_tool_empty_list(self): + """Should handle an empty candidate list gracefully.""" + path, name = _detect_tool([]) + self.assertIsNone(path) + self.assertIsNone(name) + + +class TestGetPlatformKey(unittest.TestCase): + """Tests for platform detection.""" + + def test_returns_valid_key(self): + """Should return 'posix' or 'windows'.""" + key = _get_platform_key() + self.assertIn(key, ["posix", "windows"]) + + @patch("concore_cli.commands.doctor.os.name", "nt") + def test_windows_detection(self): + """Should return 'windows' when os.name is 'nt'.""" + key = _get_platform_key() + self.assertEqual(key, "windows") + + @patch("concore_cli.commands.doctor.os.name", "posix") + def test_posix_detection(self): + """Should return 'posix' when os.name is 'posix'.""" + key = _get_platform_key() + self.assertEqual(key, "posix") + + +class TestCheckPackage(unittest.TestCase): + """Tests for package checking.""" + + def test_check_installed_package(self): + """Should detect an installed package.""" + found, version = _check_package("click") + self.assertTrue(found) + self.assertIsNotNone(version) + + def test_check_missing_package(self): + """Should return False for a package that isn't installed.""" + found, version = _check_package("nonexistent_package_abc123") + self.assertFalse(found) + self.assertIsNone(version) + + def test_check_package_with_import_name_map(self): + """Should use the correct import name for beautifulsoup4 (bs4).""" + found, version = _check_package("beautifulsoup4") + self.assertTrue(found) + + def test_check_pyzmq_import_name(self): + """Should use 'zmq' as import name for pyzmq.""" + found, version = _check_package("pyzmq") + self.assertTrue(found) + + +class TestResolveConCorePath(unittest.TestCase): + """Tests for CONCOREPATH resolution.""" + + def test_resolves_to_existing_path(self): + """Should return a Path object.""" + result = _resolve_concore_path() + self.assertIsInstance(result, Path) + + +class TestDoctorWithConfig(unittest.TestCase): + """Tests for doctor command with config files present.""" + + def setUp(self): + self.runner = CliRunner() + self.temp_dir = tempfile.mkdtemp() + + def tearDown(self): + shutil.rmtree(self.temp_dir) + + @patch("concore_cli.commands.doctor._resolve_concore_path") + def test_doctor_with_concore_tools(self, mock_path): + """Doctor should detect and report concore.tools.""" + mock_path.return_value = Path(self.temp_dir) + tools_file = Path(self.temp_dir) / "concore.tools" + tools_file.write_text("CPPEXE=/usr/bin/g++\nPYTHONEXE=/usr/bin/python3\n") + + from rich.console import Console + import io + + console = Console(file=io.StringIO(), force_terminal=True) + result = doctor_check(console) + # Just verify it doesn't crash + self.assertIsInstance(result, bool) + + @patch("concore_cli.commands.doctor._resolve_concore_path") + def test_doctor_with_concore_octave(self, mock_path): + """Doctor should detect concore.octave flag.""" + mock_path.return_value = Path(self.temp_dir) + octave_file = Path(self.temp_dir) / "concore.octave" + octave_file.write_text("") + + from rich.console import Console + import io + + console = Console(file=io.StringIO(), force_terminal=True) + result = doctor_check(console) + self.assertIsInstance(result, bool) + + @patch("concore_cli.commands.doctor._resolve_concore_path") + def test_doctor_with_concore_sudo(self, mock_path): + """Doctor should detect concore.sudo config.""" + mock_path.return_value = Path(self.temp_dir) + sudo_file = Path(self.temp_dir) / "concore.sudo" + sudo_file.write_text("docker") + + from rich.console import Console + import io + + console = Console(file=io.StringIO(), force_terminal=True) + result = doctor_check(console) + self.assertIsInstance(result, bool) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_graph.py b/tests/test_graph.py new file mode 100644 index 00000000..efebe3e5 --- /dev/null +++ b/tests/test_graph.py @@ -0,0 +1,314 @@ +import unittest +import tempfile +import shutil +from pathlib import Path +from click.testing import CliRunner +from concore_cli.cli import cli + + +class TestGraphValidation(unittest.TestCase): + def setUp(self): + self.runner = CliRunner() + self.temp_dir = tempfile.mkdtemp() + + def tearDown(self): + if Path(self.temp_dir).exists(): + shutil.rmtree(self.temp_dir) + + def create_graph_file(self, filename, content): + filepath = Path(self.temp_dir) / filename + with open(filepath, "w") as f: + f.write(content) + return str(filepath) + + def test_validate_corrupted_xml(self): + content = '' + filepath = self.create_graph_file("corrupted.graphml", content) + + result = self.runner.invoke(cli, ["validate", filepath]) + + self.assertIn("Validation failed", result.output) + self.assertIn("Invalid XML", result.output) + + def test_validate_empty_file(self): + filepath = self.create_graph_file("empty.graphml", "") + + result = self.runner.invoke(cli, ["validate", filepath]) + + self.assertIn("Validation failed", result.output) + self.assertIn("File is empty", result.output) + + def test_validate_missing_node_id(self): + content = """ + + + + n0:script.py + + + + """ + filepath = self.create_graph_file("missing_id.graphml", content) + result = self.runner.invoke(cli, ["validate", filepath]) + self.assertIn("Validation failed", result.output) + self.assertIn("Node missing required 'id' attribute", result.output) + + def test_validate_missing_edgedefault(self): + content = """ + + + + n0:script.py + + + + """ + filepath = self.create_graph_file("missing_default.graphml", content) + result = self.runner.invoke(cli, ["validate", filepath]) + self.assertIn("Validation failed", result.output) + self.assertIn("Graph missing required 'edgedefault'", result.output) + + def test_validate_missing_root_element(self): + content = '' + filepath = self.create_graph_file("not_graphml.xml", content) + + result = self.runner.invoke(cli, ["validate", filepath]) + + self.assertIn("Validation failed", result.output) + self.assertIn("missing root element", result.output) + + def test_validate_broken_edges(self): + content = """ + + + + n0:script.py + + + + + """ + filepath = self.create_graph_file("bad_edge.graphml", content) + + result = self.runner.invoke(cli, ["validate", filepath]) + + self.assertIn("Validation failed", result.output) + self.assertIn("Edge references non-existent target node", result.output) + + def test_validate_node_missing_filename(self): + content = """ + + + + n0: + + + + """ + filepath = self.create_graph_file("bad_node.graphml", content) + + result = self.runner.invoke(cli, ["validate", filepath]) + + self.assertIn("Validation failed", result.output) + self.assertIn("has no filename", result.output) + + def test_validate_unsafe_node_label(self): + content = """ + + + + n0;rm -rf /:script.py + + + + """ + filepath = self.create_graph_file("injection.graphml", content) + + result = self.runner.invoke(cli, ["validate", filepath]) + + self.assertIn("Validation failed", result.output) + self.assertIn("unsafe shell characters", result.output) + + def test_validate_valid_graph(self): + content = """ + + + + n0:script.py + + + + """ + filepath = self.create_graph_file("valid.graphml", content) + + result = self.runner.invoke(cli, ["validate", filepath]) + + self.assertIn("Validation passed", result.output) + self.assertIn("Workflow is valid", result.output) + + def test_validate_missing_source_file(self): + content = """ + + + + n0:missing.py + + + + """ + filepath = self.create_graph_file("workflow.graphml", content) + source_dir = Path(self.temp_dir) / "src" + source_dir.mkdir() + + result = self.runner.invoke( + cli, ["validate", filepath, "--source", str(source_dir)] + ) + + self.assertIn("Validation failed", result.output) + self.assertIn("Missing source file", result.output) + + def test_validate_with_existing_source_file(self): + content = """ + + + + n0:exists.py + + + + """ + filepath = self.create_graph_file("workflow.graphml", content) + source_dir = Path(self.temp_dir) / "src" + source_dir.mkdir() + (source_dir / "exists.py").write_text('print("hello")') + + result = self.runner.invoke( + cli, ["validate", filepath, "--source", str(source_dir)] + ) + + self.assertIn("Validation passed", result.output) + + def test_validate_zmq_port_conflict(self): + content = """ + + + + n0:script1.py + + + n1:script2.py + + + 0x1234_portA + + + 0x1234_portB + + + + """ + filepath = self.create_graph_file("conflict.graphml", content) + + result = self.runner.invoke(cli, ["validate", filepath]) + + self.assertIn("Validation failed", result.output) + self.assertIn("Port conflict", result.output) + + def test_validate_reserved_port(self): + content = """ + + + + n0:script1.py + + + n1:script2.py + + + 0x50_data + + + + """ + filepath = self.create_graph_file("reserved.graphml", content) + + result = self.runner.invoke(cli, ["validate", filepath]) + + self.assertIn("Port 80", result.output) + self.assertIn("reserved range", result.output) + + def test_validate_cycle_detection(self): + content = """ + + + + n0:controller.py + + + n1:plant.py + + + control_signal + + + sensor_data + + + + """ + filepath = self.create_graph_file("cycle.graphml", content) + + result = self.runner.invoke(cli, ["validate", filepath]) + + self.assertIn("cycles", result.output) + self.assertIn("control loops", result.output) + + def test_validate_port_zero(self): + content = """ + + + + n0:script1.py + + + n1:script2.py + + + 0x0_invalid + + + + """ + filepath = self.create_graph_file("port_zero.graphml", content) + + result = self.runner.invoke(cli, ["validate", filepath]) + + self.assertIn("Validation failed", result.output) + self.assertIn("must be at least 1", result.output) + + def test_validate_port_exceeds_maximum(self): + content = """ + + + + n0:script1.py + + + n1:script2.py + + + 0x10000_toobig + + + + """ + filepath = self.create_graph_file("port_max.graphml", content) + + result = self.runner.invoke(cli, ["validate", filepath]) + + self.assertIn("Validation failed", result.output) + self.assertIn("exceeds maximum (65535)", result.output) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_openjupyter_security.py b/tests/test_openjupyter_security.py new file mode 100644 index 00000000..06b29065 --- /dev/null +++ b/tests/test_openjupyter_security.py @@ -0,0 +1,164 @@ +"""Tests for the secured /openJupyter/ and /stopJupyter/ endpoints.""" + +import os +import sys +import pytest +from unittest.mock import patch, MagicMock + +# Ensure the project root is on the path +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +# Skip entire module if flask is not installed (e.g. in CI with minimal deps) +pytest.importorskip( + "flask", reason="flask not installed — skipping server endpoint tests" +) + +# Set a test API key before importing the app module +TEST_API_KEY = "test-secret-key-12345" + + +@pytest.fixture(autouse=True) +def reset_jupyter_process(): + """Reset the module-level jupyter_process before each test.""" + with patch.dict( + os.environ, + { + "CONCORE_API_KEY": TEST_API_KEY, + "FLASK_SECRET_KEY": "test-flask-secret-key", + }, + clear=False, + ): + import fri.server.main as mod + + mod.API_KEY = TEST_API_KEY + mod.jupyter_process = None + yield + mod.jupyter_process = None + + +@pytest.fixture +def client(): + """Create a Flask test client with the API key configured.""" + with patch.dict( + os.environ, + { + "CONCORE_API_KEY": TEST_API_KEY, + "FLASK_SECRET_KEY": "test-flask-secret-key", + }, + clear=False, + ): + # Re-read env var after patching + import fri.server.main as mod + + mod.API_KEY = TEST_API_KEY + mod.app.config["TESTING"] = True + with mod.app.test_client() as c: + yield c + + +@pytest.fixture +def client_no_key(): + """Create a Flask test client without API key configured.""" + import fri.server.main as mod + + mod.API_KEY = None + mod.app.config["TESTING"] = True + with mod.app.test_client() as c: + yield c + + +class TestOpenJupyterAuth: + """Test authentication on /openJupyter/ endpoint.""" + + def test_missing_api_key_header_returns_403(self, client): + """Request without X-API-KEY header should be rejected.""" + resp = client.post("/openJupyter/") + assert resp.status_code == 403 + + def test_wrong_api_key_returns_403(self, client): + """Request with wrong key should be rejected.""" + resp = client.post("/openJupyter/", headers={"X-API-KEY": "wrong-key"}) + assert resp.status_code == 403 + + def test_server_without_api_key_configured_returns_500(self, client_no_key): + """If CONCORE_API_KEY is not set on server, return 500.""" + resp = client_no_key.post("/openJupyter/", headers={"X-API-KEY": "anything"}) + assert resp.status_code == 500 + + +class TestOpenJupyterProcess: + """Test process control on /openJupyter/ endpoint.""" + + @patch("fri.server.main.subprocess.Popen") + def test_authorized_request_starts_jupyter(self, mock_popen, client): + """Valid API key should start Jupyter Lab.""" + mock_proc = MagicMock() + mock_proc.poll.return_value = None # process running + mock_popen.return_value = mock_proc + + resp = client.post("/openJupyter/", headers={"X-API-KEY": TEST_API_KEY}) + assert resp.status_code == 200 + data = resp.get_json() + assert data["message"] == "Jupyter Lab started" + + # Verify Popen was called with --no-browser and DEVNULL + call_args = mock_popen.call_args + assert "--no-browser" in call_args[0][0] + assert call_args[1].get("shell") is False + + @patch("fri.server.main.subprocess.Popen") + def test_duplicate_launch_returns_409(self, mock_popen, client): + """Second launch while first is still running should return 409.""" + mock_proc = MagicMock() + mock_proc.poll.return_value = None # still running + mock_popen.return_value = mock_proc + + # First launch + resp1 = client.post("/openJupyter/", headers={"X-API-KEY": TEST_API_KEY}) + assert resp1.status_code == 200 + + # Second launch should be rejected + resp2 = client.post("/openJupyter/", headers={"X-API-KEY": TEST_API_KEY}) + assert resp2.status_code == 409 + data = resp2.get_json() + assert data["message"] == "Jupyter already running" + + @patch("fri.server.main.subprocess.Popen", side_effect=OSError("fail")) + def test_popen_failure_returns_500(self, mock_popen, client): + """If Popen raises, return 500.""" + resp = client.post("/openJupyter/", headers={"X-API-KEY": TEST_API_KEY}) + assert resp.status_code == 500 + data = resp.get_json() + assert "error" in data + + +class TestStopJupyter: + """Test /stopJupyter/ endpoint.""" + + def test_stop_without_auth_returns_403(self, client): + """Request without API key should be rejected.""" + resp = client.post("/stopJupyter/") + assert resp.status_code == 403 + + def test_stop_when_no_process_returns_404(self, client): + """Stop with no running process returns 404.""" + resp = client.post("/stopJupyter/", headers={"X-API-KEY": TEST_API_KEY}) + assert resp.status_code == 404 + + @patch("fri.server.main.subprocess.Popen") + def test_stop_running_process_returns_200(self, mock_popen, client): + """Stop a running Jupyter instance returns 200.""" + mock_proc = MagicMock() + mock_proc.poll.return_value = None # running + mock_popen.return_value = mock_proc + + # Start first + client.post("/openJupyter/", headers={"X-API-KEY": TEST_API_KEY}) + + # Stop + resp = client.post("/stopJupyter/", headers={"X-API-KEY": TEST_API_KEY}) + assert resp.status_code == 200 + data = resp.get_json() + assert data["message"] == "Jupyter stopped" + mock_proc.terminate.assert_called_once() + mock_proc.wait.assert_called() diff --git a/tests/test_protocol_conformance.py b/tests/test_protocol_conformance.py new file mode 100644 index 00000000..e12ad2b7 --- /dev/null +++ b/tests/test_protocol_conformance.py @@ -0,0 +1,161 @@ +import json +import os +from pathlib import Path +import tempfile + +import pytest + +import concore + + +FIXTURE_DIR = Path(__file__).parent / "protocol_fixtures" +SCHEMA_PATH = FIXTURE_DIR / "schema.phase1.json" +CASES_PATH = FIXTURE_DIR / "python_phase1_cases.json" +SUPPORTED_TARGETS = {"parse_params", "initval", "write_zmq", "read_file"} + + +def _load_json(path): + with path.open("r", encoding="utf-8") as f: + return json.load(f) + + +def _validate_fixture_document_shape(doc): + required_top = {"schema_version", "runtime", "mode", "cases"} + missing = required_top - set(doc.keys()) + if missing: + raise AssertionError( + f"Fixture document missing required top-level keys: {sorted(missing)}" + ) + if doc["runtime"] != "python": + raise AssertionError( + f"Phase-1 fixture runtime must be 'python', found: {doc['runtime']}" + ) + if doc["mode"] != "report_only": + raise AssertionError( + f"Phase-1 fixture mode must be 'report_only', found: {doc['mode']}" + ) + if not isinstance(doc["cases"], list) or not doc["cases"]: + raise AssertionError("Fixture document must contain a non-empty 'cases' list") + + for idx, case in enumerate(doc["cases"]): + for key in ("id", "target", "input", "expected"): + if key not in case: + raise AssertionError(f"Case index {idx} missing required key '{key}'") + if case["target"] not in SUPPORTED_TARGETS: + raise AssertionError( + f"Case '{case['id']}' has unsupported target '{case['target']}'" + ) + + +def _run_parse_params_case(case): + result = concore.parse_params(case["input"]["sparams"]) + assert result == case["expected"]["result"] + + +def _run_initval_case(case): + old_simtime = concore.simtime + try: + concore.simtime = case["input"]["initial_simtime"] + result = concore.initval(case["input"]["simtime_val_str"]) + assert result == case["expected"]["result"] + assert concore.simtime == case["expected"]["simtime_after"] + finally: + concore.simtime = old_simtime + + +def _run_write_zmq_case(case): + class DummyPort: + def __init__(self): + self.sent_payload = None + + def send_json_with_retry(self, message): + self.sent_payload = message + + old_simtime = concore.simtime + port_name = f"fixture_{case['id'].replace('/', '_')}" + existing_port = concore.zmq_ports.get(port_name) + dummy_port = DummyPort() + + try: + concore.simtime = case["input"]["initial_simtime"] + concore.zmq_ports[port_name] = dummy_port + concore.write( + port_name, + case["input"]["name"], + case["input"]["value"], + delta=case["input"]["delta"], + ) + assert dummy_port.sent_payload == case["expected"]["sent_payload"] + assert concore.simtime == case["expected"]["simtime_after"] + finally: + concore.simtime = old_simtime + if existing_port is None: + concore.zmq_ports.pop(port_name, None) + else: + concore.zmq_ports[port_name] = existing_port + + +def _run_read_file_case(case): + old_simtime = concore.simtime + old_inpath = concore.inpath + old_delay = concore.delay + try: + with tempfile.TemporaryDirectory() as temp_dir: + concore.simtime = case["input"]["initial_simtime"] + concore.inpath = os.path.join(temp_dir, "in") + concore.delay = 0 + + port_dir = os.path.join(temp_dir, f"in{case['input']['port']}") + os.makedirs(port_dir, exist_ok=True) + + if "file_content" in case["input"]: + with open( + os.path.join(port_dir, case["input"]["name"]), + "w", + encoding="utf-8", + ) as f: + f.write(case["input"]["file_content"]) + + result, ok = concore.read( + case["input"]["port"], + case["input"]["name"], + case["input"]["initstr_val"], + ) + + assert result == case["expected"]["result"] + assert ok == case["expected"]["ok"] + assert concore.simtime == case["expected"]["simtime_after"] + finally: + concore.simtime = old_simtime + concore.inpath = old_inpath + concore.delay = old_delay + + +def _run_case(case): + if case["target"] == "parse_params": + _run_parse_params_case(case) + elif case["target"] == "initval": + _run_initval_case(case) + elif case["target"] == "write_zmq": + _run_write_zmq_case(case) + elif case["target"] == "read_file": + _run_read_file_case(case) + else: + raise AssertionError(f"Unsupported target: {case['target']}") + + +def _load_cases(): + doc = _load_json(CASES_PATH) + _validate_fixture_document_shape(doc) + return doc["cases"] + + +def test_phase1_schema_file_present_and_basic_shape(): + schema = _load_json(SCHEMA_PATH) + assert schema["title"] == "Concore Protocol Conformance Fixtures (Phase 1)" + assert "cases" in schema["properties"] + + +@pytest.mark.parametrize("case", _load_cases(), ids=lambda case: case["id"]) +def test_phase1_python_protocol_conformance(case): + _run_case(case) diff --git a/tests/test_protocol_conformance_phase2.py b/tests/test_protocol_conformance_phase2.py new file mode 100644 index 00000000..787e918c --- /dev/null +++ b/tests/test_protocol_conformance_phase2.py @@ -0,0 +1,68 @@ +import json +from pathlib import Path + + +FIXTURE_DIR = Path(__file__).parent / "protocol_fixtures" +PHASE1_CASES_PATH = FIXTURE_DIR / "python_phase1_cases.json" +PHASE2_MATRIX_PATH = FIXTURE_DIR / "cross_runtime_matrix.phase2.json" + +EXPECTED_RUNTIMES = {"python", "cpp", "java", "matlab", "octave", "verilog"} +EXPECTED_CLASSIFICATIONS = {"required", "implementation_defined", "known_deviation"} +EXPECTED_STATUSES = {"observed_pass", "observed_fail", "not_audited"} + + +def _load_json(path): + with path.open("r", encoding="utf-8") as f: + return json.load(f) + + +def _phase1_cases(): + doc = _load_json(PHASE1_CASES_PATH) + return {case["id"]: case for case in doc["cases"]} + + +def _phase2_matrix(): + return _load_json(PHASE2_MATRIX_PATH) + + +def test_phase2_matrix_metadata_and_enums(): + doc = _phase2_matrix() + assert doc["phase"] == "2" + assert doc["mode"] == "report_only" + assert doc["source_fixture"] == "python_phase1_cases.json" + assert "java" in doc["runtimes"] + assert len(doc["runtimes"]) == len(set(doc["runtimes"])) + assert set(doc["runtimes"]) == EXPECTED_RUNTIMES + assert set(doc["classifications"]) == EXPECTED_CLASSIFICATIONS + assert set(doc["statuses"]) == EXPECTED_STATUSES + + +def test_phase2_matrix_covers_all_phase1_cases(): + phase1 = _phase1_cases() + matrix_cases = _phase2_matrix()["cases"] + matrix_ids = {case["id"] for case in matrix_cases} + assert matrix_ids == set(phase1.keys()) + + +def test_phase2_matrix_rows_have_consistent_shape(): + phase1 = _phase1_cases() + for row in _phase2_matrix()["cases"]: + assert row["id"] in phase1 + assert row["target"] == phase1[row["id"]]["target"] + assert set(row["runtime_results"].keys()) == EXPECTED_RUNTIMES + + for runtime, result in row["runtime_results"].items(): + assert result["status"] in EXPECTED_STATUSES + assert result["classification"] in EXPECTED_CLASSIFICATIONS + assert isinstance(result["note"], str) and result["note"].strip() + if runtime == "python": + assert result["status"] == "observed_pass" + + +def test_phase2_matrix_java_status_is_recorded_for_each_case(): + for row in _phase2_matrix()["cases"]: + java_result = row["runtime_results"]["java"] + assert java_result["status"] in EXPECTED_STATUSES + assert java_result["classification"] in EXPECTED_CLASSIFICATIONS + assert isinstance(java_result["note"], str) and java_result["note"].strip() + assert java_result["status"] == "observed_pass" diff --git a/tests/test_read_status.py b/tests/test_read_status.py new file mode 100644 index 00000000..b9fe33d9 --- /dev/null +++ b/tests/test_read_status.py @@ -0,0 +1,282 @@ +"""Tests for read() error signalling (Issue #390). + +read() now returns (data, success_flag) and sets +concore.last_read_status / concore_base.last_read_status. +""" + +import os +import pytest + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +class DummyZMQPort: + """Minimal stand-in for ZeroMQPort used in ZMQ read tests.""" + + def __init__(self, response=None, raise_on_recv=None): + self._response = response + self._raise_on_recv = raise_on_recv + + def send_json_with_retry(self, message): + self._response = message + + def recv_json_with_retry(self): + if self._raise_on_recv: + raise self._raise_on_recv + return self._response + + +# --------------------------------------------------------------------------- +# File-based read tests +# --------------------------------------------------------------------------- + + +class TestReadFileSuccess: + """read() on a valid file returns (data, True) with SUCCESS status.""" + + @pytest.fixture(autouse=True) + def setup(self, temp_dir, monkeypatch): + import concore + + self.concore = concore + monkeypatch.setattr(concore, "delay", 0) + + # Create ./in1/ym with valid data: [simtime, value] + in_dir = os.path.join(temp_dir, "in1") + os.makedirs(in_dir, exist_ok=True) + with open(os.path.join(in_dir, "ym"), "w") as f: + f.write("[10, 3.14]") + + monkeypatch.setattr(concore, "inpath", os.path.join(temp_dir, "in")) + + def test_returns_data_and_true(self): + data, ok = self.concore.read(1, "ym", "[0, 0.0]") + assert ok is True + assert data == [3.14] + + def test_last_read_status_is_success(self): + self.concore.read(1, "ym", "[0, 0.0]") + assert self.concore.last_read_status == "SUCCESS" + + +class TestReadFileMissing: + """read() on a missing file returns (default, False) with FILE_NOT_FOUND.""" + + @pytest.fixture(autouse=True) + def setup(self, temp_dir, monkeypatch): + import concore + + self.concore = concore + monkeypatch.setattr(concore, "delay", 0) + # Point to a directory that does NOT have the file + monkeypatch.setattr(concore, "inpath", os.path.join(temp_dir, "in")) + + def test_returns_default_and_false(self): + data, ok = self.concore.read(1, "nonexistent", "[0, 0.0]") + assert ok is False + + def test_last_read_status_is_file_not_found(self): + self.concore.read(1, "nonexistent", "[0, 0.0]") + assert self.concore.last_read_status == "FILE_NOT_FOUND" + + +class TestReadFileParseError: + """read() returns (default, False) with PARSE_ERROR on malformed content.""" + + @pytest.fixture(autouse=True) + def setup(self, temp_dir, monkeypatch): + import concore + + self.concore = concore + monkeypatch.setattr(concore, "delay", 0) + + in_dir = os.path.join(temp_dir, "in1") + os.makedirs(in_dir, exist_ok=True) + with open(os.path.join(in_dir, "ym"), "w") as f: + f.write("NOT_VALID_PYTHON{{{") + + monkeypatch.setattr(concore, "inpath", os.path.join(temp_dir, "in")) + + def test_returns_default_and_false(self): + data, ok = self.concore.read(1, "ym", "[0, 0.0]") + assert ok is False + + def test_last_read_status_is_parse_error(self): + self.concore.read(1, "ym", "[0, 0.0]") + assert self.concore.last_read_status == "PARSE_ERROR" + + +class TestReadFileRetriesExceeded: + """read() returns (default, False) with RETRIES_EXCEEDED when file is empty.""" + + @pytest.fixture(autouse=True) + def setup(self, temp_dir, monkeypatch): + import concore + + self.concore = concore + monkeypatch.setattr(concore, "delay", 0) + + # Create an empty file + in_dir = os.path.join(temp_dir, "in1") + os.makedirs(in_dir, exist_ok=True) + with open(os.path.join(in_dir, "ym"), "w") as _f: + pass # empty + + monkeypatch.setattr(concore, "inpath", os.path.join(temp_dir, "in")) + + def test_returns_default_and_false(self): + data, ok = self.concore.read(1, "ym", "[0, 0.0]") + assert ok is False + + def test_last_read_status_is_retries_exceeded(self): + self.concore.read(1, "ym", "[0, 0.0]") + assert self.concore.last_read_status == "RETRIES_EXCEEDED" + + +# --------------------------------------------------------------------------- +# ZMQ read tests +# --------------------------------------------------------------------------- + + +class TestReadZMQSuccess: + """Successful ZMQ read returns (data, True).""" + + @pytest.fixture(autouse=True) + def setup(self, monkeypatch): + import concore + + self.concore = concore + self.original_ports = concore.zmq_ports.copy() + yield + concore.zmq_ports.clear() + concore.zmq_ports.update(self.original_ports) + + def test_zmq_read_returns_data_and_true(self): + dummy = DummyZMQPort(response=[5, 1.1, 2.2]) + self.concore.zmq_ports["test_port"] = dummy + self.concore.simtime = 0 + + data, ok = self.concore.read("test_port", "ym", "[]") + assert ok is True + assert data == [1.1, 2.2] + assert self.concore.last_read_status == "SUCCESS" + + +class TestReadZMQTimeout: + """ZMQ read that returns None (timeout) yields (default, False).""" + + @pytest.fixture(autouse=True) + def setup(self, monkeypatch): + import concore + + self.concore = concore + self.original_ports = concore.zmq_ports.copy() + yield + concore.zmq_ports.clear() + concore.zmq_ports.update(self.original_ports) + + def test_zmq_timeout_returns_default_and_false(self): + dummy = DummyZMQPort( + raise_on_recv=TimeoutError("ZMQ recv failed after 5 retries") + ) + self.concore.zmq_ports["test_port"] = dummy + + data, ok = self.concore.read("test_port", "ym", "[]") + assert ok is False + assert self.concore.last_read_status == "TIMEOUT" + + +class TestReadZMQError: + """ZMQ read that raises ZMQError yields (default, False).""" + + @pytest.fixture(autouse=True) + def setup(self, monkeypatch): + import concore + + self.concore = concore + self.original_ports = concore.zmq_ports.copy() + yield + concore.zmq_ports.clear() + concore.zmq_ports.update(self.original_ports) + + def test_zmq_error_returns_default_and_false(self): + import zmq + + dummy = DummyZMQPort(raise_on_recv=zmq.error.ZMQError("test error")) + self.concore.zmq_ports["test_port"] = dummy + + data, ok = self.concore.read("test_port", "ym", "[]") + assert ok is False + assert self.concore.last_read_status == "TIMEOUT" + + +# --------------------------------------------------------------------------- +# Backward compatibility +# --------------------------------------------------------------------------- + + +class TestReadBackwardCompatibility: + """Legacy callers can use isinstance check on the result.""" + + @pytest.fixture(autouse=True) + def setup(self, temp_dir, monkeypatch): + import concore + + self.concore = concore + monkeypatch.setattr(concore, "delay", 0) + + in_dir = os.path.join(temp_dir, "in1") + os.makedirs(in_dir, exist_ok=True) + with open(os.path.join(in_dir, "ym"), "w") as f: + f.write("[10, 42.0]") + + monkeypatch.setattr(concore, "inpath", os.path.join(temp_dir, "in")) + + def test_legacy_unpack_pattern(self): + """The recommended migration pattern works correctly.""" + result = self.concore.read(1, "ym", "[0, 0.0]") + + if isinstance(result, tuple): + value, ok = result + else: + value = result + ok = True + + assert value == [42.0] + assert ok is True + + def test_tuple_unpack(self): + """New-style callers can unpack directly.""" + value, ok = self.concore.read(1, "ym", "[0, 0.0]") + assert value == [42.0] + assert ok is True + + +# --------------------------------------------------------------------------- +# last_read_status exposed on module +# --------------------------------------------------------------------------- + + +class TestLastReadStatusExposed: + """concore.last_read_status is publicly accessible.""" + + def test_attribute_exists(self): + import concore + + assert hasattr(concore, "last_read_status") + + def test_initial_value_is_success(self): + import concore + + # Before any read, default is SUCCESS + assert concore.last_read_status in ( + "SUCCESS", + "FILE_NOT_FOUND", + "TIMEOUT", + "PARSE_ERROR", + "EMPTY_DATA", + "RETRIES_EXCEEDED", + ) diff --git a/tests/test_setup.py b/tests/test_setup.py new file mode 100644 index 00000000..42a05fe2 --- /dev/null +++ b/tests/test_setup.py @@ -0,0 +1,139 @@ +import tempfile +import shutil +import unittest +from pathlib import Path +from unittest.mock import patch + +from click.testing import CliRunner +from concore_cli.cli import cli + + +class TestSetupCommand(unittest.TestCase): + def setUp(self): + self.runner = CliRunner() + self.temp_dir = tempfile.mkdtemp() + + def tearDown(self): + shutil.rmtree(self.temp_dir) + + @patch("concore_cli.commands.setup._resolve_concore_path") + @patch("concore_cli.commands.setup._detect_tool") + @patch("concore_cli.commands.setup._get_platform_key") + def test_setup_dry_run_does_not_write(self, mock_plat, mock_detect, mock_path): + mock_plat.return_value = "posix" + mock_path.return_value = Path(self.temp_dir) + + def detect_side_effect(names): + if "g++" in names: + return "/usr/bin/g++", "g++" + if "python3" in names: + return "/usr/bin/python3", "python3" + if "iverilog" in names: + return "/usr/bin/iverilog", "iverilog" + if "octave" in names: + return "/usr/bin/octave", "octave" + if "docker" in names: + return "/usr/bin/docker", "docker" + return None, None + + mock_detect.side_effect = detect_side_effect + + result = self.runner.invoke(cli, ["setup", "--dry-run"]) + self.assertEqual(result.exit_code, 0) + + self.assertFalse((Path(self.temp_dir) / "concore.tools").exists()) + self.assertFalse((Path(self.temp_dir) / "concore.sudo").exists()) + self.assertFalse((Path(self.temp_dir) / "concore.octave").exists()) + + @patch("concore_cli.commands.setup._resolve_concore_path") + @patch("concore_cli.commands.setup._detect_tool") + @patch("concore_cli.commands.setup._get_platform_key") + def test_setup_writes_files(self, mock_plat, mock_detect, mock_path): + mock_plat.return_value = "posix" + mock_path.return_value = Path(self.temp_dir) + + def detect_side_effect(names): + if "g++" in names: + return "/usr/bin/g++", "g++" + if "python3" in names: + return "/usr/bin/python3", "python3" + if "iverilog" in names: + return "/usr/bin/iverilog", "iverilog" + if "octave" in names: + return "/usr/bin/octave", "octave" + if "docker" in names: + return "/usr/bin/docker", "docker" + return None, None + + mock_detect.side_effect = detect_side_effect + + result = self.runner.invoke(cli, ["setup"]) + self.assertEqual(result.exit_code, 0) + + tools_file = Path(self.temp_dir) / "concore.tools" + sudo_file = Path(self.temp_dir) / "concore.sudo" + octave_file = Path(self.temp_dir) / "concore.octave" + + self.assertTrue(tools_file.exists()) + self.assertTrue(sudo_file.exists()) + self.assertTrue(octave_file.exists()) + + tools_content = tools_file.read_text() + self.assertIn("CPPEXE=/usr/bin/g++", tools_content) + self.assertIn("PYTHONEXE=/usr/bin/python3", tools_content) + self.assertIn("VEXE=/usr/bin/iverilog", tools_content) + self.assertIn("OCTAVEEXE=/usr/bin/octave", tools_content) + self.assertEqual(sudo_file.read_text().strip(), "docker") + + @patch("concore_cli.commands.setup._resolve_concore_path") + @patch("concore_cli.commands.setup._detect_tool") + @patch("concore_cli.commands.setup._get_platform_key") + def test_setup_no_force_keeps_existing(self, mock_plat, mock_detect, mock_path): + mock_plat.return_value = "posix" + mock_path.return_value = Path(self.temp_dir) + + tools_file = Path(self.temp_dir) / "concore.tools" + tools_file.write_text("CPPEXE=/old/path\n") + + def detect_side_effect(names): + if "g++" in names: + return "/usr/bin/g++", "g++" + if "python3" in names: + return "/usr/bin/python3", "python3" + return None, None + + mock_detect.side_effect = detect_side_effect + + result = self.runner.invoke(cli, ["setup"]) + self.assertEqual(result.exit_code, 0) + self.assertEqual(tools_file.read_text(), "CPPEXE=/old/path\n") + + @patch("concore_cli.commands.setup._resolve_concore_path") + @patch("concore_cli.commands.setup._detect_tool") + @patch("concore_cli.commands.setup._get_platform_key") + def test_setup_force_overwrites_existing(self, mock_plat, mock_detect, mock_path): + mock_plat.return_value = "posix" + mock_path.return_value = Path(self.temp_dir) + + tools_file = Path(self.temp_dir) / "concore.tools" + tools_file.write_text("CPPEXE=/old/path\n") + + def detect_side_effect(names): + if "g++" in names: + return "/usr/bin/g++", "g++" + if "python3" in names: + return "/usr/bin/python3", "python3" + return None, None + + mock_detect.side_effect = detect_side_effect + + result = self.runner.invoke(cli, ["setup", "--force"]) + self.assertEqual(result.exit_code, 0) + + content = tools_file.read_text() + self.assertIn("CPPEXE=/usr/bin/g++", content) + self.assertIn("PYTHONEXE=/usr/bin/python3", content) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_tool_config.py b/tests/test_tool_config.py new file mode 100644 index 00000000..7baa25e3 --- /dev/null +++ b/tests/test_tool_config.py @@ -0,0 +1,75 @@ +import os + + +# can't import mkconcore directly (sys.argv at module level), so we duplicate the parser +def _load_tool_config(filepath): + tools = {} + with open(filepath, "r") as f: + for line in f: + line = line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + k, v = line.split("=", 1) + k, v = k.strip(), v.strip() + if v: + tools[k] = v + return tools + + +class TestLoadToolConfig: + def test_basic_overrides(self, temp_dir): + cfg = os.path.join(temp_dir, "concore.tools") + with open(cfg, "w") as f: + f.write("CPPEXE=/usr/local/bin/g++-12\n") + f.write("PYTHONEXE=/usr/bin/python3.11\n") + + tools = _load_tool_config(cfg) + assert tools["CPPEXE"] == "/usr/local/bin/g++-12" + assert tools["PYTHONEXE"] == "/usr/bin/python3.11" + assert "VEXE" not in tools + + def test_comments_and_blanks_ignored(self, temp_dir): + cfg = os.path.join(temp_dir, "concore.tools") + with open(cfg, "w") as f: + f.write("# custom tool paths\n") + f.write("\n") + f.write("OCTAVEEXE = /snap/bin/octave\n") + f.write("# MATLABEXE = /opt/matlab/bin/matlab\n") + + tools = _load_tool_config(cfg) + assert tools["OCTAVEEXE"] == "/snap/bin/octave" + assert "MATLABEXE" not in tools + + def test_empty_value_skipped(self, temp_dir): + cfg = os.path.join(temp_dir, "concore.tools") + with open(cfg, "w") as f: + f.write("CPPWIN=\n") + f.write("VEXE = \n") + + tools = _load_tool_config(cfg) + assert "CPPWIN" not in tools + assert "VEXE" not in tools + + def test_value_with_equals_sign(self, temp_dir): + cfg = os.path.join(temp_dir, "concore.tools") + with open(cfg, "w") as f: + f.write("CPPEXE=C:\\Program Files\\g++=fast\n") + + tools = _load_tool_config(cfg) + assert tools["CPPEXE"] == "C:\\Program Files\\g++=fast" + + def test_whitespace_around_key_value(self, temp_dir): + cfg = os.path.join(temp_dir, "concore.tools") + with open(cfg, "w") as f: + f.write(" VWIN = C:\\iverilog\\bin\\iverilog.exe \n") + + tools = _load_tool_config(cfg) + assert tools["VWIN"] == "C:\\iverilog\\bin\\iverilog.exe" + + def test_empty_file(self, temp_dir): + cfg = os.path.join(temp_dir, "concore.tools") + with open(cfg, "w") as _: + pass + + tools = _load_tool_config(cfg) + assert tools == {} diff --git a/testsou/ccpymat.dir/concore.py b/testsou/ccpymat.dir/concore.py deleted file mode 100644 index eab7e448..00000000 --- a/testsou/ccpymat.dir/concore.py +++ /dev/null @@ -1,79 +0,0 @@ -import time -import os -from ast import literal_eval -import sys - -#if windows, create script to kill this process -# because batch files don't provide easy way to know pid of last command -# ignored for posix!=windows, because "concorepid" is handled by script -# ignored for docker (linux!=windows), because handled by docker stop -if hasattr(sys, 'getwindowsversion'): - with open("concorekill.bat","w") as fpid: - fpid.write("taskkill /F /PID "+str(os.getpid())+"\n") - -try: - iport = literal_eval(open("concore.iport").read()) -except: - iport = dict() -try: - oport = literal_eval(open("concore.oport").read()) -except: - oport = dict() - - - -s = '' -olds = '' -delay = 1 -retrycount = 0 -inpath = "./in" #must be rel path for local -outpath = "./out" - -def unchanged(): - global olds,s - if olds==s: - s = '' - return True - else: - olds = s - return False - -def read(port, name, initstr): - global s,simtime,retrycount - time.sleep(delay) - try: - infile = open(inpath+str(port)+"/"+name); - ins = infile.read() - except: - ins = initstr - while len(ins)==0: - time.sleep(delay) - ins = infile.read() - retrycount += 1 - s += ins - inval = literal_eval(ins) - simtime = max(simtime,inval[0]) - return inval[1:] - -def write(port, name, val, delta=0): - global outpath,simtime - if isinstance(val,str): - time.sleep(2*delay) - elif isinstance(val,list)==False: - print("mywrite must have list or str") - quit() - try: - with open(outpath+str(port)+"/"+name,"w") as outfile: - if isinstance(val,list): - outfile.write(str([simtime+delta]+val)) - else: - outfile.write(val) - except: - print("skipping"+outpath+str(port)+"/"+name); - -def initval(simtime_val): - global simtime - val = literal_eval(simtime_val) - simtime = val[0] - return val[1:] - diff --git a/testsou/concore.py b/testsou/concore.py deleted file mode 100644 index eab7e448..00000000 --- a/testsou/concore.py +++ /dev/null @@ -1,79 +0,0 @@ -import time -import os -from ast import literal_eval -import sys - -#if windows, create script to kill this process -# because batch files don't provide easy way to know pid of last command -# ignored for posix!=windows, because "concorepid" is handled by script -# ignored for docker (linux!=windows), because handled by docker stop -if hasattr(sys, 'getwindowsversion'): - with open("concorekill.bat","w") as fpid: - fpid.write("taskkill /F /PID "+str(os.getpid())+"\n") - -try: - iport = literal_eval(open("concore.iport").read()) -except: - iport = dict() -try: - oport = literal_eval(open("concore.oport").read()) -except: - oport = dict() - - - -s = '' -olds = '' -delay = 1 -retrycount = 0 -inpath = "./in" #must be rel path for local -outpath = "./out" - -def unchanged(): - global olds,s - if olds==s: - s = '' - return True - else: - olds = s - return False - -def read(port, name, initstr): - global s,simtime,retrycount - time.sleep(delay) - try: - infile = open(inpath+str(port)+"/"+name); - ins = infile.read() - except: - ins = initstr - while len(ins)==0: - time.sleep(delay) - ins = infile.read() - retrycount += 1 - s += ins - inval = literal_eval(ins) - simtime = max(simtime,inval[0]) - return inval[1:] - -def write(port, name, val, delta=0): - global outpath,simtime - if isinstance(val,str): - time.sleep(2*delay) - elif isinstance(val,list)==False: - print("mywrite must have list or str") - quit() - try: - with open(outpath+str(port)+"/"+name,"w") as outfile: - if isinstance(val,list): - outfile.write(str([simtime+delta]+val)) - else: - outfile.write(val) - except: - print("skipping"+outpath+str(port)+"/"+name); - -def initval(simtime_val): - global simtime - val = literal_eval(simtime_val) - simtime = val[0] - return val[1:] - diff --git a/testsou/concoredocker.py b/testsou/concoredocker.py deleted file mode 100644 index 6fb582a2..00000000 --- a/testsou/concoredocker.py +++ /dev/null @@ -1,58 +0,0 @@ -import time -from ast import literal_eval - -s = '' -olds = '' -delay = 1 -retrycount = 0 -inpath = "/in" -outpath = "/out" - -def unchanged(): - global olds,s - if olds==s: - s = '' - return True - else: - olds = s - return False - -def read(port, name, initstr): - global s,simtime,retrycount - time.sleep(delay) - try: - infile = open(inpath+str(port)+"/"+name); - ins = infile.read() - except: - ins = initstr - while len(ins)==0: - time.sleep(delay) - ins = infile.read() - retrycount += 1 - s += ins - inval = literal_eval(ins) - simtime = max(simtime,inval[0]) - return inval[1:] - -def write(port, name, val, delta=0): - global outpath,simtime - if isinstance(val,str): - time.sleep(2*delay) - elif isinstance(val,list)==False: - print("mywrite must have list or str") - quit() - try: - with open(outpath+str(port)+"/"+name,"w") as outfile: - if isinstance(val,list): - outfile.write(str([simtime+delta]+val)) - else: - outfile.write(val) - except: - print("skipping"+outpath+str(port)+"/"+name); - -def initval(simtime_val): - global simtime - val = literal_eval(simtime_val) - simtime = val[0] - return val[1:] - diff --git a/testsou/concoreold.py b/testsou/concoreold.py deleted file mode 100644 index 310d2b53..00000000 --- a/testsou/concoreold.py +++ /dev/null @@ -1,58 +0,0 @@ -import time -from ast import literal_eval - -s = '' -olds = '' -delay = 1 -retrycount = 0 -inpath = "./in" -outpath = "./out" - -def unchanged(): - global olds,s - if olds==s: - s = '' - return True - else: - olds = s - return False - -def read(port, name, initstr): - global s,simtime,retrycount - time.sleep(delay) - try: - infile = open(inpath+str(port)+"/"+name); - ins = infile.read() - except: - ins = initstr - while len(ins)==0: - time.sleep(delay) - ins = infile.read() - retrycount += 1 - s += ins - inval = literal_eval(ins) - simtime = max(simtime,inval[0]) - return inval[1:] - -def write(port, name, val, delta=0): - global outpath,simtime - if isinstance(val,str): - time.sleep(2*delay) - elif isinstance(val,list)==False: - print("mywrite must have list or str") - quit() - try: - with open(outpath+str(port)+"/"+name,"w") as outfile: - if isinstance(val,list): - outfile.write(str([simtime+delta]+val)) - else: - outfile.write(val) - except: - print("skipping"+outpath+str(port)+"/"+name); - -def initval(simtime_val): - global simtime - val = literal_eval(simtime_val) - simtime = val[0] - return val[1:] - diff --git a/testsou/cpymat.cpp b/testsou/cpymat.cpp index 55a9d04c..aac38391 100644 --- a/testsou/cpymat.cpp +++ b/testsou/cpymat.cpp @@ -21,7 +21,7 @@ int main() auto wallclock1 = chrono::high_resolution_clock::now(); vector ym; - while(concore.simtime ym = concore.initval(init_simtime_ym); vector u; - while(concore.simtime ym = concore.initval(init_simtime_ym); vector u; - while(concore.simtime +.\params "realtime=True" +.\run or .\debug +``` \ No newline at end of file diff --git a/tools/bangbang.py b/tools/bangbang.py index eed72b83..9b723822 100644 --- a/tools/bangbang.py +++ b/tools/bangbang.py @@ -1,5 +1,6 @@ import numpy as np import concore +import logging def bangbang_controller(ym): @@ -8,9 +9,8 @@ def bangbang_controller(ym): amp = 3 elif ym[1]<65: amp = 1 - - - ustar = np.array([amp,30]) + + ustar = np.array([amp,30]) return ustar @@ -19,6 +19,9 @@ def bangbang_controller(ym): init_simtime_u = "[0.0, 0.0,0.0]" init_simtime_ym = "[0.0, 70.0,91]" u = np.array([concore.initval(init_simtime_u)]).T + +logging.info("Starting Bang-Bang Controller") + while(concore.simtime 1.1*timeout_max: #timeout_count>100: - print("timeout or bad POST request "+str(r.status_code)) + logging.error(f"timeout or bad POST request {r.status_code}") quit() if len(r.text)!=0: try: t=literal_eval(r.text)[0] - except: - print("bad eval "+r.text) + except Exception: + logging.error(f"bad eval {r.text}") oldt = t oldym = r.text - print("CW: oldym="+oldym+" t="+str(concore.simtime)) + logging.debug(f"CW: oldym={oldym} t={concore.simtime}") concore.write(1,name2,oldym) #concore.write(1,"ym",init_simtime_ym) -print("retry="+str(concore.retrycount)) - - +logging.info(f"retry={concore.retrycount}") \ No newline at end of file diff --git a/tools/learn.py b/tools/learn.py index a29e1e66..53bb50e7 100644 --- a/tools/learn.py +++ b/tools/learn.py @@ -2,6 +2,7 @@ import numpy as np import matplotlib.pyplot as plt import time +import logging GENERATE_PLOT = 1 concore.delay = 0.002 @@ -21,7 +22,7 @@ ut[int(concore.simtime)] = np.array(u).T ymt[int(concore.simtime)] = np.array(ym).T oldsimtime = concore.simtime -print("retry="+str(concore.retrycount)) +logging.info(f"retry={concore.retrycount}") ################# # plot inputs and outputs diff --git a/tools/pid2.py b/tools/pid2.py index 6f88285d..225e87c0 100644 --- a/tools/pid2.py +++ b/tools/pid2.py @@ -1,5 +1,7 @@ import numpy as np import concore +import logging + setpoint = 67.5 setpointF = 75.0 KpF = 0.1 @@ -42,7 +44,7 @@ def pid_controller(ym): if freq<10: freq = 10 Prev_ErrorF = ErrorF - ustar = np.array([amp,30]) + ustar = np.array([amp,freq]) return ustar @@ -51,7 +53,9 @@ def pid_controller(ym): init_simtime_u = "[0.0, 0.0,0.0]" init_simtime_ym = "[0.0, 70.0,91]" u = np.array([concore.initval(init_simtime_u)]).T -print("Shannon's PID controller: setpoint is "+str(setpoint)) + +logging.info(f"Shannon's PID controller: setpoint is {setpoint}") + while(concore.simtime= size: + logging.warning( + "Requested lag (%d) exceeds buffer size (%d). Clamping to %d.", + lag, size, size - 1 + ) + lag = size - 1 +logging.info(f"plot ym with lag={lag}") concore.delay = 0.005 concore.default_maxtime(150) @@ -15,33 +22,68 @@ ymt = [] ym = [] for i in range(0,size): - ym.append(concore.initval(init_simtime_ym)) + ym.append(concore.initval(init_simtime_ym)) cur = 0 + +# --- Real-time plotting setup --- +realtime = concore.tryparam('realtime', False) +if realtime: + plt.ion() + fig, axs = plt.subplots(2, 1) + lines = [ax.plot([], [])[0] for ax in axs] + + axs[0].set_ylabel('MAP (mmHg)') + axs[0].legend(['MAP'], loc=0) + + axs[1].set_xlabel('Cycles ' + str(concore.params)) + axs[1].set_ylabel('HR (bpm)') + axs[1].legend(['HR'], loc=0) + plt.tight_layout() + plt.show(block=False) +# -------------------------------- + while(concore.simtime 1.1*timeout_max: #timeout_count>200: - print("timeout or bad POST request "+str(r.status_code)) + logging.error(f"timeout or bad POST request {r.status_code}") quit() if len(r.text)!=0: try: t=literal_eval(r.text)[0] except: - print("bad eval "+r.text) + logging.error(f"bad eval {r.text}") oldt = t oldu = r.text - print("PW: oldu="+oldu+" t="+str(concore.simtime)) + logging.debug(f"PW: oldu={oldu} t={concore.simtime}") concore.write(1,name1,oldu) #concore.write(1,"u",init_simtime_u) -print("retry="+str(concore.retrycount)) +logging.info(f"retry={concore.retrycount}") \ No newline at end of file diff --git a/tools/shannon.py b/tools/shannon.py index ba371af7..9da6f51a 100644 --- a/tools/shannon.py +++ b/tools/shannon.py @@ -1,5 +1,6 @@ import numpy as np import concore +import logging setpoint = 67.5 Kp = 0.1 Ki = 0.01 @@ -27,7 +28,7 @@ def pid_controller(ym): init_simtime_u = "[0.0, 0.0,0.0]" init_simtime_ym = "[0.0, 70.0,91]" u = np.array([concore.initval(init_simtime_u)]).T -print("Shannon's PID controller: setpoint is "+str(setpoint)) +logging.info(f"Shannon's PID controller: setpoint is {setpoint}") while(concore.simtime