From cc21fb66f1806fc3fa135b78636d036e5e20b967 Mon Sep 17 00:00:00 2001 From: Prasanna Date: Tue, 20 Jan 2026 17:45:04 +0530 Subject: [PATCH 001/275] Fix path concatenation in parameter Signed-off-by: Prasanna --- concore.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/concore.py b/concore.py index 6d71f0fc..e382501e 100644 --- a/concore.py +++ b/concore.py @@ -131,7 +131,7 @@ def safe_literal_eval(filename, defaultValue): # Parameter Parsing # =================================================================== try: - sparams_path = os.path.join(inpath + "1", "concore.params") + sparams_path = os.path.join(inpath, "1", "concore.params") if os.path.exists(sparams_path): with open(sparams_path, "r") as f: sparams = f.read() @@ -171,7 +171,7 @@ def tryparam(n, i): def default_maxtime(default): """Read maximum simulation time from file or use default.""" global maxtime - maxtime_path = os.path.join(inpath + "1", "concore.maxtime") + maxtime_path = os.path.join(inpath, "1", "concore.maxtime") maxtime = safe_literal_eval(maxtime_path, default) default_maxtime(100) @@ -220,7 +220,7 @@ def read(port_identifier, name, initstr_val): return default_return_val time.sleep(delay) - file_path = os.path.join(inpath+str(file_port_num), name) + file_path = os.path.join(inpath, str(file_port_num), name) ins = "" try: @@ -287,10 +287,10 @@ def write(port_identifier, name, val, delta=0): # Case 2: File-based port try: if isinstance(port_identifier, str) and port_identifier in zmq_ports: - file_path = os.path.join("../"+port_identifier, name) + file_path = os.path.join("..", port_identifier, name) else: file_port_num = int(port_identifier) - file_path = os.path.join(outpath+str(file_port_num), name) + file_path = os.path.join(outpath, str(file_port_num), name) except ValueError: print(f"Error: Invalid port identifier '{port_identifier}' for file operation. Must be integer or ZMQ name.") return From 4ef989f1d285d98a2acd6c2ea1be82d684046e4e Mon Sep 17 00:00:00 2001 From: Prasanna Date: Tue, 20 Jan 2026 17:45:29 +0530 Subject: [PATCH 002/275] Refactor file path concatenation Signed-off-by: Prasanna --- concoredocker.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/concoredocker.py b/concoredocker.py index 5c5689d7..a167aac2 100644 --- a/concoredocker.py +++ b/concoredocker.py @@ -24,7 +24,7 @@ def safe_literal_eval(filename, defaultValue): #9/21/22 try: - sparams = open(inpath+"1/concore.params").read() + sparams = open(os.path.join(inpath, "1", "concore.params")).read() if sparams[0] == '"': #windows keeps "" need to remove sparams = sparams[1:] sparams = sparams[0:sparams.find('"')] @@ -62,7 +62,7 @@ def read(port, name, initstr): global s, simtime, retrycount max_retries=5 time.sleep(delay) - file_path = os.path.join(inpath+str(port), name) + file_path = os.path.join(inpath, str(port), name) try: with open(file_path, "r") as infile: @@ -101,7 +101,7 @@ def read(port, name, initstr): def write(port, name, val, delta=0): global simtime - file_path = os.path.join(outpath+str(port), name) + file_path = os.path.join(outpath, str(port), name) if isinstance(val, str): time.sleep(2 * delay) From 11acdce92594589ca68b2da5d064394e17e98716 Mon Sep 17 00:00:00 2001 From: Prasanna Date: Tue, 20 Jan 2026 17:53:28 +0530 Subject: [PATCH 003/275] Refactor parameter file handling Signed-off-by: Prasanna --- concore.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/concore.py b/concore.py index e382501e..e9fa88ae 100644 --- a/concore.py +++ b/concore.py @@ -125,13 +125,15 @@ def safe_literal_eval(filename, defaultValue): inpath = "./in" #must be rel path for local outpath = "./out" simtime = 0 +concore_params_file = os.path.join(inpath, "1", "concore.params") +concore_maxtime_file = os.path.join(inpath, "1", "concore.maxtime") #9/21/22 # =================================================================== # Parameter Parsing # =================================================================== try: - sparams_path = os.path.join(inpath, "1", "concore.params") + sparams_path = concore_params_file if os.path.exists(sparams_path): with open(sparams_path, "r") as f: sparams = f.read() @@ -171,8 +173,7 @@ def tryparam(n, i): def default_maxtime(default): """Read maximum simulation time from file or use default.""" global maxtime - maxtime_path = os.path.join(inpath, "1", "concore.maxtime") - maxtime = safe_literal_eval(maxtime_path, default) + maxtime = safe_literal_eval(concore_maxtime_file, default) default_maxtime(100) @@ -283,14 +284,12 @@ def write(port_identifier, name, val, delta=0): print(f"ZMQ write error on port {port_identifier} (name: {name}): {e}") except Exception as e: print(f"Unexpected error during ZMQ write on port {port_identifier} (name: {name}): {e}") + return # Case 2: File-based port try: - if isinstance(port_identifier, str) and port_identifier in zmq_ports: - file_path = os.path.join("..", port_identifier, name) - else: - file_port_num = int(port_identifier) - file_path = os.path.join(outpath, str(file_port_num), name) + file_port_num = int(port_identifier) + file_path = os.path.join(outpath, str(file_port_num), name) except ValueError: print(f"Error: Invalid port identifier '{port_identifier}' for file operation. Must be integer or ZMQ name.") return From ebf80936ee122e730c6746a5894a402c49a98d03 Mon Sep 17 00:00:00 2001 From: Prasanna Date: Tue, 20 Jan 2026 17:53:39 +0530 Subject: [PATCH 004/275] Refactor file path usage for parameter and maxtime files Signed-off-by: Prasanna --- concoredocker.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/concoredocker.py b/concoredocker.py index a167aac2..161ad1bf 100644 --- a/concoredocker.py +++ b/concoredocker.py @@ -21,10 +21,12 @@ def safe_literal_eval(filename, defaultValue): inpath = os.path.abspath("/in") outpath = os.path.abspath("/out") simtime = 0 +concore_params_file = os.path.join(inpath, "1", "concore.params") +concore_maxtime_file = os.path.join(inpath, "1", "concore.maxtime") #9/21/22 try: - sparams = open(os.path.join(inpath, "1", "concore.params")).read() + sparams = open(concore_params_file).read() if sparams[0] == '"': #windows keeps "" need to remove sparams = sparams[1:] sparams = sparams[0:sparams.find('"')] @@ -46,7 +48,7 @@ def tryparam(n, i): #9/12/21 def default_maxtime(default): global maxtime - maxtime = safe_literal_eval(os.path.join(inpath, "1", "concore.maxtime"), default) + maxtime = safe_literal_eval(concore_maxtime_file, default) default_maxtime(100) From 0c1b0dfdc1d93b5b3db00d9d6562dc565205b0c7 Mon Sep 17 00:00:00 2001 From: GREENRAT-K405 Date: Mon, 26 Jan 2026 19:00:58 +0530 Subject: [PATCH 005/275] fix: broken file path --- concore.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/concore.py b/concore.py index e9fa88ae..e1bfb5e7 100644 --- a/concore.py +++ b/concore.py @@ -125,8 +125,8 @@ def safe_literal_eval(filename, defaultValue): inpath = "./in" #must be rel path for local outpath = "./out" simtime = 0 -concore_params_file = os.path.join(inpath, "1", "concore.params") -concore_maxtime_file = os.path.join(inpath, "1", "concore.maxtime") +concore_params_file = os.path.join(inpath + "1", "concore.params") +concore_maxtime_file = os.path.join(inpath + "1", "concore.maxtime") #9/21/22 # =================================================================== @@ -221,7 +221,7 @@ def read(port_identifier, name, initstr_val): return default_return_val time.sleep(delay) - file_path = os.path.join(inpath, str(file_port_num), name) + file_path = os.path.join(inpath + str(file_port_num), name) ins = "" try: @@ -289,7 +289,7 @@ def write(port_identifier, name, val, delta=0): # Case 2: File-based port try: file_port_num = int(port_identifier) - file_path = os.path.join(outpath, str(file_port_num), name) + file_path = os.path.join(outpath + str(file_port_num), name) except ValueError: print(f"Error: Invalid port identifier '{port_identifier}' for file operation. Must be integer or ZMQ name.") return From c5561eaf8c0f4969e44a8dece3fab8d1ec3b966a Mon Sep 17 00:00:00 2001 From: Sahil Lenka Date: Tue, 27 Jan 2026 23:37:06 +0530 Subject: [PATCH 006/275] Add initial unit tests for core Python utilities - Set up a tests/ directory with a basic pytest structure - Added unit tests for key concore.py helpers (safe_literal_eval, tryparam, and core API functions) - Included pytest configuration and dev dependencies - All tests passing locally This adds a starting test foundation and helps improve confidence as the codebase evolves. --- pytest.ini | 6 ++ requirements-dev.txt | 2 + tests/__init__.py | 0 tests/__pycache__/__init__.cpython-312.pyc | Bin 0 -> 137 bytes .../conftest.cpython-312-pytest-9.0.2.pyc | Bin 0 -> 1710 bytes .../test_concore.cpython-312-pytest-9.0.2.pyc | Bin 0 -> 11504 bytes tests/conftest.py | 25 +++++ tests/test_concore.py | 95 ++++++++++++++++++ 8 files changed, 128 insertions(+) create mode 100644 pytest.ini create mode 100644 requirements-dev.txt create mode 100644 tests/__init__.py create mode 100644 tests/__pycache__/__init__.cpython-312.pyc create mode 100644 tests/__pycache__/conftest.cpython-312-pytest-9.0.2.pyc create mode 100644 tests/__pycache__/test_concore.cpython-312-pytest-9.0.2.pyc create mode 100644 tests/conftest.py create mode 100644 tests/test_concore.py diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 00000000..13bc1da9 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,6 @@ +[pytest] +testpaths = tests +python_files = test_*.py +python_classes = Test* +python_functions = test_* +addopts = -v --tb=short \ No newline at end of file diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 00000000..7571d09b --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,2 @@ +pytest>=7.0.0 +pytest-cov>=4.0.0 \ No newline at end of file diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/__pycache__/__init__.cpython-312.pyc b/tests/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5181b3a1b70f61f57e98c51ee0861cd3815a4fea GIT binary patch literal 137 zcmX@j%ge<81WbP_GC}lX5P=Rpvj9b=GgLBYGWxA#C}INgK7-W!(r~tl2`x@7Dvk+G z%*f1%NzTtp&M!)hDM>9ZDUOMc&&7%Q6n?Y2-d!92BymYoeq5tUD857@6*8)#x>1pGfQCcFp_V1%+UsVG|1`6a zW8_qEiR45a>Kq^((+dSvRP@vSjLIIFM1wh{=Yq69-rk&R^%?$EY`m`c1bJ( zmR^cW@o%J{Y{A{3CO!EdKI$=_sU}HizlwPz;4>s|;eC7H408e~4YN2N z;N7|bE%=1&5dx0p*-N@7jd{sGmOZI8s+UeX&w1KX(38e|>`9=Bzc1AWNs#4)!A-}i z)Qfi6;zs5Dg4l4S#<*6`IjdZ^)@|l6uCvuO$1Zc7Rvc0HM?E7%6 z7mjZ(Jqk?TymDmtJB@aubEAEud;P$O9hu=7Rt zQ6+jVP#H`i=^!54P$UJiWp0obS%HMIoIehWI3+6F?;-h^AbiN(brO6fpJyHO9Sz$& zJaa>F4A0V13dGaD4jXDqJrPXe4X=En_)NVR0xe|)r|UIfq=C=)Dvp<{TUW0F0;w!> z(n$EIsDX-WxmK##Rj$?QR+al2)OM_dkDCJKYOaD%BO;DfbzGH%niM4Hh)5IUJEoe|fFe!mxYlb!fydh`Xf3jEmw63pf|P4mt40}!0~PCGq79VDXMytZbS z9lOeS@KU{*9qrC!>rFYfIfkR{*tO~$wPZ87f287urwd@PY)p=F-4!JwTxCs0#p9;p z4Z8s8cpaW8-;oPR?i!Oom|5A`Y?d2Ed)-;1mRqeTg^G_a6<<}iU~{2TE3B2Ri!^~c zG5xHKq^}Y}egxQp4?~CGKZL*`L=GYJ7}Sp0Han4aq)WQ<5|`9&YJdJt@^UQd@ z`3K4S$p@+XsjuJYoqxM`?n*DZd;stCbx_PstQ~uzXyoib1!bZyq<(m$*U603KT%MU GDgFhC$6x^f literal 0 HcmV?d00001 diff --git a/tests/__pycache__/test_concore.cpython-312-pytest-9.0.2.pyc b/tests/__pycache__/test_concore.cpython-312-pytest-9.0.2.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a3e1c4a169d515844619f8b50820e7625b1395d4 GIT binary patch literal 11504 zcmeHNU2GKB6}~e&vwL^(dSfsSrm24cdkHuJLQH5VP)KQ%K$3>E&5Cxkc*d}s{c&ez zX$SJ?GBMo!Rvm zn>I=Fkn#G=x%ZxX=g!>oopbKbU*ho?f$RFeCo(?(?r)fIFIEDHg-Ze<7l}$#AxBaI zyNmhIh$xV}l#&IzB%BJfl1M7T?leW&J(`NLdn^^xXtu*`+`DHxJ>KP1qD2B_zvW7J z_y1Y)+g@8PrzPHocgAbcU9|HBqJ^_PHt(mZc!s>~WTe_ea)_v*Geni%cIu^8vpgB{ z!aQ#c%ZorB<$3KaFA8}vp0}3et%AH3p4TxcB;%jq6OWAeI}(CKKm0HJ7RVDsfDtsE zcOoR~^u}Mo?RABe$uEVAf(UQ-`j`+Fm()JrX@TetP`u%coyHe_}S$b)B}p9)C4{ z{=^*Zx)CC@`(Flruvhjc_O}TQqvDRhLKl=@BpSn0N>qh2Bo#v4NlB83^1_v6*RzE*)8?%WkD5ORjP<4w#Tlr zsETg>PErY#LqWr>hFo|sBLIfLtpjOEm2+}FT#*DKlCo45GrY7St6?>w(o13`T$an> z7X}HbM9LB4S7yTPLG~V(^X_W292phV7}gmT-*(hfU1hFuWwPxq=iOb!evue!eeDOB zqO$D1kM5UT=e6Z>o@7CQ=X<>lk`249sx6nqDbS1xRpV-FnPvo*ikjT4`C7WqeU^7G zN8Og(`zv6MCxl9*5}pLKZzF~<4rcjzfV)6m`(PsUJONlGj!eZs*+(-chRg}m62_;x zUK%K*^V-1fo&f~Jz?Pl?P^}VXY}>x0qN9#o5v(=?bWh3I%BN@`Vo50f*-KEK6fFrG zv1C^DitxF(XU`R3YW2p!vB?LMM$eu-J;?4z0$g>?DCNv6q7{NNBs)HDO&fqulgaD^ z8OYq3Oy~$49re(Xvc*inl8R$m!IH;y=!F$=FstK$Td{p(lh_?4ty^u%NII8ObZtx@ zDdxx0x@NUNv2whW$(fk~bRky+3By{a7$=LRoXSL%6S?B?bWUOK5moSYu?4GjKkHkj zSa?j=i@LQ&87=Dhw5ezlW4Ux8Z9>VFm?arnZj_k}E6O^;%$6|9x~JoyTC1=V_M2|^ zC$`xMyV-~QiH8leKRx`?(Ry0ltbJWes|MJq5$r%(pHxQmVqU?gPVG3l*j5ZJQwhOh z{0_(|a=pFd;-L$NEs<7DyzP8)`q|ld@8xH%w!~jQ^y;B%`$MxW zJ8y?bYu`V@Bs%nATi@kJ-a1}y+jcJeN#EeRJ8GT7RXTj`#cJnpEipV#@i*sW*eOnH zm~E%(9y?2ip)MEV%nTjA9f!wOI@4tBx?!p<~Ud_Cm41+_hFPnT|5+Lq`Fp$I~M(=x8eS^+*sYI@&wE z8%Yn4WK7?HJOW+ch-4Fz0VLoph@M2U8Oi-fwg5>+^daO~9oFVHOm0WA1Ia@`4Ac(? zTsE{eIVdqpdI7+tji$j=DdQ)#0_(Ms&lpCgaAIor9T2umUG(_r2r?8XWZ4ake)0F1-fl?~8+z#ql|k)}9+dRm+f(A@?U zdhG{944!uf6nc_(j2HlgbQxKW9W~11fS9_<7YFdEkgp{~NYizfaljv(IH2XoR20H# za}t55BjP=@Li}ti{xYy^^&GU`i!JmaxofEETcKK@qw5buBlt;$b=`J$!z+ejV@2$z z@f#q`4(~hl_TgrScTcmM%hcO;Az=Hev=0E=SxfZIQ~b@@X*h%Va1iSuHpu zNA*bvz0&z$hg@!Q$a%2>!Kw-lSe3BH%ss=jfg6U98mdU;P+4N`SuV@qp3!`>D@$F@ zyMtknnPGrBKKD%TJQr;+3|{*IhNsLODHsM%GGMCWNixH5XIaU0rra=uRWuA?$2~{9 z36AfnL53^hw&dO&_Y8&?+;fZVo*{ClrZZAjm<^a=wTzSu6M{(=%v-Viq+)0$M9wFy zxa$&G(K7a7m2C?S!kFlfAz7*Krnc|)DlX~3aoimT-qRQy+g_N(9MExvy_p6x=cz4z z<#eOsrKB-vFwi-kA|KN&SG%{K3!f|3=zV`}>zv;5R>Cm?164WzMxd*f7?`K{o3qn) zg3}u2+Nrw7&eDPDE-u5F89KmF7!03p;#n9Ic6s3XYU0@|f{uD=#hp;l)+~J(@|J;v z6U8#EUMUisD3;Y(5(H>a8lu?EOZv@*PBS<#_vzZ|MmV@wN58qKBKi{}3k1X14Ics? z9(W6)*oVKwD3;;jNK)4K;K%+hlD$YCM{*d+J|v!}`yM8dJOKX&IzRMuVR>-j9`JOw zAQlHZIdqTBPHs|6hOAb!ASq2R9)9NOqHYEwG}Ywba8;}#mW8rd5^xP|+1Vw?fcuiL zhR=P;U)bzYaNWYPvr8@s?o0N~34@#>pNXnrC$5!nb+0Tr?o0OKa$idU5xN(5yZv5A-vUA+^C;b^1)+HvGtZ1ds4BJkM)j>DW zJh#NfD;t%zV_M5VIg*0~ASJ`*mYIU4PHkvZCL4jxQmCZH&wwDV9(*}?L*>W>nThk!F)E6K0_8H zQQl*S=pvg@@3;&;i z7WTp8&OnQBpix*9|Du5w<3I~%3b?5q8q1}N7_ zLFY;)IAiGGgG;H94X(vC(1CDx^ovM56EbX4| z;4++5NsY<_twm+Dug#*t#lWNIQ0Jpimvk&i*QmQZ9?beD?E>a0YO*EJ|I zcF)!1Z7b7cUlk04xGMKwUEf=!z30ZK2WpAld5XU|J8dU8tzoX6s(b7#?VX0{j8l^{ zGql(76pKQ!6n?^=BH4$;a~D6xB$7k$&qFa$?pdIs+-D17@qgtr;`o`KE*;NhM)p7b zcre0#%J{OU2-_h4n&;kbE3k{6LsI$`lLugGuiev+l`;d5x z_)$CeEJzJuuOgo0&_z#N+TnhVT_h?xyh-a1}(9D!5!rBMI0nuqa9bf z*h=5{v|9DA^g)_0+u01R^Z}A^DG*os92Qslnk7L3R{9n@1q>Wf2^L!A=1nWuzzV?z zmc3R63IKel5`xniQIKbsr%JF|CK(^7u)V(G6|YX;mUDm_m|V$SHPHPv0)qI(H9&1a z1y|f7|JHjzkG!`9;BaLQTJZNIp$Cy=WW6;wdMRXYk(q1VFYK#($4vnD8{xF2BKby$ z!}4o9LeVd5g!tPoJ3?F%Mkwn0$^m*+{&Gi%uk3z}M=17%jSzp^Wk-lh!U)B_`Vnf{ zRMwvWNWy8agd5Q3(-USUuT5?7ohE@3C$X2DqbEJ~Arv@|GKe|bl14ICu_zuENgvN? zY_B*O((w!+oc>ji8>+N;f6 z8*WImQ)2%0f==ep@SvJ9S}L$p9tNB_fg^8I0|A~aNbo1v?g0pV*j6(v(|ttm|1@!L zbzQQa*j%NXYl+SC)ceiR&96<^8Fi245;Jr&)a2!ypN0I!hCl{dzn-{vn${B-f%53n zjv*li#Rp+r3DKs@`s2%K%8OkVg3+oQ#Bxz+8y}OXCkCr@u$CB{r`~Ul4r1GwQTJFb zF+&HZ6TH}EXX#+Iu^AAl(ZQu6-ZT#ND3X0hyogu9BoYjZd=YQ|0*%N?TM&zr_gd6F znZ)$S>QwBrOhdLF!||RWU9%58MR}GLaq^hP!PX5s{={}r70(L?)c8tm1C&ZN}w#>tVclJI_FKW?8?I`NI zx(>piZN&IBkQ*UE5I)8w*I&f1k?1uNyG9b%Nc5+JwxBw^D{0_1ip75Z)xAh$c>Lh=>?!an|g0QoUS Aga7~l literal 0 HcmV?d00001 diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 00000000..f6048f4f --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,25 @@ +import pytest +import os +import sys +import tempfile +import shutil + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + + +@pytest.fixture +def temp_dir(): + dirpath = tempfile.mkdtemp() + yield dirpath + if os.path.exists(dirpath): + shutil.rmtree(dirpath) + + +@pytest.fixture +def create_test_file(temp_dir): + def _create_file(filename, content): + filepath = os.path.join(temp_dir, filename) + with open(filepath, "w") as f: + f.write(content) + return filepath + return _create_file \ No newline at end of file diff --git a/tests/test_concore.py b/tests/test_concore.py new file mode 100644 index 00000000..e8ddf9e6 --- /dev/null +++ b/tests/test_concore.py @@ -0,0 +1,95 @@ +import pytest +import os +import sys +import tempfile +import shutil + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + + +@pytest.fixture +def temp_dir(): + dirpath = tempfile.mkdtemp() + yield dirpath + if os.path.exists(dirpath): + shutil.rmtree(dirpath) + + +class TestSafeLiteralEval: + + def test_reads_dictionary_from_file(self, temp_dir): + test_file = os.path.join(temp_dir, "config.txt") + with open(test_file, "w") as f: + f.write("{'name': 'test', 'value': 123}") + + from concore import safe_literal_eval + result = safe_literal_eval(test_file, {}) + + assert result == {'name': 'test', 'value': 123} + + def test_returns_default_when_file_missing(self): + from concore import safe_literal_eval + result = safe_literal_eval("nonexistent_file.txt", "fallback") + + assert result == "fallback" + + def test_returns_default_for_empty_file(self, temp_dir): + test_file = os.path.join(temp_dir, "empty.txt") + with open(test_file, "w") as f: + pass + + from concore import safe_literal_eval + result = safe_literal_eval(test_file, "default") + + assert result == "default" + + +class TestTryparam: + + def test_returns_existing_parameter(self): + from concore import tryparam, params + params['my_setting'] = 'custom_value' + + result = tryparam('my_setting', 'default_value') + + assert result == 'custom_value' + + def test_returns_default_for_missing_parameter(self): + from concore import tryparam + result = tryparam('missing_param', 'fallback') + + assert result == 'fallback' + + +class TestZeroMQPort: + + def test_class_is_defined(self): + from concore import ZeroMQPort + assert ZeroMQPort is not None + + +class TestDefaultConfiguration: + + def test_default_input_path(self): + from concore import inpath + assert inpath == "./in" + + def test_default_output_path(self): + from concore import outpath + assert outpath == "./out" + + +class TestPublicAPI: + + def test_module_imports_successfully(self): + import concore + assert concore is not None + + def test_core_functions_exist(self): + from concore import safe_literal_eval + from concore import tryparam + from concore import default_maxtime + + assert callable(safe_literal_eval) + assert callable(tryparam) + assert callable(default_maxtime) \ No newline at end of file From 2103fc7dc27d12501ed4f2165ce96c43b8d10a7d Mon Sep 17 00:00:00 2001 From: Sahil Lenka Date: Tue, 27 Jan 2026 23:38:20 +0530 Subject: [PATCH 007/275] Remove __pycache__ files and update .gitignore --- .gitignore | 22 +++++++++++++++++- tests/__pycache__/__init__.cpython-312.pyc | Bin 137 -> 0 bytes .../conftest.cpython-312-pytest-9.0.2.pyc | Bin 1710 -> 0 bytes .../test_concore.cpython-312-pytest-9.0.2.pyc | Bin 11504 -> 0 bytes 4 files changed, 21 insertions(+), 1 deletion(-) delete mode 100644 tests/__pycache__/__init__.cpython-312.pyc delete mode 100644 tests/__pycache__/conftest.cpython-312-pytest-9.0.2.pyc delete mode 100644 tests/__pycache__/test_concore.cpython-312-pytest-9.0.2.pyc diff --git a/.gitignore b/.gitignore index 79b5594d..82494f13 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,21 @@ -**/.DS_Store +# Python +__pycache__/ +*.py[cod] +*.class +*.so +.Python +venv/ +env/ +ENV/ + +# IDE +.vscode/ +.idea/ + +# Testing +.pytest_cache/ +htmlcov/ +.coverage + +# Concore specific +concorekill.bat diff --git a/tests/__pycache__/__init__.cpython-312.pyc b/tests/__pycache__/__init__.cpython-312.pyc deleted file mode 100644 index 5181b3a1b70f61f57e98c51ee0861cd3815a4fea..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 137 zcmX@j%ge<81WbP_GC}lX5P=Rpvj9b=GgLBYGWxA#C}INgK7-W!(r~tl2`x@7Dvk+G z%*f1%NzTtp&M!)hDM>9ZDUOMc&&7%Q6n?Y2-d!92BymYoeq5tUD857@6*8)#x>1pGfQCcFp_V1%+UsVG|1`6a zW8_qEiR45a>Kq^((+dSvRP@vSjLIIFM1wh{=Yq69-rk&R^%?$EY`m`c1bJ( zmR^cW@o%J{Y{A{3CO!EdKI$=_sU}HizlwPz;4>s|;eC7H408e~4YN2N z;N7|bE%=1&5dx0p*-N@7jd{sGmOZI8s+UeX&w1KX(38e|>`9=Bzc1AWNs#4)!A-}i z)Qfi6;zs5Dg4l4S#<*6`IjdZ^)@|l6uCvuO$1Zc7Rvc0HM?E7%6 z7mjZ(Jqk?TymDmtJB@aubEAEud;P$O9hu=7Rt zQ6+jVP#H`i=^!54P$UJiWp0obS%HMIoIehWI3+6F?;-h^AbiN(brO6fpJyHO9Sz$& zJaa>F4A0V13dGaD4jXDqJrPXe4X=En_)NVR0xe|)r|UIfq=C=)Dvp<{TUW0F0;w!> z(n$EIsDX-WxmK##Rj$?QR+al2)OM_dkDCJKYOaD%BO;DfbzGH%niM4Hh)5IUJEoe|fFe!mxYlb!fydh`Xf3jEmw63pf|P4mt40}!0~PCGq79VDXMytZbS z9lOeS@KU{*9qrC!>rFYfIfkR{*tO~$wPZ87f287urwd@PY)p=F-4!JwTxCs0#p9;p z4Z8s8cpaW8-;oPR?i!Oom|5A`Y?d2Ed)-;1mRqeTg^G_a6<<}iU~{2TE3B2Ri!^~c zG5xHKq^}Y}egxQp4?~CGKZL*`L=GYJ7}Sp0Han4aq)WQ<5|`9&YJdJt@^UQd@ z`3K4S$p@+XsjuJYoqxM`?n*DZd;stCbx_PstQ~uzXyoib1!bZyq<(m$*U603KT%MU GDgFhC$6x^f diff --git a/tests/__pycache__/test_concore.cpython-312-pytest-9.0.2.pyc b/tests/__pycache__/test_concore.cpython-312-pytest-9.0.2.pyc deleted file mode 100644 index a3e1c4a169d515844619f8b50820e7625b1395d4..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 11504 zcmeHNU2GKB6}~e&vwL^(dSfsSrm24cdkHuJLQH5VP)KQ%K$3>E&5Cxkc*d}s{c&ez zX$SJ?GBMo!Rvm zn>I=Fkn#G=x%ZxX=g!>oopbKbU*ho?f$RFeCo(?(?r)fIFIEDHg-Ze<7l}$#AxBaI zyNmhIh$xV}l#&IzB%BJfl1M7T?leW&J(`NLdn^^xXtu*`+`DHxJ>KP1qD2B_zvW7J z_y1Y)+g@8PrzPHocgAbcU9|HBqJ^_PHt(mZc!s>~WTe_ea)_v*Geni%cIu^8vpgB{ z!aQ#c%ZorB<$3KaFA8}vp0}3et%AH3p4TxcB;%jq6OWAeI}(CKKm0HJ7RVDsfDtsE zcOoR~^u}Mo?RABe$uEVAf(UQ-`j`+Fm()JrX@TetP`u%coyHe_}S$b)B}p9)C4{ z{=^*Zx)CC@`(Flruvhjc_O}TQqvDRhLKl=@BpSn0N>qh2Bo#v4NlB83^1_v6*RzE*)8?%WkD5ORjP<4w#Tlr zsETg>PErY#LqWr>hFo|sBLIfLtpjOEm2+}FT#*DKlCo45GrY7St6?>w(o13`T$an> z7X}HbM9LB4S7yTPLG~V(^X_W292phV7}gmT-*(hfU1hFuWwPxq=iOb!evue!eeDOB zqO$D1kM5UT=e6Z>o@7CQ=X<>lk`249sx6nqDbS1xRpV-FnPvo*ikjT4`C7WqeU^7G zN8Og(`zv6MCxl9*5}pLKZzF~<4rcjzfV)6m`(PsUJONlGj!eZs*+(-chRg}m62_;x zUK%K*^V-1fo&f~Jz?Pl?P^}VXY}>x0qN9#o5v(=?bWh3I%BN@`Vo50f*-KEK6fFrG zv1C^DitxF(XU`R3YW2p!vB?LMM$eu-J;?4z0$g>?DCNv6q7{NNBs)HDO&fqulgaD^ z8OYq3Oy~$49re(Xvc*inl8R$m!IH;y=!F$=FstK$Td{p(lh_?4ty^u%NII8ObZtx@ zDdxx0x@NUNv2whW$(fk~bRky+3By{a7$=LRoXSL%6S?B?bWUOK5moSYu?4GjKkHkj zSa?j=i@LQ&87=Dhw5ezlW4Ux8Z9>VFm?arnZj_k}E6O^;%$6|9x~JoyTC1=V_M2|^ zC$`xMyV-~QiH8leKRx`?(Ry0ltbJWes|MJq5$r%(pHxQmVqU?gPVG3l*j5ZJQwhOh z{0_(|a=pFd;-L$NEs<7DyzP8)`q|ld@8xH%w!~jQ^y;B%`$MxW zJ8y?bYu`V@Bs%nATi@kJ-a1}y+jcJeN#EeRJ8GT7RXTj`#cJnpEipV#@i*sW*eOnH zm~E%(9y?2ip)MEV%nTjA9f!wOI@4tBx?!p<~Ud_Cm41+_hFPnT|5+Lq`Fp$I~M(=x8eS^+*sYI@&wE z8%Yn4WK7?HJOW+ch-4Fz0VLoph@M2U8Oi-fwg5>+^daO~9oFVHOm0WA1Ia@`4Ac(? zTsE{eIVdqpdI7+tji$j=DdQ)#0_(Ms&lpCgaAIor9T2umUG(_r2r?8XWZ4ake)0F1-fl?~8+z#ql|k)}9+dRm+f(A@?U zdhG{944!uf6nc_(j2HlgbQxKW9W~11fS9_<7YFdEkgp{~NYizfaljv(IH2XoR20H# za}t55BjP=@Li}ti{xYy^^&GU`i!JmaxofEETcKK@qw5buBlt;$b=`J$!z+ejV@2$z z@f#q`4(~hl_TgrScTcmM%hcO;Az=Hev=0E=SxfZIQ~b@@X*h%Va1iSuHpu zNA*bvz0&z$hg@!Q$a%2>!Kw-lSe3BH%ss=jfg6U98mdU;P+4N`SuV@qp3!`>D@$F@ zyMtknnPGrBKKD%TJQr;+3|{*IhNsLODHsM%GGMCWNixH5XIaU0rra=uRWuA?$2~{9 z36AfnL53^hw&dO&_Y8&?+;fZVo*{ClrZZAjm<^a=wTzSu6M{(=%v-Viq+)0$M9wFy zxa$&G(K7a7m2C?S!kFlfAz7*Krnc|)DlX~3aoimT-qRQy+g_N(9MExvy_p6x=cz4z z<#eOsrKB-vFwi-kA|KN&SG%{K3!f|3=zV`}>zv;5R>Cm?164WzMxd*f7?`K{o3qn) zg3}u2+Nrw7&eDPDE-u5F89KmF7!03p;#n9Ic6s3XYU0@|f{uD=#hp;l)+~J(@|J;v z6U8#EUMUisD3;Y(5(H>a8lu?EOZv@*PBS<#_vzZ|MmV@wN58qKBKi{}3k1X14Ics? z9(W6)*oVKwD3;;jNK)4K;K%+hlD$YCM{*d+J|v!}`yM8dJOKX&IzRMuVR>-j9`JOw zAQlHZIdqTBPHs|6hOAb!ASq2R9)9NOqHYEwG}Ywba8;}#mW8rd5^xP|+1Vw?fcuiL zhR=P;U)bzYaNWYPvr8@s?o0N~34@#>pNXnrC$5!nb+0Tr?o0OKa$idU5xN(5yZv5A-vUA+^C;b^1)+HvGtZ1ds4BJkM)j>DW zJh#NfD;t%zV_M5VIg*0~ASJ`*mYIU4PHkvZCL4jxQmCZH&wwDV9(*}?L*>W>nThk!F)E6K0_8H zQQl*S=pvg@@3;&;i z7WTp8&OnQBpix*9|Du5w<3I~%3b?5q8q1}N7_ zLFY;)IAiGGgG;H94X(vC(1CDx^ovM56EbX4| z;4++5NsY<_twm+Dug#*t#lWNIQ0Jpimvk&i*QmQZ9?beD?E>a0YO*EJ|I zcF)!1Z7b7cUlk04xGMKwUEf=!z30ZK2WpAld5XU|J8dU8tzoX6s(b7#?VX0{j8l^{ zGql(76pKQ!6n?^=BH4$;a~D6xB$7k$&qFa$?pdIs+-D17@qgtr;`o`KE*;NhM)p7b zcre0#%J{OU2-_h4n&;kbE3k{6LsI$`lLugGuiev+l`;d5x z_)$CeEJzJuuOgo0&_z#N+TnhVT_h?xyh-a1}(9D!5!rBMI0nuqa9bf z*h=5{v|9DA^g)_0+u01R^Z}A^DG*os92Qslnk7L3R{9n@1q>Wf2^L!A=1nWuzzV?z zmc3R63IKel5`xniQIKbsr%JF|CK(^7u)V(G6|YX;mUDm_m|V$SHPHPv0)qI(H9&1a z1y|f7|JHjzkG!`9;BaLQTJZNIp$Cy=WW6;wdMRXYk(q1VFYK#($4vnD8{xF2BKby$ z!}4o9LeVd5g!tPoJ3?F%Mkwn0$^m*+{&Gi%uk3z}M=17%jSzp^Wk-lh!U)B_`Vnf{ zRMwvWNWy8agd5Q3(-USUuT5?7ohE@3C$X2DqbEJ~Arv@|GKe|bl14ICu_zuENgvN? zY_B*O((w!+oc>ji8>+N;f6 z8*WImQ)2%0f==ep@SvJ9S}L$p9tNB_fg^8I0|A~aNbo1v?g0pV*j6(v(|ttm|1@!L zbzQQa*j%NXYl+SC)ceiR&96<^8Fi245;Jr&)a2!ypN0I!hCl{dzn-{vn${B-f%53n zjv*li#Rp+r3DKs@`s2%K%8OkVg3+oQ#Bxz+8y}OXCkCr@u$CB{r`~Ul4r1GwQTJFb zF+&HZ6TH}EXX#+Iu^AAl(ZQu6-ZT#ND3X0hyogu9BoYjZd=YQ|0*%N?TM&zr_gd6F znZ)$S>QwBrOhdLF!||RWU9%58MR}GLaq^hP!PX5s{={}r70(L?)c8tm1C&ZN}w#>tVclJI_FKW?8?I`NI zx(>piZN&IBkQ*UE5I)8w*I&f1k?1uNyG9b%Nc5+JwxBw^D{0_1ip75Z)xAh$c>Lh=>?!an|g0QoUS Aga7~l From 33a603feb8150db72a9013073fdc97294288c265 Mon Sep 17 00:00:00 2001 From: Sahil Lenka Date: Tue, 27 Jan 2026 23:57:07 +0530 Subject: [PATCH 008/275] Fix code review feedback from Copilot - Remove duplicate temp_dir fixture from test_concore.py - Remove duplicate sys.path.insert statement - Remove unused imports (tempfile, shutil) - Remove unused create_test_file fixture - Add fixture to reset params state for test isolation - Simplify imports to avoid duplication --- tests/conftest.py | 12 +----------- tests/test_concore.py | 26 +++++++++----------------- 2 files changed, 10 insertions(+), 28 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index f6048f4f..10d20ac0 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -12,14 +12,4 @@ def temp_dir(): dirpath = tempfile.mkdtemp() yield dirpath if os.path.exists(dirpath): - shutil.rmtree(dirpath) - - -@pytest.fixture -def create_test_file(temp_dir): - def _create_file(filename, content): - filepath = os.path.join(temp_dir, filename) - with open(filepath, "w") as f: - f.write(content) - return filepath - return _create_file \ No newline at end of file + shutil.rmtree(dirpath) \ No newline at end of file diff --git a/tests/test_concore.py b/tests/test_concore.py index e8ddf9e6..b1b41261 100644 --- a/tests/test_concore.py +++ b/tests/test_concore.py @@ -1,19 +1,5 @@ import pytest import os -import sys -import tempfile -import shutil - -sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - - -@pytest.fixture -def temp_dir(): - dirpath = tempfile.mkdtemp() - yield dirpath - if os.path.exists(dirpath): - shutil.rmtree(dirpath) - class TestSafeLiteralEval: @@ -46,6 +32,14 @@ def test_returns_default_for_empty_file(self, temp_dir): class TestTryparam: + @pytest.fixture(autouse=True) + def reset_params(self): + from concore import params + original_params = params.copy() + yield + params.clear() + params.update(original_params) + def test_returns_existing_parameter(self): from concore import tryparam, params params['my_setting'] = 'custom_value' @@ -86,9 +80,7 @@ def test_module_imports_successfully(self): assert concore is not None def test_core_functions_exist(self): - from concore import safe_literal_eval - from concore import tryparam - from concore import default_maxtime + from concore import safe_literal_eval, tryparam, default_maxtime assert callable(safe_literal_eval) assert callable(tryparam) From 738c5ca83e3916737bdf1f39fd326cf3cef56993 Mon Sep 17 00:00:00 2001 From: Pradeeban Kathiravelu Date: Tue, 27 Jan 2026 20:08:07 -0900 Subject: [PATCH 009/275] Update tests/test_concore.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- tests/test_concore.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_concore.py b/tests/test_concore.py index b1b41261..862554c8 100644 --- a/tests/test_concore.py +++ b/tests/test_concore.py @@ -76,8 +76,8 @@ def test_default_output_path(self): class TestPublicAPI: def test_module_imports_successfully(self): - import concore - assert concore is not None + from concore import safe_literal_eval + assert safe_literal_eval is not None def test_core_functions_exist(self): from concore import safe_literal_eval, tryparam, default_maxtime From 1e893ecd507aa76759f6915934372275d1f1e826 Mon Sep 17 00:00:00 2001 From: Sahil Lenka Date: Wed, 28 Jan 2026 22:33:54 +0530 Subject: [PATCH 010/275] Add GitHub Actions CI for automated testing - Run tests on push and pull requests - Test on Python 3.9, 3.10, 3.11, 3.12 - Uses ubuntu-latest for fast execution --- .github/workflows/tests.yml | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 .github/workflows/tests.yml diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 00000000..912cc04e --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,32 @@ +name: Tests + +on: + push: + branches: [main, dev] + pull_request: + branches: [main, dev] + +jobs: + test: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ['3.9', '3.10', '3.11', '3.12'] + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements-dev.txt + pip install pyzmq + + - name: Run tests + run: pytest -v \ No newline at end of file From 152359fe2a33747f37c0a9185a3b505be996f812 Mon Sep 17 00:00:00 2001 From: GREENRAT-K405 Date: Thu, 29 Jan 2026 01:34:19 +0530 Subject: [PATCH 011/275] fix: infinite loop problem during the running and debugging of concore study --- concore.py | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/concore.py b/concore.py index e1bfb5e7..4c549e21 100644 --- a/concore.py +++ b/concore.py @@ -4,6 +4,7 @@ import sys import re import zmq # Added for ZeroMQ +import numpy as np # Added for numpy type conversion # if windows, create script to kill this process # because batch files don't provide easy way to know pid of last command @@ -100,6 +101,24 @@ def terminate_zmq(): print(f"Error while terminating ZMQ port {port.address}: {e}") # --- ZeroMQ Integration End --- + +# NumPy Type Conversion Helper +def convert_numpy_to_python(obj): + #Recursively convert numpy types to native Python types. + #This is necessary because literal_eval cannot parse numpy representations + #like np.float64(1.0), but can parse native Python types like 1.0. + if isinstance(obj, np.generic): + # Convert numpy scalar types to Python native types + return obj.item() + elif isinstance(obj, list): + return [convert_numpy_to_python(item) for item in obj] + elif isinstance(obj, tuple): + return tuple(convert_numpy_to_python(item) for item in obj) + elif isinstance(obj, dict): + return {key: convert_numpy_to_python(value) for key, value in obj.items()} + else: + return obj + # =================================================================== # File & Parameter Handling # =================================================================== @@ -229,8 +248,10 @@ def read(port_identifier, name, initstr_val): ins = infile.read() except FileNotFoundError: ins = str(initstr_val) + s += ins # Update s to break unchanged() loop except Exception as e: print(f"Error reading {file_path}: {e}. Using default value.") + s += str(initstr_val) # Update s to break unchanged() loop return default_return_val # Retry logic if file is empty @@ -248,6 +269,7 @@ def read(port_identifier, name, initstr_val): if len(ins) == 0: print(f"Max retries reached for {file_path}, using default value.") + s += str(initstr_val) # Update s to break unchanged() loop return default_return_val s += ins @@ -304,7 +326,9 @@ def write(port_identifier, name, val, delta=0): try: with open(file_path, "w") as outfile: if isinstance(val, list): - data_to_write = [simtime + delta] + val + # Convert numpy types to native Python types + val_converted = convert_numpy_to_python(val) + data_to_write = [simtime + delta] + val_converted outfile.write(str(data_to_write)) simtime += delta else: From de1fcc3cb88b35984238ba83d3b1b4cb96ce9671 Mon Sep 17 00:00:00 2001 From: Sahil Lenka Date: Fri, 30 Jan 2026 01:34:16 +0530 Subject: [PATCH 012/275] Fix bare exceptions and replace print with logging mkconcore.py: - Replace 13 bare except: with except (FileNotFoundError, IOError) as e: - Adds actual error message to logs for better debugging concore.py: - Add import logging with basic configuration - Replace 25+ print() statements with appropriate logging levels - logging.info() for status messages - logging.warning() for retries and non-critical issues - logging.error() for errors and failures This improves debugging experience and follows Python best practices. --- concore.py | 69 ++++++++++++++++++++++++++++------------------------ mkconcore.py | 48 ++++++++++++++++++------------------ 2 files changed, 61 insertions(+), 56 deletions(-) diff --git a/concore.py b/concore.py index 6d71f0fc..26550ebb 100644 --- a/concore.py +++ b/concore.py @@ -1,9 +1,14 @@ import time +import logging import os from ast import literal_eval import sys import re -import zmq # Added for ZeroMQ +import zmq +logging.basicConfig( + level=logging.INFO, + format='%(levelname)s - %(message)s' +) # Added for ZeroMQ # if windows, create script to kill this process # because batch files don't provide easy way to know pid of last command @@ -36,10 +41,10 @@ def __init__(self, port_type, address, zmq_socket_type): # Bind or connect if self.port_type == "bind": self.socket.bind(address) - print(f"ZMQ Port bound to {address}") + logging.info(f"ZMQ Port bound to {address}") else: self.socket.connect(address) - print(f"ZMQ Port connected to {address}") + logging.info(f"ZMQ Port connected to {address}") def send_json_with_retry(self, message): """Send JSON message with retries if timeout occurs.""" @@ -48,9 +53,9 @@ def send_json_with_retry(self, message): self.socket.send_json(message) return except zmq.Again: - print(f"Send timeout (attempt {attempt + 1}/5)") + logging.warning(f"Send timeout (attempt {attempt + 1}/5)") time.sleep(0.5) - print("Failed to send after retries.") + logging.error("Failed to send after retries.") return def recv_json_with_retry(self): @@ -59,9 +64,9 @@ def recv_json_with_retry(self): try: return self.socket.recv_json() except zmq.Again: - print(f"Receive timeout (attempt {attempt + 1}/5)") + logging.warning(f"Receive timeout (attempt {attempt + 1}/5)") time.sleep(0.5) - print("Failed to receive after retries.") + logging.error("Failed to receive after retries.") return None # Global ZeroMQ ports registry @@ -76,20 +81,20 @@ def init_zmq_port(port_name, port_type, address, socket_type_str): socket_type_str (str): String representation of ZMQ socket type (e.g., "REQ", "REP", "PUB", "SUB"). """ if port_name in zmq_ports: - print(f"ZMQ Port {port_name} already initialized.") + logging.info(f"ZMQ Port {port_name} already initialized.") return # Avoid reinitialization try: # Map socket type string to actual ZMQ constant (e.g., zmq.REQ, zmq.REP) zmq_socket_type = getattr(zmq, socket_type_str.upper()) zmq_ports[port_name] = ZeroMQPort(port_type, address, zmq_socket_type) - print(f"Initialized ZMQ port: {port_name} ({socket_type_str}) on {address}") + logging.info(f"Initialized ZMQ port: {port_name} ({socket_type_str}) on {address}") except AttributeError: - print(f"Error: Invalid ZMQ socket type string '{socket_type_str}'.") + logging.error(f"Error: Invalid ZMQ socket type string '{socket_type_str}'.") except zmq.error.ZMQError as e: - print(f"Error initializing ZMQ port {port_name} on {address}: {e}") + logging.error(f"Error initializing ZMQ port {port_name} on {address}: {e}") except Exception as e: - print(f"An unexpected error occurred during ZMQ port initialization for {port_name}: {e}") + logging.error(f"An unexpected error occurred during ZMQ port initialization for {port_name}: {e}") def terminate_zmq(): for port in zmq_ports.values(): @@ -97,7 +102,7 @@ def terminate_zmq(): port.socket.close() port.context.term() except Exception as e: - print(f"Error while terminating ZMQ port {port.address}: {e}") + logging.error(f"Error while terminating ZMQ port {port.address}: {e}") # --- ZeroMQ Integration End --- # =================================================================== @@ -142,13 +147,13 @@ def safe_literal_eval(filename, defaultValue): # Convert key=value;key2=value2 to Python dict format if sparams != '{' and not (sparams.startswith('{') and sparams.endswith('}')): # Check if it needs conversion - print("converting sparams: "+sparams) + logging.debug("converting sparams: "+sparams) sparams = "{'"+re.sub(';',",'",re.sub('=',"':",re.sub(' ','',sparams)))+"}" - print("converted sparams: " + sparams) + logging.debug("converted sparams: " + sparams) try: params = literal_eval(sparams) except Exception as e: - print(f"bad params content: {sparams}, error: {e}") + logging.warning(f"bad params content: {sparams}, error: {e}") params = dict() else: params = dict() @@ -206,17 +211,17 @@ def read(port_identifier, name, initstr_val): message = zmq_p.recv_json_with_retry() return message except zmq.error.ZMQError as e: - print(f"ZMQ read error on port {port_identifier} (name: {name}): {e}. Returning default.") + logging.error(f"ZMQ read error on port {port_identifier} (name: {name}): {e}. Returning default.") return default_return_val except Exception as e: - print(f"Unexpected error during ZMQ read on port {port_identifier} (name: {name}): {e}. Returning default.") + logging.error(f"Unexpected error during ZMQ read on port {port_identifier} (name: {name}): {e}. Returning default.") return default_return_val # Case 2: File-based port try: file_port_num = int(port_identifier) except ValueError: - print(f"Error: Invalid port identifier '{port_identifier}' for file operation. Must be integer or ZMQ name.") + logging.error(f"Error: Invalid port identifier '{port_identifier}' for file operation. Must be integer or ZMQ name.") return default_return_val time.sleep(delay) @@ -229,7 +234,7 @@ def read(port_identifier, name, initstr_val): except FileNotFoundError: ins = str(initstr_val) except Exception as e: - print(f"Error reading {file_path}: {e}. Using default value.") + logging.error(f"Error reading {file_path}: {e}. Using default value.") return default_return_val # Retry logic if file is empty @@ -241,12 +246,12 @@ def read(port_identifier, name, initstr_val): with open(file_path, "r") as infile: ins = infile.read() except Exception as e: - print(f"Retry {attempts + 1}: Error reading {file_path} - {e}") + logging.warning(f"Retry {attempts + 1}: Error reading {file_path} - {e}") attempts += 1 retrycount += 1 if len(ins) == 0: - print(f"Max retries reached for {file_path}, using default value.") + logging.error(f"Max retries reached for {file_path}, using default value.") return default_return_val s += ins @@ -260,10 +265,10 @@ def read(port_identifier, name, initstr_val): simtime = max(simtime, current_simtime_from_file) return inval[1:] else: - print(f"Warning: Unexpected data format in {file_path}: {ins}. Returning raw content or default.") + logging.warning(f"Warning: Unexpected data format in {file_path}: {ins}. Returning raw content or default.") return inval except Exception as e: - print(f"Error parsing content from {file_path} ('{ins}'): {e}. Returning default.") + logging.error(f"Error parsing content from {file_path} ('{ins}'): {e}. Returning default.") return default_return_val @@ -280,9 +285,9 @@ def write(port_identifier, name, val, delta=0): try: zmq_p.send_json_with_retry(val) except zmq.error.ZMQError as e: - print(f"ZMQ write error on port {port_identifier} (name: {name}): {e}") + logging.error(f"ZMQ write error on port {port_identifier} (name: {name}): {e}") except Exception as e: - print(f"Unexpected error during ZMQ write on port {port_identifier} (name: {name}): {e}") + logging.error(f"Unexpected error during ZMQ write on port {port_identifier} (name: {name}): {e}") # Case 2: File-based port try: @@ -292,14 +297,14 @@ def write(port_identifier, name, val, delta=0): file_port_num = int(port_identifier) file_path = os.path.join(outpath+str(file_port_num), name) except ValueError: - print(f"Error: Invalid port identifier '{port_identifier}' for file operation. Must be integer or ZMQ name.") + logging.error(f"Error: Invalid port identifier '{port_identifier}' for file operation. Must be integer or ZMQ name.") return # File writing rules if isinstance(val, str): time.sleep(2 * delay) # string writes wait longer elif not isinstance(val, list): - print(f"File write to {file_path} must have list or str value, got {type(val)}") + logging.error(f"File write to {file_path} must have list or str value, got {type(val)}") return try: @@ -311,7 +316,7 @@ def write(port_identifier, name, val, delta=0): else: outfile.write(val) except Exception as e: - print(f"Error writing to {file_path}: {e}") + logging.error(f"Error writing to {file_path}: {e}") def initval(simtime_val_str): """ @@ -327,12 +332,12 @@ def initval(simtime_val_str): simtime = first_element return val[1:] else: - print(f"Error: First element in initval string '{simtime_val_str}' is not a number. Using data part as is or empty.") + logging.error(f"Error: First element in initval string '{simtime_val_str}' is not a number. Using data part as is or empty.") return val[1:] if len(val) > 1 else [] else: - print(f"Error: initval string '{simtime_val_str}' is not a list or is empty. Returning empty list.") + logging.error(f"Error: initval string '{simtime_val_str}' is not a list or is empty. Returning empty list.") return [] except Exception as e: - print(f"Error parsing simtime_val_str '{simtime_val_str}': {e}. Returning empty list.") + logging.error(f"Error parsing simtime_val_str '{simtime_val_str}': {e}. Returning empty list.") return [] \ No newline at end of file diff --git a/mkconcore.py b/mkconcore.py index b243c230..10135dc9 100644 --- a/mkconcore.py +++ b/mkconcore.py @@ -413,8 +413,8 @@ fsource = open(CONCOREPATH+"/concoredocker.py") else: fsource = open(CONCOREPATH+"/concore.py") -except: - logging.error(f"{CONCOREPATH} is not correct path to concore") +except (FileNotFoundError, IOError) as e: + logging.error(f"{CONCOREPATH} is not correct path to concore: {e}") quit() with open(outdir+"/src/concore.py","w") as fcopy: fcopy.write(fsource.read()) @@ -426,8 +426,8 @@ fsource = open(CONCOREPATH+"/concoredocker.hpp") else: fsource = open(CONCOREPATH+"/concore.hpp") -except: - logging.error(f"{CONCOREPATH} is not correct path to concore") +except (FileNotFoundError, IOError) as e: + logging.error(f"{CONCOREPATH} is not correct path to concore: {e}") quit() with open(outdir+"/src/concore.hpp","w") as fcopy: fcopy.write(fsource.read()) @@ -439,8 +439,8 @@ fsource = open(CONCOREPATH+"/concoredocker.v") else: fsource = open(CONCOREPATH+"/concore.v") -except: - logging.error(f"{CONCOREPATH} is not correct path to concore") +except (FileNotFoundError, IOError) as e: + logging.error(f"{CONCOREPATH} is not correct path to concore: {e}") quit() with open(outdir+"/src/concore.v","w") as fcopy: fcopy.write(fsource.read()) @@ -449,8 +449,8 @@ #copy mkcompile into /src 5/27/21 try: fsource = open(CONCOREPATH+"/mkcompile") -except: - logging.error(f"{CONCOREPATH} is not correct path to concore") +except (FileNotFoundError, IOError) as e: + logging.error(f"{CONCOREPATH} is not correct path to concore: {e}") quit() with open(outdir+"/src/mkcompile","w") as fcopy: fcopy.write(fsource.read()) @@ -460,56 +460,56 @@ #copy concore*.m into /src 4/2/21 try: #maxtime in matlab 11/22/21 fsource = open(CONCOREPATH+"/concore_default_maxtime.m") -except: - logging.error(f"{CONCOREPATH} is not correct path to concore") +except (FileNotFoundError, IOError) as e: + logging.error(f"{CONCOREPATH} is not correct path to concore: {e}") quit() with open(outdir+"/src/concore_default_maxtime.m","w") as fcopy: fcopy.write(fsource.read()) fsource.close() try: fsource = open(CONCOREPATH+"/concore_unchanged.m") -except: - logging.error(f"{CONCOREPATH} is not correct path to concore") +except (FileNotFoundError, IOError) as e: + logging.error(f"{CONCOREPATH} is not correct path to concore: {e}") quit() with open(outdir+"/src/concore_unchanged.m","w") as fcopy: fcopy.write(fsource.read()) fsource.close() try: fsource = open(CONCOREPATH+"/concore_read.m") -except: - logging.error(f"{CONCOREPATH} is not correct path to concore") +except (FileNotFoundError, IOError) as e: + logging.error(f"{CONCOREPATH} is not correct path to concore: {e}") quit() with open(outdir+"/src/concore_read.m","w") as fcopy: fcopy.write(fsource.read()) fsource.close() try: fsource = open(CONCOREPATH+"/concore_write.m") -except: - logging.error(f"{CONCOREPATH} is not correct path to concore") +except (FileNotFoundError, IOError) as e: + logging.error(f"{CONCOREPATH} is not correct path to concore: {e}") quit() with open(outdir+"/src/concore_write.m","w") as fcopy: fcopy.write(fsource.read()) fsource.close() try: #4/9/21 fsource = open(CONCOREPATH+"/concore_initval.m") -except: - logging.error(f"{CONCOREPATH} is not correct path to concore") +except (FileNotFoundError, IOError) as e: + logging.error(f"{CONCOREPATH} is not correct path to concore: {e}") quit() with open(outdir+"/src/concore_initval.m","w") as fcopy: fcopy.write(fsource.read()) fsource.close() try: #11/19/21 fsource = open(CONCOREPATH+"/concore_iport.m") -except: - logging.error(f"{CONCOREPATH} is not correct path to concore") +except (FileNotFoundError, IOError) as e: + logging.error(f"{CONCOREPATH} is not correct path to concore: {e}") quit() with open(outdir+"/src/concore_iport.m","w") as fcopy: fcopy.write(fsource.read()) fsource.close() try: #11/19/21 fsource = open(CONCOREPATH+"/concore_oport.m") -except: - logging.error(f"{CONCOREPATH} is not correct path to concore") +except (FileNotFoundError, IOError) as e: + logging.error(f"{CONCOREPATH} is not correct path to concore: {e}") quit() with open(outdir+"/src/concore_oport.m","w") as fcopy: fcopy.write(fsource.read()) @@ -519,8 +519,8 @@ fsource = open(CONCOREPATH+"/import_concoredocker.m") else: fsource = open(CONCOREPATH+"/import_concore.m") -except: - logging.error(f"{CONCOREPATH} is not correct path to concore") +except (FileNotFoundError, IOError) as e: + logging.error(f"{CONCOREPATH} is not correct path to concore: {e}") quit() with open(outdir+"/src/import_concore.m","w") as fcopy: fcopy.write(fsource.read()) From 89059928aa66cea11f44baf357e78e60248a3edd Mon Sep 17 00:00:00 2001 From: GREENRAT-K405 Date: Sun, 1 Feb 2026 13:12:10 +0530 Subject: [PATCH 013/275] Add scripts to start the flask server --- startserver | 14 ++++++++++++++ startserver.bat | 8 ++++++++ 2 files changed, 22 insertions(+) create mode 100644 startserver create mode 100644 startserver.bat diff --git a/startserver b/startserver new file mode 100644 index 00000000..73bd6de6 --- /dev/null +++ b/startserver @@ -0,0 +1,14 @@ +#!/bin/bash +cd fri/server +if ! command -v python3 >/dev/null 2>&1; then + python main.py +else + python3 main.py +fi + +if [ $? -ne 0 ]; then + echo "" + echo "Error: Make sure modules are installed from fri/requirements.txt" + echo "Run: pip install -r fri/requirements.txt" + echo "" +fi diff --git a/startserver.bat b/startserver.bat new file mode 100644 index 00000000..856824a4 --- /dev/null +++ b/startserver.bat @@ -0,0 +1,8 @@ +cd fri\server +python main.py +if errorlevel 1 ( + echo. + echo Error: Make sure modules are installed from fri/requirements.txt + echo Run: pip install -r fri/requirements.txt + echo. +) \ No newline at end of file From 553f83bb86586a875155cdfc1eb427667324499b Mon Sep 17 00:00:00 2001 From: Prasanna Date: Sun, 1 Feb 2026 14:42:10 +0530 Subject: [PATCH 014/275] fix: import numpy library as np Signed-off-by: Prasanna --- concore.py | 1 + 1 file changed, 1 insertion(+) diff --git a/concore.py b/concore.py index 924364db..2da12506 100644 --- a/concore.py +++ b/concore.py @@ -5,6 +5,7 @@ import sys import re import zmq +import numpy as np logging.basicConfig( level=logging.INFO, format='%(levelname)s - %(message)s' From 14ee3a9bbe7c137cbee7cb278e6e96e221ada26c Mon Sep 17 00:00:00 2001 From: Prasanna Date: Sun, 1 Feb 2026 14:49:11 +0530 Subject: [PATCH 015/275] fix: github action flow Signed-off-by: Prasanna --- .github/workflows/tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 912cc04e..6b7cdf04 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -25,6 +25,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip + pip install -r requirements.txt pip install -r requirements-dev.txt pip install pyzmq From 2c6ae723daaa853debc8e931914e7da2a0acb93a Mon Sep 17 00:00:00 2001 From: Sahil Lenka Date: Mon, 2 Feb 2026 00:28:45 +0530 Subject: [PATCH 016/275] Add CLI for concore (#183) Features: - concore init: Create new project with templates - concore run: Execute workflows with auto-build option - concore validate: Check GraphML files before running - concore status: Show running processes with details - concore stop: Stop all concore processes safely Tech: - Built with Click and Rich for beautiful output - Cross-platform support (Windows, Linux, macOS) - All 9 tests passing Closes #183 --- README.md | 29 +++++ concore_cli/README.md | 180 +++++++++++++++++++++++++++++++ concore_cli/__init__.py | 3 + concore_cli/cli.py | 78 ++++++++++++++ concore_cli/commands/__init__.py | 7 ++ concore_cli/commands/init.py | 93 ++++++++++++++++ concore_cli/commands/run.py | 96 +++++++++++++++++ concore_cli/commands/status.py | 89 +++++++++++++++ concore_cli/commands/stop.py | 92 ++++++++++++++++ concore_cli/commands/validate.py | 144 +++++++++++++++++++++++++ requirements.txt | 5 +- setup.py | 41 +++++++ tests/test_cli.py | 85 +++++++++++++++ 13 files changed, 941 insertions(+), 1 deletion(-) create mode 100644 concore_cli/README.md create mode 100644 concore_cli/__init__.py create mode 100644 concore_cli/cli.py create mode 100644 concore_cli/commands/__init__.py create mode 100644 concore_cli/commands/init.py create mode 100644 concore_cli/commands/run.py create mode 100644 concore_cli/commands/status.py create mode 100644 concore_cli/commands/stop.py create mode 100644 concore_cli/commands/validate.py create mode 100644 setup.py create mode 100644 tests/test_cli.py diff --git a/README.md b/README.md index 815627c7..103ecb2e 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,35 @@ Please follow the [ReadTheDocs](https://control-core.readthedocs.io/en/latest/in Installation instructions for concore can be found [here](https://control-core.readthedocs.io/en/latest/installation.html). Usage instructions can be found [here](https://control-core.readthedocs.io/en/latest/usage.html). +## Command-Line Interface (CLI) + +_concore_ now includes a command-line interface for easier workflow management. Install it with: + +```bash +pip install -e . +``` + +Quick start with the CLI: + +```bash +# Create a new project +concore init my-project + +# Validate your workflow +concore validate workflow.graphml + +# Run your workflow +concore run workflow.graphml --auto-build + +# Monitor running processes +concore status + +# Stop all processes +concore stop +``` + +For detailed CLI documentation, see [concore_cli/README.md](concore_cli/README.md). + For a detailed and more scientific documentation, please read our extensive [open-access research paper on CONTROL-CORE](https://doi.org/10.1109/ACCESS.2022.3161471). This paper has a complete discussion on the CONTROL-CORE architecture and deployment, together with the commands to execute the studies in different programming languages and programming environments (Ubuntu, Windows, MacOS, Docker, and distributed execution). diff --git a/concore_cli/README.md b/concore_cli/README.md new file mode 100644 index 00000000..e29c7657 --- /dev/null +++ b/concore_cli/README.md @@ -0,0 +1,180 @@ +# Concore CLI + +A command-line interface for managing concore neuromodulation workflows. + +## Installation + +```bash +pip install -e . +``` + +## Quick Start + +```bash +# Create a new project +concore init my-project + +# Navigate to your project +cd my-project + +# Validate your workflow +concore validate workflow.graphml + +# Run your workflow +concore run workflow.graphml + +# Check running processes +concore status + +# Stop all processes +concore stop +``` + +## Commands + +### `concore init ` + +Creates a new concore project with a basic structure. + +**Options:** +- `--template` - Template type to use (default: basic) + +**Example:** +```bash +concore init my-workflow +``` + +Creates: +``` +my-workflow/ +├── workflow.graphml # Sample workflow definition +├── src/ +│ └── script.py # Sample processing script +└── README.md # Project documentation +``` + +### `concore run ` + +Generates and optionally builds a workflow from a GraphML file. + +**Options:** +- `-s, --source ` - Source directory (default: src) +- `-o, --output ` - Output directory (default: out) +- `-t, --type ` - Execution type: windows, posix, or docker (default: windows) +- `--auto-build` - Automatically run build script after generation + +**Example:** +```bash +concore run workflow.graphml --source ./src --output ./build --auto-build +``` + +### `concore validate ` + +Validates a GraphML workflow file before running. + +Checks: +- Valid XML structure +- GraphML format compliance +- Node and edge definitions +- File references and naming conventions +- ZMQ vs file-based communication + +**Example:** +```bash +concore validate workflow.graphml +``` + +### `concore status` + +Shows all currently running concore processes with details: +- Process ID (PID) +- Process name +- Uptime +- Memory usage +- Command + +**Example:** +```bash +concore status +``` + +### `concore stop` + +Stops all running concore processes. Prompts for confirmation before proceeding. + +**Example:** +```bash +concore stop +``` + +## Development Workflow + +1. **Create a new project** + ```bash + concore init my-neuro-study + cd my-neuro-study + ``` + +2. **Edit your workflow** + - Open `workflow.graphml` in yEd or similar GraphML editor + - Add nodes for your processing steps + - Connect nodes with edges to define data flow + +3. **Add processing scripts** + - Place your Python/C++/MATLAB/Verilog files in the `src/` directory + - Reference them in your workflow nodes + +4. **Validate before running** + ```bash + concore validate workflow.graphml + ``` + +5. **Generate and run** + ```bash + concore run workflow.graphml --auto-build + cd out + ./run.bat # or ./run on Linux/Mac + ``` + +6. **Monitor execution** + ```bash + concore status + ``` + +7. **Stop when done** + ```bash + concore stop + ``` + +## Workflow File Format + +Nodes should follow the format: `ID:filename.ext` + +Example: +``` +N1:controller.py +N2:processor.cpp +M1:analyzer.m +``` + +Supported file types: +- `.py` - Python +- `.cpp` - C++ +- `.m` - MATLAB/Octave +- `.v` - Verilog +- `.java` - Java + +## Troubleshooting + +**Issue: "Output directory already exists"** +- Remove the existing output directory or choose a different name +- Use `concore stop` to terminate any running processes first + +**Issue: Validation fails** +- Check that your GraphML file is properly formatted +- Ensure all nodes have labels in the format `ID:filename.ext` +- Verify that edge connections reference valid nodes + +**Issue: Processes won't stop** +- Try running `concore stop` with administrator/sudo privileges +- Manually kill processes using Task Manager (Windows) or `kill` command (Linux/Mac) diff --git a/concore_cli/__init__.py b/concore_cli/__init__.py new file mode 100644 index 00000000..658e35e4 --- /dev/null +++ b/concore_cli/__init__.py @@ -0,0 +1,3 @@ +from .cli import cli + +__all__ = ['cli'] diff --git a/concore_cli/cli.py b/concore_cli/cli.py new file mode 100644 index 00000000..f7144533 --- /dev/null +++ b/concore_cli/cli.py @@ -0,0 +1,78 @@ +import click +from rich.console import Console +from rich.table import Table +from rich.panel import Panel +from rich import print as rprint +import sys +import os +from pathlib import Path + +from .commands.init import init_project +from .commands.run import run_workflow +from .commands.validate import validate_workflow +from .commands.status import show_status +from .commands.stop import stop_all + +console = Console() + +@click.group() +@click.version_option(version='1.0.0', prog_name='concore') +def cli(): + pass + +@cli.command() +@click.argument('name', required=True) +@click.option('--template', default='basic', help='Template type to use') +def init(name, template): + """Create a new concore project""" + try: + init_project(name, template, console) + except Exception as e: + console.print(f"[red]Error:[/red] {str(e)}") + sys.exit(1) + +@cli.command() +@click.argument('workflow_file', type=click.Path(exists=True)) +@click.option('--source', '-s', default='src', help='Source directory') +@click.option('--output', '-o', default='out', help='Output directory') +@click.option('--type', '-t', default='windows', type=click.Choice(['windows', 'posix', 'docker']), help='Execution type') +@click.option('--auto-build', is_flag=True, help='Automatically run build after generation') +def run(workflow_file, source, output, type, auto_build): + """Run a concore workflow""" + try: + run_workflow(workflow_file, source, output, type, auto_build, console) + except Exception as e: + console.print(f"[red]Error:[/red] {str(e)}") + sys.exit(1) + +@cli.command() +@click.argument('workflow_file', type=click.Path(exists=True)) +def validate(workflow_file): + """Validate a workflow file""" + try: + validate_workflow(workflow_file, console) + except Exception as e: + console.print(f"[red]Error:[/red] {str(e)}") + sys.exit(1) + +@cli.command() +def status(): + """Show running concore processes""" + try: + show_status(console) + except Exception as e: + console.print(f"[red]Error:[/red] {str(e)}") + sys.exit(1) + +@cli.command() +@click.confirmation_option(prompt='Stop all running concore processes?') +def stop(): + """Stop all running concore processes""" + try: + stop_all(console) + except Exception as e: + console.print(f"[red]Error:[/red] {str(e)}") + sys.exit(1) + +if __name__ == '__main__': + cli() diff --git a/concore_cli/commands/__init__.py b/concore_cli/commands/__init__.py new file mode 100644 index 00000000..dd9bae05 --- /dev/null +++ b/concore_cli/commands/__init__.py @@ -0,0 +1,7 @@ +from .init import init_project +from .run import run_workflow +from .validate import validate_workflow +from .status import show_status +from .stop import stop_all + +__all__ = ['init_project', 'run_workflow', 'validate_workflow', 'show_status', 'stop_all'] diff --git a/concore_cli/commands/init.py b/concore_cli/commands/init.py new file mode 100644 index 00000000..5b0a9981 --- /dev/null +++ b/concore_cli/commands/init.py @@ -0,0 +1,93 @@ +import os +import shutil +from pathlib import Path +from rich.panel import Panel + +SAMPLE_GRAPHML = ''' + + + + + + + + + + + N1:script.py + + + + + + +''' + +SAMPLE_PYTHON = '''import concore + +while not concore.concore_unchanged(): + data = concore.concore_read() + result = data * 2 + concore.concore_write(result) +''' + +README_TEMPLATE = '''# {project_name} + +A concore workflow project. + +## Getting Started + +1. Edit your workflow in `workflow.graphml` using yEd or similar GraphML editor +2. Add your processing scripts to the `src/` directory +3. Run your workflow: + ``` + concore run workflow.graphml + ``` + +## Project Structure + +- `workflow.graphml` - Your workflow definition +- `src/` - Source files for your nodes +- `README.md` - This file + +## Next Steps + +- Modify `workflow.graphml` to define your processing pipeline +- Add Python/C++/MATLAB scripts to `src/` +- Use `concore validate workflow.graphml` to check your workflow +- Use `concore status` to monitor running processes +''' + +def init_project(name, template, console): + project_path = Path(name) + + if project_path.exists(): + raise FileExistsError(f"Directory '{name}' already exists") + + console.print(f"[cyan]Creating project:[/cyan] {name}") + + project_path.mkdir() + (project_path / 'src').mkdir() + + workflow_file = project_path / 'workflow.graphml' + with open(workflow_file, 'w') as f: + f.write(SAMPLE_GRAPHML) + + sample_script = project_path / 'src' / 'script.py' + with open(sample_script, 'w') as f: + f.write(SAMPLE_PYTHON) + + readme_file = project_path / 'README.md' + with open(readme_file, 'w') as f: + f.write(README_TEMPLATE.format(project_name=name)) + + console.print() + console.print(Panel.fit( + f"[green]✓[/green] Project created successfully!\n\n" + f"Next steps:\n" + f" cd {name}\n" + f" concore validate workflow.graphml\n" + f" concore run workflow.graphml", + title="Success", + border_style="green" + )) diff --git a/concore_cli/commands/run.py b/concore_cli/commands/run.py new file mode 100644 index 00000000..c6aaed2f --- /dev/null +++ b/concore_cli/commands/run.py @@ -0,0 +1,96 @@ +import os +import sys +import subprocess +from pathlib import Path +from rich.panel import Panel +from rich.progress import Progress, SpinnerColumn, TextColumn + +def run_workflow(workflow_file, source, output, exec_type, auto_build, console): + workflow_path = Path(workflow_file).resolve() + source_path = Path(source).resolve() + output_path = Path(output) + + if not source_path.exists(): + raise FileNotFoundError(f"Source directory '{source}' not found") + + if output_path.exists(): + console.print(f"[yellow]Warning:[/yellow] Output directory '{output}' already exists") + console.print("Remove it first or choose a different output directory") + return + + console.print(f"[cyan]Workflow:[/cyan] {workflow_path.name}") + console.print(f"[cyan]Source:[/cyan] {source_path}") + console.print(f"[cyan]Output:[/cyan] {output_path}") + console.print(f"[cyan]Type:[/cyan] {exec_type}") + console.print() + + with Progress( + SpinnerColumn(), + TextColumn("[progress.description]{task.description}"), + console=console + ) as progress: + task = progress.add_task("Generating workflow...", total=None) + + try: + result = subprocess.run( + [sys.executable, 'mkconcore.py', str(workflow_path), str(source_path), str(output_path), exec_type], + capture_output=True, + text=True, + check=True + ) + + progress.update(task, completed=True) + + if result.stdout: + console.print(result.stdout) + + console.print(f"[green]✓[/green] Workflow generated in [cyan]{output_path}[/cyan]") + + except subprocess.CalledProcessError as e: + progress.stop() + console.print(f"[red]Generation failed:[/red]") + if e.stdout: + console.print(e.stdout) + if e.stderr: + console.print(e.stderr) + raise + + if auto_build: + console.print() + build_script = output_path / ('build.bat' if exec_type == 'windows' else 'build') + + if build_script.exists(): + with Progress( + SpinnerColumn(), + TextColumn("[progress.description]{task.description}"), + console=console + ) as progress: + task = progress.add_task("Building workflow...", total=None) + + try: + result = subprocess.run( + [str(build_script)], + cwd=output_path, + capture_output=True, + text=True, + shell=True, + check=True + ) + progress.update(task, completed=True) + console.print(f"[green]✓[/green] Build completed") + except subprocess.CalledProcessError as e: + progress.stop() + console.print(f"[yellow]Build failed[/yellow]") + if e.stderr: + console.print(e.stderr) + + console.print() + console.print(Panel.fit( + f"[green]✓[/green] Workflow ready!\n\n" + f"To run your workflow:\n" + f" cd {output_path}\n" + f" {'build.bat' if exec_type == 'windows' else './build'}\n" + f" {'run.bat' if exec_type == 'windows' else './run'}", + title="Next Steps", + border_style="green" + )) diff --git a/concore_cli/commands/status.py b/concore_cli/commands/status.py new file mode 100644 index 00000000..e4072246 --- /dev/null +++ b/concore_cli/commands/status.py @@ -0,0 +1,89 @@ +import psutil +import os +from pathlib import Path +from rich.table import Table +from rich.panel import Panel +from datetime import datetime + +def show_status(console): + console.print("[cyan]Scanning for concore processes...[/cyan]\n") + + concore_processes = [] + + try: + current_pid = os.getpid() + + for proc in psutil.process_iter(['pid', 'name', 'cmdline', 'create_time', 'memory_info', 'cpu_percent']): + try: + cmdline = proc.info.get('cmdline') or [] + name = proc.info.get('name', '').lower() + + if proc.info['pid'] == current_pid: + continue + + cmdline_str = ' '.join(cmdline) if cmdline else '' + + is_concore = ( + 'concore' in cmdline_str.lower() or + 'concore.py' in cmdline_str.lower() or + any('concorekill.bat' in str(item) for item in cmdline) or + (name in ['python.exe', 'python', 'python3'] and 'concore' in cmdline_str) + ) + + if is_concore: + try: + create_time = datetime.fromtimestamp(proc.info['create_time']) + uptime = datetime.now() - create_time + hours, remainder = divmod(int(uptime.total_seconds()), 3600) + minutes, seconds = divmod(remainder, 60) + uptime_str = f"{hours}h {minutes}m {seconds}s" + except: + uptime_str = "unknown" + + try: + mem_mb = proc.info['memory_info'].rss / 1024 / 1024 + mem_str = f"{mem_mb:.1f} MB" + except: + mem_str = "unknown" + + command = ' '.join(cmdline[:3]) if len(cmdline) >= 3 else cmdline_str[:50] + + concore_processes.append({ + 'pid': proc.info['pid'], + 'name': proc.info.get('name', 'unknown'), + 'command': command, + 'uptime': uptime_str, + 'memory': mem_str + }) + except (psutil.NoSuchProcess, psutil.AccessDenied): + continue + + except Exception as e: + console.print(f"[red]Error scanning processes:[/red] {str(e)}") + return + + if not concore_processes: + console.print(Panel.fit( + "[yellow]No concore processes currently running[/yellow]", + border_style="yellow" + )) + else: + table = Table(title=f"Concore Processes ({len(concore_processes)} running)", show_header=True) + table.add_column("PID", style="cyan", justify="right") + table.add_column("Name", style="green") + table.add_column("Uptime", style="yellow") + table.add_column("Memory", style="magenta") + table.add_column("Command", style="white") + + for proc in concore_processes: + table.add_row( + str(proc['pid']), + proc['name'], + proc['uptime'], + proc['memory'], + proc['command'] + ) + + console.print(table) + console.print() + console.print(f"[dim]Use 'concore stop' to terminate all processes[/dim]") diff --git a/concore_cli/commands/stop.py b/concore_cli/commands/stop.py new file mode 100644 index 00000000..ad1f38f3 --- /dev/null +++ b/concore_cli/commands/stop.py @@ -0,0 +1,92 @@ +import psutil +import os +import subprocess +import sys +from pathlib import Path +from rich.panel import Panel + +def stop_all(console): + console.print("[cyan]Finding concore processes...[/cyan]\n") + + processes_to_kill = [] + current_pid = os.getpid() + + try: + for proc in psutil.process_iter(['pid', 'name', 'cmdline']): + try: + if proc.info['pid'] == current_pid: + continue + + cmdline = proc.info.get('cmdline') or [] + name = proc.info.get('name', '').lower() + cmdline_str = ' '.join(cmdline) if cmdline else '' + + is_concore = ( + 'concore' in cmdline_str.lower() or + 'concore.py' in cmdline_str.lower() or + any('concorekill.bat' in str(item) for item in cmdline) or + (name in ['python.exe', 'python', 'python3'] and 'concore' in cmdline_str) + ) + + if is_concore: + processes_to_kill.append(proc) + except (psutil.NoSuchProcess, psutil.AccessDenied): + continue + + except Exception as e: + console.print(f"[red]Error:[/red] {str(e)}") + return + + if not processes_to_kill: + console.print(Panel.fit( + "[yellow]No concore processes found[/yellow]", + border_style="yellow" + )) + return + + console.print(f"[yellow]Stopping {len(processes_to_kill)} process(es)...[/yellow]\n") + + killed_count = 0 + failed_count = 0 + + for proc in processes_to_kill: + try: + pid = proc.info['pid'] + name = proc.info.get('name', 'unknown') + + if sys.platform == 'win32': + subprocess.run(['taskkill', '/F', '/PID', str(pid)], + capture_output=True, + check=False) + else: + proc.terminate() + proc.wait(timeout=3) + + console.print(f" [green]✓[/green] Stopped {name} (PID: {pid})") + killed_count += 1 + + except psutil.TimeoutExpired: + try: + proc.kill() + console.print(f" [yellow]⚠[/yellow] Force killed {name} (PID: {pid})") + killed_count += 1 + except: + console.print(f" [red]✗[/red] Failed to stop {name} (PID: {pid})") + failed_count += 1 + except Exception as e: + console.print(f" [red]✗[/red] Failed to stop PID {pid}: {str(e)}") + failed_count += 1 + + console.print() + + if failed_count == 0: + console.print(Panel.fit( + f"[green]✓[/green] Successfully stopped all {killed_count} process(es)", + border_style="green" + )) + else: + console.print(Panel.fit( + f"[yellow]Stopped {killed_count} process(es)\n" + f"Failed to stop {failed_count} process(es)[/yellow]", + border_style="yellow" + )) diff --git a/concore_cli/commands/validate.py b/concore_cli/commands/validate.py new file mode 100644 index 00000000..e920f119 --- /dev/null +++ b/concore_cli/commands/validate.py @@ -0,0 +1,144 @@ +from pathlib import Path +from bs4 import BeautifulSoup +from rich.panel import Panel +from rich.table import Table +import re + +def validate_workflow(workflow_file, console): + workflow_path = Path(workflow_file) + + console.print(f"[cyan]Validating:[/cyan] {workflow_path.name}") + console.print() + + errors = [] + warnings = [] + info = [] + + try: + with open(workflow_path, 'r') as f: + content = f.read() + + if not content.strip(): + errors.append("File is empty") + return show_results(console, errors, warnings, info) + + try: + soup = BeautifulSoup(content, 'xml') + except Exception as e: + errors.append(f"Invalid XML: {str(e)}") + return show_results(console, errors, warnings, info) + + if not soup.find('graphml'): + errors.append("Not a valid GraphML file - missing root element") + return show_results(console, errors, warnings, info) + + nodes = soup.find_all('node') + edges = soup.find_all('edge') + + if len(nodes) == 0: + warnings.append("No nodes found in workflow") + else: + info.append(f"Found {len(nodes)} node(s)") + + if len(edges) == 0: + warnings.append("No edges found in workflow") + else: + info.append(f"Found {len(edges)} edge(s)") + + node_labels = [] + for node in nodes: + try: + label_tag = node.find('y:NodeLabel') + if label_tag and label_tag.text: + label = label_tag.text.strip() + node_labels.append(label) + + if ':' not in label: + warnings.append(f"Node '{label}' missing format 'ID:filename'") + else: + parts = label.split(':') + if len(parts) != 2: + warnings.append(f"Node '{label}' has invalid format") + else: + node_id, filename = parts + if not filename: + errors.append(f"Node '{label}' has no filename") + elif not any(filename.endswith(ext) for ext in ['.py', '.cpp', '.m', '.v', '.java']): + warnings.append(f"Node '{label}' has unusual file extension") + else: + warnings.append(f"Node {node.get('id', 'unknown')} has no label") + except Exception as e: + warnings.append(f"Error parsing node: {str(e)}") + + node_ids = {node.get('id') for node in nodes if node.get('id')} + for edge in edges: + source = edge.get('source') + target = edge.get('target') + + if not source or not target: + errors.append("Edge missing source or target") + continue + + if source not in node_ids: + errors.append(f"Edge references non-existent source node: {source}") + if target not in node_ids: + errors.append(f"Edge references non-existent target node: {target}") + + edge_label_regex = re.compile(r"0x([a-fA-F0-9]+)_(\S+)") + zmq_edges = 0 + file_edges = 0 + + for edge in edges: + try: + label_tag = edge.find('y:EdgeLabel') + if label_tag and label_tag.text: + if edge_label_regex.match(label_tag.text.strip()): + zmq_edges += 1 + else: + file_edges += 1 + except: + pass + + if zmq_edges > 0: + info.append(f"ZMQ-based edges: {zmq_edges}") + if file_edges > 0: + info.append(f"File-based edges: {file_edges}") + + show_results(console, errors, warnings, info) + + except FileNotFoundError: + console.print(f"[red]Error:[/red] File not found: {workflow_path}") + except Exception as e: + console.print(f"[red]Validation failed:[/red] {str(e)}") + +def show_results(console, errors, warnings, info): + if errors: + console.print("[red]✗ Validation failed[/red]\n") + for error in errors: + console.print(f" [red]✗[/red] {error}") + else: + console.print("[green]✓ Validation passed[/green]\n") + + if warnings: + console.print() + for warning in warnings: + console.print(f" [yellow]⚠[/yellow] {warning}") + + if info: + console.print() + for item in info: + console.print(f" [blue]ℹ[/blue] {item}") + + console.print() + + if not errors: + console.print(Panel.fit( + "[green]✓[/green] Workflow is valid and ready to run", + border_style="green" + )) + else: + console.print(Panel.fit( + f"[red]Found {len(errors)} error(s)[/red]\n" + "Fix the errors above before running the workflow", + border_style="red" + )) diff --git a/requirements.txt b/requirements.txt index 067a3ec1..897bdf6c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,4 +5,7 @@ numpy scipy matplotlib cvxopt -PyGithub \ No newline at end of file +PyGithub +click>=8.0.0 +rich>=10.0.0 +psutil>=5.8.0 \ No newline at end of file diff --git a/setup.py b/setup.py new file mode 100644 index 00000000..74951343 --- /dev/null +++ b/setup.py @@ -0,0 +1,41 @@ +from setuptools import setup, find_packages + +with open("README.md", "r", encoding="utf-8") as fh: + long_description = fh.read() + +setup( + name="concore", + version="1.0.0", + author="ControlCore Project", + description="A command-line interface for concore neuromodulation workflows", + long_description=long_description, + long_description_content_type="text/markdown", + url="https://github.com/ControlCore-Project/concore", + packages=find_packages(), + classifiers=[ + "Development Status :: 4 - Beta", + "Intended Audience :: Science/Research", + "Topic :: Scientific/Engineering", + "License :: OSI Approved :: MIT License", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + ], + python_requires=">=3.9", + install_requires=[ + "beautifulsoup4", + "lxml", + "numpy", + "scipy", + "matplotlib", + "click>=8.0.0", + "rich>=10.0.0", + "psutil>=5.8.0", + ], + entry_points={ + "console_scripts": [ + "concore=concore_cli.cli:cli", + ], + }, +) diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 00000000..af31a36a --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,85 @@ +import unittest +import tempfile +import shutil +from pathlib import Path +from click.testing import CliRunner +from concore_cli.cli import cli + +class TestConcoreCLI(unittest.TestCase): + + def setUp(self): + self.runner = CliRunner() + self.temp_dir = tempfile.mkdtemp() + + def tearDown(self): + if Path(self.temp_dir).exists(): + shutil.rmtree(self.temp_dir) + + def test_version(self): + result = self.runner.invoke(cli, ['--version']) + self.assertEqual(result.exit_code, 0) + self.assertIn('1.0.0', result.output) + + def test_help(self): + result = self.runner.invoke(cli, ['--help']) + self.assertEqual(result.exit_code, 0) + self.assertIn('Usage:', result.output) + self.assertIn('Commands:', result.output) + + def test_init_command(self): + with self.runner.isolated_filesystem(temp_dir=self.temp_dir): + result = self.runner.invoke(cli, ['init', 'test-project']) + self.assertEqual(result.exit_code, 0) + + project_path = Path('test-project') + self.assertTrue(project_path.exists()) + self.assertTrue((project_path / 'workflow.graphml').exists()) + self.assertTrue((project_path / 'src').exists()) + self.assertTrue((project_path / 'README.md').exists()) + self.assertTrue((project_path / 'src' / 'script.py').exists()) + + def test_init_existing_directory(self): + with self.runner.isolated_filesystem(temp_dir=self.temp_dir): + Path('existing').mkdir() + result = self.runner.invoke(cli, ['init', 'existing']) + self.assertNotEqual(result.exit_code, 0) + self.assertIn('already exists', result.output) + + def test_validate_missing_file(self): + result = self.runner.invoke(cli, ['validate', 'nonexistent.graphml']) + self.assertNotEqual(result.exit_code, 0) + + def test_validate_valid_file(self): + with self.runner.isolated_filesystem(temp_dir=self.temp_dir): + result = self.runner.invoke(cli, ['init', 'test-project']) + self.assertEqual(result.exit_code, 0) + + result = self.runner.invoke(cli, ['validate', 'test-project/workflow.graphml']) + self.assertEqual(result.exit_code, 0) + self.assertIn('Validation passed', result.output) + + def test_status_command(self): + result = self.runner.invoke(cli, ['status']) + self.assertEqual(result.exit_code, 0) + + def test_run_command_missing_source(self): + with self.runner.isolated_filesystem(temp_dir=self.temp_dir): + result = self.runner.invoke(cli, ['init', 'test-project']) + result = self.runner.invoke(cli, ['run', 'test-project/workflow.graphml', '--source', 'nonexistent']) + self.assertNotEqual(result.exit_code, 0) + + def test_run_command_existing_output(self): + with self.runner.isolated_filesystem(temp_dir=self.temp_dir): + result = self.runner.invoke(cli, ['init', 'test-project']) + Path('output').mkdir() + + result = self.runner.invoke(cli, [ + 'run', + 'test-project/workflow.graphml', + '--source', 'test-project/src', + '--output', 'output' + ]) + self.assertIn('already exists', result.output.lower()) + +if __name__ == '__main__': + unittest.main() From decfd37fdda2d5fd27de5ff529bd23c5eb7798ea Mon Sep 17 00:00:00 2001 From: GREENRAT-K405 Date: Wed, 4 Feb 2026 00:06:15 +0530 Subject: [PATCH 017/275] fix: broken java literaleval, update the Dockerfile for java --- Dockerfile.java | 4 ++-- concoredocker.java | 53 ++++++++++++++++++++++++++++++++++++++-------- 2 files changed, 46 insertions(+), 11 deletions(-) diff --git a/Dockerfile.java b/Dockerfile.java index 178c4512..1b32be00 100644 --- a/Dockerfile.java +++ b/Dockerfile.java @@ -1,9 +1,9 @@ -FROM openjdk:17-jdk-alpine +FROM eclipse-temurin:17-jdk-alpine WORKDIR /app # Only copy the JAR if it exists -COPY ./target/concore-0.0.1-SNAPSHOT.jar /app/concore.jar || true +COPY ./target/concore-0.0.1-SNAPSHOT.jar /app/concore.jar # Ensure the JAR file is executable if present RUN [ -f /app/concore.jar ] && chmod +x /app/concore.jar || true diff --git a/concoredocker.java b/concoredocker.java index dde521c8..b9813dec 100644 --- a/concoredocker.java +++ b/concoredocker.java @@ -3,6 +3,8 @@ import java.nio.file.Paths; import java.util.HashMap; import java.util.Map; +import java.util.ArrayList; +import java.util.List; public class concoredocker { private static Map iport = new HashMap<>(); @@ -40,7 +42,8 @@ public static void main(String[] args) { System.out.println("converted sparams: " + sparams); } try { - params = literalEval(sparams); + // literalEval returns a proper Map for "{...}" + params = (Map) literalEval(sparams); } catch (Exception e) { System.out.println("bad params: " + sparams); } @@ -51,15 +54,17 @@ public static void main(String[] args) { defaultMaxTime(100); } + @SuppressWarnings("unchecked") private static Map parseFile(String filename) throws IOException { String content = new String(Files.readAllBytes(Paths.get(filename))); - return literalEval(content); + return (Map) literalEval(content); // Casted to Map } private static void defaultMaxTime(int defaultValue) { try { String content = new String(Files.readAllBytes(Paths.get(inpath + "1/concore.maxtime"))); - maxtime = literalEval(content).size(); + // changed assumption from map to list for maxtime, as it usually represents a list of time steps + maxtime = ((List) literalEval(content)).size(); } catch (IOException e) { maxtime = defaultValue; } @@ -90,10 +95,10 @@ private static Object read(int port, String name, String initstr) { retrycount++; } s += ins; - Object[] inval = new Map[] { literalEval(ins) }; + Object[] inval = ((List) literalEval(ins)).toArray(); // FIXED: Casted to List, converted to Array int simtime = Math.max((int) inval[0], 0); // assuming simtime is an integer return inval[1]; - } catch (IOException | InterruptedException e) { + } catch (IOException | InterruptedException | ClassCastException e) { return initstr; } } @@ -132,7 +137,7 @@ private static Object[] initVal(String simtimeVal) { int simtime = 0; Object[] val = new Object[] {}; try { - Object[] arrayVal = new Map[] { literalEval(simtimeVal) }; + Object[] arrayVal = ((List) literalEval(simtimeVal)).toArray(); // FIXED: Casted to List, converted to Array simtime = (int) arrayVal[0]; // assuming simtime is an integer val = new Object[arrayVal.length - 1]; System.arraycopy(arrayVal, 1, val, 0, val.length); @@ -142,8 +147,38 @@ private static Object[] initVal(String simtimeVal) { return val; } - private static Map literalEval(String s) { + // custom parser + private static Object literalEval(String s) { + s = s.trim(); + if (s.startsWith("{") && s.endsWith("}")) { + Map map = new HashMap<>(); + String content = s.substring(1, s.length() - 1); + if (content.isEmpty()) return map; + for (String pair : content.split(",")) { + String[] kv = pair.split(":"); + if (kv.length == 2) map.put((String) parseVal(kv[0]), parseVal(kv[1])); + } + return map; + } else if (s.startsWith("[") && s.endsWith("]")) { + List list = new ArrayList<>(); + String content = s.substring(1, s.length() - 1); + if (content.isEmpty()) return list; + for (String val : content.split(",")) { + list.add(parseVal(val)); + } + return list; + } + return parseVal(s); + } - return new HashMap<>(); + // helper: Converts Python types to Java primitives + private static Object parseVal(String s) { + s = s.trim().replace("'", "").replace("\"", ""); + if (s.equalsIgnoreCase("True")) return true; + if (s.equalsIgnoreCase("False")) return false; + if (s.equalsIgnoreCase("None")) return null; + try { return Integer.parseInt(s); } catch (NumberFormatException e1) { + try { return Double.parseDouble(s); } catch (NumberFormatException e2) { return s; } + } } -} +} \ No newline at end of file From a607e75e181209f65241834a20981a3c80db2cb1 Mon Sep 17 00:00:00 2001 From: GREENRAT-K405 Date: Wed, 4 Feb 2026 16:23:21 +0530 Subject: [PATCH 018/275] add robust parameter parsing, fixes #184 --- concore.py | 41 +++++++++++++++++++++++++++++++---------- concoredocker.py | 45 +++++++++++++++++++++++++++++++++------------ 2 files changed, 64 insertions(+), 22 deletions(-) diff --git a/concore.py b/concore.py index 2da12506..28fc2fd6 100644 --- a/concore.py +++ b/concore.py @@ -156,6 +156,33 @@ def safe_literal_eval(filename, defaultValue): # =================================================================== # Parameter Parsing # =================================================================== +def parse_params(sparams: str) -> dict: + """Parse semicolon-delimited key=value pairs into a dictionary. + + Args: + sparams: String in format "key1=value1;key2=value2" + + Returns: + Dictionary with parsed key-value pairs + """ + params = {} + if not sparams: + return params + + for item in sparams.split(";"): + if "=" in item: + key, value = item.split("=", 1) # split only once + key=key.strip() + value=value.strip() + #try to convert to python type (int, float, list, etc.) + # Use literal_eval to preserve backward compatibility (integers/lists) + # Fallback to string for unquoted values (paths, URLs) + try: + params[key] = literal_eval(value) + except (ValueError, SyntaxError): + params[key] = value + return params + try: sparams_path = concore_params_file if os.path.exists(sparams_path): @@ -166,16 +193,10 @@ def safe_literal_eval(filename, defaultValue): if sparams[0] == '"' and sparams[-1] == '"': #windows keeps "" need to remove sparams = sparams[1:-1] - # Convert key=value;key2=value2 to Python dict format - if sparams != '{' and not (sparams.startswith('{') and sparams.endswith('}')): # Check if it needs conversion - logging.debug("converting sparams: "+sparams) - sparams = "{'"+re.sub(';',",'",re.sub('=',"':",re.sub(' ','',sparams)))+"}" - logging.debug("converted sparams: " + sparams) - try: - params = literal_eval(sparams) - except Exception as e: - logging.warning(f"bad params content: {sparams}, error: {e}") - params = dict() + # Parse params using clean function instead of regex + logging.debug("parsing sparams: "+sparams) + params = parse_params(sparams) + logging.debug("parsed params: " + str(params)) else: params = dict() else: diff --git a/concoredocker.py b/concoredocker.py index 161ad1bf..925fd9b6 100644 --- a/concoredocker.py +++ b/concoredocker.py @@ -25,20 +25,41 @@ def safe_literal_eval(filename, defaultValue): concore_maxtime_file = os.path.join(inpath, "1", "concore.maxtime") #9/21/22 +def parse_params(sparams): + params = {} + if not sparams: + return params + + # keep backward compatibility: comma-separated params + for item in sparams.split(","): + if "=" in item: + key, value = item.split("=", 1) + key = key.strip() + value = value.strip() + #try to convert to python type (int, float, list, etc.) + # Use literal_eval to preserve backward compatibility (integers/lists) + # Fallback to string for unquoted values (paths, URLs) + try: + params[key] = literal_eval(value) + except (ValueError, SyntaxError): + params[key] = value + return params + try: - sparams = open(concore_params_file).read() - if sparams[0] == '"': #windows keeps "" need to remove + sparams = open(concore_params_file).read().strip() + + if sparams and sparams[0] == '"': # windows keeps quotes sparams = sparams[1:] - sparams = sparams[0:sparams.find('"')] - if sparams != '{': - print("converting sparams: "+sparams) - sparams = "{'"+re.sub(',',",'",re.sub('=',"':",re.sub(' ','',sparams)))+"}" - print("converted sparams: " + sparams) - try: - params = literal_eval(sparams) - except: - print("bad params: "+sparams) -except: + if '"' in sparams: + sparams = sparams[:sparams.find('"')] + + if sparams: + print("parsing sparams:", sparams) + params = parse_params(sparams) + else: + params = dict() +except Exception as e: + print(f"Error reading concore.params: {e}") params = dict() #9/30/22 From 07910563a2ba89dc883689e82d07ccdb3ed257a2 Mon Sep 17 00:00:00 2001 From: GREENRAT-K405 Date: Wed, 4 Feb 2026 17:32:28 +0530 Subject: [PATCH 019/275] fix dict issue, delimiter causing break in line, and file remains opened --- concore.py | 23 +++++++++++++---------- concoredocker.py | 16 ++++++++++++++-- 2 files changed, 27 insertions(+), 12 deletions(-) diff --git a/concore.py b/concore.py index 28fc2fd6..bf804e4e 100644 --- a/concore.py +++ b/concore.py @@ -157,19 +157,22 @@ def safe_literal_eval(filename, defaultValue): # Parameter Parsing # =================================================================== def parse_params(sparams: str) -> dict: - """Parse semicolon-delimited key=value pairs into a dictionary. - - Args: - sparams: String in format "key1=value1;key2=value2" - - Returns: - Dictionary with parsed key-value pairs - """ params = {} if not sparams: return params - for item in sparams.split(";"): + s = sparams.strip() + + #full dict literal + if s.startswith("{") and s.endswith("}"): + try: + val = literal_eval(s) + if isinstance(val, dict): + return val + except (ValueError, SyntaxError): + pass + + for item in s.split(";"): if "=" in item: key, value = item.split("=", 1) # split only once key=key.strip() @@ -187,7 +190,7 @@ def parse_params(sparams: str) -> dict: sparams_path = concore_params_file if os.path.exists(sparams_path): with open(sparams_path, "r") as f: - sparams = f.read() + sparams = f.read().strip() if sparams: # Ensure sparams is not empty # Windows sometimes keeps quotes if sparams[0] == '"' and sparams[-1] == '"': #windows keeps "" need to remove diff --git a/concoredocker.py b/concoredocker.py index 925fd9b6..65463c99 100644 --- a/concoredocker.py +++ b/concoredocker.py @@ -30,8 +30,19 @@ def parse_params(sparams): if not sparams: return params + s = sparams.strip() + + # full dict literal + if s.startswith("{") and s.endswith("}"): + try: + val = literal_eval(s) + if isinstance(val, dict): + return val + except (ValueError, SyntaxError): + pass + # keep backward compatibility: comma-separated params - for item in sparams.split(","): + for item in s.split(","): if "=" in item: key, value = item.split("=", 1) key = key.strip() @@ -46,7 +57,8 @@ def parse_params(sparams): return params try: - sparams = open(concore_params_file).read().strip() + with open(concore_params_file, "r") as f: + sparams = f.read().strip() if sparams and sparams[0] == '"': # windows keeps quotes sparams = sparams[1:] From 6877489969da821fafa9400fa41fae4715a1f947 Mon Sep 17 00:00:00 2001 From: GREENRAT-K405 Date: Wed, 4 Feb 2026 17:54:26 +0530 Subject: [PATCH 020/275] add test class TestParseParams --- tests/test_concore.py | 39 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/tests/test_concore.py b/tests/test_concore.py index 862554c8..5eb3129e 100644 --- a/tests/test_concore.py +++ b/tests/test_concore.py @@ -84,4 +84,41 @@ def test_core_functions_exist(self): assert callable(safe_literal_eval) assert callable(tryparam) - assert callable(default_maxtime) \ No newline at end of file + assert callable(default_maxtime) + + +class TestParseParams: + + def test_simple_key_value_pairs(self): + from concore import parse_params + params = parse_params("a=1;b=2") + assert params == {"a": 1, "b": 2} + + def test_preserves_whitespace_in_values(self): + from concore import parse_params + params = parse_params("label = hello world ; x = 5") + assert params["label"] == "hello world" + assert params["x"] == 5 + + def test_embedded_equals_in_value(self): + from concore import parse_params + params = parse_params("url=https://example.com?a=1&b=2") + assert params["url"] == "https://example.com?a=1&b=2" + + def test_numeric_and_list_coercion(self): + from concore import parse_params + params = parse_params("delay=5;coeffs=[1,2,3]") + assert params["delay"] == 5 + assert params["coeffs"] == [1, 2, 3] + + def test_dict_literal_backward_compatibility(self): + from concore import parse_params + params = parse_params("{'a': 1, 'b': 2}") + assert params == {"a": 1, "b": 2} + + def test_windows_quoted_input(self): + from concore import parse_params + s = "\"a=1;b=2\"" + s = s[1:-1] # simulate quote stripping before parse_params + params = parse_params(s) + assert params == {"a": 1, "b": 2} \ No newline at end of file From 8a43659759e547a403545cf40d3a5f8eb9210ec9 Mon Sep 17 00:00:00 2001 From: Avinash Kumar Deepak Date: Thu, 5 Feb 2026 00:07:40 +0530 Subject: [PATCH 021/275] Move bot token to environment variable --- contribute.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/contribute.py b/contribute.py index 43972f28..ef5d8ceb 100644 --- a/contribute.py +++ b/contribute.py @@ -3,8 +3,7 @@ import os,sys,platform,base64,time # Intializing the Variables -# Hashed token -BOT_TOKEN = "Z2l0aHViX3BhdF8xMUFYS0pGVFkwU2VhNW9ORjRyN0E5X053WDAwTVBUUU5RVUNTa2lNNlFYZHJET1lZa3B4cTIxS091YVhkeVhUYmRQMzdVUkZaRWpFMjlRRXM5" +BOT_TOKEN = os.environ.get('CONCORE_BOT_TOKEN', '') BOT_ACCOUNT = 'concore-bot' #bot account name REPO_NAME = 'concore-studies' #study repo name UPSTREAM_ACCOUNT = 'ControlCore-Project' #upstream account name @@ -113,7 +112,7 @@ def decode_token(encoded_token): PR_BODY = f"Study Name: {STUDY_NAME}\nAuthor Name: {AUTHOR_NAME}" if PR_BODY == "#" else PR_BODY DIR_PATH = STUDY_NAME DIR_PATH = DIR_PATH.replace(" ","_") - g = Github(decode_token(BOT_TOKEN)) + g = Github(BOT_TOKEN) repo = g.get_user(BOT_ACCOUNT).get_repo(REPO_NAME) upstream_repo = g.get_repo(f'{UPSTREAM_ACCOUNT}/{REPO_NAME}') #controlcore-Project/concore-studies base_ref = upstream_repo.get_branch(repo.default_branch) From e1cebd91ebc3ad571781fef8c360cb804f207777 Mon Sep 17 00:00:00 2001 From: GREENRAT-K405 Date: Thu, 5 Feb 2026 00:44:15 +0530 Subject: [PATCH 022/275] add loggings to tools --- concore.py | 14 +++++++++++--- tools/bangbang.py | 6 +++++- tools/cardiac_pm.py | 6 +++++- tools/cwrap.py | 35 +++++++++++++++++------------------ tools/learn.py | 3 ++- tools/pid2.py | 8 ++++++-- tools/pidmayuresh.py | 9 +++++---- tools/pidsig.py | 9 +++++---- tools/plotu.py | 7 ++++--- tools/plotym.py | 7 ++++--- tools/plotymlag.py | 7 ++++--- tools/pwrap.py | 31 ++++++++++++++++--------------- tools/shannon.py | 5 +++-- 13 files changed, 87 insertions(+), 60 deletions(-) diff --git a/concore.py b/concore.py index 2da12506..7a99b8f3 100644 --- a/concore.py +++ b/concore.py @@ -7,9 +7,17 @@ import zmq import numpy as np logging.basicConfig( - level=logging.INFO, - format='%(levelname)s - %(message)s' -) # Added for ZeroMQ + level=logging.DEBUG, + format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', + force=True +) + +#these lines mute the noisy library +logging.getLogger('matplotlib').setLevel(logging.WARNING) +logging.getLogger('PIL').setLevel(logging.WARNING) +logging.getLogger('urllib3').setLevel(logging.WARNING) +logging.getLogger('requests').setLevel(logging.WARNING) + # if windows, create script to kill this process # because batch files don't provide easy way to know pid of last command diff --git a/tools/bangbang.py b/tools/bangbang.py index eed72b83..3bf9e5a2 100644 --- a/tools/bangbang.py +++ b/tools/bangbang.py @@ -1,5 +1,6 @@ import numpy as np import concore +import logging def bangbang_controller(ym): @@ -19,6 +20,9 @@ def bangbang_controller(ym): init_simtime_u = "[0.0, 0.0,0.0]" init_simtime_ym = "[0.0, 70.0,91]" u = np.array([concore.initval(init_simtime_u)]).T + +logging.info("Starting Bang-Bang Controller") + while(concore.simtime 1.1*timeout_max: #timeout_count>100: - print("timeout or bad POST request "+str(r.status_code)) + logging.error(f"timeout or bad POST request {r.status_code}") quit() if len(r.text)!=0: try: t=literal_eval(r.text)[0] except: - print("bad eval "+r.text) + logging.error(f"bad eval {r.text}") oldt = t oldym = r.text - print("CW: oldym="+oldym+" t="+str(concore.simtime)) + logging.debug(f"CW: oldym={oldym} t={concore.simtime}") concore.write(1,name2,oldym) #concore.write(1,"ym",init_simtime_ym) -print("retry="+str(concore.retrycount)) - - +logging.info(f"retry={concore.retrycount}") \ No newline at end of file diff --git a/tools/learn.py b/tools/learn.py index a29e1e66..53bb50e7 100644 --- a/tools/learn.py +++ b/tools/learn.py @@ -2,6 +2,7 @@ import numpy as np import matplotlib.pyplot as plt import time +import logging GENERATE_PLOT = 1 concore.delay = 0.002 @@ -21,7 +22,7 @@ ut[int(concore.simtime)] = np.array(u).T ymt[int(concore.simtime)] = np.array(ym).T oldsimtime = concore.simtime -print("retry="+str(concore.retrycount)) +logging.info(f"retry={concore.retrycount}") ################# # plot inputs and outputs diff --git a/tools/pid2.py b/tools/pid2.py index 6f88285d..1c720f85 100644 --- a/tools/pid2.py +++ b/tools/pid2.py @@ -1,5 +1,7 @@ import numpy as np import concore +import logging + setpoint = 67.5 setpointF = 75.0 KpF = 0.1 @@ -51,7 +53,9 @@ def pid_controller(ym): init_simtime_u = "[0.0, 0.0,0.0]" init_simtime_ym = "[0.0, 70.0,91]" u = np.array([concore.initval(init_simtime_u)]).T -print("Shannon's PID controller: setpoint is "+str(setpoint)) + +logging.info(f"Shannon's PID controller: setpoint is {setpoint}") + while(concore.simtime 1.1*timeout_max: #timeout_count>200: - print("timeout or bad POST request "+str(r.status_code)) + logging.error(f"timeout or bad POST request {r.status_code}") quit() if len(r.text)!=0: try: t=literal_eval(r.text)[0] except: - print("bad eval "+r.text) + logging.error(f"bad eval {r.text}") oldt = t oldu = r.text - print("PW: oldu="+oldu+" t="+str(concore.simtime)) + logging.debug(f"PW: oldu={oldu} t={concore.simtime}") concore.write(1,name1,oldu) #concore.write(1,"u",init_simtime_u) -print("retry="+str(concore.retrycount)) +logging.info(f"retry={concore.retrycount}") \ No newline at end of file diff --git a/tools/shannon.py b/tools/shannon.py index ba371af7..9da6f51a 100644 --- a/tools/shannon.py +++ b/tools/shannon.py @@ -1,5 +1,6 @@ import numpy as np import concore +import logging setpoint = 67.5 Kp = 0.1 Ki = 0.01 @@ -27,7 +28,7 @@ def pid_controller(ym): init_simtime_u = "[0.0, 0.0,0.0]" init_simtime_ym = "[0.0, 70.0,91]" u = np.array([concore.initval(init_simtime_u)]).T -print("Shannon's PID controller: setpoint is "+str(setpoint)) +logging.info(f"Shannon's PID controller: setpoint is {setpoint}") while(concore.simtime Date: Thu, 5 Feb 2026 00:58:57 +0530 Subject: [PATCH 023/275] Update tools/plotu.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- tools/plotu.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/plotu.py b/tools/plotu.py index bae0d50d..73edc6ec 100644 --- a/tools/plotu.py +++ b/tools/plotu.py @@ -16,7 +16,7 @@ while concore.unchanged(): u = concore.read(1,"u",init_simtime_u) concore.write(1,"u",u) - logging.info(f"u={u}") + logging.debug(f"u={u}") ut.append(np.array(u).T) logging.info(f"retry={concore.retrycount}") From e7d32e8ba0e0e7fba9546d11c170a275c7e21059 Mon Sep 17 00:00:00 2001 From: PARAM KANADA Date: Thu, 5 Feb 2026 00:59:26 +0530 Subject: [PATCH 024/275] Update tools/pidsig.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- tools/pidsig.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/pidsig.py b/tools/pidsig.py index 67378fff..42b3dda8 100644 --- a/tools/pidsig.py +++ b/tools/pidsig.py @@ -73,7 +73,7 @@ def pid_controller(ym): else: ustar = pid_controller(ym) - logging.info(f"{concore.simtime} u={ustar} ym={ym}") + logging.debug(f"{concore.simtime} u={ustar} ym={ym}") concore.write(1,"u",list(ustar),delta=0) From 69141e5fdff05a67a6d6fe069367d870893a1fc7 Mon Sep 17 00:00:00 2001 From: Sahil Lenka Date: Wed, 4 Feb 2026 21:26:36 +0530 Subject: [PATCH 025/275] add workflow inspect command with rich output and json export --- concore_cli/cli.py | 18 ++- concore_cli/commands/inspect.py | 259 ++++++++++++++++++++++++++++++++ concore_cli/commands/status.py | 1 + concore_cli/commands/stop.py | 1 + pyproject.toml | 30 ++++ 5 files changed, 304 insertions(+), 5 deletions(-) create mode 100644 concore_cli/commands/inspect.py create mode 100644 pyproject.toml diff --git a/concore_cli/cli.py b/concore_cli/cli.py index f7144533..49c78cb6 100644 --- a/concore_cli/cli.py +++ b/concore_cli/cli.py @@ -1,17 +1,13 @@ import click from rich.console import Console -from rich.table import Table -from rich.panel import Panel -from rich import print as rprint import sys -import os -from pathlib import Path from .commands.init import init_project from .commands.run import run_workflow from .commands.validate import validate_workflow from .commands.status import show_status from .commands.stop import stop_all +from .commands.inspect import inspect_workflow console = Console() @@ -55,6 +51,18 @@ def validate(workflow_file): console.print(f"[red]Error:[/red] {str(e)}") sys.exit(1) +@cli.command() +@click.argument('workflow_file', type=click.Path(exists=True)) +@click.option('--source', '-s', default='src', help='Source directory') +@click.option('--json', 'output_json', is_flag=True, help='Output in JSON format') +def inspect(workflow_file, source, output_json): + """Inspect a workflow file and show its structure""" + try: + inspect_workflow(workflow_file, source, output_json, console) + except Exception as e: + console.print(f"[red]Error:[/red] {str(e)}") + sys.exit(1) + @cli.command() def status(): """Show running concore processes""" diff --git a/concore_cli/commands/inspect.py b/concore_cli/commands/inspect.py new file mode 100644 index 00000000..e5ce383f --- /dev/null +++ b/concore_cli/commands/inspect.py @@ -0,0 +1,259 @@ +from pathlib import Path +from bs4 import BeautifulSoup +from rich.table import Table +from rich.tree import Tree +from rich.panel import Panel +from collections import defaultdict +import re + +def inspect_workflow(workflow_file, console, output_json=False): + workflow_path = Path(workflow_file) + + if output_json: + return _inspect_json(workflow_path) + + _inspect_rich(workflow_path, console) + +def _inspect_rich(workflow_path, console): + console.print() + console.print(f"[bold cyan]Workflow:[/bold cyan] {workflow_path.name}") + console.print() + + try: + with open(workflow_path, 'r') as f: + content = f.read() + + soup = BeautifulSoup(content, 'xml') + + if not soup.find('graphml'): + console.print("[red]Not a valid GraphML file[/red]") + return + + nodes = soup.find_all('node') + edges = soup.find_all('edge') + + tree = Tree("📊 [bold]Workflow Overview[/bold]") + + lang_counts = defaultdict(int) + node_files = [] + missing_files = [] + + for node in nodes: + label_tag = node.find('y:NodeLabel') + if label_tag and label_tag.text: + label = label_tag.text.strip() + if ':' in label: + _, filename = label.split(':', 1) + node_files.append(filename) + + ext = Path(filename).suffix + if ext == '.py': + lang_counts['Python'] += 1 + elif ext == '.m': + lang_counts['MATLAB'] += 1 + elif ext == '.java': + lang_counts['Java'] += 1 + elif ext == '.cpp' or ext == '.hpp': + lang_counts['C++'] += 1 + elif ext == '.v': + lang_counts['Verilog'] += 1 + else: + lang_counts['Other'] += 1 + + src_dir = workflow_path.parent / 'src' + if not (src_dir / filename).exists(): + missing_files.append(filename) + + nodes_branch = tree.add(f"Nodes: [bold]{len(nodes)}[/bold]") + if lang_counts: + for lang, count in sorted(lang_counts.items(), key=lambda x: -x[1]): + nodes_branch.add(f"{lang}: {count}") + + edges_branch = tree.add(f"Edges: [bold]{len(edges)}[/bold]") + + edge_label_regex = re.compile(r"0x([a-fA-F0-9]+)_(\S+)") + zmq_count = 0 + file_count = 0 + + for edge in edges: + label_tag = edge.find('y:EdgeLabel') + if label_tag and label_tag.text: + if edge_label_regex.match(label_tag.text.strip()): + zmq_count += 1 + else: + file_count += 1 + + if zmq_count > 0: + edges_branch.add(f"ZMQ: {zmq_count}") + if file_count > 0: + edges_branch.add(f"File-based: {file_count}") + + comm_type = "ZMQ (0mq)" if zmq_count > 0 else "File-based" if file_count > 0 else "None" + tree.add(f"Communication: [bold]{comm_type}[/bold]") + + if missing_files: + missing_branch = tree.add(f"[yellow]Missing files: {len(missing_files)}[/yellow]") + for f in missing_files[:5]: + missing_branch.add(f"[yellow]{f}[/yellow]") + if len(missing_files) > 5: + missing_branch.add(f"[dim]...and {len(missing_files) - 5} more[/dim]") + + console.print(tree) + console.print() + + if nodes: + table = Table(title="Node Details", show_header=True, header_style="bold magenta") + table.add_column("ID", style="cyan", width=12) + table.add_column("File", style="white") + table.add_column("Language", style="green") + table.add_column("Status", style="yellow") + + for node in nodes[:10]: + label_tag = node.find('y:NodeLabel') + if label_tag and label_tag.text: + label = label_tag.text.strip() + if ':' in label: + node_id, filename = label.split(':', 1) + + ext = Path(filename).suffix + lang_map = { + '.py': 'Python', + '.m': 'MATLAB', + '.java': 'Java', + '.cpp': 'C++', + '.hpp': 'C++', + '.v': 'Verilog' + } + lang = lang_map.get(ext, 'Other') + + src_dir = workflow_path.parent / 'src' + status = "✓" if (src_dir / filename).exists() else "✗" + + table.add_row(node_id, filename, lang, status) + + if len(nodes) > 10: + table.caption = f"Showing 10 of {len(nodes)} nodes" + + console.print(table) + console.print() + + if edges: + edge_table = Table(title="Edge Connections", show_header=True, header_style="bold magenta") + edge_table.add_column("From", style="cyan", width=12) + edge_table.add_column("To", style="cyan", width=12) + edge_table.add_column("Type", style="green") + + for edge in edges[:10]: + source = edge.get('source', 'unknown') + target = edge.get('target', 'unknown') + + label_tag = edge.find('y:EdgeLabel') + edge_type = "File" + if label_tag and label_tag.text: + if edge_label_regex.match(label_tag.text.strip()): + edge_type = "ZMQ" + + edge_table.add_row(source, target, edge_type) + + if len(edges) > 10: + edge_table.caption = f"Showing 10 of {len(edges)} edges" + + console.print(edge_table) + console.print() + + except FileNotFoundError: + console.print(f"[red]File not found:[/red] {workflow_path}") + except Exception as e: + console.print(f"[red]Inspection failed:[/red] {str(e)}") + +def _inspect_json(workflow_path): + import json + + try: + with open(workflow_path, 'r') as f: + content = f.read() + + soup = BeautifulSoup(content, 'xml') + + nodes = soup.find_all('node') + edges = soup.find_all('edge') + + lang_counts = defaultdict(int) + node_list = [] + edge_list = [] + missing_files = [] + + for node in nodes: + label_tag = node.find('y:NodeLabel') + if label_tag and label_tag.text: + label = label_tag.text.strip() + if ':' in label: + node_id, filename = label.split(':', 1) + + ext = Path(filename).suffix + lang_map = { + '.py': 'python', + '.m': 'matlab', + '.java': 'java', + '.cpp': 'cpp', + '.hpp': 'cpp', + '.v': 'verilog' + } + lang = lang_map.get(ext, 'other') + lang_counts[lang] += 1 + + src_dir = workflow_path.parent / 'src' + exists = (src_dir / filename).exists() + if not exists: + missing_files.append(filename) + + node_list.append({ + 'id': node_id, + 'file': filename, + 'language': lang, + 'exists': exists + }) + + edge_label_regex = re.compile(r"0x([a-fA-F0-9]+)_(\S+)") + zmq_count = 0 + file_count = 0 + + for edge in edges: + source = edge.get('source') + target = edge.get('target') + + label_tag = edge.find('y:EdgeLabel') + edge_type = 'file' + if label_tag and label_tag.text: + if edge_label_regex.match(label_tag.text.strip()): + edge_type = 'zmq' + zmq_count += 1 + else: + file_count += 1 + + edge_list.append({ + 'source': source, + 'target': target, + 'type': edge_type + }) + + result = { + 'workflow': str(workflow_path.name), + 'nodes': { + 'total': len(nodes), + 'by_language': dict(lang_counts), + 'list': node_list + }, + 'edges': { + 'total': len(edges), + 'zmq': zmq_count, + 'file': file_count, + 'list': edge_list + }, + 'missing_files': missing_files + } + + print(json.dumps(result, indent=2)) + + except Exception as e: + print(json.dumps({'error': str(e)}, indent=2)) diff --git a/concore_cli/commands/status.py b/concore_cli/commands/status.py index e4072246..7d5e526b 100644 --- a/concore_cli/commands/status.py +++ b/concore_cli/commands/status.py @@ -56,6 +56,7 @@ def show_status(console): 'memory': mem_str }) except (psutil.NoSuchProcess, psutil.AccessDenied): + # Process may have exited or be inaccessible; safe to ignore continue except Exception as e: diff --git a/concore_cli/commands/stop.py b/concore_cli/commands/stop.py index ad1f38f3..1943a1e0 100644 --- a/concore_cli/commands/stop.py +++ b/concore_cli/commands/stop.py @@ -31,6 +31,7 @@ def stop_all(console): if is_concore: processes_to_kill.append(proc) except (psutil.NoSuchProcess, psutil.AccessDenied): + # Process may have exited or be inaccessible; safe to ignore continue except Exception as e: diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..8c79b669 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,30 @@ +[build-system] +requires = ["setuptools>=45", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "concore" +version = "1.0.0" +description = "Concore workflow management CLI" +readme = "README.md" +requires-python = ">=3.8" +license = {text = "MIT"} +dependencies = [ + "click>=8.0.0", + "rich>=10.0.0", + "beautifulsoup4>=4.9.0", + "lxml>=4.6.0", + "psutil>=5.8.0", +] + +[project.optional-dependencies] +dev = [ + "pytest>=6.0.0", + "pytest-cov>=2.10.0", +] + +[project.scripts] +concore = "concore_cli.cli:cli" + +[tool.setuptools] +packages = ["concore_cli", "concore_cli.commands"] From 673feed9abe5d4a960c836d550263107a5c27e67 Mon Sep 17 00:00:00 2001 From: Sahil Lenka Date: Wed, 4 Feb 2026 23:10:15 +0530 Subject: [PATCH 026/275] fix copilot review issues: add init files, fix function signatures, add comments to except clauses, improve edge counting --- concore_cli/commands/inspect.py | 42 ++++++++++++++++++--------------- concore_cli/commands/status.py | 2 ++ concore_cli/commands/stop.py | 2 +- pyproject.toml | 1 + 4 files changed, 27 insertions(+), 20 deletions(-) diff --git a/concore_cli/commands/inspect.py b/concore_cli/commands/inspect.py index e5ce383f..c84fbed4 100644 --- a/concore_cli/commands/inspect.py +++ b/concore_cli/commands/inspect.py @@ -6,15 +6,15 @@ from collections import defaultdict import re -def inspect_workflow(workflow_file, console, output_json=False): +def inspect_workflow(workflow_file, source_dir, output_json, console): workflow_path = Path(workflow_file) if output_json: - return _inspect_json(workflow_path) + return _inspect_json(workflow_path, source_dir) - _inspect_rich(workflow_path, console) + _inspect_rich(workflow_path, source_dir, console) -def _inspect_rich(workflow_path, console): +def _inspect_rich(workflow_path, source_dir, console): console.print() console.print(f"[bold cyan]Workflow:[/bold cyan] {workflow_path.name}") console.print() @@ -60,7 +60,7 @@ def _inspect_rich(workflow_path, console): else: lang_counts['Other'] += 1 - src_dir = workflow_path.parent / 'src' + src_dir = workflow_path.parent / source_dir if not (src_dir / filename).exists(): missing_files.append(filename) @@ -77,11 +77,11 @@ def _inspect_rich(workflow_path, console): for edge in edges: label_tag = edge.find('y:EdgeLabel') - if label_tag and label_tag.text: - if edge_label_regex.match(label_tag.text.strip()): - zmq_count += 1 - else: - file_count += 1 + label_text = label_tag.text.strip() if label_tag and label_tag.text else "" + if label_text and edge_label_regex.match(label_text): + zmq_count += 1 + else: + file_count += 1 if zmq_count > 0: edges_branch.add(f"ZMQ: {zmq_count}") @@ -126,7 +126,7 @@ def _inspect_rich(workflow_path, console): } lang = lang_map.get(ext, 'Other') - src_dir = workflow_path.parent / 'src' + src_dir = workflow_path.parent / source_dir status = "✓" if (src_dir / filename).exists() else "✗" table.add_row(node_id, filename, lang, status) @@ -166,7 +166,7 @@ def _inspect_rich(workflow_path, console): except Exception as e: console.print(f"[red]Inspection failed:[/red] {str(e)}") -def _inspect_json(workflow_path): +def _inspect_json(workflow_path, source_dir): import json try: @@ -175,6 +175,10 @@ def _inspect_json(workflow_path): soup = BeautifulSoup(content, 'xml') + if not soup.find('graphml'): + print(json.dumps({'error': 'Not a valid GraphML file'}, indent=2)) + return + nodes = soup.find_all('node') edges = soup.find_all('edge') @@ -202,7 +206,7 @@ def _inspect_json(workflow_path): lang = lang_map.get(ext, 'other') lang_counts[lang] += 1 - src_dir = workflow_path.parent / 'src' + src_dir = workflow_path.parent / source_dir exists = (src_dir / filename).exists() if not exists: missing_files.append(filename) @@ -223,13 +227,13 @@ def _inspect_json(workflow_path): target = edge.get('target') label_tag = edge.find('y:EdgeLabel') + label_text = label_tag.text.strip() if label_tag and label_tag.text else "" edge_type = 'file' - if label_tag and label_tag.text: - if edge_label_regex.match(label_tag.text.strip()): - edge_type = 'zmq' - zmq_count += 1 - else: - file_count += 1 + if label_text and edge_label_regex.match(label_text): + edge_type = 'zmq' + zmq_count += 1 + else: + file_count += 1 edge_list.append({ 'source': source, diff --git a/concore_cli/commands/status.py b/concore_cli/commands/status.py index 7d5e526b..6eaf6c8b 100644 --- a/concore_cli/commands/status.py +++ b/concore_cli/commands/status.py @@ -38,12 +38,14 @@ def show_status(console): minutes, seconds = divmod(remainder, 60) uptime_str = f"{hours}h {minutes}m {seconds}s" except: + # Failed to calculate uptime uptime_str = "unknown" try: mem_mb = proc.info['memory_info'].rss / 1024 / 1024 mem_str = f"{mem_mb:.1f} MB" except: + # Failed to get memory info mem_str = "unknown" command = ' '.join(cmdline[:3]) if len(cmdline) >= 3 else cmdline_str[:50] diff --git a/concore_cli/commands/stop.py b/concore_cli/commands/stop.py index 1943a1e0..0b2f530e 100644 --- a/concore_cli/commands/stop.py +++ b/concore_cli/commands/stop.py @@ -31,7 +31,7 @@ def stop_all(console): if is_concore: processes_to_kill.append(proc) except (psutil.NoSuchProcess, psutil.AccessDenied): - # Process may have exited or be inaccessible; safe to ignore + # Process already exited or access denied; continue continue except Exception as e: diff --git a/pyproject.toml b/pyproject.toml index 8c79b669..63a8f6b7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,3 +28,4 @@ concore = "concore_cli.cli:cli" [tool.setuptools] packages = ["concore_cli", "concore_cli.commands"] +py-modules = ["mkconcore"] From ff3206ab17535d0f35879f0a60f0743383b1850d Mon Sep 17 00:00:00 2001 From: Sahil Lenka Date: Thu, 5 Feb 2026 02:19:11 +0530 Subject: [PATCH 027/275] fix: add atexit and signal handlers for ZMQ cleanup - Register terminate_zmq() with atexit to ensure cleanup on normal exit - Add signal handlers for SIGINT (Ctrl+C) and SIGTERM - Improve terminate_zmq() with better logging and error handling - Clear zmq_ports dict after cleanup to prevent double-cleanup --- concore.py | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/concore.py b/concore.py index 2da12506..4072505d 100644 --- a/concore.py +++ b/concore.py @@ -6,6 +6,9 @@ import re import zmq import numpy as np +import atexit +import signal + logging.basicConfig( level=logging.INFO, format='%(levelname)s - %(message)s' @@ -98,12 +101,31 @@ def init_zmq_port(port_name, port_type, address, socket_type_str): logging.error(f"An unexpected error occurred during ZMQ port initialization for {port_name}: {e}") def terminate_zmq(): - for port in zmq_ports.values(): + """Clean up all ZMQ sockets and contexts before exit.""" + if not zmq_ports: + return # No ports to clean up + + print("\nCleaning up ZMQ resources...") + for port_name, port in zmq_ports.items(): try: port.socket.close() port.context.term() + print(f"Closed ZMQ port: {port_name}") except Exception as e: logging.error(f"Error while terminating ZMQ port {port.address}: {e}") + zmq_ports.clear() + +def signal_handler(sig, frame): + """Handle interrupt signals gracefully.""" + print(f"\nReceived signal {sig}, shutting down gracefully...") + terminate_zmq() + sys.exit(0) + +# Register cleanup handlers +atexit.register(terminate_zmq) +signal.signal(signal.SIGINT, signal_handler) # Handle Ctrl+C +signal.signal(signal.SIGTERM, signal_handler) # Handle termination + # --- ZeroMQ Integration End --- From b3268f371a8c281e396f9d36de5d2f153a7e6227 Mon Sep 17 00:00:00 2001 From: Sahil Lenka Date: Thu, 5 Feb 2026 02:38:31 +0530 Subject: [PATCH 028/275] fix: address Copilot review, prevent reentrant cleanup and platform-specific SIGTERM --- concore.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/concore.py b/concore.py index 4072505d..6d1763c5 100644 --- a/concore.py +++ b/concore.py @@ -75,6 +75,7 @@ def recv_json_with_retry(self): # Global ZeroMQ ports registry zmq_ports = {} +_cleanup_in_progress = False def init_zmq_port(port_name, port_type, address, socket_type_str): """ @@ -102,9 +103,15 @@ def init_zmq_port(port_name, port_type, address, socket_type_str): def terminate_zmq(): """Clean up all ZMQ sockets and contexts before exit.""" + global _cleanup_in_progress + + if _cleanup_in_progress: + return # Already cleaning up, prevent reentrant calls + if not zmq_ports: return # No ports to clean up + _cleanup_in_progress = True print("\nCleaning up ZMQ resources...") for port_name, port in zmq_ports.items(): try: @@ -114,17 +121,25 @@ def terminate_zmq(): except Exception as e: logging.error(f"Error while terminating ZMQ port {port.address}: {e}") zmq_ports.clear() + _cleanup_in_progress = False def signal_handler(sig, frame): """Handle interrupt signals gracefully.""" print(f"\nReceived signal {sig}, shutting down gracefully...") + # Prevent terminate_zmq from being called twice: once here and once via atexit + try: + atexit.unregister(terminate_zmq) + except Exception: + # If unregister fails for any reason, proceed with explicit cleanup anyway + pass terminate_zmq() sys.exit(0) # Register cleanup handlers atexit.register(terminate_zmq) signal.signal(signal.SIGINT, signal_handler) # Handle Ctrl+C -signal.signal(signal.SIGTERM, signal_handler) # Handle termination +if not hasattr(sys, 'getwindowsversion'): + signal.signal(signal.SIGTERM, signal_handler) # Handle termination (Unix only) # --- ZeroMQ Integration End --- From dc93fd97fe486d06dfde846949a912afedfac5ac Mon Sep 17 00:00:00 2001 From: GREENRAT-K405 Date: Thu, 5 Feb 2026 03:07:14 +0530 Subject: [PATCH 029/275] add numpy and simulation related tests --- tests/test_concore.py | 94 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 93 insertions(+), 1 deletion(-) diff --git a/tests/test_concore.py b/tests/test_concore.py index 862554c8..79fed389 100644 --- a/tests/test_concore.py +++ b/tests/test_concore.py @@ -1,5 +1,6 @@ import pytest import os +import numpy as np class TestSafeLiteralEval: @@ -84,4 +85,95 @@ def test_core_functions_exist(self): assert callable(safe_literal_eval) assert callable(tryparam) - assert callable(default_maxtime) \ No newline at end of file + assert callable(default_maxtime) + + +class TestNumpyConversion: + def test_convert_scalar(self): + from concore import convert_numpy_to_python + val = np.float64(3.14) + res = convert_numpy_to_python(val) + assert type(res) == float + assert res == 3.14 + + def test_convert_list_and_dict(self): + from concore import convert_numpy_to_python + data = { + 'a': np.int32(10), + 'b': [np.float64(1.1), np.float64(2.2)] + } + res = convert_numpy_to_python(data) + assert type(res['a']) == int + assert type(res['b'][0]) == float + assert res['b'][1] == 2.2 + +class TestInitVal: + @pytest.fixture(autouse=True) + def reset_simtime(self): + import concore + old_simtime = concore.simtime + yield + concore.simtime = old_simtime + + def test_initval_updates_simtime(self): + import concore + concore.simtime = 0 + # initval takes string repr of a list [time, val1, val2...] + result = concore.initval("[100, 'data']") + + assert concore.simtime == 100 + assert result == ['data'] + + def test_initval_handles_bad_input(self): + import concore + concore.simtime = 0 + # Input that isn't a list + result = concore.initval("not_a_list") + assert concore.simtime == 0 + assert result == [] + +class TestDefaultMaxTime: + def test_uses_file_value(self, temp_dir, monkeypatch): + import concore + # Mock the path to maxtime file + maxtime_file = os.path.join(temp_dir, "concore.maxtime") + with open(maxtime_file, "w") as f: + f.write("500") + + monkeypatch.setattr(concore, 'concore_maxtime_file', maxtime_file) + concore.default_maxtime(100) + + assert concore.maxtime == 500 + + def test_uses_default_when_missing(self, monkeypatch): + import concore + monkeypatch.setattr(concore, 'concore_maxtime_file', "missing_file") + concore.default_maxtime(999) + assert concore.maxtime == 999 + +class TestUnchanged: + @pytest.fixture(autouse=True) + def reset_globals(self): + import concore + old_s = concore.s + old_olds = concore.olds + yield + concore.s = old_s + concore.olds = old_olds + + def test_unchanged_returns_true_if_same(self): + import concore + concore.s = "same" + concore.olds = "same" + + # Should return True and reset s to empty + assert concore.unchanged() is True + assert concore.s == '' + + def test_unchanged_returns_false_if_diff(self): + import concore + concore.s = "new" + concore.olds = "old" + + assert concore.unchanged() is False + assert concore.olds == "new" \ No newline at end of file From 58fc130c776fb275d79c98f27873b35a5f8b202c Mon Sep 17 00:00:00 2001 From: GREENRAT-K405 Date: Fri, 6 Feb 2026 00:29:38 +0530 Subject: [PATCH 030/275] fix harmful shell injections from graphml --- mkconcore.py | 191 ++++++++++++++++++++++++++++++++------------------- 1 file changed, 121 insertions(+), 70 deletions(-) diff --git a/mkconcore.py b/mkconcore.py index 10135dc9..fc225bb8 100644 --- a/mkconcore.py +++ b/mkconcore.py @@ -71,6 +71,21 @@ import stat import copy_with_port_portname import numpy as np +import shlex # Added for POSIX shell escaping + +# --- SECURITY FIX: Input Validation Helper --- +def safe_name(value, context): + """ + Validates that the input string does not contain characters dangerous + for filesystem paths or shell command injection. + """ + if not value: + raise ValueError(f"{context} cannot be empty") + # Blocks path traversal (/, \\) and shell metacharacters (*, ?, <, >, |, ;, &, $, `) + if re.search(r'[\\/:*?"<>|;&`$]', value): + raise ValueError(f"Unsafe {context}: '{value}' contains illegal characters.") + return value +# --------------------------------------------- MKCONCORE_VER = "22-09-18" @@ -200,7 +215,19 @@ node_label_tag = data.find('y:NodeLabel') if node_label_tag: node_label = prefixedgenode + node_label_tag.text - nodes_dict[node['id']] = re.sub(r'(\s+|\n)', ' ', node_label) + node_label = re.sub(r'(\s+|\n)', ' ', node_label) + + # --- SECURITY FIX: Validate Node Labels --- + # Check for malicious characters in container name and source file + if ':' in node_label: + container_part, source_part = node_label.split(':', 1) + safe_name(container_part, f"Node container name '{container_part}'") + safe_name(source_part, f"Node source file '{source_part}'") + else: + safe_name(node_label, f"Node label '{node_label}'") + # ------------------------------------------ + + nodes_dict[node['id']] = node_label node_id_to_label_map[node['id']] = node_label.split(':')[0] except (IndexError, AttributeError): logging.debug('A node with no valid properties encountered and ignored') @@ -215,6 +242,12 @@ edge_label = prefixedgenode + raw_label # Filter out ZMQ edges from the file-based edge dictionary by checking the raw label if not edge_label_regex.match(raw_label): + + # --- SECURITY FIX: Validate Edge Labels --- + # These labels become directory/volume names. Strict validation required. + safe_name(edge_label, f"Edge label '{edge_label}'") + # ------------------------------------------ + if edge_label not in edges_dict: edges_dict[edge_label] = [nodes_dict[edge['source']], []] edges_dict[edge_label][1].append(nodes_dict[edge['target']]) @@ -660,19 +693,16 @@ for node in nodes_dict: containername,sourcecode = nodes_dict[node].split(':') if len(sourcecode)!=0: + safe_container = shlex.quote(containername) # Added safety if sourcecode.find(".")==-1: logging.debug(f"Generating Docker run command: {DOCKEREXE} run --name={containername+volswr[i]+volsro[i]} {DOCKEREPO}/docker- {sourcecode}") - frun.write(DOCKEREXE+' run --name='+containername+volswr[i]+volsro[i]+" "+DOCKEREPO+"/docker-"+sourcecode+"&\n") + # Use safe_container + frun.write(DOCKEREXE+' run --name='+safe_container+volswr[i]+volsro[i]+" "+DOCKEREPO+"/docker-"+shlex.quote(sourcecode)+"&\n") else: dockername,langext = sourcecode.split(".") logging.debug(f"Generating Docker run command for {dockername}: {DOCKEREXE} run --name={containername+volswr[i]+volsro[i]} docker-{dockername}") - frun.write(DOCKEREXE+' run --name='+containername+volswr[i]+volsro[i]+" docker-"+dockername+"&\n") - #if langext != "m": #3/27/21 - # print(DOCKEREXE+' run --name='+containername+volswr[i]+volsro[i]+" docker-"+dockername) - # frun.write(DOCKEREXE+' run --name='+containername+volswr[i]+volsro[i]+" docker-"+dockername+"&\n") - #else: - # print(DOCKEREXE+' run --name='+containername+volswr[i]+volsro[i]+" docker-"+dockername+' octave -qf --eval "run('+"'"+sourcecode+"'"+')"'+"&\n") - # frun.write(DOCKEREXE+' run --name='+containername+volswr[i]+volsro[i]+" docker-"+dockername+' octave -qf --eval "run('+"'"+sourcecode+"'"+')"'+"&\n") + # Use safe_container + frun.write(DOCKEREXE+' run --name='+safe_container+volswr[i]+volsro[i]+" docker-"+shlex.quote(dockername)+"&\n") i=i+1 frun.close() @@ -683,8 +713,9 @@ if len(sourcecode)!=0: #dockername,langext = sourcecode.split(".") dockername = sourcecode.split(".")[0] # 3/28/21 - fstop.write(DOCKEREXE+' stop '+containername+"\n") - fstop.write(DOCKEREXE+' rm '+containername+"\n") + safe_container = shlex.quote(containername) # Added safety + fstop.write(DOCKEREXE+' stop '+safe_container+"\n") + fstop.write(DOCKEREXE+' rm '+safe_container+"\n") i=i+1 fstop.close() @@ -696,7 +727,7 @@ dockername = sourcecode.split(".")[0] #3/28/21 writeedges = volswr[i] while writeedges.find(":") != -1: - fclear.write(DOCKEREXE+' volume rm ' +writeedges.split(":")[0].split("-v")[1]+"\n") + fclear.write(DOCKEREXE+' volume rm ' +writeedges.split(":")[0].split("-v")[1].strip()+"\n") # Added strip() writeedges = writeedges[writeedges.find(":")+1:] i=i+1 fclear.close() @@ -715,8 +746,8 @@ writeedges = volswr[i] while writeedges.find(":") != -1: fmaxtime.write(' -v ') - fmaxtime.write(writeedges.split(":")[0].split("-v ")[1]+":/") - fmaxtime.write(writeedges.split(":")[0].split("-v ")[1]) + fmaxtime.write(writeedges.split(":")[0].split("-v ")[1].strip()+":/") # Added strip() + fmaxtime.write(writeedges.split(":")[0].split("-v ")[1].strip()) # Added strip() writeedges = writeedges[writeedges.find(":")+1:] i=i+1 fmaxtime.write(' docker-concore >/dev/null &\n') @@ -730,7 +761,7 @@ writeedges = volswr[i] while writeedges.find(":") != -1: fmaxtime.write('sudo docker cp concore.maxtime concore:/') - fmaxtime.write(writeedges.split(":")[0].split("-v ")[1]+"/concore.maxtime\n") + fmaxtime.write(writeedges.split(":")[0].split("-v ")[1].strip()+"/concore.maxtime\n") # Added strip() writeedges = writeedges[writeedges.find(":")+1:] i=i+1 fmaxtime.write('sudo docker stop concore \n') @@ -754,8 +785,8 @@ writeedges = volswr[i] while writeedges.find(":") != -1: fparams.write(' -v ') - fparams.write(writeedges.split(":")[0].split("-v ")[1]+":/") - fparams.write(writeedges.split(":")[0].split("-v ")[1]) + fparams.write(writeedges.split(":")[0].split("-v ")[1].strip()+":/") # Added strip() + fparams.write(writeedges.split(":")[0].split("-v ")[1].strip()) # Added strip() writeedges = writeedges[writeedges.find(":")+1:] i=i+1 fparams.write(' docker-concore >/dev/null &\n') @@ -769,7 +800,7 @@ writeedges = volswr[i] while writeedges.find(":") != -1: fparams.write('sudo docker cp concore.params concore:/') - fparams.write(writeedges.split(":")[0].split("-v ")[1]+"/concore.params\n") + fparams.write(writeedges.split(":")[0].split("-v ")[1].strip()+"/concore.params\n") # Added strip() writeedges = writeedges[writeedges.find(":")+1:] i=i+1 fparams.write('sudo docker stop concore \n') @@ -792,8 +823,8 @@ writeedges = volswr[i] while writeedges.find(":") != -1: funlock.write(' -v ') - funlock.write(writeedges.split(":")[0].split("-v ")[1]+":/") - funlock.write(writeedges.split(":")[0].split("-v ")[1]) + funlock.write(writeedges.split(":")[0].split("-v ")[1].strip()+":/") # Added strip() + funlock.write(writeedges.split(":")[0].split("-v ")[1].strip()) # Added strip() writeedges = writeedges[writeedges.find(":")+1:] i=i+1 funlock.write(' docker-concore >/dev/null &\n') @@ -807,7 +838,7 @@ writeedges = volswr[i] while writeedges.find(":") != -1: funlock.write('sudo docker cp ~/concore.apikey concore:/') - funlock.write(writeedges.split(":")[0].split("-v ")[1]+"/concore.apikey\n") + funlock.write(writeedges.split(":")[0].split("-v ")[1].strip()+"/concore.apikey\n") # Added strip() writeedges = writeedges[writeedges.find(":")+1:] i=i+1 funlock.write('sudo docker stop concore \n') @@ -822,7 +853,9 @@ containername,sourcecode = nodes_dict[node].split(':') if len(sourcecode)!=0 and sourcecode.find(".")!=-1: #3/28/21 dockername,langext = sourcecode.split(".") - fdebug.write(DOCKEREXE+' run -it --name='+containername+volswr[i]+volsro[i]+" docker-"+dockername+"&\n") + # safe_container added to debug line (POSIX) + safe_container = shlex.quote(containername) + fdebug.write(DOCKEREXE+' run -it --name='+safe_container+volswr[i]+volsro[i]+" docker-"+dockername+"&\n") i=i+1 fdebug.close() os.chmod(outdir+"/build",stat.S_IRWXU) @@ -937,91 +970,103 @@ logging.error(f"Extension .{langext} is unsupported") quit() if concoretype=="windows": + # manual double quoting for Windows + Input validation above prevents breakout + q_container = f'"{containername}"' + q_source = f'"{sourcecode}"' + if langext=="py": - frun.write('start /B /D '+containername+" "+PYTHONWIN+" "+sourcecode+" >"+containername+"\\concoreout.txt\n") - fdebug.write('start /D '+containername+" cmd /K "+PYTHONWIN+" "+sourcecode+"\n") + frun.write('start /B /D '+q_container+" "+PYTHONWIN+" "+q_source+" >"+q_container+"\\concoreout.txt\n") + fdebug.write('start /D '+q_container+" cmd /K "+PYTHONWIN+" "+q_source+"\n") elif langext=="cpp": #6/25/21 - frun.write('cd '+containername+'\n') - frun.write(CPPWIN+' '+sourcecode+'\n') + frun.write('cd '+q_container+'\n') + frun.write(CPPWIN+' '+q_source+'\n') frun.write('cd ..\n') - frun.write('start /B /D '+containername+' cmd /c a >'+containername+'\\concoreout.txt\n') + frun.write('start /B /D '+q_container+' cmd /c a >'+q_container+'\\concoreout.txt\n') #frun.write('start /B /D '+containername+' "'+CPPWIN+' '+sourcecode+'|a >'+containername+'\\concoreout.txt"\n') - fdebug.write('cd '+containername+'\n') - fdebug.write(CPPWIN+' '+sourcecode+'\n') + fdebug.write('cd '+q_container+'\n') + fdebug.write(CPPWIN+' '+q_source+'\n') fdebug.write('cd ..\n') - fdebug.write('start /D '+containername+' cmd /K a\n') + fdebug.write('start /D '+q_container+' cmd /K a\n') #fdebug.write('start /D '+containername+' cmd /K "'+CPPWIN+' '+sourcecode+'|a"\n') elif langext=="v": #6/25/21 - frun.write('cd '+containername+'\n') - frun.write(VWIN+' '+sourcecode+'\n') + frun.write('cd '+q_container+'\n') + frun.write(VWIN+' '+q_source+'\n') frun.write('cd ..\n') - frun.write('start /B /D '+containername+' cmd /c vvp a.out >'+containername+'\\concoreout.txt\n') - fdebug.write('cd '+containername+'\n') - fdebug.write(VWIN+' '+sourcecode+'\n') + frun.write('start /B /D '+q_container+' cmd /c vvp a.out >'+q_container+'\\concoreout.txt\n') + fdebug.write('cd '+q_container+'\n') + fdebug.write(VWIN+' '+q_source+'\n') fdebug.write('cd ..\n') - fdebug.write('start /D '+containername+' cmd /K vvp a.out\n') + fdebug.write('start /D '+q_container+' cmd /K vvp a.out\n') #fdebug.write('start /D '+containername+' cmd /K "'+CPPWIN+' '+sourcecode+'|a"\n') elif langext=="m": #3/23/21 if M_IS_OCTAVE: - frun.write('start /B /D '+containername+" "+OCTAVEWIN+' -qf --eval "run('+"'"+sourcecode+"'"+')"'+" >"+containername+"\\concoreout.txt\n") - fdebug.write('start /D '+containername+" cmd /K " +OCTAVEWIN+' -qf --eval "run('+"'"+sourcecode+"'"+')"'+"\n") + frun.write('start /B /D '+q_container+" "+OCTAVEWIN+' -qf --eval "run('+"'"+sourcecode+"'"+')"'+" >"+q_container+"\\concoreout.txt\n") + fdebug.write('start /D '+q_container+" cmd /K " +OCTAVEWIN+' -qf --eval "run('+"'"+sourcecode+"'"+')"'+"\n") else: # 4/2/21 - frun.write('start /B /D '+containername+" "+MATLABWIN+' -batch "run('+"'"+sourcecode+"'"+')"'+" >"+containername+"\\concoreout.txt\n") - fdebug.write('start /D '+containername+" cmd /K " +MATLABWIN+' -batch "run('+"'"+sourcecode+"'"+')"'+"\n") + frun.write('start /B /D '+q_container+" "+MATLABWIN+' -batch "run('+"'"+sourcecode+"'"+')"'+" >"+q_container+"\\concoreout.txt\n") + fdebug.write('start /D '+q_container+" cmd /K " +MATLABWIN+' -batch "run('+"'"+sourcecode+"'"+')"'+"\n") else: + #use shlex.quote for POSIX systems + safe_container = shlex.quote(containername) + safe_source = shlex.quote(sourcecode) + if langext == "py": - frun.write('(cd "' + containername + '"; ' + PYTHONEXE + ' ' + sourcecode + ' >concoreout.txt & echo $! >concorepid) &\n') + frun.write('(cd ' + safe_container + '; ' + PYTHONEXE + ' ' + safe_source + ' >concoreout.txt & echo $! >concorepid) &\n') if ubuntu: fdebug.write('concorewd="$(pwd)"\n') - fdebug.write('xterm -e bash -c "cd \\"$concorewd/' + containername + '\\"; ' + PYTHONEXE + ' ' + sourcecode + '; bash" &\n') + # quote the directory path inside the inner bash command + fdebug.write('xterm -e bash -c "cd \\"$concorewd/' + safe_container + '\\"; ' + PYTHONEXE + ' ' + safe_source + '; bash" &\n') else: fdebug.write('concorewd="$(pwd)"\n') - fdebug.write('osascript -e "tell application \\"Terminal\\" to do script \\"cd \\\\\\"$concorewd/' + containername + '\\\\\\"; ' + PYTHONEXE + ' ' + sourcecode + '\\"" \n') + fdebug.write('osascript -e "tell application \\"Terminal\\" to do script \\"cd \\\\\\"$concorewd/' + safe_container + '\\\\\\"; ' + PYTHONEXE + ' ' + safe_source + '\\"" \n') elif langext == "cpp": # 6/22/21 - frun.write('(cd "' + containername + '"; ' + CPPEXE + ' ' + sourcecode + '; ./a.out >concoreout.txt & echo $! >concorepid) &\n') + frun.write('(cd ' + safe_container + '; ' + CPPEXE + ' ' + safe_source + '; ./a.out >concoreout.txt & echo $! >concorepid) &\n') if ubuntu: fdebug.write('concorewd="$(pwd)"\n') - fdebug.write('xterm -e bash -c "cd \\"$concorewd/' + containername + '\\"; ' + CPPEXE + ' ' + sourcecode + '; ./a.out; bash" &\n') + fdebug.write('xterm -e bash -c "cd \\"$concorewd/' + safe_container + '\\"; ' + CPPEXE + ' ' + safe_source + '; ./a.out; bash" &\n') else: fdebug.write('concorewd="$(pwd)"\n') - fdebug.write('osascript -e "tell application \\"Terminal\\" to do script \\"cd \\\\\\"$concorewd/' + containername + '\\\\\\"; ' + CPPEXE + ' ' + sourcecode + '; ./a.out\\"" \n') + fdebug.write('osascript -e "tell application \\"Terminal\\" to do script \\"cd \\\\\\"$concorewd/' + safe_container + '\\\\\\"; ' + CPPEXE + ' ' + safe_source + '; ./a.out\\"" \n') elif langext == "v": # 6/25/21 - frun.write('(cd "' + containername + '"; ' + VEXE + ' ' + sourcecode + '; ./a.out >concoreout.txt & echo $! >concorepid) &\n') + frun.write('(cd ' + safe_container + '; ' + VEXE + ' ' + safe_source + '; ./a.out >concoreout.txt & echo $! >concorepid) &\n') if ubuntu: fdebug.write('concorewd="$(pwd)"\n') - fdebug.write('xterm -e bash -c "cd \\"$concorewd/' + containername + '\\"; ' + VEXE + ' ' + sourcecode + '; ./a.out; bash" &\n') + fdebug.write('xterm -e bash -c "cd \\"$concorewd/' + safe_container + '\\"; ' + VEXE + ' ' + safe_source + '; ./a.out; bash" &\n') else: fdebug.write('concorewd="$(pwd)"\n') - fdebug.write('osascript -e "tell application \\"Terminal\\" to do script \\"cd \\\\\\"$concorewd/' + containername + '\\\\\\"; ' + VEXE + ' ' + sourcecode + '; vvp a.out\\"" \n') + fdebug.write('osascript -e "tell application \\"Terminal\\" to do script \\"cd \\\\\\"$concorewd/' + safe_container + '\\\\\\"; ' + VEXE + ' ' + safe_source + '; vvp a.out\\"" \n') elif langext == "sh": # 5/19/21 - frun.write('(cd "' + containername + '"; ./' + sourcecode + ' ' + MCRPATH + ' >concoreout.txt & echo $! >concorepid) &\n') + frun.write('(cd ' + safe_container + '; ./' + safe_source + ' ' + MCRPATH + ' >concoreout.txt & echo $! >concorepid) &\n') if ubuntu: fdebug.write('concorewd="$(pwd)"\n') - fdebug.write('xterm -e bash -c "cd \\"$concorewd/' + containername + '\\"; ./' + sourcecode + ' ' + MCRPATH + '; bash" &\n') + fdebug.write('xterm -e bash -c "cd \\"$concorewd/' + safe_container + '\\"; ./' + safe_source + ' ' + MCRPATH + '; bash" &\n') else: fdebug.write('concorewd="$(pwd)"\n') - fdebug.write('osascript -e "tell application \\"Terminal\\" to do script \\"cd \\\\\\"$concorewd/' + containername + '\\\\\\"; ./' + sourcecode + ' ' + MCRPATH + '\\"" \n') + fdebug.write('osascript -e "tell application \\"Terminal\\" to do script \\"cd \\\\\\"$concorewd/' + safe_container + '\\\\\\"; ./' + safe_source + ' ' + MCRPATH + '\\"" \n') elif langext == "m": #3/23/21 + # construct safe eval command + safe_eval_cmd = shlex.quote(f"run('{sourcecode}')") if M_IS_OCTAVE: - frun.write('(cd "' + containername + '"; ' + OCTAVEEXE + ' -qf --eval run(\\\'' + sourcecode + '\\\') >concoreout.txt & echo $! >concorepid) &\n') + frun.write('(cd ' + safe_container + '; ' + OCTAVEEXE + ' -qf --eval ' + safe_eval_cmd + ' >concoreout.txt & echo $! >concorepid) &\n') if ubuntu: fdebug.write('concorewd="$(pwd)"\n') - fdebug.write('xterm -e bash -c "cd \\"$concorewd/' + containername + '\\"; ' + OCTAVEEXE + ' -qf --eval run(\\\'' + sourcecode + '\\\'); bash" &\n') + fdebug.write('xterm -e bash -c "cd \\"$concorewd/' + safe_container + '\\"; ' + OCTAVEEXE + ' -qf --eval ' + safe_eval_cmd + '; bash" &\n') else: fdebug.write('concorewd="$(pwd)"\n') - fdebug.write('osascript -e "tell application \\"Terminal\\" to do script \\"cd \\\\\\"$concorewd/' + containername + '\\\\\\"; ' + OCTAVEEXE + ' -qf --eval run(\\\\\\\'' + sourcecode + '\\\\\\\')\\"" \n') + #osascript quoting is very complex; minimal safe_container applied + fdebug.write('osascript -e "tell application \\"Terminal\\" to do script \\"cd \\\\\\"$concorewd/' + safe_container + '\\\\\\"; ' + OCTAVEEXE + ' -qf --eval run(\\\\\\\'' + sourcecode + '\\\\\\\')\\"" \n') else: - frun.write('(cd "' + containername + '"; ' + MATLABEXE + ' -batch run(\\\'' + sourcecode + '\\\') >concoreout.txt & echo $! >concorepid) &\n') + frun.write('(cd ' + safe_container + '; ' + MATLABEXE + ' -batch ' + safe_eval_cmd + ' >concoreout.txt & echo $! >concorepid) &\n') if ubuntu: fdebug.write('concorewd="$(pwd)"\n') - fdebug.write('xterm -e bash -c "cd \\"$concorewd/' + containername + '\\"; ' + MATLABEXE + ' -batch run(\\\'' + sourcecode + '\\\'); bash" &\n') + fdebug.write('xterm -e bash -c "cd \\"$concorewd/' + safe_container + '\\"; ' + MATLABEXE + ' -batch ' + safe_eval_cmd + '; bash" &\n') else: fdebug.write('concorewd="$(pwd)"\n') - fdebug.write('osascript -e "tell application \\"Terminal\\" to do script \\"cd \\\\\\"$concorewd/' + containername + '\\\\\\"; ' + MATLABEXE + ' -batch run(\\\\\\\'' + sourcecode + '\\\\\\\')\\"" \n') + fdebug.write('osascript -e "tell application \\"Terminal\\" to do script \\"cd \\\\\\"$concorewd/' + safe_container + '\\\\\\"; ' + MATLABEXE + ' -batch run(\\\\\\\'' + sourcecode + '\\\\\\\')\\"" \n') if concoretype=="posix": fstop.write('#!/bin/bash' + "\n") i=0 # 3/30/21 @@ -1030,11 +1075,13 @@ if len(sourcecode)!=0: dockername = sourcecode.split(".")[0] # 3/28/21 if concoretype=="windows": - fstop.write('cmd /C '+containername+"\\concorekill\n") - fstop.write('del '+containername+"\\concorekill.bat\n") + q_container = f'"{containername}"' + fstop.write('cmd /C '+q_container+"\\concorekill\n") + fstop.write('del '+q_container+"\\concorekill.bat\n") else: - fstop.write('kill -9 `cat '+containername+"/concorepid`\n") - fstop.write('rm '+containername+"/concorepid\n") + safe_pidfile = shlex.quote(f"{containername}/concorepid") + fstop.write('kill -9 `cat '+safe_pidfile+'`\n') + fstop.write('rm '+safe_pidfile+'\n') i=i+1 fstop.close() @@ -1047,10 +1094,11 @@ dockername = sourcecode.split(".")[0] #3/28/21 writeedges = volswr[i] while writeedges.find(":") != -1: + path_part = writeedges.split(":")[0].split("-v")[1].strip() if concoretype=="windows": - fclear.write('del /Q' + writeedges.split(":")[0].split("-v")[1]+ "\\*\n") + fclear.write('del /Q "' + path_part + '\\*"\n') else: - fclear.write('rm ' + writeedges.split(":")[0].split("-v")[1]+ "/*\n") + fclear.write('rm ' + shlex.quote(path_part) + '/*\n') writeedges = writeedges[writeedges.find(":")+1:] i=i+1 fclear.close() @@ -1064,10 +1112,11 @@ dockername = sourcecode.split(".")[0] #3/28/21 writeedges = volswr[i] while writeedges.find(":") != -1: + path_part = writeedges.split(":")[0].split("-v")[1].strip() if concoretype=="windows": - fmaxtime.write('echo %1 >' + writeedges.split(":")[0].split("-v")[1]+ "\\concore.maxtime\n") + fmaxtime.write('echo %1 >"' + path_part + '\\concore.maxtime"\n') else: - fmaxtime.write('echo "$1" >' + writeedges.split(":")[0].split("-v")[1]+ "/concore.maxtime\n") + fmaxtime.write('echo "$1" >' + shlex.quote(path_part + "/concore.maxtime") + '\n') writeedges = writeedges[writeedges.find(":")+1:] i=i+1 fmaxtime.close() @@ -1081,10 +1130,11 @@ dockername = sourcecode.split(".")[0] #3/28/21 writeedges = volswr[i] while writeedges.find(":") != -1: + path_part = writeedges.split(":")[0].split("-v")[1].strip() if concoretype=="windows": - fparams.write('echo %1 >' + writeedges.split(":")[0].split("-v")[1]+ "\\concore.params\n") + fparams.write('echo %1 >"' + path_part + '\\concore.params"\n') else: - fparams.write('echo "$1" >' + writeedges.split(":")[0].split("-v")[1]+ "/concore.params\n") + fparams.write('echo "$1" >' + shlex.quote(path_part + "/concore.params") + '\n') writeedges = writeedges[writeedges.find(":")+1:] i=i+1 fparams.close() @@ -1098,10 +1148,11 @@ dockername = sourcecode.split(".")[0] #3/28/21 writeedges = volswr[i] while writeedges.find(":") != -1: + path_part = writeedges.split(":")[0].split("-v")[1].strip() if concoretype=="windows": - funlock.write('copy %HOMEDRIVE%%HOMEPATH%\\concore.apikey' + writeedges.split(":")[0].split("-v")[1]+ "\\concore.apikey\n") + funlock.write('copy %HOMEDRIVE%%HOMEPATH%\\concore.apikey "' + path_part + '\\concore.apikey"\n') else: - funlock.write('cp ~/concore.apikey ' + writeedges.split(":")[0].split("-v")[1]+ "/concore.apikey\n") + funlock.write('cp ~/concore.apikey ' + shlex.quote(path_part + "/concore.apikey") + '\n') writeedges = writeedges[writeedges.find(":")+1:] i=i+1 funlock.close() From 83193b542a8ebefb0bf74a1d9df464f57a14b526 Mon Sep 17 00:00:00 2001 From: GREENRAT-K405 Date: Fri, 6 Feb 2026 00:41:34 +0530 Subject: [PATCH 031/275] fix harmful shell injections from graphml --- mkconcore.py | 38 ++++++++++++++++---------------------- 1 file changed, 16 insertions(+), 22 deletions(-) diff --git a/mkconcore.py b/mkconcore.py index fc225bb8..355c1df3 100644 --- a/mkconcore.py +++ b/mkconcore.py @@ -73,7 +73,7 @@ import numpy as np import shlex # Added for POSIX shell escaping -# --- SECURITY FIX: Input Validation Helper --- +# input validation helper def safe_name(value, context): """ Validates that the input string does not contain characters dangerous @@ -81,11 +81,10 @@ def safe_name(value, context): """ if not value: raise ValueError(f"{context} cannot be empty") - # Blocks path traversal (/, \\) and shell metacharacters (*, ?, <, >, |, ;, &, $, `) + #blocks path traversal (/, \\) and shell metacharacters (*, ?, <, >, |, ;, &, $, `) if re.search(r'[\\/:*?"<>|;&`$]', value): raise ValueError(f"Unsafe {context}: '{value}' contains illegal characters.") return value -# --------------------------------------------- MKCONCORE_VER = "22-09-18" @@ -217,15 +216,13 @@ def safe_name(value, context): node_label = prefixedgenode + node_label_tag.text node_label = re.sub(r'(\s+|\n)', ' ', node_label) - # --- SECURITY FIX: Validate Node Labels --- - # Check for malicious characters in container name and source file + #Validate node labels if ':' in node_label: container_part, source_part = node_label.split(':', 1) safe_name(container_part, f"Node container name '{container_part}'") safe_name(source_part, f"Node source file '{source_part}'") else: safe_name(node_label, f"Node label '{node_label}'") - # ------------------------------------------ nodes_dict[node['id']] = node_label node_id_to_label_map[node['id']] = node_label.split(':')[0] @@ -243,10 +240,8 @@ def safe_name(value, context): # Filter out ZMQ edges from the file-based edge dictionary by checking the raw label if not edge_label_regex.match(raw_label): - # --- SECURITY FIX: Validate Edge Labels --- - # These labels become directory/volume names. Strict validation required. + #Validate edge labels safe_name(edge_label, f"Edge label '{edge_label}'") - # ------------------------------------------ if edge_label not in edges_dict: edges_dict[edge_label] = [nodes_dict[edge['source']], []] @@ -693,7 +688,7 @@ def safe_name(value, context): for node in nodes_dict: containername,sourcecode = nodes_dict[node].split(':') if len(sourcecode)!=0: - safe_container = shlex.quote(containername) # Added safety + safe_container = shlex.quote(containername) if sourcecode.find(".")==-1: logging.debug(f"Generating Docker run command: {DOCKEREXE} run --name={containername+volswr[i]+volsro[i]} {DOCKEREPO}/docker- {sourcecode}") # Use safe_container @@ -713,7 +708,7 @@ def safe_name(value, context): if len(sourcecode)!=0: #dockername,langext = sourcecode.split(".") dockername = sourcecode.split(".")[0] # 3/28/21 - safe_container = shlex.quote(containername) # Added safety + safe_container = shlex.quote(containername) fstop.write(DOCKEREXE+' stop '+safe_container+"\n") fstop.write(DOCKEREXE+' rm '+safe_container+"\n") i=i+1 @@ -746,8 +741,8 @@ def safe_name(value, context): writeedges = volswr[i] while writeedges.find(":") != -1: fmaxtime.write(' -v ') - fmaxtime.write(writeedges.split(":")[0].split("-v ")[1].strip()+":/") # Added strip() - fmaxtime.write(writeedges.split(":")[0].split("-v ")[1].strip()) # Added strip() + fmaxtime.write(writeedges.split(":")[0].split("-v ")[1].strip()+":/") + fmaxtime.write(writeedges.split(":")[0].split("-v ")[1].strip()) writeedges = writeedges[writeedges.find(":")+1:] i=i+1 fmaxtime.write(' docker-concore >/dev/null &\n') @@ -761,7 +756,7 @@ def safe_name(value, context): writeedges = volswr[i] while writeedges.find(":") != -1: fmaxtime.write('sudo docker cp concore.maxtime concore:/') - fmaxtime.write(writeedges.split(":")[0].split("-v ")[1].strip()+"/concore.maxtime\n") # Added strip() + fmaxtime.write(writeedges.split(":")[0].split("-v ")[1].strip()+"/concore.maxtime\n") writeedges = writeedges[writeedges.find(":")+1:] i=i+1 fmaxtime.write('sudo docker stop concore \n') @@ -785,8 +780,8 @@ def safe_name(value, context): writeedges = volswr[i] while writeedges.find(":") != -1: fparams.write(' -v ') - fparams.write(writeedges.split(":")[0].split("-v ")[1].strip()+":/") # Added strip() - fparams.write(writeedges.split(":")[0].split("-v ")[1].strip()) # Added strip() + fparams.write(writeedges.split(":")[0].split("-v ")[1].strip()+":/") + fparams.write(writeedges.split(":")[0].split("-v ")[1].strip()) writeedges = writeedges[writeedges.find(":")+1:] i=i+1 fparams.write(' docker-concore >/dev/null &\n') @@ -800,7 +795,7 @@ def safe_name(value, context): writeedges = volswr[i] while writeedges.find(":") != -1: fparams.write('sudo docker cp concore.params concore:/') - fparams.write(writeedges.split(":")[0].split("-v ")[1].strip()+"/concore.params\n") # Added strip() + fparams.write(writeedges.split(":")[0].split("-v ")[1].strip()+"/concore.params\n") writeedges = writeedges[writeedges.find(":")+1:] i=i+1 fparams.write('sudo docker stop concore \n') @@ -823,8 +818,8 @@ def safe_name(value, context): writeedges = volswr[i] while writeedges.find(":") != -1: funlock.write(' -v ') - funlock.write(writeedges.split(":")[0].split("-v ")[1].strip()+":/") # Added strip() - funlock.write(writeedges.split(":")[0].split("-v ")[1].strip()) # Added strip() + funlock.write(writeedges.split(":")[0].split("-v ")[1].strip()+":/") + funlock.write(writeedges.split(":")[0].split("-v ")[1].strip()) writeedges = writeedges[writeedges.find(":")+1:] i=i+1 funlock.write(' docker-concore >/dev/null &\n') @@ -838,7 +833,7 @@ def safe_name(value, context): writeedges = volswr[i] while writeedges.find(":") != -1: funlock.write('sudo docker cp ~/concore.apikey concore:/') - funlock.write(writeedges.split(":")[0].split("-v ")[1].strip()+"/concore.apikey\n") # Added strip() + funlock.write(writeedges.split(":")[0].split("-v ")[1].strip()+"/concore.apikey\n") writeedges = writeedges[writeedges.find(":")+1:] i=i+1 funlock.write('sudo docker stop concore \n') @@ -1173,5 +1168,4 @@ def safe_name(value, context): os.chmod(outdir+"/clear",stat.S_IRWXU) os.chmod(outdir+"/maxtime",stat.S_IRWXU) os.chmod(outdir+"/params",stat.S_IRWXU) - os.chmod(outdir+"/unlock",stat.S_IRWXU) - + os.chmod(outdir+"/unlock",stat.S_IRWXU) \ No newline at end of file From 991373c53b789cb812dbb194279d6f45879ca9ec Mon Sep 17 00:00:00 2001 From: GREENRAT-K405 Date: Fri, 6 Feb 2026 01:33:19 +0530 Subject: [PATCH 032/275] complete shell escaping --- mkconcore.py | 73 +++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 52 insertions(+), 21 deletions(-) diff --git a/mkconcore.py b/mkconcore.py index 355c1df3..77f849fd 100644 --- a/mkconcore.py +++ b/mkconcore.py @@ -81,8 +81,8 @@ def safe_name(value, context): """ if not value: raise ValueError(f"{context} cannot be empty") - #blocks path traversal (/, \\) and shell metacharacters (*, ?, <, >, |, ;, &, $, `) - if re.search(r'[\\/:*?"<>|;&`$]', value): + # blocks path traversal (/, \), control characters, and shell metacharacters (*, ?, <, >, |, ;, &, $, `, ', ", (, )) + if re.search(r'[\x00-\x1F\x7F\\/:*?"<>|;&`$\'()]', value): raise ValueError(f"Unsafe {context}: '{value}' contains illegal characters.") return value @@ -130,6 +130,10 @@ def safe_name(value, context): prefixedgenode = "" sourcedir = sys.argv[2] outdir = sys.argv[3] + +# Validate outdir argument +safe_name(outdir, "Output directory argument") + if not os.path.isdir(sourcedir): logging.error(f"{sourcedir} does not exist") quit() @@ -223,6 +227,8 @@ def safe_name(value, context): safe_name(source_part, f"Node source file '{source_part}'") else: safe_name(node_label, f"Node label '{node_label}'") + # Explicitly reject incorrect format to prevent later crashes and ambiguity + raise ValueError(f"Invalid node label '{node_label}': expected format 'container:source' with a ':' separator.") nodes_dict[node['id']] = node_label node_id_to_label_map[node['id']] = node_label.split(':')[0] @@ -722,7 +728,9 @@ def safe_name(value, context): dockername = sourcecode.split(".")[0] #3/28/21 writeedges = volswr[i] while writeedges.find(":") != -1: - fclear.write(DOCKEREXE+' volume rm ' +writeedges.split(":")[0].split("-v")[1].strip()+"\n") # Added strip() + #scape volume path using shlex.quote for Docker commands (defense-in-depth) + volume_path = writeedges.split(":")[0].split("-v")[1].strip() + fclear.write(DOCKEREXE+' volume rm ' + shlex.quote(volume_path) +"\n") # Added strip() and quote writeedges = writeedges[writeedges.find(":")+1:] i=i+1 fclear.close() @@ -741,8 +749,10 @@ def safe_name(value, context): writeedges = volswr[i] while writeedges.find(":") != -1: fmaxtime.write(' -v ') - fmaxtime.write(writeedges.split(":")[0].split("-v ")[1].strip()+":/") - fmaxtime.write(writeedges.split(":")[0].split("-v ")[1].strip()) + # escape volume paths in Docker run + vol_path = writeedges.split(":")[0].split("-v ")[1].strip() + fmaxtime.write(shlex.quote(vol_path)+":/") + fmaxtime.write(shlex.quote(vol_path)) writeedges = writeedges[writeedges.find(":")+1:] i=i+1 fmaxtime.write(' docker-concore >/dev/null &\n') @@ -756,7 +766,9 @@ def safe_name(value, context): writeedges = volswr[i] while writeedges.find(":") != -1: fmaxtime.write('sudo docker cp concore.maxtime concore:/') - fmaxtime.write(writeedges.split(":")[0].split("-v ")[1].strip()+"/concore.maxtime\n") + # escape destination path in docker cp + vol_path = writeedges.split(":")[0].split("-v ")[1].strip() + fmaxtime.write(shlex.quote(vol_path+"/concore.maxtime")+"\n") writeedges = writeedges[writeedges.find(":")+1:] i=i+1 fmaxtime.write('sudo docker stop concore \n') @@ -780,8 +792,10 @@ def safe_name(value, context): writeedges = volswr[i] while writeedges.find(":") != -1: fparams.write(' -v ') - fparams.write(writeedges.split(":")[0].split("-v ")[1].strip()+":/") - fparams.write(writeedges.split(":")[0].split("-v ")[1].strip()) + #escape volume paths + vol_path = writeedges.split(":")[0].split("-v ")[1].strip() + fparams.write(shlex.quote(vol_path)+":/") + fparams.write(shlex.quote(vol_path)) writeedges = writeedges[writeedges.find(":")+1:] i=i+1 fparams.write(' docker-concore >/dev/null &\n') @@ -795,7 +809,9 @@ def safe_name(value, context): writeedges = volswr[i] while writeedges.find(":") != -1: fparams.write('sudo docker cp concore.params concore:/') - fparams.write(writeedges.split(":")[0].split("-v ")[1].strip()+"/concore.params\n") + # escape destination path + vol_path = writeedges.split(":")[0].split("-v ")[1].strip() + fparams.write(shlex.quote(vol_path+"/concore.params")+"\n") writeedges = writeedges[writeedges.find(":")+1:] i=i+1 fparams.write('sudo docker stop concore \n') @@ -818,8 +834,10 @@ def safe_name(value, context): writeedges = volswr[i] while writeedges.find(":") != -1: funlock.write(' -v ') - funlock.write(writeedges.split(":")[0].split("-v ")[1].strip()+":/") - funlock.write(writeedges.split(":")[0].split("-v ")[1].strip()) + # escape volume paths + vol_path = writeedges.split(":")[0].split("-v ")[1].strip() + funlock.write(shlex.quote(vol_path)+":/") + funlock.write(shlex.quote(vol_path)) writeedges = writeedges[writeedges.find(":")+1:] i=i+1 funlock.write(' docker-concore >/dev/null &\n') @@ -833,7 +851,9 @@ def safe_name(value, context): writeedges = volswr[i] while writeedges.find(":") != -1: funlock.write('sudo docker cp ~/concore.apikey concore:/') - funlock.write(writeedges.split(":")[0].split("-v ")[1].strip()+"/concore.apikey\n") + # escape destination path + vol_path = writeedges.split(":")[0].split("-v ")[1].strip() + funlock.write(shlex.quote(vol_path+"/concore.apikey")+"\n") writeedges = writeedges[writeedges.find(":")+1:] i=i+1 funlock.write('sudo docker stop concore \n') @@ -850,7 +870,8 @@ def safe_name(value, context): dockername,langext = sourcecode.split(".") # safe_container added to debug line (POSIX) safe_container = shlex.quote(containername) - fdebug.write(DOCKEREXE+' run -it --name='+safe_container+volswr[i]+volsro[i]+" docker-"+dockername+"&\n") + safe_image = shlex.quote("docker-" + dockername) # escape docker image name + fdebug.write(DOCKEREXE+' run -it --name='+safe_container+volswr[i]+volsro[i]+" "+safe_image+"&\n") i=i+1 fdebug.close() os.chmod(outdir+"/build",stat.S_IRWXU) @@ -994,12 +1015,13 @@ def safe_name(value, context): fdebug.write('start /D '+q_container+' cmd /K vvp a.out\n') #fdebug.write('start /D '+containername+' cmd /K "'+CPPWIN+' '+sourcecode+'|a"\n') elif langext=="m": #3/23/21 + # Use q_source in Windows commands to ensure quoting consistency if M_IS_OCTAVE: - frun.write('start /B /D '+q_container+" "+OCTAVEWIN+' -qf --eval "run('+"'"+sourcecode+"'"+')"'+" >"+q_container+"\\concoreout.txt\n") - fdebug.write('start /D '+q_container+" cmd /K " +OCTAVEWIN+' -qf --eval "run('+"'"+sourcecode+"'"+')"'+"\n") + frun.write('start /B /D '+q_container+" "+OCTAVEWIN+' -qf --eval "run('+q_source+')"'+" >"+q_container+"\\concoreout.txt\n") + fdebug.write('start /D '+q_container+" cmd /K " +OCTAVEWIN+' -qf --eval "run('+q_source+')"'+"\n") else: # 4/2/21 - frun.write('start /B /D '+q_container+" "+MATLABWIN+' -batch "run('+"'"+sourcecode+"'"+')"'+" >"+q_container+"\\concoreout.txt\n") - fdebug.write('start /D '+q_container+" cmd /K " +MATLABWIN+' -batch "run('+"'"+sourcecode+"'"+')"'+"\n") + frun.write('start /B /D '+q_container+" "+MATLABWIN+' -batch "run('+q_source+')"'+" >"+q_container+"\\concoreout.txt\n") + fdebug.write('start /D '+q_container+" cmd /K " +MATLABWIN+' -batch "run('+q_source+')"'+"\n") else: #use shlex.quote for POSIX systems safe_container = shlex.quote(containername) @@ -1034,15 +1056,22 @@ def safe_name(value, context): fdebug.write('osascript -e "tell application \\"Terminal\\" to do script \\"cd \\\\\\"$concorewd/' + safe_container + '\\\\\\"; ' + VEXE + ' ' + safe_source + '; vvp a.out\\"" \n') elif langext == "sh": # 5/19/21 - frun.write('(cd ' + safe_container + '; ./' + safe_source + ' ' + MCRPATH + ' >concoreout.txt & echo $! >concorepid) &\n') + # FIX: Escape MCRPATH to prevent shell injection + safe_mcr = shlex.quote(MCRPATH) + frun.write('(cd ' + safe_container + '; ./' + safe_source + ' ' + safe_mcr + ' >concoreout.txt & echo $! >concorepid) &\n') if ubuntu: fdebug.write('concorewd="$(pwd)"\n') - fdebug.write('xterm -e bash -c "cd \\"$concorewd/' + safe_container + '\\"; ./' + safe_source + ' ' + MCRPATH + '; bash" &\n') + fdebug.write('xterm -e bash -c "cd \\"$concorewd/' + safe_container + '\\"; ./' + safe_source + ' ' + safe_mcr + '; bash" &\n') else: fdebug.write('concorewd="$(pwd)"\n') - fdebug.write('osascript -e "tell application \\"Terminal\\" to do script \\"cd \\\\\\"$concorewd/' + safe_container + '\\\\\\"; ./' + safe_source + ' ' + MCRPATH + '\\"" \n') + fdebug.write('osascript -e "tell application \\"Terminal\\" to do script \\"cd \\\\\\"$concorewd/' + safe_container + '\\\\\\"; ./' + safe_source + ' ' + safe_mcr + '\\"" \n') elif langext == "m": #3/23/21 + # FIX: Verify filename safety for MATLAB to prevent injection in run() + # MATLAB/Octave run('filename') is vulnerable if filename contains quotes or metachars. + if not re.match(r'^[A-Za-z0-9_./\-]+$', sourcecode): + raise ValueError(f"Invalid MATLAB/Octave source file name: {sourcecode!r}") + # construct safe eval command safe_eval_cmd = shlex.quote(f"run('{sourcecode}')") if M_IS_OCTAVE: @@ -1093,7 +1122,9 @@ def safe_name(value, context): if concoretype=="windows": fclear.write('del /Q "' + path_part + '\\*"\n') else: - fclear.write('rm ' + shlex.quote(path_part) + '/*\n') + # FIX: Safer wildcard removal. + # Avoid quoting the wildcard itself ('path/*'). Instead cd into directory and remove contents. + fclear.write(f'cd {shlex.quote(path_part)} && rm -f *\n') writeedges = writeedges[writeedges.find(":")+1:] i=i+1 fclear.close() From b76aa456a2e80b278a5ee77d4589b9e7ff9cde6b Mon Sep 17 00:00:00 2001 From: Titas-Ghosh Date: Sat, 7 Feb 2026 01:32:32 +0530 Subject: [PATCH 033/275] Fix: Resolve absolute paths in run.py to support execution outside repo root --- concore_cli/commands/run.py | 16 ++++++++++++++-- mkconcore.py | 21 ++++++++++++++++----- tests/test_cli.py | 17 ++++++++++++++++- 3 files changed, 46 insertions(+), 8 deletions(-) diff --git a/concore_cli/commands/run.py b/concore_cli/commands/run.py index c6aaed2f..91a876b7 100644 --- a/concore_cli/commands/run.py +++ b/concore_cli/commands/run.py @@ -5,10 +5,17 @@ from rich.panel import Panel from rich.progress import Progress, SpinnerColumn, TextColumn +def _find_mkconcore_path(): + for parent in Path(__file__).resolve().parents: + candidate = parent / "mkconcore.py" + if candidate.exists(): + return candidate + return None + def run_workflow(workflow_file, source, output, exec_type, auto_build, console): workflow_path = Path(workflow_file).resolve() source_path = Path(source).resolve() - output_path = Path(output) + output_path = Path(output).resolve() if not source_path.exists(): raise FileNotFoundError(f"Source directory '{source}' not found") @@ -24,6 +31,10 @@ def run_workflow(workflow_file, source, output, exec_type, auto_build, console): console.print(f"[cyan]Type:[/cyan] {exec_type}") console.print() + mkconcore_path = _find_mkconcore_path() + if mkconcore_path is None: + raise FileNotFoundError("mkconcore.py not found. Please install concore from source.") + with Progress( SpinnerColumn(), TextColumn("[progress.description]{task.description}"), @@ -33,7 +44,8 @@ def run_workflow(workflow_file, source, output, exec_type, auto_build, console): try: result = subprocess.run( - [sys.executable, 'mkconcore.py', str(workflow_path), str(source_path), str(output_path), exec_type], + [sys.executable, str(mkconcore_path), str(workflow_path), str(source_path), str(output_path), exec_type], + cwd=mkconcore_path.parent, capture_output=True, text=True, check=True diff --git a/mkconcore.py b/mkconcore.py index 10135dc9..e630dd9c 100644 --- a/mkconcore.py +++ b/mkconcore.py @@ -72,11 +72,22 @@ import copy_with_port_portname import numpy as np -MKCONCORE_VER = "22-09-18" - -GRAPHML_FILE = sys.argv[1] -TRIMMED_LOGS = True -CONCOREPATH = "." +MKCONCORE_VER = "22-09-18" + +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) + +def _resolve_concore_path(): + script_concore = os.path.join(SCRIPT_DIR, "concore.py") + if os.path.exists(script_concore): + return SCRIPT_DIR + cwd_concore = os.path.join(os.getcwd(), "concore.py") + if os.path.exists(cwd_concore): + return os.getcwd() + return SCRIPT_DIR + +GRAPHML_FILE = sys.argv[1] +TRIMMED_LOGS = True +CONCOREPATH = _resolve_concore_path() CPPWIN = "g++" #Windows C++ 6/22/21 CPPEXE = "g++" #Ubuntu/macOS C++ 6/22/21 VWIN = "iverilog" #Windows verilog 6/25/21 diff --git a/tests/test_cli.py b/tests/test_cli.py index af31a36a..a90cc3fe 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -67,7 +67,22 @@ def test_run_command_missing_source(self): result = self.runner.invoke(cli, ['init', 'test-project']) result = self.runner.invoke(cli, ['run', 'test-project/workflow.graphml', '--source', 'nonexistent']) self.assertNotEqual(result.exit_code, 0) - + + def test_run_command_from_project_dir(self): + with self.runner.isolated_filesystem(temp_dir=self.temp_dir): + result = self.runner.invoke(cli, ['init', 'test-project']) + self.assertEqual(result.exit_code, 0) + + result = self.runner.invoke(cli, [ + 'run', + 'test-project/workflow.graphml', + '--source', 'test-project/src', + '--output', 'out', + '--type', 'posix' + ]) + self.assertEqual(result.exit_code, 0) + self.assertTrue(Path('out/src/concore.py').exists()) + def test_run_command_existing_output(self): with self.runner.isolated_filesystem(temp_dir=self.temp_dir): result = self.runner.invoke(cli, ['init', 'test-project']) From a42d3ff75a4d5f7765a8332c3186e4fc36cddd8f Mon Sep 17 00:00:00 2001 From: GREENRAT-K405 Date: Sun, 8 Feb 2026 00:30:27 +0530 Subject: [PATCH 034/275] add tests for gaphml with strict xml checks --- concore_cli/commands/validate.py | 40 +++++++++- tests/test_graph.py | 133 +++++++++++++++++++++++++++++++ 2 files changed, 170 insertions(+), 3 deletions(-) create mode 100644 tests/test_graph.py diff --git a/concore_cli/commands/validate.py b/concore_cli/commands/validate.py index e920f119..fa1ea184 100644 --- a/concore_cli/commands/validate.py +++ b/concore_cli/commands/validate.py @@ -3,6 +3,7 @@ from rich.panel import Panel from rich.table import Table import re +import xml.etree.ElementTree as ET def validate_workflow(workflow_file, console): workflow_path = Path(workflow_file) @@ -22,15 +23,34 @@ def validate_workflow(workflow_file, console): errors.append("File is empty") return show_results(console, errors, warnings, info) + # strict XML syntax check + try: + ET.fromstring(content) + except ET.ParseError as e: + errors.append(f"Invalid XML: {str(e)}") + return show_results(console, errors, warnings, info) + try: soup = BeautifulSoup(content, 'xml') except Exception as e: errors.append(f"Invalid XML: {str(e)}") return show_results(console, errors, warnings, info) - if not soup.find('graphml'): + root = soup.find('graphml') + if not root: errors.append("Not a valid GraphML file - missing root element") return show_results(console, errors, warnings, info) + + # check the graph attributes + graph = soup.find('graph') + if not graph: + errors.append("Missing element") + else: + edgedefault = graph.get('edgedefault') + if not edgedefault: + errors.append("Graph missing required 'edgedefault' attribute") + elif edgedefault not in ['directed', 'undirected']: + errors.append(f"Invalid edgedefault value '{edgedefault}' (must be 'directed' or 'undirected')") nodes = soup.find_all('node') edges = soup.find_all('edge') @@ -47,8 +67,19 @@ def validate_workflow(workflow_file, console): node_labels = [] for node in nodes: + #check the node id + node_id = node.get('id') + if not node_id: + errors.append("Node missing required 'id' attribute") + #skip further checks for this node to avoid noise + continue + try: + #robust find: try with namespace prefix first, then without label_tag = node.find('y:NodeLabel') + if not label_tag: + label_tag = node.find('NodeLabel') + if label_tag and label_tag.text: label = label_tag.text.strip() node_labels.append(label) @@ -60,13 +91,13 @@ def validate_workflow(workflow_file, console): if len(parts) != 2: warnings.append(f"Node '{label}' has invalid format") else: - node_id, filename = parts + nodeId_part, filename = parts if not filename: errors.append(f"Node '{label}' has no filename") elif not any(filename.endswith(ext) for ext in ['.py', '.cpp', '.m', '.v', '.java']): warnings.append(f"Node '{label}' has unusual file extension") else: - warnings.append(f"Node {node.get('id', 'unknown')} has no label") + warnings.append(f"Node {node_id} has no label") except Exception as e: warnings.append(f"Error parsing node: {str(e)}") @@ -91,6 +122,9 @@ def validate_workflow(workflow_file, console): for edge in edges: try: label_tag = edge.find('y:EdgeLabel') + if not label_tag: + label_tag = edge.find('EdgeLabel') + if label_tag and label_tag.text: if edge_label_regex.match(label_tag.text.strip()): zmq_edges += 1 diff --git a/tests/test_graph.py b/tests/test_graph.py new file mode 100644 index 00000000..97102dce --- /dev/null +++ b/tests/test_graph.py @@ -0,0 +1,133 @@ +import unittest +import tempfile +import shutil +from pathlib import Path +from click.testing import CliRunner +from concore_cli.cli import cli + +class TestGraphValidation(unittest.TestCase): + + def setUp(self): + self.runner = CliRunner() + self.temp_dir = tempfile.mkdtemp() + + def tearDown(self): + if Path(self.temp_dir).exists(): + shutil.rmtree(self.temp_dir) + + def create_graph_file(self, filename, content): + filepath = Path(self.temp_dir) / filename + with open(filepath, 'w') as f: + f.write(content) + return str(filepath) + + def test_validate_corrupted_xml(self): + content = '' + filepath = self.create_graph_file('corrupted.graphml', content) + + result = self.runner.invoke(cli, ['validate', filepath]) + + self.assertIn('Validation failed', result.output) + self.assertIn('Invalid XML', result.output) + + def test_validate_empty_file(self): + filepath = self.create_graph_file('empty.graphml', '') + + result = self.runner.invoke(cli, ['validate', filepath]) + + self.assertIn('Validation failed', result.output) + self.assertIn('File is empty', result.output) + + def test_validate_missing_node_id(self): + content = ''' + + + + n0:script.py + + + + ''' + filepath = self.create_graph_file('missing_id.graphml', content) + result = self.runner.invoke(cli, ['validate', filepath]) + self.assertIn('Validation failed', result.output) + self.assertIn("Node missing required 'id' attribute", result.output) + + def test_validate_missing_edgedefault(self): + content = ''' + + + + n0:script.py + + + + ''' + filepath = self.create_graph_file('missing_default.graphml', content) + result = self.runner.invoke(cli, ['validate', filepath]) + self.assertIn('Validation failed', result.output) + self.assertIn("Graph missing required 'edgedefault'", result.output) + + def test_validate_missing_root_element(self): + content = '' + filepath = self.create_graph_file('not_graphml.xml', content) + + result = self.runner.invoke(cli, ['validate', filepath]) + + self.assertIn('Validation failed', result.output) + self.assertIn('missing root element', result.output) + + def test_validate_broken_edges(self): + content = ''' + + + + n0:script.py + + + + + ''' + filepath = self.create_graph_file('bad_edge.graphml', content) + + result = self.runner.invoke(cli, ['validate', filepath]) + + self.assertIn('Validation failed', result.output) + self.assertIn('Edge references non-existent target node', result.output) + + def test_validate_node_missing_filename(self): + content = ''' + + + + n0: + + + + ''' + filepath = self.create_graph_file('bad_node.graphml', content) + + result = self.runner.invoke(cli, ['validate', filepath]) + + self.assertIn('Validation failed', result.output) + self.assertIn('has no filename', result.output) + + def test_validate_valid_graph(self): + content = ''' + + + + n0:script.py + + + + ''' + filepath = self.create_graph_file('valid.graphml', content) + + result = self.runner.invoke(cli, ['validate', filepath]) + + self.assertIn('Validation passed', result.output) + self.assertIn('Workflow is valid', result.output) + +if __name__ == '__main__': + unittest.main() \ No newline at end of file From dcb8de061fcdbec7d90b24cc0f24620c4a6e3d80 Mon Sep 17 00:00:00 2001 From: Avinash Kumar Deepak Date: Sun, 8 Feb 2026 12:54:35 +0530 Subject: [PATCH 035/275] Add test coverage for concoredocker.py --- tests/test_concoredocker.py | 153 ++++++++++++++++++++++++++++++++++++ 1 file changed, 153 insertions(+) create mode 100644 tests/test_concoredocker.py diff --git a/tests/test_concoredocker.py b/tests/test_concoredocker.py new file mode 100644 index 00000000..7677f0ae --- /dev/null +++ b/tests/test_concoredocker.py @@ -0,0 +1,153 @@ +import os + + +class TestSafeLiteralEval: + + def test_reads_dictionary_from_file(self, temp_dir): + test_file = os.path.join(temp_dir, "ports.txt") + with open(test_file, "w") as f: + f.write("{'a': 1, 'b': 2}") + + from concoredocker import safe_literal_eval + result = safe_literal_eval(test_file, {}) + + assert result == {'a': 1, 'b': 2} + + def test_reads_list_from_file(self, temp_dir): + test_file = os.path.join(temp_dir, "data.txt") + with open(test_file, "w") as f: + f.write("[1, 2, 3]") + + from concoredocker import safe_literal_eval + result = safe_literal_eval(test_file, []) + + assert result == [1, 2, 3] + + def test_returns_default_when_file_missing(self): + from concoredocker import safe_literal_eval + result = safe_literal_eval("/nonexistent.txt", {'default': True}) + + assert result == {'default': True} + + def test_returns_default_for_bad_syntax(self, temp_dir): + test_file = os.path.join(temp_dir, "bad.txt") + with open(test_file, "w") as f: + f.write("not valid {{{") + + from concoredocker import safe_literal_eval + result = safe_literal_eval(test_file, "fallback") + + assert result == "fallback" + + +class TestUnchanged: + + def test_returns_true_when_unchanged(self): + import concoredocker + concoredocker.s = "abc" + concoredocker.olds = "abc" + + assert concoredocker.unchanged() == True + assert concoredocker.s == '' + + def test_returns_false_when_changed(self): + import concoredocker + concoredocker.s = "new" + concoredocker.olds = "old" + + assert concoredocker.unchanged() == False + assert concoredocker.olds == "new" + + +class TestInitval: + + def test_parses_simtime_and_values(self): + import concoredocker + concoredocker.simtime = 0 + result = concoredocker.initval("[5.0, 1.0, 2.0]") + + assert result == [1.0, 2.0] + assert concoredocker.simtime == 5.0 + + def test_parses_single_value(self): + import concoredocker + concoredocker.simtime = 0 + result = concoredocker.initval("[10.0, 99]") + + assert result == [99] + assert concoredocker.simtime == 10.0 + + +class TestWrite: + + def test_writes_list_with_simtime(self, temp_dir): + import concoredocker + old_outpath = concoredocker.outpath + outdir = os.path.join(temp_dir, "1") + os.makedirs(outdir) + concoredocker.outpath = temp_dir + concoredocker.simtime = 5.0 + + concoredocker.write(1, "testfile", [1.0, 2.0], delta=0) + + with open(os.path.join(outdir, "testfile")) as f: + content = f.read() + assert content == "[5.0, 1.0, 2.0]" + concoredocker.outpath = old_outpath + + def test_writes_with_delta(self, temp_dir): + import concoredocker + old_outpath = concoredocker.outpath + outdir = os.path.join(temp_dir, "1") + os.makedirs(outdir) + concoredocker.outpath = temp_dir + concoredocker.simtime = 10.0 + + concoredocker.write(1, "testfile", [3.0], delta=2) + + with open(os.path.join(outdir, "testfile")) as f: + content = f.read() + assert content == "[12.0, 3.0]" + assert concoredocker.simtime == 12.0 + concoredocker.outpath = old_outpath + + +class TestRead: + + def test_reads_and_parses_data(self, temp_dir): + import concoredocker + old_inpath = concoredocker.inpath + old_delay = concoredocker.delay + indir = os.path.join(temp_dir, "1") + os.makedirs(indir) + concoredocker.inpath = temp_dir + concoredocker.delay = 0.001 + + with open(os.path.join(indir, "data"), 'w') as f: + f.write("[7.0, 100, 200]") + + concoredocker.s = '' + concoredocker.simtime = 0 + result = concoredocker.read(1, "data", "[0, 0, 0]") + + assert result == [100, 200] + assert concoredocker.simtime == 7.0 + concoredocker.inpath = old_inpath + concoredocker.delay = old_delay + + def test_returns_default_when_file_missing(self, temp_dir): + import concoredocker + old_inpath = concoredocker.inpath + old_delay = concoredocker.delay + indir = os.path.join(temp_dir, "1") + os.makedirs(indir) + concoredocker.inpath = temp_dir + concoredocker.delay = 0.001 + + concoredocker.s = '' + concoredocker.simtime = 0 + result = concoredocker.read(1, "nofile", "[0, 5, 5]") + + assert result == [5, 5] + concoredocker.inpath = old_inpath + concoredocker.delay = old_delay From 1c009f2f2c70b512b2145a1bf169e1fbc6ee8a26 Mon Sep 17 00:00:00 2001 From: Avinash Kumar Deepak Date: Sun, 8 Feb 2026 14:43:49 +0530 Subject: [PATCH 036/275] Register terminate_zmq with atexit --- concore.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/concore.py b/concore.py index bf804e4e..6bf253e4 100644 --- a/concore.py +++ b/concore.py @@ -1,6 +1,7 @@ import time import logging import os +import atexit from ast import literal_eval import sys import re @@ -104,6 +105,8 @@ def terminate_zmq(): port.context.term() except Exception as e: logging.error(f"Error while terminating ZMQ port {port.address}: {e}") + +atexit.register(terminate_zmq) # --- ZeroMQ Integration End --- From 048029f437e881fbb743e5f44efee9f181906eb4 Mon Sep 17 00:00:00 2001 From: Avinash Kumar Deepak Date: Sun, 8 Feb 2026 14:50:53 +0530 Subject: [PATCH 037/275] Add pyzmq dependency --- requirements.txt | 1 + setup.py | 1 + 2 files changed, 2 insertions(+) diff --git a/requirements.txt b/requirements.txt index 897bdf6c..9a3554f4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,6 +2,7 @@ beautifulsoup4 lxml tensorflow numpy +pyzmq scipy matplotlib cvxopt diff --git a/setup.py b/setup.py index 74951343..f7bbb866 100644 --- a/setup.py +++ b/setup.py @@ -27,6 +27,7 @@ "beautifulsoup4", "lxml", "numpy", + "pyzmq", "scipy", "matplotlib", "click>=8.0.0", From 71b496832da11acf800a6804195000e3a3b5c829 Mon Sep 17 00:00:00 2001 From: GREENRAT-K405 Date: Sun, 8 Feb 2026 16:34:53 +0530 Subject: [PATCH 038/275] update global logging level to info, change api logging to debug --- concore.py | 2 +- tools/cwrap.py | 2 +- tools/pwrap.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/concore.py b/concore.py index 7a99b8f3..32e81de4 100644 --- a/concore.py +++ b/concore.py @@ -7,7 +7,7 @@ import zmq import numpy as np logging.basicConfig( - level=logging.DEBUG, + level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', force=True ) diff --git a/tools/cwrap.py b/tools/cwrap.py index 3523d00c..d1fda305 100644 --- a/tools/cwrap.py +++ b/tools/cwrap.py @@ -59,7 +59,7 @@ except: init_simtime_ym = "[0.0, 0.0, 0.0]" -logging.info(f"API Key: {apikey}") +logging.debug(f"API Key: {apikey}") logging.info(f"Yuyu: {yuyu}") logging.info(f"{name1}={init_simtime_u}") logging.info(f"{name2}={init_simtime_ym}") diff --git a/tools/pwrap.py b/tools/pwrap.py index 73f16513..82aea7da 100644 --- a/tools/pwrap.py +++ b/tools/pwrap.py @@ -62,7 +62,7 @@ except: init_simtime_ym = "[0.0, 0.0, 0.0]" -logging.info(f"API Key: {apikey}") +logging.debug(f"API Key: {apikey}") logging.info(f"Yuyu: {yuyu}") logging.info(f"{name1}={init_simtime_u}") logging.info(f"{name2}={init_simtime_ym}") From f534d711ce990d4d00c54283b1727be250e3f2dd Mon Sep 17 00:00:00 2001 From: GREENRAT-K405 Date: Mon, 9 Feb 2026 00:48:14 +0530 Subject: [PATCH 039/275] update concoredocker.py to be up-to-date with concore.py --- concoredocker.py | 214 ++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 185 insertions(+), 29 deletions(-) diff --git a/concoredocker.py b/concoredocker.py index 65463c99..31ce6664 100644 --- a/concoredocker.py +++ b/concoredocker.py @@ -2,13 +2,112 @@ from ast import literal_eval import re import os +import logging +import zmq +import numpy as np + +logging.basicConfig( + level=logging.INFO, + format='%(levelname)s - %(message)s' +) + +class ZeroMQPort: + def __init__(self, port_type, address, zmq_socket_type): + self.context = zmq.Context() + self.socket = self.context.socket(zmq_socket_type) + self.port_type = port_type + self.address = address + + # Configure timeouts & immediate close on failure + self.socket.setsockopt(zmq.RCVTIMEO, 2000) # 2 sec receive timeout + self.socket.setsockopt(zmq.SNDTIMEO, 2000) # 2 sec send timeout + self.socket.setsockopt(zmq.LINGER, 0) # Drop pending messages on close + + # Bind or connect + if self.port_type == "bind": + self.socket.bind(address) + logging.info(f"ZMQ Port bound to {address}") + else: + self.socket.connect(address) + logging.info(f"ZMQ Port connected to {address}") + + def send_json_with_retry(self, message): + """Send JSON message with retries if timeout occurs.""" + for attempt in range(5): + try: + self.socket.send_json(message) + return + except zmq.Again: + logging.warning(f"Send timeout (attempt {attempt + 1}/5)") + time.sleep(0.5) + logging.error("Failed to send after retries.") + return + + def recv_json_with_retry(self): + """Receive JSON message with retries if timeout occurs.""" + for attempt in range(5): + try: + return self.socket.recv_json() + except zmq.Again: + logging.warning(f"Receive timeout (attempt {attempt + 1}/5)") + time.sleep(0.5) + logging.error("Failed to receive after retries.") + return None + +# Global ZeroMQ ports registry +zmq_ports = {} + +def init_zmq_port(port_name, port_type, address, socket_type_str): + """ + Initializes and registers a ZeroMQ port. + port_name (str): A unique name for this ZMQ port. + port_type (str): "bind" or "connect". + address (str): The ZMQ address (e.g., "tcp://*:5555", "tcp://localhost:5555"). + socket_type_str (str): String representation of ZMQ socket type (e.g., "REQ", "REP", "PUB", "SUB"). + """ + if port_name in zmq_ports: + logging.info(f"ZMQ Port {port_name} already initialized.") + return#avoid reinstallation + try: + # Map socket type string to actual ZMQ constant (e.g., zmq.REQ, zmq.REP) + zmq_socket_type = getattr(zmq, socket_type_str.upper()) + zmq_ports[port_name] = ZeroMQPort(port_type, address, zmq_socket_type) + logging.info(f"Initialized ZMQ port: {port_name} ({socket_type_str}) on {address}") + except AttributeError: + logging.error(f"Error: Invalid ZMQ socket type string '{socket_type_str}'.") + except zmq.error.ZMQError as e: + logging.error(f"Error initializing ZMQ port {port_name} on {address}: {e}") + except Exception as e: + logging.error(f"An unexpected error occurred during ZMQ port initialization for {port_name}: {e}") + +def terminate_zmq(): + for port in zmq_ports.values(): + try: + port.socket.close() + port.context.term() + except Exception as e: + logging.error(f"Error while terminating ZMQ port {port.address}: {e}") +# --- ZeroMQ Integration End --- + +# NumPy Type Conversion Helper +def convert_numpy_to_python(obj): + if isinstance(obj, np.generic): + return obj.item() + elif isinstance(obj, list): + return [convert_numpy_to_python(item) for item in obj] + elif isinstance(obj, tuple): + return tuple(convert_numpy_to_python(item) for item in obj) + elif isinstance(obj, dict): + return {key: convert_numpy_to_python(value) for key, value in obj.items()} + else: + return obj def safe_literal_eval(filename, defaultValue): try: with open(filename, "r") as file: return literal_eval(file.read()) except (FileNotFoundError, SyntaxError, ValueError, Exception) as e: - print(f"Error reading {filename}: {e}") + logging.error(f"Error reading {filename}: {e}") return defaultValue iport = safe_literal_eval("concore.iport", {}) @@ -21,8 +120,8 @@ def safe_literal_eval(filename, defaultValue): inpath = os.path.abspath("/in") outpath = os.path.abspath("/out") simtime = 0 -concore_params_file = os.path.join(inpath, "1", "concore.params") -concore_maxtime_file = os.path.join(inpath, "1", "concore.maxtime") +concore_params_file = os.path.join(inpath + "1", "concore.params") +concore_maxtime_file = os.path.join(inpath + "1", "concore.maxtime") #9/21/22 def parse_params(sparams): @@ -42,7 +141,7 @@ def parse_params(sparams): pass # keep backward compatibility: comma-separated params - for item in s.split(","): + for item in s.split(";"): if "=" in item: key, value = item.split("=", 1) key = key.strip() @@ -66,12 +165,13 @@ def parse_params(sparams): sparams = sparams[:sparams.find('"')] if sparams: - print("parsing sparams:", sparams) + logging.debug("parsing sparams: "+sparams) params = parse_params(sparams) + logging.debug("parsed params: " + str(params)) else: params = dict() except Exception as e: - print(f"Error reading concore.params: {e}") + logging.error(f"Error reading concore.params: {e}") params = dict() #9/30/22 @@ -93,74 +193,130 @@ def unchanged(): olds = s return False -def read(port, name, initstr): +def read(port_identifier, name, initstr_val): global s, simtime, retrycount - max_retries=5 + + default_return_val = initstr_val + if isinstance(initstr_val, str): + try: + default_return_val = literal_eval(initstr_val) + except (SyntaxError, ValueError): + pass + + if isinstance(port_identifier, str) and port_identifier in zmq_ports: + zmq_p = zmq_ports[port_identifier] + try: + message = zmq_p.recv_json_with_retry() + return message + except zmq.error.ZMQError as e: + logging.error(f"ZMQ read error on port {port_identifier} (name: {name}): {e}. Returning default.") + return default_return_val + except Exception as e: + logging.error(f"Unexpected error during ZMQ read on port {port_identifier} (name: {name}): {e}. Returning default.") + return default_return_val + + try: + file_port_num = int(port_identifier) + except ValueError: + logging.error(f"Error: Invalid port identifier '{port_identifier}' for file operation. Must be integer or ZMQ name.") + return default_return_val + time.sleep(delay) - file_path = os.path.join(inpath, str(port), name) + file_path = os.path.join(inpath, str(file_port_num), name) try: with open(file_path, "r") as infile: ins = infile.read() except FileNotFoundError: - print(f"File {file_path} not found, using default value.") - ins = initstr + ins = str(initstr_val) + s += ins except Exception as e: - print(f"Error reading {file_path}: {e}") - return initstr + logging.error(f"Error reading {file_path}: {e}") + return default_return_val attempts = 0 + max_retries = 5 while len(ins) == 0 and attempts < max_retries: time.sleep(delay) try: with open(file_path, "r") as infile: ins = infile.read() except Exception as e: - print(f"Retry {attempts + 1}: Error reading {file_path} - {e}") + logging.warning(f"Retry {attempts + 1}: Error reading {file_path} - {e}") attempts += 1 retrycount += 1 if len(ins) == 0: - print(f"Max retries reached for {file_path}, using default value.") - return initstr + logging.error(f"Max retries reached for {file_path}, using default value.") + return default_return_val s += ins try: inval = literal_eval(ins) - simtime = max(simtime, inval[0]) - return inval[1:] + if isinstance(inval, list) and len(inval) > 0: + current_simtime_from_file = inval[0] + if isinstance(current_simtime_from_file, (int, float)): + simtime = max(simtime, current_simtime_from_file) + return inval[1:] + else: + logging.warning(f"Warning: Unexpected data format in {file_path}: {ins}. Returning raw content or default.") + return inval except Exception as e: - print(f"Error parsing {ins}: {e}") - return initstr + logging.error(f"Error parsing {ins}: {e}") + return default_return_val - -def write(port, name, val, delta=0): +def write(port_identifier, name, val, delta=0): global simtime - file_path = os.path.join(outpath, str(port), name) + + if isinstance(port_identifier, str) and port_identifier in zmq_ports: + zmq_p = zmq_ports[port_identifier] + try: + zmq_p.send_json_with_retry(val) + except zmq.error.ZMQError as e: + logging.error(f"ZMQ write error on port {port_identifier} (name: {name}): {e}") + except Exception as e: + logging.error(f"Unexpected error during ZMQ write on port {port_identifier} (name: {name}): {e}") + + try: + file_port_num = int(port_identifier) + file_path = os.path.join(outpath, str(file_port_num), name) + except ValueError: + logging.error(f"Error: Invalid port identifier '{port_identifier}' for file operation. Must be integer or ZMQ name.") + return if isinstance(val, str): time.sleep(2 * delay) elif not isinstance(val, list): - print("write must have list or str") + logging.error("write must have list or str") return try: with open(file_path, "w") as outfile: if isinstance(val, list): - outfile.write(str([simtime + delta] + val)) + val_converted = convert_numpy_to_python(val) + outfile.write(str([simtime + delta] + val_converted)) simtime += delta else: outfile.write(val) except Exception as e: - print(f"Error writing to {file_path}: {e}") + logging.error(f"Error writing to {file_path}: {e}") def initval(simtime_val): global simtime try: val = literal_eval(simtime_val) - simtime = val[0] - return val[1:] + if isinstance(val, list) and len(val) > 0: + first_element = val[0] + if isinstance(first_element, (int, float)): + simtime = first_element + return val[1:] + else: + logging.error(f"Error: First element in initval string '{simtime_val}' is not a number.") + return val[1:] if len(val) > 1 else [] + else: + logging.error(f"Error: initval string '{simtime_val}' is not a list or is empty.") + return [] except Exception as e: - print(f"Error parsing simtime_val: {e}") + logging.error(f"Error parsing simtime_val: {e}") return [] From 2bad98f9476f707ea3d996b47d38e9b2c500d8f1 Mon Sep 17 00:00:00 2001 From: GREENRAT-K405 Date: Mon, 9 Feb 2026 00:55:18 +0530 Subject: [PATCH 040/275] keep the logs consistent with concore.py --- concoredocker.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/concoredocker.py b/concoredocker.py index 31ce6664..880a774a 100644 --- a/concoredocker.py +++ b/concoredocker.py @@ -231,7 +231,7 @@ def read(port_identifier, name, initstr_val): ins = str(initstr_val) s += ins except Exception as e: - logging.error(f"Error reading {file_path}: {e}") + logging.error(f"Error reading {file_path}: {e}. Using default value.") return default_return_val attempts = 0 @@ -262,7 +262,7 @@ def read(port_identifier, name, initstr_val): logging.warning(f"Warning: Unexpected data format in {file_path}: {ins}. Returning raw content or default.") return inval except Exception as e: - logging.error(f"Error parsing {ins}: {e}") + logging.error(f"Error parsing content from {file_path} ('{ins}'): {e}. Returning default.") return default_return_val def write(port_identifier, name, val, delta=0): @@ -287,14 +287,15 @@ def write(port_identifier, name, val, delta=0): if isinstance(val, str): time.sleep(2 * delay) elif not isinstance(val, list): - logging.error("write must have list or str") + logging.error(f"File write to {file_path} must have list or str value, got {type(val)}") return try: with open(file_path, "w") as outfile: if isinstance(val, list): val_converted = convert_numpy_to_python(val) - outfile.write(str([simtime + delta] + val_converted)) + data_to_write = [simtime + delta] + val_converted + outfile.write(str(data_to_write)) simtime += delta else: outfile.write(val) @@ -311,12 +312,12 @@ def initval(simtime_val): simtime = first_element return val[1:] else: - logging.error(f"Error: First element in initval string '{simtime_val}' is not a number.") + logging.error(f"Error: First element in initval string '{simtime_val}' is not a number. Using data part as is or empty.") return val[1:] if len(val) > 1 else [] else: - logging.error(f"Error: initval string '{simtime_val}' is not a list or is empty.") + logging.error(f"Error: initval string '{simtime_val}' is not a list or is empty. Returning empty list.") return [] except Exception as e: - logging.error(f"Error parsing simtime_val: {e}") + logging.error(f"Error parsing simtime_val_str '{simtime_val}': {e}. Returning empty list.") return [] From 073c6a6038bbeab607fc19697805a2c6976034af Mon Sep 17 00:00:00 2001 From: PARAM KANADA Date: Mon, 9 Feb 2026 01:13:13 +0530 Subject: [PATCH 041/275] Update concoredocker.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- concoredocker.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/concoredocker.py b/concoredocker.py index 880a774a..3e296f5e 100644 --- a/concoredocker.py +++ b/concoredocker.py @@ -222,7 +222,8 @@ def read(port_identifier, name, initstr_val): return default_return_val time.sleep(delay) - file_path = os.path.join(inpath, str(file_port_num), name) + # Construct file path consistent with other components (e.g., /in1/) + file_path = os.path.join(inpath + str(file_port_num), name) try: with open(file_path, "r") as infile: From b6bae58b48ba32d1992c526ad373a8be7cbeb63d Mon Sep 17 00:00:00 2001 From: PARAM KANADA Date: Mon, 9 Feb 2026 01:15:31 +0530 Subject: [PATCH 042/275] Update concoredocker.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- concoredocker.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/concoredocker.py b/concoredocker.py index 3e296f5e..1a3f6b43 100644 --- a/concoredocker.py +++ b/concoredocker.py @@ -201,7 +201,11 @@ def read(port_identifier, name, initstr_val): try: default_return_val = literal_eval(initstr_val) except (SyntaxError, ValueError): - pass + # Failed to parse initstr_val; fall back to the original string value. + logging.debug( + "Could not parse initstr_val %r with literal_eval; using raw string as default.", + initstr_val + ) if isinstance(port_identifier, str) and port_identifier in zmq_ports: zmq_p = zmq_ports[port_identifier] From f597ad0cce9164daf6cebe7874de7b8e118272c4 Mon Sep 17 00:00:00 2001 From: PARAM KANADA Date: Mon, 9 Feb 2026 01:21:21 +0530 Subject: [PATCH 043/275] Update concoredocker.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- concoredocker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/concoredocker.py b/concoredocker.py index 1a3f6b43..c95a1782 100644 --- a/concoredocker.py +++ b/concoredocker.py @@ -284,7 +284,7 @@ def write(port_identifier, name, val, delta=0): try: file_port_num = int(port_identifier) - file_path = os.path.join(outpath, str(file_port_num), name) + file_path = os.path.join(outpath + str(file_port_num), name) except ValueError: logging.error(f"Error: Invalid port identifier '{port_identifier}' for file operation. Must be integer or ZMQ name.") return From f9dd68880a4b4bdea5a936119e916e821dea7107 Mon Sep 17 00:00:00 2001 From: PARAM KANADA Date: Mon, 9 Feb 2026 01:25:12 +0530 Subject: [PATCH 044/275] Change error logging to info level for file read --- concoredocker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/concoredocker.py b/concoredocker.py index c95a1782..18e00785 100644 --- a/concoredocker.py +++ b/concoredocker.py @@ -107,7 +107,7 @@ def safe_literal_eval(filename, defaultValue): with open(filename, "r") as file: return literal_eval(file.read()) except (FileNotFoundError, SyntaxError, ValueError, Exception) as e: - logging.error(f"Error reading {filename}: {e}") + logging.info(f"Error reading {filename}: {e}") return defaultValue iport = safe_literal_eval("concore.iport", {}) From 1e765584f443f42796deda865d5b607ccea9ac83 Mon Sep 17 00:00:00 2001 From: PARAM KANADA Date: Mon, 9 Feb 2026 02:04:30 +0530 Subject: [PATCH 045/275] Update regarding moving away from backward compatibility for params Updated comment to reflect potential breaking change regarding comma-separated parameters. --- concoredocker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/concoredocker.py b/concoredocker.py index 18e00785..d337c388 100644 --- a/concoredocker.py +++ b/concoredocker.py @@ -140,7 +140,7 @@ def parse_params(sparams): except (ValueError, SyntaxError): pass - # keep backward compatibility: comma-separated params + # Potentially breaking backward compatibility: moving away from the comma-separated params for item in s.split(";"): if "=" in item: key, value = item.split("=", 1) From 00e659a6d34e6c3948bfbc49827fb926886455da Mon Sep 17 00:00:00 2001 From: Avinash Kumar Deepak Date: Sun, 8 Feb 2026 12:34:54 +0530 Subject: [PATCH 046/275] Fix Java concoredocker to match Python behavior --- concoredocker.java | 41 ++++++++++++++++++++++++----------------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/concoredocker.java b/concoredocker.java index b9813dec..ec5f9a79 100644 --- a/concoredocker.java +++ b/concoredocker.java @@ -16,6 +16,7 @@ public class concoredocker { private static String inpath = "/in"; private static String outpath = "/out"; private static Map params = new HashMap<>(); + private static int simtime = 0; private static int maxtime; public static void main(String[] args) { @@ -63,19 +64,19 @@ private static Map parseFile(String filename) throws IOException private static void defaultMaxTime(int defaultValue) { try { String content = new String(Files.readAllBytes(Paths.get(inpath + "1/concore.maxtime"))); - // changed assumption from map to list for maxtime, as it usually represents a list of time steps - maxtime = ((List) literalEval(content)).size(); - } catch (IOException e) { + maxtime = ((Number) literalEval(content)).intValue(); + } catch (Exception e) { maxtime = defaultValue; } } - private static void unchanged() { + private static boolean unchanged() { if (olds.equals(s)) { s = ""; - } else { - olds = s; + return true; } + olds = s; + return false; } private static Object tryParam(String n, Object i) { @@ -95,9 +96,13 @@ private static Object read(int port, String name, String initstr) { retrycount++; } s += ins; - Object[] inval = ((List) literalEval(ins)).toArray(); // FIXED: Casted to List, converted to Array - int simtime = Math.max((int) inval[0], 0); // assuming simtime is an integer - return inval[1]; + List inval = (List) literalEval(ins); + simtime = Math.max(simtime, ((Number) inval.get(0)).intValue()); + Object[] val = new Object[inval.size() - 1]; + for (int i = 1; i < inval.size(); i++) { + val[i - 1] = inval.get(i); + } + return val; } catch (IOException | InterruptedException | ClassCastException e) { return initstr; } @@ -110,13 +115,13 @@ private static void write(int port, String name, Object val, int delta) { if (val instanceof String) { Thread.sleep(2 * delay); } else if (!(val instanceof Object[])) { - System.out.println("mywrite must have list or str"); - System.exit(1); + System.out.println("write must have list or str"); + return; } if (val instanceof Object[]) { Object[] arrayVal = (Object[]) val; content.append("[") - .append(maxtime + delta) + .append(simtime + delta) .append(",") .append(arrayVal[0]); for (int i = 1; i < arrayVal.length; i++) { @@ -124,6 +129,7 @@ private static void write(int port, String name, Object val, int delta) { .append(arrayVal[i]); } content.append("]"); + simtime += delta; } else { content.append(val); } @@ -134,13 +140,14 @@ private static void write(int port, String name, Object val, int delta) { } private static Object[] initVal(String simtimeVal) { - int simtime = 0; Object[] val = new Object[] {}; try { - Object[] arrayVal = ((List) literalEval(simtimeVal)).toArray(); // FIXED: Casted to List, converted to Array - simtime = (int) arrayVal[0]; // assuming simtime is an integer - val = new Object[arrayVal.length - 1]; - System.arraycopy(arrayVal, 1, val, 0, val.length); + List inval = (List) literalEval(simtimeVal); + simtime = ((Number) inval.get(0)).intValue(); + val = new Object[inval.size() - 1]; + for (int i = 1; i < inval.size(); i++) { + val[i - 1] = inval.get(i); + } } catch (Exception e) { e.printStackTrace(); } From c9706e57b03a7564beeab9309431eb254e2a8178 Mon Sep 17 00:00:00 2001 From: Avinash Kumar Deepak Date: Mon, 9 Feb 2026 11:53:36 +0530 Subject: [PATCH 047/275] Use targeted exceptions instead of generic Exception in defaultMaxTime --- concoredocker.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/concoredocker.java b/concoredocker.java index ec5f9a79..345f8b3e 100644 --- a/concoredocker.java +++ b/concoredocker.java @@ -65,7 +65,7 @@ private static void defaultMaxTime(int defaultValue) { try { String content = new String(Files.readAllBytes(Paths.get(inpath + "1/concore.maxtime"))); maxtime = ((Number) literalEval(content)).intValue(); - } catch (Exception e) { + } catch (IOException | ClassCastException | NumberFormatException e) { maxtime = defaultValue; } } From ad7a68829c13bb402ee4554eba9ee71ce7ea1bf5 Mon Sep 17 00:00:00 2001 From: Avinash Kumar Deepak Date: Mon, 9 Feb 2026 12:23:19 +0530 Subject: [PATCH 048/275] Fix path construction in read() and write() functions --- concoredocker.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/concoredocker.py b/concoredocker.py index d337c388..c81e7303 100644 --- a/concoredocker.py +++ b/concoredocker.py @@ -227,7 +227,7 @@ def read(port_identifier, name, initstr_val): time.sleep(delay) # Construct file path consistent with other components (e.g., /in1/) - file_path = os.path.join(inpath + str(file_port_num), name) + file_path = os.path.join(inpath, str(file_port_num), name) try: with open(file_path, "r") as infile: @@ -284,7 +284,7 @@ def write(port_identifier, name, val, delta=0): try: file_port_num = int(port_identifier) - file_path = os.path.join(outpath + str(file_port_num), name) + file_path = os.path.join(outpath, str(file_port_num), name) except ValueError: logging.error(f"Error: Invalid port identifier '{port_identifier}' for file operation. Must be integer or ZMQ name.") return From d7d54c0ae1e602afce9fed4c2bf38de6fb3a797a Mon Sep 17 00:00:00 2001 From: Avinash Kumar Deepak Date: Mon, 9 Feb 2026 15:23:10 +0530 Subject: [PATCH 049/275] Fix sample script to use correct concore function names --- concore_cli/commands/init.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/concore_cli/commands/init.py b/concore_cli/commands/init.py index 5b0a9981..0b6badc3 100644 --- a/concore_cli/commands/init.py +++ b/concore_cli/commands/init.py @@ -25,10 +25,17 @@ SAMPLE_PYTHON = '''import concore -while not concore.concore_unchanged(): - data = concore.concore_read() - result = data * 2 - concore.concore_write(result) +concore.default_maxtime(100) +concore.delay = 0.02 + +init_simtime_val = "[0.0, 0.0]" +val = concore.initval(init_simtime_val) + +while(concore.simtime Date: Mon, 9 Feb 2026 17:21:34 +0530 Subject: [PATCH 050/275] ci: add basic testing and linting GitHub Actions workflow --- .github/workflows/ci.yml | 60 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..e6614fb4 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,60 @@ +name: CI + +on: + push: + branches: [main, master] + pull_request: + branches: [main, master] + +jobs: + test: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + cache: 'pip' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install pytest ruff + + - name: Run linter (ruff) + run: | + ruff check . --select=E9,F63,F7,F82 --output-format=github \ + --exclude="Dockerfile.*" \ + --exclude="linktest/" \ + --exclude="measurements/" \ + --exclude="0mq/" \ + --exclude="ratc/" + # E9: Runtime errors (syntax errors, etc.) + # F63: Invalid print syntax + # F7: Syntax errors in type comments + # F82: Undefined names in __all__ + # Excludes: Dockerfiles (not Python), linktest (symlinks), + # measurements/0mq/ratc (config-dependent experimental scripts) + + - name: Run tests (pytest) + run: | + pytest --tb=short -q \ + --ignore=measurements/ \ + --ignore=0mq/ \ + --ignore=ratc/ \ + --ignore=linktest/ \ + || true + # Allow pytest to pass even if no tests are collected yet + # Remove "|| true" once proper tests are added + # Ignores: experimental/config-dependent scripts + + - name: Validate Dockerfile build + run: | + docker build -f Dockerfile.py -t concore-py-test . + # Validates that Dockerfile.py can be built successfully + # Does not push the image From d78da9de6e231fb612f4c44ac5dae830f508ae5f Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Mon, 9 Feb 2026 17:41:37 +0530 Subject: [PATCH 051/275] ci: address review feedback - optimize CI pipeline - Use minimal requirements-ci.txt (no tensorflow/heavy deps) - Handle pytest exit code 5 (no tests) vs real failures - Move Dockerfile build to separate job with path filter - Add pyzmq to CI deps (required by concore.py) --- .github/workflows/ci.yml | 40 ++++++++++++++++++++++++++++++++-------- requirements-ci.txt | 6 ++++++ 2 files changed, 38 insertions(+), 8 deletions(-) create mode 100644 requirements-ci.txt diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e6614fb4..6f140943 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,7 +7,7 @@ on: branches: [main, master] jobs: - test: + lint-and-test: runs-on: ubuntu-latest steps: @@ -23,8 +23,8 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install -r requirements.txt - pip install pytest ruff + pip install -r requirements-ci.txt + # Uses minimal CI requirements (no tensorflow/heavy packages) - name: Run linter (ruff) run: | @@ -47,13 +47,37 @@ jobs: --ignore=measurements/ \ --ignore=0mq/ \ --ignore=ratc/ \ - --ignore=linktest/ \ - || true - # Allow pytest to pass even if no tests are collected yet - # Remove "|| true" once proper tests are added - # Ignores: experimental/config-dependent scripts + --ignore=linktest/ + status=$? + # Allow success if no tests are collected (pytest exit code 5) + if [ "$status" -ne 0 ] && [ "$status" -ne 5 ]; then + exit "$status" + fi + # Fails on real test failures, passes on no tests collected + + docker-build: + runs-on: ubuntu-latest + # Only run when Dockerfile.py or related files change + if: | + github.event_name == 'push' || + (github.event_name == 'pull_request' && + contains(github.event.pull_request.changed_files, 'Dockerfile')) + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Check if Dockerfile.py changed + uses: dorny/paths-filter@v3 + id: filter + with: + filters: | + dockerfile: + - 'Dockerfile.py' + - 'requirements.txt' - name: Validate Dockerfile build + if: steps.filter.outputs.dockerfile == 'true' run: | docker build -f Dockerfile.py -t concore-py-test . # Validates that Dockerfile.py can be built successfully diff --git a/requirements-ci.txt b/requirements-ci.txt new file mode 100644 index 00000000..1dc06cfb --- /dev/null +++ b/requirements-ci.txt @@ -0,0 +1,6 @@ +# Minimal dependencies for CI (linting and testing) +# Does not include heavyweight packages like tensorflow +pytest +ruff +pyzmq +numpy From bef87a5a7ea923aee8d1d83ce9ba4add3dca4176 Mon Sep 17 00:00:00 2001 From: Avinash Kumar Deepak Date: Mon, 9 Feb 2026 21:32:32 +0530 Subject: [PATCH 052/275] Fix broken functions in concoredocker.hpp to match Python behavior - safe_literal_eval now reads and parses file content instead of always returning default - load_params populates the params map from concore.params file - read() parses list data, extracts simtime from first element, returns rest - Added initval() to parse simtime string and extract data portion - Fixed naming to match codebase style (no underscores in function names) Fixes #224 --- concoredocker.hpp | 80 +++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 78 insertions(+), 2 deletions(-) diff --git a/concoredocker.hpp b/concoredocker.hpp index cbce7184..2471e3ec 100644 --- a/concoredocker.hpp +++ b/concoredocker.hpp @@ -12,6 +12,7 @@ #include #include #include +#include class Concore { public: @@ -26,6 +27,52 @@ class Concore { int maxtime = 100; std::unordered_map params; + std::string stripstr(const std::string& str) { + size_t start = str.find_first_not_of(" \t\n\r"); + if (start == std::string::npos) return ""; + size_t end = str.find_last_not_of(" \t\n\r"); + return str.substr(start, end - start + 1); + } + + std::string stripquotes(const std::string& str) { + if (str.size() >= 2 && ((str.front() == '\'' && str.back() == '\'') || (str.front() == '"' && str.back() == '"'))) + return str.substr(1, str.size() - 2); + return str; + } + + std::unordered_map parsedict(const std::string& str) { + std::unordered_map result; + std::string trimmed = stripstr(str); + if (trimmed.size() < 2 || trimmed.front() != '{' || trimmed.back() != '}') + return result; + std::string inner = trimmed.substr(1, trimmed.size() - 2); + std::stringstream ss(inner); + std::string token; + while (std::getline(ss, token, ',')) { + size_t colon = token.find(':'); + if (colon == std::string::npos) continue; + std::string key = stripquotes(stripstr(token.substr(0, colon))); + std::string val = stripquotes(stripstr(token.substr(colon + 1))); + if (!key.empty()) result[key] = val; + } + return result; + } + + std::vector parselist(const std::string& str) { + std::vector result; + std::string trimmed = stripstr(str); + if (trimmed.size() < 2 || trimmed.front() != '[' || trimmed.back() != ']') + return result; + std::string inner = trimmed.substr(1, trimmed.size() - 2); + std::stringstream ss(inner); + std::string token; + while (std::getline(ss, token, ',')) { + std::string val = stripstr(token); + if (!val.empty()) result.push_back(val); + } + return result; + } + Concore() { iport = safe_literal_eval("concore.iport", {}); oport = safe_literal_eval("concore.oport", {}); @@ -39,7 +86,14 @@ class Concore { std::cerr << "Error reading " << filename << "\n"; return defaultValue; } - return defaultValue; + std::stringstream buf; + buf << file.rdbuf(); + std::string content = buf.str(); + try { + return parsedict(content); + } catch (...) { + return defaultValue; + } } void load_params() { @@ -54,8 +108,11 @@ class Concore { } if (!sparams.empty() && sparams[0] != '{') { - sparams = "{'" + std::regex_replace(std::regex_replace(std::regex_replace(sparams, std::regex(","), ", '"), std::regex("="), "':"), std::regex(" "), "") + "}"; + sparams = "{\"" + std::regex_replace(std::regex_replace(std::regex_replace(sparams, std::regex(","), ",\""), std::regex("="), "\":"), std::regex(" "), "") + "}"; } + try { + params = parsedict(sparams); + } catch (...) {} } std::string tryparam(const std::string& n, const std::string& i) { @@ -106,6 +163,14 @@ class Concore { } s += ins; + try { + std::vector inval = parselist(ins); + if (!inval.empty()) { + int file_simtime = (int)std::stod(inval[0]); + simtime = std::max(simtime, file_simtime); + return std::vector(inval.begin() + 1, inval.end()); + } + } catch (...) {} return {ins}; } @@ -125,6 +190,17 @@ class Concore { simtime += delta; } } + + std::vector initval(const std::string& simtime_val) { + try { + std::vector val = parselist(simtime_val); + if (!val.empty()) { + simtime = (int)std::stod(val[0]); + return std::vector(val.begin() + 1, val.end()); + } + } catch (...) {} + return {}; + } }; #endif From 3c0201aac2b1456fa79159056b7287692dbb772e Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Tue, 10 Feb 2026 00:07:46 +0530 Subject: [PATCH 053/275] ci: fix pytest exit code handling, add dev branch trigger - Add set +e before pytest to prevent bash -e from exiting on code 5 - Add dev branch to push/pull_request triggers per maintainer request --- .github/workflows/ci.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6f140943..1100ba9c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,9 +2,9 @@ name: CI on: push: - branches: [main, master] + branches: [main, master, dev] pull_request: - branches: [main, master] + branches: [main, master, dev] jobs: lint-and-test: @@ -43,12 +43,14 @@ jobs: - name: Run tests (pytest) run: | + set +e pytest --tb=short -q \ --ignore=measurements/ \ --ignore=0mq/ \ --ignore=ratc/ \ --ignore=linktest/ status=$? + set -e # Allow success if no tests are collected (pytest exit code 5) if [ "$status" -ne 0 ] && [ "$status" -ne 5 ]; then exit "$status" From 5af526f738e1684c5572c781af9c7b1349eb4a93 Mon Sep 17 00:00:00 2001 From: Avinash Kumar Deepak Date: Tue, 10 Feb 2026 17:06:06 +0530 Subject: [PATCH 054/275] Fix: add missing return after ZMQ send in write() - Fixes #238 --- concore.py | 1 + 1 file changed, 1 insertion(+) diff --git a/concore.py b/concore.py index bf804e4e..75d96bd8 100644 --- a/concore.py +++ b/concore.py @@ -333,6 +333,7 @@ def write(port_identifier, name, val, delta=0): logging.error(f"ZMQ write error on port {port_identifier} (name: {name}): {e}") except Exception as e: logging.error(f"Unexpected error during ZMQ write on port {port_identifier} (name: {name}): {e}") + return # Case 2: File-based port try: From 0949fc0b81185d92a84da6370c20e626aadfa865 Mon Sep 17 00:00:00 2001 From: Avinash Kumar Deepak Date: Tue, 10 Feb 2026 17:19:15 +0530 Subject: [PATCH 055/275] Fix: add max retry limit and error handling to MATLAB read loop - Fixes #239 --- concore_read.m | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/concore_read.m b/concore_read.m index 2103255d..ba32c7b1 100644 --- a/concore_read.m +++ b/concore_read.m @@ -8,12 +8,21 @@ catch exc ins = inistr; end - while length(ins) == 0 + maxretries = 5; + attempts = 0; + while length(ins) == 0 && attempts < maxretries pause(concore.delay); - input1 = fopen(strcat(concore.inpath,num2str(port),'/',name)); - ins = fscanf(input1,'%c'); - fclose(input1); + try + input1 = fopen(strcat(concore.inpath,num2str(port),'/',name)); + ins = fscanf(input1,'%c'); + fclose(input1); + catch exc + end concore.retrycount = concore.retrycount + 1; + attempts = attempts + 1; + end + if length(ins) == 0 + ins = inistr; end concore.s = strcat(concore.s, ins); result = eval(ins); From ac714cc69380754adce633a920c139a007863439 Mon Sep 17 00:00:00 2001 From: GREENRAT-K405 Date: Wed, 11 Feb 2026 15:22:59 +0530 Subject: [PATCH 056/275] add gemini PR review --- .github/workflows/PR-review.yaml | 46 ++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 .github/workflows/PR-review.yaml diff --git a/.github/workflows/PR-review.yaml b/.github/workflows/PR-review.yaml new file mode 100644 index 00000000..83d480d9 --- /dev/null +++ b/.github/workflows/PR-review.yaml @@ -0,0 +1,46 @@ +name: AI Code Reviewer + +on: + issue_comment: + types: [created] + +permissions: + contents: read + pull-requests: write + issues: write + +jobs: + gemini-code-review: + runs-on: ubuntu-latest + if: | + github.event.issue.pull_request && + contains(github.event.comment.body, '/gemini-review') + steps: + - name: PR Info + run: | + echo "Comment: ${{ github.event.comment.body }}" + echo "Issue Number: ${{ github.event.issue.number }}" + echo "Repository: ${{ github.repository }}" + + - name: Checkout Repo + uses: actions/checkout@v3 + with: + fetch-depth: 0 + ref: refs/pull/${{ github.event.issue.number }}/head + + - name: Get PR Details + id: pr + run: | + PR_JSON=$(gh api repos/${{ github.repository }}/pulls/${{ github.event.issue.number }}) + echo "head_sha=$(echo $PR_JSON | jq -r .head.sha)" >> $GITHUB_OUTPUT + echo "base_sha=$(echo $PR_JSON | jq -r .base.sha)" >> $GITHUB_OUTPUT + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - uses: truongnh1992/gemini-ai-code-reviewer@main + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} + GEMINI_MODEL: gemini-2.5-flash + EXCLUDE: "*.md,*.txt,package-lock.json" + \ No newline at end of file From 64e483c30598186d29187bb7bbf77e17b770e389 Mon Sep 17 00:00:00 2001 From: Avinash Kumar Deepak Date: Wed, 11 Feb 2026 17:04:33 +0530 Subject: [PATCH 057/275] wrap bare open() calls in context managers to prevent file handle leaks (#241) --- mkconcore.py | 123 ++++++++++++++++++++++++++++----------------------- 1 file changed, 67 insertions(+), 56 deletions(-) diff --git a/mkconcore.py b/mkconcore.py index 10135dc9..e552ff98 100644 --- a/mkconcore.py +++ b/mkconcore.py @@ -63,6 +63,7 @@ # - Sets the executable permission (`stat.S_IRWXU`) for the generated scripts on POSIX systems. from bs4 import BeautifulSoup +import atexit import logging import re import sys @@ -104,13 +105,16 @@ M_IS_OCTAVE = True #treat .m as octave 9/27/21 if os.path.exists(CONCOREPATH+"/concore.mcr"): # 11/12/21 - MCRPATH = open(CONCOREPATH+"/concore.mcr", "r").readline().strip() #path to local Ubunta Matlab Compiler Runtime + with open(CONCOREPATH+"/concore.mcr", "r") as f: + MCRPATH = f.readline().strip() #path to local Ubunta Matlab Compiler Runtime if os.path.exists(CONCOREPATH+"/concore.sudo"): # 12/04/21 - DOCKEREXE = open(CONCOREPATH+"/concore.sudo", "r").readline().strip() #to omit sudo in docker + with open(CONCOREPATH+"/concore.sudo", "r") as f: + DOCKEREXE = f.readline().strip() #to omit sudo in docker if os.path.exists(CONCOREPATH+"/concore.repo"): # 12/04/21 - DOCKEREPO = open(CONCOREPATH+"/concore.repo", "r").readline().strip() #docker id for repo + with open(CONCOREPATH+"/concore.repo", "r") as f: + DOCKEREPO = f.readline().strip() #docker id for repo prefixedgenode = "" @@ -166,6 +170,12 @@ funlock = open("unlock", "w") # 12/4/21 fparams = open("params", "w") # 9/18/22 +def cleanup_script_files(): + for fh in [fbuild, frun, fdebug, fstop, fclear, fmaxtime, funlock, fparams]: + if not fh.closed: + fh.close() +atexit.register(cleanup_script_files) + os.mkdir("src") os.chdir("..") @@ -179,8 +189,8 @@ logging.info(f"MCR path: {MCRPATH}") logging.info(f"Docker repository: {DOCKEREPO}") -f = open(GRAPHML_FILE, "r") -text_str = f.read() +with open(GRAPHML_FILE, "r") as f: + text_str = f.read() soup = BeautifulSoup(text_str, 'xml') @@ -410,121 +420,125 @@ #copy proper concore.py into /src try: if concoretype=="docker": - fsource = open(CONCOREPATH+"/concoredocker.py") + with open(CONCOREPATH+"/concoredocker.py") as fsource: + source_content = fsource.read() else: - fsource = open(CONCOREPATH+"/concore.py") + with open(CONCOREPATH+"/concore.py") as fsource: + source_content = fsource.read() except (FileNotFoundError, IOError) as e: logging.error(f"{CONCOREPATH} is not correct path to concore: {e}") quit() with open(outdir+"/src/concore.py","w") as fcopy: - fcopy.write(fsource.read()) -fsource.close() + fcopy.write(source_content) #copy proper concore.hpp into /src 6/22/21 try: if concoretype=="docker": - fsource = open(CONCOREPATH+"/concoredocker.hpp") + with open(CONCOREPATH+"/concoredocker.hpp") as fsource: + source_content = fsource.read() else: - fsource = open(CONCOREPATH+"/concore.hpp") + with open(CONCOREPATH+"/concore.hpp") as fsource: + source_content = fsource.read() except (FileNotFoundError, IOError) as e: logging.error(f"{CONCOREPATH} is not correct path to concore: {e}") quit() with open(outdir+"/src/concore.hpp","w") as fcopy: - fcopy.write(fsource.read()) -fsource.close() + fcopy.write(source_content) #copy proper concore.v into /src 6/25/21 try: if concoretype=="docker": - fsource = open(CONCOREPATH+"/concoredocker.v") + with open(CONCOREPATH+"/concoredocker.v") as fsource: + source_content = fsource.read() else: - fsource = open(CONCOREPATH+"/concore.v") + with open(CONCOREPATH+"/concore.v") as fsource: + source_content = fsource.read() except (FileNotFoundError, IOError) as e: logging.error(f"{CONCOREPATH} is not correct path to concore: {e}") quit() with open(outdir+"/src/concore.v","w") as fcopy: - fcopy.write(fsource.read()) -fsource.close() + fcopy.write(source_content) #copy mkcompile into /src 5/27/21 try: - fsource = open(CONCOREPATH+"/mkcompile") + with open(CONCOREPATH+"/mkcompile") as fsource: + source_content = fsource.read() except (FileNotFoundError, IOError) as e: logging.error(f"{CONCOREPATH} is not correct path to concore: {e}") quit() with open(outdir+"/src/mkcompile","w") as fcopy: - fcopy.write(fsource.read()) -fsource.close() + fcopy.write(source_content) os.chmod(outdir+"/src/mkcompile",stat.S_IRWXU) #copy concore*.m into /src 4/2/21 try: #maxtime in matlab 11/22/21 - fsource = open(CONCOREPATH+"/concore_default_maxtime.m") + with open(CONCOREPATH+"/concore_default_maxtime.m") as fsource: + source_content = fsource.read() except (FileNotFoundError, IOError) as e: logging.error(f"{CONCOREPATH} is not correct path to concore: {e}") quit() with open(outdir+"/src/concore_default_maxtime.m","w") as fcopy: - fcopy.write(fsource.read()) -fsource.close() + fcopy.write(source_content) try: - fsource = open(CONCOREPATH+"/concore_unchanged.m") + with open(CONCOREPATH+"/concore_unchanged.m") as fsource: + source_content = fsource.read() except (FileNotFoundError, IOError) as e: logging.error(f"{CONCOREPATH} is not correct path to concore: {e}") quit() with open(outdir+"/src/concore_unchanged.m","w") as fcopy: - fcopy.write(fsource.read()) -fsource.close() + fcopy.write(source_content) try: - fsource = open(CONCOREPATH+"/concore_read.m") + with open(CONCOREPATH+"/concore_read.m") as fsource: + source_content = fsource.read() except (FileNotFoundError, IOError) as e: logging.error(f"{CONCOREPATH} is not correct path to concore: {e}") quit() with open(outdir+"/src/concore_read.m","w") as fcopy: - fcopy.write(fsource.read()) -fsource.close() + fcopy.write(source_content) try: - fsource = open(CONCOREPATH+"/concore_write.m") + with open(CONCOREPATH+"/concore_write.m") as fsource: + source_content = fsource.read() except (FileNotFoundError, IOError) as e: logging.error(f"{CONCOREPATH} is not correct path to concore: {e}") quit() with open(outdir+"/src/concore_write.m","w") as fcopy: - fcopy.write(fsource.read()) -fsource.close() + fcopy.write(source_content) try: #4/9/21 - fsource = open(CONCOREPATH+"/concore_initval.m") + with open(CONCOREPATH+"/concore_initval.m") as fsource: + source_content = fsource.read() except (FileNotFoundError, IOError) as e: logging.error(f"{CONCOREPATH} is not correct path to concore: {e}") quit() with open(outdir+"/src/concore_initval.m","w") as fcopy: - fcopy.write(fsource.read()) -fsource.close() + fcopy.write(source_content) try: #11/19/21 - fsource = open(CONCOREPATH+"/concore_iport.m") + with open(CONCOREPATH+"/concore_iport.m") as fsource: + source_content = fsource.read() except (FileNotFoundError, IOError) as e: logging.error(f"{CONCOREPATH} is not correct path to concore: {e}") quit() with open(outdir+"/src/concore_iport.m","w") as fcopy: - fcopy.write(fsource.read()) -fsource.close() + fcopy.write(source_content) try: #11/19/21 - fsource = open(CONCOREPATH+"/concore_oport.m") + with open(CONCOREPATH+"/concore_oport.m") as fsource: + source_content = fsource.read() except (FileNotFoundError, IOError) as e: logging.error(f"{CONCOREPATH} is not correct path to concore: {e}") quit() with open(outdir+"/src/concore_oport.m","w") as fcopy: - fcopy.write(fsource.read()) -fsource.close() + fcopy.write(source_content) try: # 4/4/21 if concoretype=="docker": - fsource = open(CONCOREPATH+"/import_concoredocker.m") + with open(CONCOREPATH+"/import_concoredocker.m") as fsource: + source_content = fsource.read() else: - fsource = open(CONCOREPATH+"/import_concore.m") + with open(CONCOREPATH+"/import_concore.m") as fsource: + source_content = fsource.read() except (FileNotFoundError, IOError) as e: logging.error(f"{CONCOREPATH} is not correct path to concore: {e}") quit() with open(outdir+"/src/import_concore.m","w") as fcopy: - fcopy.write(fsource.read()) -fsource.close() + fcopy.write(source_content) # --- Generate iport and oport mappings --- logging.info("Generating iport/oport mappings...") @@ -593,25 +607,27 @@ if not os.path.exists(outdir+"/src/Dockerfile."+dockername): # 3/30/21 try: if langext=="py": - fsource = open(CONCOREPATH+"/Dockerfile.py") + src_path = CONCOREPATH+"/Dockerfile.py" logging.info("assuming .py extension for Dockerfile") elif langext == "cpp": # 6/22/21 - fsource = open(CONCOREPATH+"/Dockerfile.cpp") + src_path = CONCOREPATH+"/Dockerfile.cpp" logging.info("assuming .cpp extension for Dockerfile") elif langext == "v": # 6/26/21 - fsource = open(CONCOREPATH+"/Dockerfile.v") + src_path = CONCOREPATH+"/Dockerfile.v" logging.info("assuming .v extension for Dockerfile") elif langext == "sh": # 5/19/21 - fsource = open(CONCOREPATH+"/Dockerfile.sh") + src_path = CONCOREPATH+"/Dockerfile.sh" logging.info("assuming .sh extension for Dockerfile") else: - fsource = open(CONCOREPATH+"/Dockerfile.m") + src_path = CONCOREPATH+"/Dockerfile.m" logging.info("assuming .m extension for Dockerfile") + with open(src_path) as fsource: + source_content = fsource.read() except: logging.error(f"{CONCOREPATH} is not correct path to concore") quit() with open(outdir+"/src/Dockerfile."+dockername,"w") as fcopy: - fcopy.write(fsource.read()) + fcopy.write(source_content) if langext=="py": fcopy.write('CMD ["python", "-i", "'+sourcecode+'"]\n') if langext=="m": @@ -622,7 +638,6 @@ if langext=="v": fcopy.write('RUN iverilog ./'+sourcecode+'\n') # 7/02/21 fcopy.write('CMD ["./a.out"]\n') # 7/02/21 - fsource.close() fbuild.write('#!/bin/bash' + "\n") for node in nodes_dict: @@ -1110,10 +1125,6 @@ frun.close() fbuild.close() fdebug.close() -fstop.close() -fclear.close() -fmaxtime.close() -fparams.close() if concoretype != "windows": os.chmod(outdir+"/build",stat.S_IRWXU) os.chmod(outdir+"/run",stat.S_IRWXU) From 003540c5ac010c99626af52d06261a8fce96d24c Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Thu, 12 Feb 2026 00:43:59 +0530 Subject: [PATCH 058/275] fix(java): rewrite literalEval() with recursive descent parser Fix #228: concoredocker.java literalEval() is fundamentally broken The original implementation used String.split(",") which failed for: - Nested structures (lists within dicts, etc.) - Strings containing commas or colons - Escape sequences in strings Changes: - Implement recursive descent parser for Python literal syntax - Add escapePythonString() for proper string escaping - Fix toPythonLiteral() to use Python True/False/None - Fix read() to return List with max retries - Fix write() delay to double (was int) with separate catch blocks - Fix simtime to double type - Add TestLiteralEval.java with 49 comprehensive tests All tests pass. --- TestLiteralEval.java | 266 ++++++++++++++++++++++ concoredocker.java | 527 +++++++++++++++++++++++++++++++++++++------ 2 files changed, 725 insertions(+), 68 deletions(-) create mode 100644 TestLiteralEval.java diff --git a/TestLiteralEval.java b/TestLiteralEval.java new file mode 100644 index 00000000..dc1c466d --- /dev/null +++ b/TestLiteralEval.java @@ -0,0 +1,266 @@ +import java.util.*; + +/** + * Test suite for concoredocker.literalEval() recursive descent parser. + * Covers: dicts, lists, tuples, numbers, strings, booleans, None, + * nested structures, escape sequences, scientific notation, + * toPythonLiteral serialization, and fractional simtime. + */ +public class TestLiteralEval { + static int passed = 0; + static int failed = 0; + + public static void main(String[] args) { + testEmptyDict(); + testSimpleDict(); + testDictWithIntValues(); + testEmptyList(); + testSimpleList(); + testListOfDoubles(); + testNestedDictWithList(); + testNestedListsDeep(); + testBooleansAndNone(); + testStringsWithCommas(); + testStringsWithColons(); + testStringEscapeSequences(); + testScientificNotation(); + testNegativeNumbers(); + testTuple(); + testTrailingComma(); + testToPythonLiteralBooleans(); + testToPythonLiteralNone(); + testToPythonLiteralString(); + testFractionalSimtime(); + testRoundTripSerialization(); + testStringEscapingSerialization(); + testUnterminatedList(); + testUnterminatedDict(); + testUnterminatedTuple(); + + System.out.println("\n=== Results: " + passed + " passed, " + failed + " failed out of " + (passed + failed) + " tests ==="); + if (failed > 0) { + System.exit(1); + } + } + + static void check(String testName, Object expected, Object actual) { + if (Objects.equals(expected, actual)) { + System.out.println("PASS: " + testName); + passed++; + } else { + System.out.println("FAIL: " + testName + " | expected: " + expected + " | actual: " + actual); + failed++; + } + } + + static void testEmptyDict() { + Object result = concoredocker.literalEval("{}"); + check("empty dict", new HashMap<>(), result); + } + + static void testSimpleDict() { + @SuppressWarnings("unchecked") + Map result = (Map) concoredocker.literalEval("{'PYM': 1}"); + check("simple dict key", true, result.containsKey("PYM")); + check("simple dict value", 1, result.get("PYM")); + } + + static void testDictWithIntValues() { + @SuppressWarnings("unchecked") + Map result = (Map) concoredocker.literalEval("{'a': 10, 'b': 20}"); + check("dict int value a", 10, result.get("a")); + check("dict int value b", 20, result.get("b")); + } + + static void testEmptyList() { + Object result = concoredocker.literalEval("[]"); + check("empty list", new ArrayList<>(), result); + } + + static void testSimpleList() { + @SuppressWarnings("unchecked") + List result = (List) concoredocker.literalEval("[1, 2, 3]"); + check("simple list size", 3, result.size()); + check("simple list[0]", 1, result.get(0)); + check("simple list[2]", 3, result.get(2)); + } + + static void testListOfDoubles() { + @SuppressWarnings("unchecked") + List result = (List) concoredocker.literalEval("[0.0, 1.5, 2.7]"); + check("list doubles[0]", 0.0, result.get(0)); + check("list doubles[1]", 1.5, result.get(1)); + } + + static void testNestedDictWithList() { + @SuppressWarnings("unchecked") + Map result = (Map) concoredocker.literalEval("{'key': [1, 2, 3]}"); + check("nested dict has key", true, result.containsKey("key")); + @SuppressWarnings("unchecked") + List inner = (List) result.get("key"); + check("nested list size", 3, inner.size()); + check("nested list[0]", 1, inner.get(0)); + } + + static void testNestedListsDeep() { + @SuppressWarnings("unchecked") + List result = (List) concoredocker.literalEval("[[1, 2], [3, 4]]"); + check("nested lists size", 2, result.size()); + @SuppressWarnings("unchecked") + List inner = (List) result.get(0); + check("inner list[0]", 1, inner.get(0)); + check("inner list[1]", 2, inner.get(1)); + } + + static void testBooleansAndNone() { + @SuppressWarnings("unchecked") + List result = (List) concoredocker.literalEval("[True, False, None]"); + check("boolean True", Boolean.TRUE, result.get(0)); + check("boolean False", Boolean.FALSE, result.get(1)); + check("None", null, result.get(2)); + } + + static void testStringsWithCommas() { + @SuppressWarnings("unchecked") + Map result = (Map) concoredocker.literalEval("{'key': 'hello, world'}"); + check("string with comma", "hello, world", result.get("key")); + } + + static void testStringsWithColons() { + @SuppressWarnings("unchecked") + Map result = (Map) concoredocker.literalEval("{'url': 'http://example.com'}"); + check("string with colon", "http://example.com", result.get("url")); + } + + static void testStringEscapeSequences() { + Object result = concoredocker.literalEval("'hello\\nworld'"); + check("escaped newline", "hello\nworld", result); + } + + static void testScientificNotation() { + Object result = concoredocker.literalEval("1.5e3"); + check("scientific notation", 1500.0, result); + } + + static void testNegativeNumbers() { + @SuppressWarnings("unchecked") + List result = (List) concoredocker.literalEval("[-1, -2.5, 3]"); + check("negative int", -1, result.get(0)); + check("negative double", -2.5, result.get(1)); + check("positive int", 3, result.get(2)); + } + + static void testTuple() { + @SuppressWarnings("unchecked") + List result = (List) concoredocker.literalEval("(1, 2, 3)"); + check("tuple size", 3, result.size()); + check("tuple[0]", 1, result.get(0)); + } + + static void testTrailingComma() { + @SuppressWarnings("unchecked") + List result = (List) concoredocker.literalEval("[1, 2, 3,]"); + check("trailing comma size", 3, result.size()); + } + + // --- Serialization tests (toPythonLiteral via write format) --- + + static void testToPythonLiteralBooleans() { + // Test that booleans serialize to Python format (True/False, not true/false) + @SuppressWarnings("unchecked") + List input = (List) concoredocker.literalEval("[True, False]"); + // Re-parse and check the values are correct Java booleans + check("parsed True is Boolean.TRUE", Boolean.TRUE, input.get(0)); + check("parsed False is Boolean.FALSE", Boolean.FALSE, input.get(1)); + } + + static void testToPythonLiteralNone() { + @SuppressWarnings("unchecked") + List input = (List) concoredocker.literalEval("[None, 1]"); + check("parsed None is null", null, input.get(0)); + check("parsed 1 is Integer 1", 1, input.get(1)); + } + + static void testToPythonLiteralString() { + Object result = concoredocker.literalEval("'hello'"); + check("parsed string", "hello", result); + } + + static void testFractionalSimtime() { + // Simtime values like [0.5, 1.0, 2.0] should preserve fractional part + @SuppressWarnings("unchecked") + List result = (List) concoredocker.literalEval("[0.5, 1.0, 2.0]"); + check("fractional simtime[0]", 0.5, result.get(0)); + check("fractional simtime[1]", 1.0, result.get(1)); + check("fractional simtime[2]", 2.0, result.get(2)); + } + + // --- Round-trip serialization tests --- + + static void testRoundTripSerialization() { + // Serialize a list with mixed types, then re-parse and verify + List original = new ArrayList<>(); + original.add(1); + original.add(2.5); + original.add(true); + original.add(false); + original.add(null); + original.add("hello"); + + // Use reflection-free approach: build the Python literal manually + // and verify round-trip through literalEval + String serialized = "[1, 2.5, True, False, None, 'hello']"; + @SuppressWarnings("unchecked") + List roundTripped = (List) concoredocker.literalEval(serialized); + check("round-trip int", 1, roundTripped.get(0)); + check("round-trip double", 2.5, roundTripped.get(1)); + check("round-trip True", Boolean.TRUE, roundTripped.get(2)); + check("round-trip False", Boolean.FALSE, roundTripped.get(3)); + check("round-trip None", null, roundTripped.get(4)); + check("round-trip string", "hello", roundTripped.get(5)); + } + + static void testStringEscapingSerialization() { + // Strings with special chars should survive parse -> serialize -> re-parse + String input = "'hello\\nworld'"; + Object parsed = concoredocker.literalEval(input); + check("escape parse", "hello\nworld", parsed); + + // Test string with embedded single quote + String input2 = "'it\\'s'"; + Object parsed2 = concoredocker.literalEval(input2); + check("escape single quote", "it's", parsed2); + } + + // --- Unterminated input tests (should throw) --- + + static void testUnterminatedList() { + try { + concoredocker.literalEval("[1, 2"); + System.out.println("FAIL: unterminated list should throw"); + failed++; + } catch (IllegalArgumentException e) { + check("unterminated list throws", true, e.getMessage().contains("Unterminated list")); + } + } + + static void testUnterminatedDict() { + try { + concoredocker.literalEval("{'a': 1"); + System.out.println("FAIL: unterminated dict should throw"); + failed++; + } catch (IllegalArgumentException e) { + check("unterminated dict throws", true, e.getMessage().contains("Unterminated dict")); + } + } + + static void testUnterminatedTuple() { + try { + concoredocker.literalEval("(1, 2"); + System.out.println("FAIL: unterminated tuple should throw"); + failed++; + } catch (IllegalArgumentException e) { + check("unterminated tuple throws", true, e.getMessage().contains("Unterminated tuple")); + } + } +} diff --git a/concoredocker.java b/concoredocker.java index 345f8b3e..ca551b59 100644 --- a/concoredocker.java +++ b/concoredocker.java @@ -6,17 +6,26 @@ import java.util.ArrayList; import java.util.List; +/** + * Java implementation of concore Docker communication. + * + * This class provides file-based inter-process communication for control systems, + * mirroring the functionality of concoredocker.py. + */ public class concoredocker { private static Map iport = new HashMap<>(); private static Map oport = new HashMap<>(); private static String s = ""; private static String olds = ""; - private static int delay = 1; + // delay in milliseconds (Python uses time.sleep(1) = 1 second) + private static int delay = 1000; private static int retrycount = 0; + private static int maxRetries = 5; private static String inpath = "/in"; private static String outpath = "/out"; private static Map params = new HashMap<>(); - private static int simtime = 0; + // simtime as double to preserve fractional values (e.g. "[0.0, ...]") + private static double simtime = 0; private static int maxtime; public static void main(String[] args) { @@ -33,7 +42,7 @@ public static void main(String[] args) { try { String sparams = new String(Files.readAllBytes(Paths.get(inpath + "1/concore.params"))); - if (sparams.charAt(0) == '"') { // windows keeps "" need to remove + if (sparams.length() > 0 && sparams.charAt(0) == '"') { // windows keeps "" need to remove sparams = sparams.substring(1); sparams = sparams.substring(0, sparams.indexOf('"')); } @@ -43,8 +52,12 @@ public static void main(String[] args) { System.out.println("converted sparams: " + sparams); } try { - // literalEval returns a proper Map for "{...}" - params = (Map) literalEval(sparams); + Object parsed = literalEval(sparams); + if (parsed instanceof Map) { + @SuppressWarnings("unchecked") + Map parsedMap = (Map) parsed; + params = parsedMap; + } } catch (Exception e) { System.out.println("bad params: " + sparams); } @@ -55,17 +68,43 @@ public static void main(String[] args) { defaultMaxTime(100); } - @SuppressWarnings("unchecked") + /** + * Parses a file containing a Python-style dictionary literal. + * Returns empty map if file is empty or malformed (matches Python safe_literal_eval). + */ private static Map parseFile(String filename) throws IOException { String content = new String(Files.readAllBytes(Paths.get(filename))); - return (Map) literalEval(content); // Casted to Map + content = content.trim(); + if (content.isEmpty()) { + return new HashMap<>(); + } + try { + Object result = literalEval(content); + if (result instanceof Map) { + @SuppressWarnings("unchecked") + Map map = (Map) result; + return map; + } + } catch (IllegalArgumentException e) { + System.err.println("Failed to parse file as map: " + filename + " (" + e.getMessage() + ")"); + } + return new HashMap<>(); } + /** + * Sets maxtime from concore.maxtime file, or uses defaultValue if file not found. + * Catches both IOException and RuntimeException to match Python safe_literal_eval. + */ private static void defaultMaxTime(int defaultValue) { try { String content = new String(Files.readAllBytes(Paths.get(inpath + "1/concore.maxtime"))); - maxtime = ((Number) literalEval(content)).intValue(); - } catch (IOException | ClassCastException | NumberFormatException e) { + Object parsed = literalEval(content.trim()); + if (parsed instanceof Number) { + maxtime = ((Number) parsed).intValue(); + } else { + maxtime = defaultValue; + } + } catch (IOException | RuntimeException e) { maxtime = defaultValue; } } @@ -87,105 +126,457 @@ private static Object tryParam(String n, Object i) { } } - private static Object read(int port, String name, String initstr) { + /** + * Reads data from a port file. Returns the values after extracting simtime. + * Input format: [simtime, val1, val2, ...] + * Returns: list of values after simtime + * Includes max retry limit to avoid infinite blocking (matches Python behavior). + */ + private static List read(int port, String name, String initstr) { + // Parse default value upfront for consistent return type + List defaultVal = new ArrayList<>(); + try { + List parsed = (List) literalEval(initstr); + if (parsed.size() > 1) { + defaultVal = new ArrayList<>(parsed.subList(1, parsed.size())); + } + } catch (Exception e) { + // initstr not parseable as list; defaultVal stays empty + } + + String filePath = inpath + port + "/" + name; + try { + Thread.sleep(delay); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + s += initstr; + return defaultVal; + } + + String ins; try { - String ins = new String(Files.readAllBytes(Paths.get(inpath + port + "/" + name))); - while (ins.length() == 0) { + ins = new String(Files.readAllBytes(Paths.get(filePath))); + } catch (IOException e) { + System.out.println("File " + filePath + " not found, using default value."); + s += initstr; + return defaultVal; + } + + int attempts = 0; + while (ins.length() == 0 && attempts < maxRetries) { + try { Thread.sleep(delay); - ins = new String(Files.readAllBytes(Paths.get(inpath + port + "/" + name))); - retrycount++; + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + s += initstr; + return defaultVal; } - s += ins; + try { + ins = new String(Files.readAllBytes(Paths.get(filePath))); + } catch (IOException e) { + System.out.println("Retry " + (attempts + 1) + ": Error reading " + filePath); + } + attempts++; + retrycount++; + } + + if (ins.length() == 0) { + System.out.println("Max retries reached for " + filePath + ", using default value."); + s += initstr; + return defaultVal; + } + + s += ins; + try { List inval = (List) literalEval(ins); - simtime = Math.max(simtime, ((Number) inval.get(0)).intValue()); - Object[] val = new Object[inval.size() - 1]; - for (int i = 1; i < inval.size(); i++) { - val[i - 1] = inval.get(i); + if (!inval.isEmpty()) { + double firstSimtime = ((Number) inval.get(0)).doubleValue(); + simtime = Math.max(simtime, firstSimtime); + return new ArrayList<>(inval.subList(1, inval.size())); } - return val; - } catch (IOException | InterruptedException | ClassCastException e) { - return initstr; + } catch (Exception e) { + System.out.println("Error parsing " + ins + ": " + e.getMessage()); } + s += initstr; + return defaultVal; } + /** + * Escapes a Java string so it can be safely used as a single-quoted Python string literal. + * At minimum, escapes backslash, single quote, newline, carriage return, and tab. + */ + private static String escapePythonString(String s) { + StringBuilder sb = new StringBuilder(s.length()); + for (int i = 0; i < s.length(); i++) { + char c = s.charAt(i); + switch (c) { + case '\\': sb.append("\\\\"); break; + case '\'': sb.append("\\'"); break; + case '\n': sb.append("\\n"); break; + case '\r': sb.append("\\r"); break; + case '\t': sb.append("\\t"); break; + default: sb.append(c); break; + } + } + return sb.toString(); + } + + /** + * Converts a Java object to its Python-literal string representation. + * True/False/None instead of true/false/null; strings single-quoted. + */ + private static String toPythonLiteral(Object obj) { + if (obj == null) return "None"; + if (obj instanceof Boolean) return ((Boolean) obj) ? "True" : "False"; + if (obj instanceof String) return "'" + escapePythonString((String) obj) + "'"; + if (obj instanceof Number) return obj.toString(); + if (obj instanceof List) { + List list = (List) obj; + StringBuilder sb = new StringBuilder("["); + for (int i = 0; i < list.size(); i++) { + if (i > 0) sb.append(", "); + sb.append(toPythonLiteral(list.get(i))); + } + sb.append("]"); + return sb.toString(); + } + if (obj instanceof Map) { + Map map = (Map) obj; + StringBuilder sb = new StringBuilder("{"); + boolean first = true; + for (Map.Entry entry : map.entrySet()) { + if (!first) sb.append(", "); + sb.append(toPythonLiteral(entry.getKey())).append(": ").append(toPythonLiteral(entry.getValue())); + first = false; + } + sb.append("}"); + return sb.toString(); + } + return obj.toString(); + } + + /** + * Writes data to a port file. + * Prepends simtime+delta to the value list, then serializes to Python-literal format. + * Accepts List or String values (matching Python implementation). + */ private static void write(int port, String name, Object val, int delta) { try { String path = outpath + port + "/" + name; StringBuilder content = new StringBuilder(); if (val instanceof String) { Thread.sleep(2 * delay); - } else if (!(val instanceof Object[])) { - System.out.println("write must have list or str"); - return; - } - if (val instanceof Object[]) { + content.append(val); + } else if (val instanceof List) { + List listVal = (List) val; + content.append("["); + content.append(toPythonLiteral(simtime + delta)); + for (int i = 0; i < listVal.size(); i++) { + content.append(", "); + content.append(toPythonLiteral(listVal.get(i))); + } + content.append("]"); + simtime += delta; + } else if (val instanceof Object[]) { + // Legacy support for Object[] arguments Object[] arrayVal = (Object[]) val; - content.append("[") - .append(simtime + delta) - .append(",") - .append(arrayVal[0]); - for (int i = 1; i < arrayVal.length; i++) { - content.append(",") - .append(arrayVal[i]); + content.append("["); + content.append(toPythonLiteral(simtime + delta)); + for (Object o : arrayVal) { + content.append(", "); + content.append(toPythonLiteral(o)); } content.append("]"); simtime += delta; } else { - content.append(val); + System.out.println("write must have list or str"); + return; } Files.write(Paths.get(path), content.toString().getBytes()); - } catch (IOException | InterruptedException e) { - System.out.println("skipping" + outpath + port + "/" + name); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + System.out.println("skipping " + outpath + port + "/" + name); + } catch (IOException e) { + System.out.println("skipping " + outpath + port + "/" + name); } } - private static Object[] initVal(String simtimeVal) { - Object[] val = new Object[] {}; + /** + * Parses an initial value string like "[0.0, 1.0, 2.0]". + * Extracts simtime from position 0 and returns the remaining values as a List. + */ + private static List initVal(String simtimeVal) { + List val = new ArrayList<>(); try { List inval = (List) literalEval(simtimeVal); - simtime = ((Number) inval.get(0)).intValue(); - val = new Object[inval.size() - 1]; - for (int i = 1; i < inval.size(); i++) { - val[i - 1] = inval.get(i); + if (!inval.isEmpty()) { + simtime = ((Number) inval.get(0)).doubleValue(); + val = new ArrayList<>(inval.subList(1, inval.size())); } } catch (Exception e) { - e.printStackTrace(); + System.out.println("Error parsing initVal: " + e.getMessage()); } return val; } - // custom parser - private static Object literalEval(String s) { + /** + * Parses a Python-literal string into Java objects using a recursive descent parser. + * Supports: dict, list, int, float, string (single/double quoted), bool, None, nested structures. + * This replaces the broken split-based parser that could not handle quoted commas or nesting. + */ + static Object literalEval(String s) { + if (s == null) throw new IllegalArgumentException("Input cannot be null"); s = s.trim(); - if (s.startsWith("{") && s.endsWith("}")) { + if (s.isEmpty()) throw new IllegalArgumentException("Input cannot be empty"); + Parser parser = new Parser(s); + Object result = parser.parseExpression(); + parser.skipWhitespace(); + if (parser.pos < parser.input.length()) { + throw new IllegalArgumentException("Unexpected trailing content at position " + parser.pos); + } + return result; + } + + /** + * Recursive descent parser for Python literal expressions. + * Handles: dicts, lists, tuples, strings, numbers, booleans, None. + */ + private static class Parser { + final String input; + int pos; + + Parser(String input) { + this.input = input; + this.pos = 0; + } + + void skipWhitespace() { + while (pos < input.length() && Character.isWhitespace(input.charAt(pos))) { + pos++; + } + } + + char peek() { + skipWhitespace(); + if (pos >= input.length()) throw new IllegalArgumentException("Unexpected end of input"); + return input.charAt(pos); + } + + char advance() { + char c = input.charAt(pos); + pos++; + return c; + } + + boolean hasMore() { + skipWhitespace(); + return pos < input.length(); + } + + Object parseExpression() { + skipWhitespace(); + if (pos >= input.length()) throw new IllegalArgumentException("Unexpected end of input"); + char c = input.charAt(pos); + + if (c == '{') return parseDict(); + if (c == '[') return parseList(); + if (c == '(') return parseTuple(); + if (c == '\'' || c == '"') return parseString(); + if (c == '-' || c == '+' || Character.isDigit(c)) return parseNumber(); + return parseKeyword(); + } + + Map parseDict() { Map map = new HashMap<>(); - String content = s.substring(1, s.length() - 1); - if (content.isEmpty()) return map; - for (String pair : content.split(",")) { - String[] kv = pair.split(":"); - if (kv.length == 2) map.put((String) parseVal(kv[0]), parseVal(kv[1])); + pos++; // skip '{' + skipWhitespace(); + if (hasMore() && input.charAt(pos) == '}') { + pos++; + return map; + } + while (true) { + skipWhitespace(); + Object key = parseExpression(); + skipWhitespace(); + if (pos >= input.length() || input.charAt(pos) != ':') { + throw new IllegalArgumentException("Expected ':' in dict at position " + pos); + } + pos++; // skip ':' + skipWhitespace(); + Object value = parseExpression(); + map.put(key.toString(), value); + skipWhitespace(); + if (pos >= input.length()) { + throw new IllegalArgumentException("Unterminated dict: missing '}'"); + } + if (input.charAt(pos) == '}') { + pos++; + break; + } + if (input.charAt(pos) == ',') { + pos++; + skipWhitespace(); + // trailing comma before close + if (hasMore() && input.charAt(pos) == '}') { + pos++; + break; + } + } else { + throw new IllegalArgumentException("Expected ',' or '}' in dict at position " + pos); + } } return map; - } else if (s.startsWith("[") && s.endsWith("]")) { + } + + List parseList() { List list = new ArrayList<>(); - String content = s.substring(1, s.length() - 1); - if (content.isEmpty()) return list; - for (String val : content.split(",")) { - list.add(parseVal(val)); + pos++; // skip '[' + skipWhitespace(); + if (hasMore() && input.charAt(pos) == ']') { + pos++; + return list; + } + while (true) { + skipWhitespace(); + list.add(parseExpression()); + skipWhitespace(); + if (pos >= input.length()) { + throw new IllegalArgumentException("Unterminated list: missing ']'"); + } + if (input.charAt(pos) == ']') { + pos++; + break; + } + if (input.charAt(pos) == ',') { + pos++; + skipWhitespace(); + // trailing comma before close + if (hasMore() && input.charAt(pos) == ']') { + pos++; + break; + } + } else { + throw new IllegalArgumentException("Expected ',' or ']' in list at position " + pos); + } } return list; } - return parseVal(s); - } - // helper: Converts Python types to Java primitives - private static Object parseVal(String s) { - s = s.trim().replace("'", "").replace("\"", ""); - if (s.equalsIgnoreCase("True")) return true; - if (s.equalsIgnoreCase("False")) return false; - if (s.equalsIgnoreCase("None")) return null; - try { return Integer.parseInt(s); } catch (NumberFormatException e1) { - try { return Double.parseDouble(s); } catch (NumberFormatException e2) { return s; } + List parseTuple() { + List list = new ArrayList<>(); + pos++; // skip '(' + skipWhitespace(); + if (hasMore() && input.charAt(pos) == ')') { + pos++; + return list; + } + while (true) { + skipWhitespace(); + list.add(parseExpression()); + skipWhitespace(); + if (pos >= input.length()) { + throw new IllegalArgumentException("Unterminated tuple: missing ')'"); + } + if (input.charAt(pos) == ')') { + pos++; + break; + } + if (input.charAt(pos) == ',') { + pos++; + skipWhitespace(); + // trailing comma before close + if (hasMore() && input.charAt(pos) == ')') { + pos++; + break; + } + } else { + throw new IllegalArgumentException("Expected ',' or ')' in tuple at position " + pos); + } + } + return list; + } + + String parseString() { + char quote = advance(); // opening quote + StringBuilder sb = new StringBuilder(); + while (pos < input.length()) { + char c = input.charAt(pos); + if (c == '\\' && pos + 1 < input.length()) { + pos++; + char escaped = input.charAt(pos); + switch (escaped) { + case 'n': sb.append('\n'); break; + case 't': sb.append('\t'); break; + case 'r': sb.append('\r'); break; + case '\\': sb.append('\\'); break; + case '\'': sb.append('\''); break; + case '"': sb.append('"'); break; + default: sb.append('\\').append(escaped); break; + } + pos++; + } else if (c == quote) { + pos++; + return sb.toString(); + } else { + sb.append(c); + pos++; + } + } + throw new IllegalArgumentException("Unterminated string starting at position " + (pos - sb.length() - 1)); + } + + Number parseNumber() { + int start = pos; + if (pos < input.length() && (input.charAt(pos) == '-' || input.charAt(pos) == '+')) { + pos++; + } + boolean hasDecimal = false; + boolean hasExponent = false; + while (pos < input.length()) { + char c = input.charAt(pos); + if (Character.isDigit(c)) { + pos++; + } else if (c == '.' && !hasDecimal && !hasExponent) { + hasDecimal = true; + pos++; + } else if ((c == 'e' || c == 'E') && !hasExponent) { + hasExponent = true; + pos++; + if (pos < input.length() && (input.charAt(pos) == '+' || input.charAt(pos) == '-')) { + pos++; + } + } else { + break; + } + } + String numStr = input.substring(start, pos); + try { + if (hasDecimal || hasExponent) { + return Double.parseDouble(numStr); + } else { + try { + return Integer.parseInt(numStr); + } catch (NumberFormatException e) { + return Long.parseLong(numStr); + } + } + } catch (NumberFormatException e) { + throw new IllegalArgumentException("Invalid number: '" + numStr + "' at position " + start); + } + } + + Object parseKeyword() { + int start = pos; + while (pos < input.length() && Character.isLetterOrDigit(input.charAt(pos)) || (pos < input.length() && input.charAt(pos) == '_')) { + pos++; + } + String word = input.substring(start, pos); + switch (word) { + case "True": return Boolean.TRUE; + case "False": return Boolean.FALSE; + case "None": return null; + default: throw new IllegalArgumentException("Unknown keyword: '" + word + "' at position " + start); + } } } -} \ No newline at end of file +} From 3d09a1031b4a8d7f911e10d2bf81e1c36c2fcae2 Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Thu, 12 Feb 2026 00:49:29 +0530 Subject: [PATCH 059/275] fix(ci): add missing dependencies to requirements-ci.txt Add click, rich, psutil, beautifulsoup4, and lxml which are required by tests/test_cli.py and tests/test_graph.py for test imports to work. --- requirements-ci.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/requirements-ci.txt b/requirements-ci.txt index 1dc06cfb..5668f6a5 100644 --- a/requirements-ci.txt +++ b/requirements-ci.txt @@ -4,3 +4,8 @@ pytest ruff pyzmq numpy +click>=8.0.0 +rich>=10.0.0 +psutil>=5.8.0 +beautifulsoup4 +lxml From c732f606549689cdbd77553591be6913d70fb8fd Mon Sep 17 00:00:00 2001 From: Titas-Ghosh Date: Thu, 12 Feb 2026 01:00:35 +0530 Subject: [PATCH 060/275] Fix ZMQ write JSON serialization for NumPy types --- concore.py | 6 ++++-- mkconcore.py | 34 ++++++++++++++++++++-------------- tests/test_concore.py | 32 ++++++++++++++++++++++++++++++++ 3 files changed, 56 insertions(+), 16 deletions(-) diff --git a/concore.py b/concore.py index 3742412a..088cc5df 100644 --- a/concore.py +++ b/concore.py @@ -373,7 +373,9 @@ def write(port_identifier, name, val, delta=0): if isinstance(port_identifier, str) and port_identifier in zmq_ports: zmq_p = zmq_ports[port_identifier] try: - zmq_p.send_json_with_retry(val) + # Keep ZMQ payloads JSON-serializable by normalizing numpy types. + zmq_val = convert_numpy_to_python(val) + zmq_p.send_json_with_retry(zmq_val) except zmq.error.ZMQError as e: logging.error(f"ZMQ write error on port {port_identifier} (name: {name}): {e}") except Exception as e: @@ -430,4 +432,4 @@ def initval(simtime_val_str): except Exception as e: logging.error(f"Error parsing simtime_val_str '{simtime_val_str}': {e}. Returning empty list.") - return [] \ No newline at end of file + return [] diff --git a/mkconcore.py b/mkconcore.py index c3995df0..3b20f8e4 100644 --- a/mkconcore.py +++ b/mkconcore.py @@ -75,17 +75,23 @@ import shlex # Added for POSIX shell escaping # input validation helper -def safe_name(value, context): - """ - Validates that the input string does not contain characters dangerous - for filesystem paths or shell command injection. - """ - if not value: - raise ValueError(f"{context} cannot be empty") - # blocks path traversal (/, \), control characters, and shell metacharacters (*, ?, <, >, |, ;, &, $, `, ', ", (, )) - if re.search(r'[\x00-\x1F\x7F\\/:*?"<>|;&`$\'()]', value): - raise ValueError(f"Unsafe {context}: '{value}' contains illegal characters.") - return value +def safe_name(value, context, allow_path=False): + """ + Validates that the input string does not contain characters dangerous + for filesystem paths or shell command injection. + """ + if not value: + raise ValueError(f"{context} cannot be empty") + # blocks control characters and shell metacharacters + # allow path separators and drive colons for full paths when needed + if allow_path: + pattern = r'[\x00-\x1F\x7F*?"<>|;&`$\'()]' + else: + # blocks path traversal (/, \, :) in addition to shell metacharacters + pattern = r'[\x00-\x1F\x7F\\/:*?"<>|;&`$\'()]' + if re.search(pattern, value): + raise ValueError(f"Unsafe {context}: '{value}' contains illegal characters.") + return value MKCONCORE_VER = "22-09-18" @@ -146,8 +152,8 @@ def _resolve_concore_path(): sourcedir = sys.argv[2] outdir = sys.argv[3] -# Validate outdir argument -safe_name(outdir, "Output directory argument") +# Validate outdir argument (allow full paths) +safe_name(outdir, "Output directory argument", allow_path=True) if not os.path.isdir(sourcedir): logging.error(f"{sourcedir} does not exist") @@ -1221,4 +1227,4 @@ def cleanup_script_files(): os.chmod(outdir+"/clear",stat.S_IRWXU) os.chmod(outdir+"/maxtime",stat.S_IRWXU) os.chmod(outdir+"/params",stat.S_IRWXU) - os.chmod(outdir+"/unlock",stat.S_IRWXU) \ No newline at end of file + os.chmod(outdir+"/unlock",stat.S_IRWXU) diff --git a/tests/test_concore.py b/tests/test_concore.py index e7db9637..1fb0182d 100644 --- a/tests/test_concore.py +++ b/tests/test_concore.py @@ -212,3 +212,35 @@ def test_windows_quoted_input(self): s = s[1:-1] # simulate quote stripping before parse_params params = parse_params(s) assert params == {"a": 1, "b": 2} + + +class TestWriteZMQ: + @pytest.fixture(autouse=True) + def reset_zmq_ports(self): + import concore + original_ports = concore.zmq_ports.copy() + yield + concore.zmq_ports.clear() + concore.zmq_ports.update(original_ports) + + def test_write_converts_numpy_types_for_zmq(self): + import concore + + class DummyPort: + def __init__(self): + self.sent = None + + def send_json_with_retry(self, message): + self.sent = message + + dummy = DummyPort() + concore.zmq_ports["test_zmq"] = dummy + + payload = [np.int64(7), np.float64(3.5), {"x": np.float32(1.25)}] + concore.write("test_zmq", "data", payload) + + assert dummy.sent is not None + assert dummy.sent == [7, 3.5, {"x": 1.25}] + assert not isinstance(dummy.sent[0], np.generic) + assert not isinstance(dummy.sent[1], np.generic) + assert not isinstance(dummy.sent[2]["x"], np.generic) From 64e321a6f995cad9d463ae70b2d4a0fe00f41035 Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Thu, 12 Feb 2026 01:01:20 +0530 Subject: [PATCH 061/275] Address Copilot code review suggestions - Remove s += initstr on max-retries and parse-error paths in read() (accumulator should not include default when returning it) - Add dict key validation: throw if key is not a non-null String - Use explicit UTF-8 charset in parseFile() and sparams reading - Align sparams parsing with Python parse_params logic: try literalEval first if input looks like dict literal - Update test comment to reflect actual coverage - Remove unused list creation in testRoundTripSerialization() - Add test for non-string dict key validation All 50 tests pass. --- TestLiteralEval.java | 24 ++++++++++++---------- concoredocker.java | 47 ++++++++++++++++++++++++++++++-------------- 2 files changed, 46 insertions(+), 25 deletions(-) diff --git a/TestLiteralEval.java b/TestLiteralEval.java index dc1c466d..97d80f3c 100644 --- a/TestLiteralEval.java +++ b/TestLiteralEval.java @@ -4,7 +4,7 @@ * Test suite for concoredocker.literalEval() recursive descent parser. * Covers: dicts, lists, tuples, numbers, strings, booleans, None, * nested structures, escape sequences, scientific notation, - * toPythonLiteral serialization, and fractional simtime. + * fractional simtime, and related round-trip parsing behavior. */ public class TestLiteralEval { static int passed = 0; @@ -36,6 +36,7 @@ public static void main(String[] args) { testUnterminatedList(); testUnterminatedDict(); testUnterminatedTuple(); + testNonStringDictKey(); System.out.println("\n=== Results: " + passed + " passed, " + failed + " failed out of " + (passed + failed) + " tests ==="); if (failed > 0) { @@ -198,15 +199,7 @@ static void testFractionalSimtime() { // --- Round-trip serialization tests --- static void testRoundTripSerialization() { - // Serialize a list with mixed types, then re-parse and verify - List original = new ArrayList<>(); - original.add(1); - original.add(2.5); - original.add(true); - original.add(false); - original.add(null); - original.add("hello"); - + // Conceptually: a list with mixed types [1, 2.5, true, false, null, "hello"] // Use reflection-free approach: build the Python literal manually // and verify round-trip through literalEval String serialized = "[1, 2.5, True, False, None, 'hello']"; @@ -263,4 +256,15 @@ static void testUnterminatedTuple() { check("unterminated tuple throws", true, e.getMessage().contains("Unterminated tuple")); } } + + static void testNonStringDictKey() { + // Dict keys must be strings - numeric keys should throw + try { + concoredocker.literalEval("{123: 'value'}"); + System.out.println("FAIL: numeric dict key should throw"); + failed++; + } catch (IllegalArgumentException e) { + check("numeric dict key throws", true, e.getMessage().contains("Dict keys must be non-null strings")); + } + } } diff --git a/concoredocker.java b/concoredocker.java index ca551b59..b77bdd54 100644 --- a/concoredocker.java +++ b/concoredocker.java @@ -41,25 +41,39 @@ public static void main(String[] args) { } try { - String sparams = new String(Files.readAllBytes(Paths.get(inpath + "1/concore.params"))); + String sparams = new String(Files.readAllBytes(Paths.get(inpath + "1/concore.params")), java.nio.charset.StandardCharsets.UTF_8); if (sparams.length() > 0 && sparams.charAt(0) == '"') { // windows keeps "" need to remove sparams = sparams.substring(1); sparams = sparams.substring(0, sparams.indexOf('"')); } - if (!sparams.equals("{")) { + // Try parsing as dict literal first (matches Python parse_params logic) + sparams = sparams.trim(); + if (sparams.startsWith("{") && sparams.endsWith("}")) { + try { + Object parsed = literalEval(sparams); + if (parsed instanceof Map) { + @SuppressWarnings("unchecked") + Map parsedMap = (Map) parsed; + params = parsedMap; + } + } catch (Exception e) { + System.out.println("bad params: " + sparams); + } + } else if (!sparams.isEmpty()) { + // Fallback: convert key=value,key=value format to dict System.out.println("converting sparams: " + sparams); sparams = "{'" + sparams.replaceAll(",", ",'").replaceAll("=", "':").replaceAll(" ", "") + "}"; System.out.println("converted sparams: " + sparams); - } - try { - Object parsed = literalEval(sparams); - if (parsed instanceof Map) { - @SuppressWarnings("unchecked") - Map parsedMap = (Map) parsed; - params = parsedMap; + try { + Object parsed = literalEval(sparams); + if (parsed instanceof Map) { + @SuppressWarnings("unchecked") + Map parsedMap = (Map) parsed; + params = parsedMap; + } + } catch (Exception e) { + System.out.println("bad params: " + sparams); } - } catch (Exception e) { - System.out.println("bad params: " + sparams); } } catch (IOException e) { params = new HashMap<>(); @@ -73,7 +87,7 @@ public static void main(String[] args) { * Returns empty map if file is empty or malformed (matches Python safe_literal_eval). */ private static Map parseFile(String filename) throws IOException { - String content = new String(Files.readAllBytes(Paths.get(filename))); + String content = new String(Files.readAllBytes(Paths.get(filename)), java.nio.charset.StandardCharsets.UTF_8); content = content.trim(); if (content.isEmpty()) { return new HashMap<>(); @@ -182,7 +196,6 @@ private static List read(int port, String name, String initstr) { if (ins.length() == 0) { System.out.println("Max retries reached for " + filePath + ", using default value."); - s += initstr; return defaultVal; } @@ -197,7 +210,6 @@ private static List read(int port, String name, String initstr) { } catch (Exception e) { System.out.println("Error parsing " + ins + ": " + e.getMessage()); } - s += initstr; return defaultVal; } @@ -404,7 +416,12 @@ Map parseDict() { pos++; // skip ':' skipWhitespace(); Object value = parseExpression(); - map.put(key.toString(), value); + if (!(key instanceof String)) { + throw new IllegalArgumentException( + "Dict keys must be non-null strings, but got: " + + (key == null ? "null" : key.getClass().getSimpleName())); + } + map.put((String) key, value); skipWhitespace(); if (pos >= input.length()) { throw new IllegalArgumentException("Unterminated dict: missing '}'"); From 7d47d9062ff00c7af67e8f45c1bfb913382e2b7b Mon Sep 17 00:00:00 2001 From: GREENRAT-K405 Date: Thu, 12 Feb 2026 01:05:17 +0530 Subject: [PATCH 062/275] add greetings bot --- .github/workflows/greetings.yml | 41 +++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 .github/workflows/greetings.yml diff --git a/.github/workflows/greetings.yml b/.github/workflows/greetings.yml new file mode 100644 index 00000000..7de235e8 --- /dev/null +++ b/.github/workflows/greetings.yml @@ -0,0 +1,41 @@ +name: Greetings + +on: + pull_request_target: + types: [opened] + issues: + types: [opened] + +jobs: + greeting: + runs-on: ubuntu-latest + permissions: + issues: write + pull-requests: write + steps: + - uses: actions/first-interaction@v3 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + + issue_message: | + 👋 Welcome to the Control-core Project, @${{ github.actor }}! Thank you for opening your first issue in concore. + We appreciate your contribution to the organization and will review it as soon as possible. + + Before we get started, please check out these resources: + - 📚 [Project Documentation](https://control-core.readthedocs.io/) + - 📘 [Contribution Guidelines](https://github.com/ControlCore-Project/concore/blob/main/CONTRIBUTING.md) + - 📜 [Code of Conduct](https://github.com/ControlCore-Project/concore/blob/main/CODE_OF_CONDUCT.md) + + pr_message: | + 🎉 Welcome aboard, @${{ github.actor }}! Thank you for your first pull request in concore. + + Please ensure that you are contributing to the **dev** branch. + + Your contribution means a lot to us. We'll review it shortly. + + Please ensure you have reviewed our: + - 📚 [Project Documentation](https://control-core.readthedocs.io/) + - 📘 [Contribution Guidelines](https://github.com/ControlCore-Project/concore/blob/main/CONTRIBUTING.md) + - 📜 [Code of Conduct](https://github.com/ControlCore-Project/concore/blob/main/CODE_OF_CONDUCT.md) + + If you have any questions, feel free to ask. Happy coding! \ No newline at end of file From f791ab0b9e0bf53bc9c2d3efbabacb2c5dfd8b3b Mon Sep 17 00:00:00 2001 From: Pradeeban Kathiravelu Date: Wed, 11 Feb 2026 10:39:45 -0900 Subject: [PATCH 063/275] Remove 'master' branch from CI workflow --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1100ba9c..3be703e1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,9 +2,9 @@ name: CI on: push: - branches: [main, master, dev] + branches: [main, dev] pull_request: - branches: [main, master, dev] + branches: [main, dev] jobs: lint-and-test: From 1f8ee7cd94d6a8c76bd86b9bc96f211a0571a8f2 Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Thu, 12 Feb 2026 01:21:23 +0530 Subject: [PATCH 064/275] Fix: undefined behavior in destructor, cross-platform guards, and infinite loop issues (Issue #236) --- concore.hpp | 77 +++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 60 insertions(+), 17 deletions(-) diff --git a/concore.hpp b/concore.hpp index b74ddd7e..65109afe 100644 --- a/concore.hpp +++ b/concore.hpp @@ -12,9 +12,11 @@ //libraries for platform independent delay. Supports C++11 upwards #include #include +#ifdef __linux__ #include #include #include +#endif #include #include @@ -32,11 +34,11 @@ class Concore{ string inpath = "./in"; string outpath = "./out"; - int shmId_create; - int shmId_get; + int shmId_create = -1; + int shmId_get = -1; - char* sharedData_create; - char* sharedData_get; + char* sharedData_create = nullptr; + char* sharedData_get = nullptr; // File sharing:- 0, Shared Memory:- 1 int communication_iport = 0; // iport refers to input port int communication_oport = 0; // oport refers to input port @@ -56,14 +58,23 @@ class Concore{ Concore(){ iport = mapParser("concore.iport"); oport = mapParser("concore.oport"); - std::map::iterator it_iport = iport.begin(); - std::map::iterator it_oport = oport.begin(); - int iport_number = ExtractNumeric(it_iport->first); - int oport_number = ExtractNumeric(it_oport->first); + + int iport_number = -1; + int oport_number = -1; + + if (!iport.empty()) { + std::map::iterator it_iport = iport.begin(); + iport_number = ExtractNumeric(it_iport->first); + } + if (!oport.empty()) { + std::map::iterator it_oport = oport.begin(); + oport_number = ExtractNumeric(it_oport->first); + } // if iport_number and oport_number is equal to -1 then it refers to File Method, // otherwise it refers to Shared Memory and the number represent the unique key. +#ifdef __linux__ if(oport_number != -1) { // oport_number is not equal to -1 so refers to SM and value is key. @@ -76,7 +87,8 @@ class Concore{ // iport_number is not equal to -1 so refers to SM and value is key. communication_iport = 1; this->getSharedMemory(iport_number); - } + } +#endif } /** @@ -85,12 +97,20 @@ class Concore{ */ ~Concore() { +#ifdef __linux__ // Detach the shared memory segment from the process - shmdt(sharedData_create); - shmdt(sharedData_get); + if (communication_oport == 1 && sharedData_create != nullptr) { + shmdt(sharedData_create); + } + if (communication_iport == 1 && sharedData_get != nullptr) { + shmdt(sharedData_get); + } // Remove the shared memory segment - shmctl(shmId_create, IPC_RMID, nullptr); + if (shmId_create != -1) { + shmctl(shmId_create, IPC_RMID, nullptr); + } +#endif } /** @@ -127,6 +147,7 @@ class Concore{ return std::stoi(numberString); } +#ifdef __linux__ /** * @brief Creates a shared memory segment with the given key. * @param key The key for the shared memory segment. @@ -143,6 +164,7 @@ class Concore{ sharedData_create = static_cast(shmat(shmId_create, NULL, 0)); if (sharedData_create == reinterpret_cast(-1)) { std::cerr << "Failed to attach shared memory segment." << std::endl; + sharedData_create = nullptr; } } @@ -153,7 +175,9 @@ class Concore{ */ void getSharedMemory(key_t key) { - while (true) { + int retry = 0; + const int MAX_RETRY = 100; + while (retry < MAX_RETRY) { // Get the shared memory segment created by Writer shmId_get = shmget(key, 256, 0666); // Check if shared memory exists @@ -163,11 +187,22 @@ class Concore{ std::cout << "Shared memory does not exist. Make sure the writer process is running." << std::endl; sleep(1); // Sleep for 1 second before checking again + retry++; + } + + if (shmId_get == -1) { + std::cerr << "Failed to get shared memory segment after max retries." << std::endl; + return; } // Attach the shared memory segment to the process's address space sharedData_get = static_cast(shmat(shmId_get, NULL, 0)); + if (sharedData_get == reinterpret_cast(-1)) { + std::cerr << "Failed to attach shared memory segment." << std::endl; + sharedData_get = nullptr; + } } +#endif /** * @brief Parses a file containing port and number mappings and returns a map of the values. @@ -187,6 +222,10 @@ class Concore{ portfile.close(); } + if (portstr.empty()) { + return ans; + } + portstr[portstr.size()-1]=','; portstr+='}'; int i=0; @@ -303,7 +342,9 @@ class Concore{ ins = initstr; } - while ((int)ins.length()==0){ + int retry = 0; + const int MAX_RETRY = 100; + while ((int)ins.length()==0 && retry < MAX_RETRY){ this_thread::sleep_for(timespan); try{ ifstream infile; @@ -324,8 +365,7 @@ class Concore{ catch(...){ cout<<"Read error"; } - - + retry++; } s += ins; @@ -368,7 +408,9 @@ class Concore{ ins = initstr; } - while ((int)ins.length()==0){ + int retry = 0; + const int MAX_RETRY = 100; + while ((int)ins.length()==0 && retry < MAX_RETRY){ this_thread::sleep_for(timespan); try{ if(shmId_get != -1) { @@ -385,6 +427,7 @@ class Concore{ catch(...){ std::cout << "Read error" << std::endl; } + retry++; } s += ins; From c3b498ea1542223cf226e209b3e5ed3597258246 Mon Sep 17 00:00:00 2001 From: PARAM KANADA Date: Thu, 12 Feb 2026 01:27:27 +0530 Subject: [PATCH 065/275] Fix project name in greetings.yml Updated project name to 'CONTROL-CORE' in issue message. --- .github/workflows/greetings.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/greetings.yml b/.github/workflows/greetings.yml index 7de235e8..e9d7e175 100644 --- a/.github/workflows/greetings.yml +++ b/.github/workflows/greetings.yml @@ -18,7 +18,7 @@ jobs: repo_token: ${{ secrets.GITHUB_TOKEN }} issue_message: | - 👋 Welcome to the Control-core Project, @${{ github.actor }}! Thank you for opening your first issue in concore. + 👋 Welcome to the CONTROL-CORE Project, @${{ github.actor }}! Thank you for opening your first issue in concore. We appreciate your contribution to the organization and will review it as soon as possible. Before we get started, please check out these resources: @@ -38,4 +38,4 @@ jobs: - 📘 [Contribution Guidelines](https://github.com/ControlCore-Project/concore/blob/main/CONTRIBUTING.md) - 📜 [Code of Conduct](https://github.com/ControlCore-Project/concore/blob/main/CODE_OF_CONDUCT.md) - If you have any questions, feel free to ask. Happy coding! \ No newline at end of file + If you have any questions, feel free to ask. Happy coding! From 0e49e15b11f13b409823ce364288a977da9761cc Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Thu, 12 Feb 2026 01:36:21 +0530 Subject: [PATCH 066/275] Fix: Prepend simtime to ZMQ write() payload for consistency (Issue #244) - ZMQ write now prepends [simtime + delta] to match file-based write behavior - ZMQ read now strips simtime prefix (mirroring file-based read behavior) - Updated docstring to clarify val is data-only - Added round-trip test to verify write+read returns original data --- concore.py | 16 ++++++++++++++-- tests/test_concore.py | 38 +++++++++++++++++++++++++++++++++++--- 2 files changed, 49 insertions(+), 5 deletions(-) diff --git a/concore.py b/concore.py index 088cc5df..cb97ab61 100644 --- a/concore.py +++ b/concore.py @@ -298,6 +298,12 @@ def read(port_identifier, name, initstr_val): zmq_p = zmq_ports[port_identifier] try: message = zmq_p.recv_json_with_retry() + # Strip simtime prefix if present (mirroring file-based read behavior) + if isinstance(message, list) and len(message) > 0: + first_element = message[0] + if isinstance(first_element, (int, float)): + simtime = max(simtime, first_element) + return message[1:] return message except zmq.error.ZMQError as e: logging.error(f"ZMQ read error on port {port_identifier} (name: {name}): {e}. Returning default.") @@ -365,7 +371,7 @@ def read(port_identifier, name, initstr_val): def write(port_identifier, name, val, delta=0): """ Write data either to ZMQ port or file. - `val` must be list (with simtime prefix) or string. + `val` is the data payload (list or string); write() prepends [simtime + delta] internally. """ global simtime @@ -375,7 +381,13 @@ def write(port_identifier, name, val, delta=0): try: # Keep ZMQ payloads JSON-serializable by normalizing numpy types. zmq_val = convert_numpy_to_python(val) - zmq_p.send_json_with_retry(zmq_val) + if isinstance(zmq_val, list): + # Prepend simtime to match file-based write behavior + payload = [simtime + delta] + zmq_val + zmq_p.send_json_with_retry(payload) + simtime += delta + else: + zmq_p.send_json_with_retry(zmq_val) except zmq.error.ZMQError as e: logging.error(f"ZMQ write error on port {port_identifier} (name: {name}): {e}") except Exception as e: diff --git a/tests/test_concore.py b/tests/test_concore.py index 1fb0182d..833edbb0 100644 --- a/tests/test_concore.py +++ b/tests/test_concore.py @@ -236,11 +236,43 @@ def send_json_with_retry(self, message): dummy = DummyPort() concore.zmq_ports["test_zmq"] = dummy + # Reset simtime for predictable test behavior + concore.simtime = 0 + payload = [np.int64(7), np.float64(3.5), {"x": np.float32(1.25)}] concore.write("test_zmq", "data", payload) assert dummy.sent is not None - assert dummy.sent == [7, 3.5, {"x": 1.25}] - assert not isinstance(dummy.sent[0], np.generic) + # ZMQ write now prepends simtime (0 in this case) to match file-based write behavior + assert dummy.sent == [0, 7, 3.5, {"x": 1.25}] + # Data values (after simtime) should be converted from numpy types assert not isinstance(dummy.sent[1], np.generic) - assert not isinstance(dummy.sent[2]["x"], np.generic) + assert not isinstance(dummy.sent[2], np.generic) + assert not isinstance(dummy.sent[3]["x"], np.generic) + + def test_zmq_write_read_roundtrip(self): + """Test that ZMQ write+read returns original data without simtime prefix.""" + import concore + + class DummyZMQPort: + def __init__(self): + self.buffer = None + + def send_json_with_retry(self, message): + self.buffer = message + + def recv_json_with_retry(self): + return self.buffer + + dummy = DummyZMQPort() + concore.zmq_ports["roundtrip_test"] = dummy + + # Reset simtime for predictable test behavior + concore.simtime = 0 + + original_data = [1.5, 2.5, 3.5] + concore.write("roundtrip_test", "data", original_data) + + # Read should return original data (simtime stripped) + result = concore.read("roundtrip_test", "data", "[]") + assert result == original_data From 7db41e4b1c661e79564287f62e536367851d043a Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Thu, 12 Feb 2026 01:54:03 +0530 Subject: [PATCH 067/275] Security: Replace unsafe eval() in MATLAB parsers with safe numeric parsing (Issue #245) --- concore_default_maxtime.m | 5 ++++- concore_initval.m | 5 ++++- concore_iport.m | 4 +++- concore_read.m | 5 ++++- 4 files changed, 15 insertions(+), 4 deletions(-) diff --git a/concore_default_maxtime.m b/concore_default_maxtime.m index 5627d5f5..81ce6931 100644 --- a/concore_default_maxtime.m +++ b/concore_default_maxtime.m @@ -3,7 +3,10 @@ function concore_default_maxtime(default) try maxfile = fopen(strcat(concore.inpath,'1/concore.maxtime')); instr = fscanf(maxfile,'%c'); - concore.maxtime = eval(instr); + % Safe numeric parsing (replaces unsafe eval) + clean_str = strtrim(instr); + clean_str = regexprep(clean_str, '[\[\]]', ''); + concore.maxtime = sscanf(clean_str, '%f'); fclose(maxfile); catch exc concore.maxtime = default; diff --git a/concore_initval.m b/concore_initval.m index 73cc1469..da3b5a07 100644 --- a/concore_initval.m +++ b/concore_initval.m @@ -1,6 +1,9 @@ function [result] = concore_initval(simtime_val) global concore; - result = eval(simtime_val); + % Safe numeric parsing (replaces unsafe eval) + clean_str = strtrim(simtime_val); + clean_str = regexprep(clean_str, '[\[\]]', ''); + result = sscanf(clean_str, '%f').'; concore.simtime = result(1); result = result(2:length(result)); end diff --git a/concore_iport.m b/concore_iport.m index 128252e2..16ca2b75 100644 --- a/concore_iport.m +++ b/concore_iport.m @@ -7,7 +7,9 @@ if isequal(s(i:i+length(target)-1),target) for j = i+length(target):length(s) if isequal(s(j),',')||isequal(s(j),'}') - result = eval(s(i+length(target):j-1)); + % Safe numeric parsing (replaces unsafe eval) + port_str = strtrim(s(i+length(target):j-1)); + result = sscanf(port_str, '%f'); return end end diff --git a/concore_read.m b/concore_read.m index ba32c7b1..2abd8e35 100644 --- a/concore_read.m +++ b/concore_read.m @@ -25,7 +25,10 @@ ins = inistr; end concore.s = strcat(concore.s, ins); - result = eval(ins); + % Safe numeric parsing (replaces unsafe eval) + clean_str = strtrim(ins); + clean_str = regexprep(clean_str, '[\[\]]', ''); + result = sscanf(clean_str, '%f').'; concore.simtime = max(concore.simtime,result(1)); result = result(2:length(result)); end From f2096895bc28c888909bd6556127cb1f7f0d6f5e Mon Sep 17 00:00:00 2001 From: Pradeeban Kathiravelu Date: Wed, 11 Feb 2026 11:42:00 -0900 Subject: [PATCH 068/275] Update concore_initval.m Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- concore_initval.m | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/concore_initval.m b/concore_initval.m index da3b5a07..8e9e175d 100644 --- a/concore_initval.m +++ b/concore_initval.m @@ -4,6 +4,16 @@ clean_str = strtrim(simtime_val); clean_str = regexprep(clean_str, '[\[\]]', ''); result = sscanf(clean_str, '%f').'; + % Guard against empty or invalid numeric input + if isempty(result) + concore.simtime = 0; + result = []; + return; + end concore.simtime = result(1); - result = result(2:length(result)); + if numel(result) >= 2 + result = result(2:end); + else + result = []; + end end From 0371a284d1cc4f3520ed0c5020ef77d9ef4babed Mon Sep 17 00:00:00 2001 From: Pradeeban Kathiravelu Date: Wed, 11 Feb 2026 11:42:14 -0900 Subject: [PATCH 069/275] Update concore_default_maxtime.m Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- concore_default_maxtime.m | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/concore_default_maxtime.m b/concore_default_maxtime.m index 81ce6931..489187d0 100644 --- a/concore_default_maxtime.m +++ b/concore_default_maxtime.m @@ -6,7 +6,14 @@ function concore_default_maxtime(default) % Safe numeric parsing (replaces unsafe eval) clean_str = strtrim(instr); clean_str = regexprep(clean_str, '[\[\]]', ''); - concore.maxtime = sscanf(clean_str, '%f'); + % Normalize commas to whitespace so sscanf can parse all tokens + clean_str = strrep(clean_str, ',', ' '); + parsed_values = sscanf(clean_str, '%f'); + if numel(parsed_values) == 1 + concore.maxtime = parsed_values; + else + concore.maxtime = default; + end fclose(maxfile); catch exc concore.maxtime = default; From 1966e9d94de8369aad3895d129e38f0625ae7998 Mon Sep 17 00:00:00 2001 From: Pradeeban Kathiravelu Date: Wed, 11 Feb 2026 11:42:24 -0900 Subject: [PATCH 070/275] Update concore_iport.m Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- concore_iport.m | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/concore_iport.m b/concore_iport.m index 16ca2b75..8a2146f1 100644 --- a/concore_iport.m +++ b/concore_iport.m @@ -10,6 +10,10 @@ % Safe numeric parsing (replaces unsafe eval) port_str = strtrim(s(i+length(target):j-1)); result = sscanf(port_str, '%f'); + if isempty(result) + % Keep the initialized default value (0) if parsing fails + result = 0; + end return end end From 23df70f8e229c12fabccb8ebe4ff5539d6a2979d Mon Sep 17 00:00:00 2001 From: Pradeeban Kathiravelu Date: Wed, 11 Feb 2026 11:43:00 -0900 Subject: [PATCH 071/275] Update concore_read.m Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- concore_read.m | 2 ++ 1 file changed, 2 insertions(+) diff --git a/concore_read.m b/concore_read.m index 2abd8e35..69de9ffa 100644 --- a/concore_read.m +++ b/concore_read.m @@ -28,6 +28,8 @@ % Safe numeric parsing (replaces unsafe eval) clean_str = strtrim(ins); clean_str = regexprep(clean_str, '[\[\]]', ''); + % Normalize comma delimiters to whitespace so sscanf parses all values + clean_str = strrep(clean_str, ',', ' '); result = sscanf(clean_str, '%f').'; concore.simtime = max(concore.simtime,result(1)); result = result(2:length(result)); From b600bedf36d182fe5004bae0a5084d775f1ec1d5 Mon Sep 17 00:00:00 2001 From: Pradeeban Kathiravelu Date: Wed, 11 Feb 2026 11:43:09 -0900 Subject: [PATCH 072/275] Update concore_read.m Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- concore_read.m | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/concore_read.m b/concore_read.m index 69de9ffa..b4ed1bc0 100644 --- a/concore_read.m +++ b/concore_read.m @@ -31,6 +31,15 @@ % Normalize comma delimiters to whitespace so sscanf parses all values clean_str = strrep(clean_str, ',', ' '); result = sscanf(clean_str, '%f').'; - concore.simtime = max(concore.simtime,result(1)); - result = result(2:length(result)); + % Guard against empty parse result to avoid indexing errors + if isempty(result) + result = []; + return; + end + concore.simtime = max(concore.simtime, result(1)); + if numel(result) > 1 + result = result(2:end); + else + result = []; + end end From ec4e9f0ce18601a83f2ccb46d2a4e7d7ad703f51 Mon Sep 17 00:00:00 2001 From: Pradeeban Kathiravelu Date: Wed, 11 Feb 2026 11:43:18 -0900 Subject: [PATCH 073/275] Update concore_initval.m Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- concore_initval.m | 1 + 1 file changed, 1 insertion(+) diff --git a/concore_initval.m b/concore_initval.m index 8e9e175d..4b92b318 100644 --- a/concore_initval.m +++ b/concore_initval.m @@ -3,6 +3,7 @@ % Safe numeric parsing (replaces unsafe eval) clean_str = strtrim(simtime_val); clean_str = regexprep(clean_str, '[\[\]]', ''); + clean_str = strrep(clean_str, ',', ' '); result = sscanf(clean_str, '%f').'; % Guard against empty or invalid numeric input if isempty(result) From 764bc1a95fec57708d02d774e99dd22c79bbe2d5 Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Thu, 12 Feb 2026 02:20:41 +0530 Subject: [PATCH 074/275] Security: Also fix eval() in concore_oport.m --- concore_oport.m | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/concore_oport.m b/concore_oport.m index 9cbe3de8..a9ed01b5 100644 --- a/concore_oport.m +++ b/concore_oport.m @@ -7,7 +7,9 @@ if isequal(s(i:i+length(target)-1),target) for j = i+length(target):length(s) if isequal(s(j),',')||isequal(s(j),'}') - result = eval(s(i+length(target):j-1)); + % Safe numeric parsing (replaces unsafe eval) + port_str = strtrim(s(i+length(target):j-1)); + result = sscanf(port_str, '%f'); return end end From af11b1d2e82e258a43c056a0367710fc930b4109 Mon Sep 17 00:00:00 2001 From: Titas-Ghosh Date: Thu, 12 Feb 2026 02:47:42 +0530 Subject: [PATCH 075/275] Fix default execution type in CLI and add test for run command --- concore_cli/cli.py | 4 +++- tests/test_cli.py | 18 ++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/concore_cli/cli.py b/concore_cli/cli.py index 49c78cb6..6e9ed076 100644 --- a/concore_cli/cli.py +++ b/concore_cli/cli.py @@ -1,5 +1,6 @@ import click from rich.console import Console +import os import sys from .commands.init import init_project @@ -10,6 +11,7 @@ from .commands.inspect import inspect_workflow console = Console() +DEFAULT_EXEC_TYPE = 'windows' if os.name == 'nt' else 'posix' @click.group() @click.version_option(version='1.0.0', prog_name='concore') @@ -31,7 +33,7 @@ def init(name, template): @click.argument('workflow_file', type=click.Path(exists=True)) @click.option('--source', '-s', default='src', help='Source directory') @click.option('--output', '-o', default='out', help='Output directory') -@click.option('--type', '-t', default='windows', type=click.Choice(['windows', 'posix', 'docker']), help='Execution type') +@click.option('--type', '-t', default=DEFAULT_EXEC_TYPE, type=click.Choice(['windows', 'posix', 'docker']), help='Execution type') @click.option('--auto-build', is_flag=True, help='Automatically run build after generation') def run(workflow_file, source, output, type, auto_build): """Run a concore workflow""" diff --git a/tests/test_cli.py b/tests/test_cli.py index a90cc3fe..6aa78109 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,6 +1,7 @@ import unittest import tempfile import shutil +import os from pathlib import Path from click.testing import CliRunner from concore_cli.cli import cli @@ -83,6 +84,23 @@ def test_run_command_from_project_dir(self): self.assertEqual(result.exit_code, 0) self.assertTrue(Path('out/src/concore.py').exists()) + def test_run_command_default_type(self): + with self.runner.isolated_filesystem(temp_dir=self.temp_dir): + result = self.runner.invoke(cli, ['init', 'test-project']) + self.assertEqual(result.exit_code, 0) + + result = self.runner.invoke(cli, [ + 'run', + 'test-project/workflow.graphml', + '--source', 'test-project/src', + '--output', 'out' + ]) + self.assertEqual(result.exit_code, 0) + if os.name == 'nt': + self.assertTrue(Path('out/build.bat').exists()) + else: + self.assertTrue(Path('out/build').exists()) + def test_run_command_existing_output(self): with self.runner.isolated_filesystem(temp_dir=self.temp_dir): result = self.runner.invoke(cli, ['init', 'test-project']) From 1ddd97c2824bab5c79f7adfe99bd5c0f124d840e Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Thu, 12 Feb 2026 03:11:42 +0530 Subject: [PATCH 076/275] Fix: Initialize ysp before use in controller2knob (Issue #266) --- demo/controller2knob.py | 1 + 1 file changed, 1 insertion(+) diff --git a/demo/controller2knob.py b/demo/controller2knob.py index d6427563..284e22e1 100644 --- a/demo/controller2knob.py +++ b/demo/controller2knob.py @@ -1,6 +1,7 @@ import numpy as np import concore +ysp = 3.0 def controller(ym): if ym[0] < ysp: From 2a53537006d325d5c8436a215dd68833dd9cf91c Mon Sep 17 00:00:00 2001 From: Avinash Kumar Deepak Date: Thu, 12 Feb 2026 11:38:41 +0530 Subject: [PATCH 077/275] Add node label validation to prevent command injection (#251) From e0b44ee989ff1a1ae1fe696fdf63b88892c9242c Mon Sep 17 00:00:00 2001 From: Avinash Kumar Deepak Date: Thu, 12 Feb 2026 12:03:50 +0530 Subject: [PATCH 078/275] Reject node labels with shell metacharacters in CLI validate (#251) --- concore_cli/commands/validate.py | 5 +++++ tests/test_graph.py | 17 +++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/concore_cli/commands/validate.py b/concore_cli/commands/validate.py index fa1ea184..f8265b90 100644 --- a/concore_cli/commands/validate.py +++ b/concore_cli/commands/validate.py @@ -84,6 +84,11 @@ def validate_workflow(workflow_file, console): label = label_tag.text.strip() node_labels.append(label) + # reject shell metacharacters to prevent command injection (#251) + if re.search(r'[;&|`$\'"()\\]', label): + errors.append(f"Node '{label}' contains unsafe shell characters") + continue + if ':' not in label: warnings.append(f"Node '{label}' missing format 'ID:filename'") else: diff --git a/tests/test_graph.py b/tests/test_graph.py index 97102dce..f6e2825d 100644 --- a/tests/test_graph.py +++ b/tests/test_graph.py @@ -112,6 +112,23 @@ def test_validate_node_missing_filename(self): self.assertIn('Validation failed', result.output) self.assertIn('has no filename', result.output) + def test_validate_unsafe_node_label(self): + content = ''' + + + + n0;rm -rf /:script.py + + + + ''' + filepath = self.create_graph_file('injection.graphml', content) + + result = self.runner.invoke(cli, ['validate', filepath]) + + self.assertIn('Validation failed', result.output) + self.assertIn('unsafe shell characters', result.output) + def test_validate_valid_graph(self): content = ''' From 11045c1ccd3452d8e21e0d896577033dafb3977d Mon Sep 17 00:00:00 2001 From: Avinash Kumar Deepak Date: Thu, 12 Feb 2026 14:14:18 +0530 Subject: [PATCH 079/275] check taskkill return code on Windows to detect failures --- concore_cli/commands/stop.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/concore_cli/commands/stop.py b/concore_cli/commands/stop.py index 0b2f530e..5b0a9a92 100644 --- a/concore_cli/commands/stop.py +++ b/concore_cli/commands/stop.py @@ -56,9 +56,11 @@ def stop_all(console): name = proc.info.get('name', 'unknown') if sys.platform == 'win32': - subprocess.run(['taskkill', '/F', '/PID', str(pid)], - capture_output=True, - check=False) + result = subprocess.run(['taskkill', '/F', '/PID', str(pid)], + capture_output=True, + check=False) + if result.returncode != 0: + raise RuntimeError(f"taskkill failed with code {result.returncode}") else: proc.terminate() proc.wait(timeout=3) From ffb504408aa7c0beaee91ae79a9b90a3eba404f8 Mon Sep 17 00:00:00 2001 From: Avinash Kumar Deepak Date: Thu, 12 Feb 2026 14:26:23 +0530 Subject: [PATCH 080/275] use module logger in copy_with_port_portname to avoid conflict --- copy_with_port_portname.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/copy_with_port_portname.py b/copy_with_port_portname.py index 7307289f..1a0a033a 100644 --- a/copy_with_port_portname.py +++ b/copy_with_port_portname.py @@ -5,12 +5,6 @@ import logging import json -logging.basicConfig( - level=logging.INFO, - format='%(message)s', - datefmt='%Y-%m-%d %H:%M:%S' -) - def run_specialization_script(template_script_path, output_dir, edge_params_list, python_exe, copy_script_path): """ Calls the copy script to generate a specialized version of a node's script. @@ -149,6 +143,12 @@ def create_modified_script(template_script_path, output_dir, edge_params_json_st sys.exit(1) if __name__ == "__main__": + logging.basicConfig( + level=logging.INFO, + format='%(message)s', + datefmt='%Y-%m-%d %H:%M:%S' + ) + if len(sys.argv) != 4: print("\nUsage: python3 copy_with_port_portname.py ''\n") print("Example JSON: '[{\"port\": \"2355\", \"port_name\": \"FUNBODY_REP_1\", \"source_node_label\": \"nodeA\", \"target_node_label\": \"nodeB\"}]'") From 3b864b47763fb2aaede9358643116a6849b95c4a Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Thu, 12 Feb 2026 17:28:18 +0530 Subject: [PATCH 081/275] Security: Fix command injection in /contribute and /library endpoints (Issue #261) --- fri/server/main.py | 63 +++++++++++++++++++++++++++++++++++++++------- 1 file changed, 54 insertions(+), 9 deletions(-) diff --git a/fri/server/main.py b/fri/server/main.py index f94bc663..8cf30cfd 100644 --- a/fri/server/main.py +++ b/fri/server/main.py @@ -7,8 +7,22 @@ from pathlib import Path import json import platform +import re from flask_cors import CORS, cross_origin +# Input validation pattern for safe names (alphanumeric, dash, underscore, slash, dot, space) +SAFE_INPUT_PATTERN = re.compile(r'^[a-zA-Z0-9_\-/. ]+$') + +def validate_input(value, field_name): + """Validate that input contains only safe characters.""" + if value is None: + return True + if not isinstance(value, str): + raise ValueError(f"Invalid {field_name}: must be a string") + if len(value) > 0 and not SAFE_INPUT_PATTERN.match(value): + raise ValueError(f"Invalid {field_name}: contains unsafe characters") + return True + cur_path = os.path.dirname(os.path.abspath(__file__)) concore_path = os.path.abspath(os.path.join(cur_path, '../../')) @@ -304,14 +318,24 @@ def contribute(): STUDY_NAME = data.get('study') STUDY_NAME_PATH = data.get('path') BRANCH_NAME = data.get('branch') + + # Validate all user inputs to prevent command injection + validate_input(STUDY_NAME, 'study') + validate_input(STUDY_NAME_PATH, 'path') + validate_input(AUTHOR_NAME, 'auth') + validate_input(BRANCH_NAME, 'branch') + validate_input(PR_TITLE, 'title') + validate_input(PR_BODY, 'desc') + if(platform.uname()[0]=='Windows'): - proc=check_output(["contribute",STUDY_NAME,STUDY_NAME_PATH,AUTHOR_NAME,BRANCH_NAME,PR_TITLE,PR_BODY],cwd=concore_path,shell=True) + proc = subprocess.run(["contribute",STUDY_NAME,STUDY_NAME_PATH,AUTHOR_NAME,BRANCH_NAME,PR_TITLE,PR_BODY],cwd=concore_path,check=True,capture_output=True,text=True) + output_string = proc.stdout else: if len(BRANCH_NAME)==0: proc = check_output([r"./contribute",STUDY_NAME,STUDY_NAME_PATH,AUTHOR_NAME],cwd=concore_path) else: proc = check_output([r"./contribute",STUDY_NAME,STUDY_NAME_PATH,AUTHOR_NAME,BRANCH_NAME,PR_TITLE,PR_BODY],cwd=concore_path) - output_string = proc.decode() + output_string = proc.decode() status=200 if output_string.find("/pulls/")!=-1: status=200 @@ -320,6 +344,11 @@ def contribute(): else: status=400 return jsonify({'message': output_string}),status + except ValueError as e: + return jsonify({'message': str(e)}), 400 + except subprocess.CalledProcessError as e: + output_string = e.stderr if hasattr(e, 'stderr') and e.stderr else "Command execution failed" + return jsonify({'message': output_string}), 501 except Exception as e: output_string = "Some Error occured.Please try after some time" status=501 @@ -365,18 +394,34 @@ def library(dir): dir_path = os.path.abspath(os.path.join(concore_path, dir_name)) filename = request.args.get('filename') library_path = request.args.get('path') + + # Validate user inputs to prevent command injection + try: + validate_input(filename, 'filename') + validate_input(library_path, 'path') + except ValueError as e: + resp = jsonify({'message': str(e)}) + resp.status_code = 400 + return resp + proc = 0 if (library_path == None or library_path == ''): library_path = r"../tools" - if(platform.uname()[0]=='Windows'): - proc = subprocess.check_output([r"..\library", library_path, filename],shell=True, cwd=dir_path) - else: - proc = subprocess.check_output([r"../library", library_path, filename], cwd=dir_path) - if(proc != 0): - resp = jsonify({'message': proc.decode("utf-8")}) + try: + if(platform.uname()[0]=='Windows'): + result = subprocess.run([r"..\library", library_path, filename], cwd=dir_path, check=True, capture_output=True, text=True) + proc = result.stdout + else: + proc = subprocess.check_output([r"../library", library_path, filename], cwd=dir_path) + proc = proc.decode("utf-8") + resp = jsonify({'message': proc}) resp.status_code = 201 return resp - else: + except subprocess.CalledProcessError as e: + resp = jsonify({'message': 'Command execution failed'}) + resp.status_code = 500 + return resp + except Exception as e: resp = jsonify({'message': 'There is an Error'}) resp.status_code = 500 return resp From 3bc84bc630bf5a2df6bc4c9176777c690f4259be Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Thu, 12 Feb 2026 17:46:28 +0530 Subject: [PATCH 082/275] Address review comments: improve validation and Windows compatibility - Add validate_filename() for strict filename validation (no path traversal) - Add validate_text_field() for PR title/body (allow punctuation, check length) - Add get_error_output() to extract error details from CalledProcessError - Use cmd.exe /c to invoke .bat scripts on Windows for shell=False compatibility - Add required parameter to validate_input() to reject None for required fields - Normalize None to empty string for optional fields - Include captured output in error responses for better diagnostics --- fri/server/main.py | 104 ++++++++++++++++++++++++++++++++++++--------- 1 file changed, 84 insertions(+), 20 deletions(-) diff --git a/fri/server/main.py b/fri/server/main.py index 8cf30cfd..cf5c7ac3 100644 --- a/fri/server/main.py +++ b/fri/server/main.py @@ -12,17 +12,75 @@ # Input validation pattern for safe names (alphanumeric, dash, underscore, slash, dot, space) SAFE_INPUT_PATTERN = re.compile(r'^[a-zA-Z0-9_\-/. ]+$') +# Pattern for filenames - no path separators or .. allowed +SAFE_FILENAME_PATTERN = re.compile(r'^[a-zA-Z0-9_\-. ]+$') -def validate_input(value, field_name): +def validate_input(value, field_name, required=False): """Validate that input contains only safe characters.""" if value is None: + if required: + raise ValueError(f"Missing required field: {field_name}") return True if not isinstance(value, str): raise ValueError(f"Invalid {field_name}: must be a string") + if required and len(value) == 0: + raise ValueError(f"Missing required field: {field_name}") if len(value) > 0 and not SAFE_INPUT_PATTERN.match(value): raise ValueError(f"Invalid {field_name}: contains unsafe characters") return True +def validate_filename(value, field_name, required=False): + """Validate filename - no path separators or .. segments allowed.""" + if value is None: + if required: + raise ValueError(f"Missing required field: {field_name}") + return True + if not isinstance(value, str): + raise ValueError(f"Invalid {field_name}: must be a string") + if required and len(value) == 0: + raise ValueError(f"Missing required field: {field_name}") + # Reject path traversal attempts + if '..' in value: + raise ValueError(f"Invalid {field_name}: path traversal not allowed") + # Use basename to strip any path components + basename = os.path.basename(value) + if basename != value: + raise ValueError(f"Invalid {field_name}: must be a filename, not a path") + if len(value) > 0 and not SAFE_FILENAME_PATTERN.match(value): + raise ValueError(f"Invalid {field_name}: contains unsafe characters") + return True + +def validate_text_field(value, field_name, max_length=None): + """Validate text fields like PR title/body - allow more characters but check type/length.""" + if value is None: + return True + if not isinstance(value, str): + raise ValueError(f"Invalid {field_name}: must be a string") + if max_length and len(value) > max_length: + raise ValueError(f"Invalid {field_name}: too long (max {max_length} characters)") + return True + +def get_error_output(e): + """Extract error output from CalledProcessError, preferring stderr then output.""" + raw_output = None + if hasattr(e, 'stderr') and e.stderr: + raw_output = e.stderr + elif hasattr(e, 'output') and e.output: + raw_output = e.output + + if raw_output is None: + return "Command execution failed" + + if isinstance(raw_output, bytes): + try: + return raw_output.decode('utf-8', errors='replace') + except Exception: + return str(raw_output) + elif isinstance(raw_output, str): + return raw_output + else: + return str(raw_output) + cur_path = os.path.dirname(os.path.abspath(__file__)) concore_path = os.path.abspath(os.path.join(cur_path, '../../')) @@ -312,23 +370,27 @@ def clear(dir): def contribute(): try: data = request.json - PR_TITLE = data.get('title') - PR_BODY = data.get('desc') - AUTHOR_NAME = data.get('auth') - STUDY_NAME = data.get('study') - STUDY_NAME_PATH = data.get('path') - BRANCH_NAME = data.get('branch') + PR_TITLE = data.get('title') or '' + PR_BODY = data.get('desc') or '' + AUTHOR_NAME = data.get('auth') or '' + STUDY_NAME = data.get('study') or '' + STUDY_NAME_PATH = data.get('path') or '' + BRANCH_NAME = data.get('branch') or '' # Validate all user inputs to prevent command injection - validate_input(STUDY_NAME, 'study') - validate_input(STUDY_NAME_PATH, 'path') - validate_input(AUTHOR_NAME, 'auth') - validate_input(BRANCH_NAME, 'branch') - validate_input(PR_TITLE, 'title') - validate_input(PR_BODY, 'desc') + # Strict validation for names/paths that go into command arguments + validate_input(STUDY_NAME, 'study', required=True) + validate_input(STUDY_NAME_PATH, 'path', required=True) + validate_input(AUTHOR_NAME, 'auth', required=True) + validate_input(BRANCH_NAME, 'branch', required=False) + + # For PR title/body, allow more characters but enforce type/length + validate_text_field(PR_TITLE, 'title', max_length=512) + validate_text_field(PR_BODY, 'desc', max_length=8192) if(platform.uname()[0]=='Windows'): - proc = subprocess.run(["contribute",STUDY_NAME,STUDY_NAME_PATH,AUTHOR_NAME,BRANCH_NAME,PR_TITLE,PR_BODY],cwd=concore_path,check=True,capture_output=True,text=True) + # Use cmd.exe /c to invoke contribute.bat on Windows + proc = subprocess.run(["cmd.exe", "/c", "contribute.bat", STUDY_NAME, STUDY_NAME_PATH, AUTHOR_NAME, BRANCH_NAME, PR_TITLE, PR_BODY], cwd=concore_path, check=True, capture_output=True, text=True) output_string = proc.stdout else: if len(BRANCH_NAME)==0: @@ -347,7 +409,7 @@ def contribute(): except ValueError as e: return jsonify({'message': str(e)}), 400 except subprocess.CalledProcessError as e: - output_string = e.stderr if hasattr(e, 'stderr') and e.stderr else "Command execution failed" + output_string = get_error_output(e) return jsonify({'message': output_string}), 501 except Exception as e: output_string = "Some Error occured.Please try after some time" @@ -397,19 +459,20 @@ def library(dir): # Validate user inputs to prevent command injection try: - validate_input(filename, 'filename') - validate_input(library_path, 'path') + # Use strict filename validation - no path separators or .. allowed + validate_filename(filename, 'filename', required=True) + validate_input(library_path, 'path', required=False) except ValueError as e: resp = jsonify({'message': str(e)}) resp.status_code = 400 return resp - proc = 0 if (library_path == None or library_path == ''): library_path = r"../tools" try: if(platform.uname()[0]=='Windows'): - result = subprocess.run([r"..\library", library_path, filename], cwd=dir_path, check=True, capture_output=True, text=True) + # Use cmd.exe /c to invoke library.bat on Windows + result = subprocess.run(["cmd.exe", "/c", r"..\library.bat", library_path, filename], cwd=dir_path, check=True, capture_output=True, text=True) proc = result.stdout else: proc = subprocess.check_output([r"../library", library_path, filename], cwd=dir_path) @@ -418,7 +481,8 @@ def library(dir): resp.status_code = 201 return resp except subprocess.CalledProcessError as e: - resp = jsonify({'message': 'Command execution failed'}) + error_output = get_error_output(e) + resp = jsonify({'message': f'Command execution failed: {error_output}'}) resp.status_code = 500 return resp except Exception as e: From 2ea8715eb3ab45eb09fd5322b0d1c51beb3df2cd Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Thu, 12 Feb 2026 17:51:08 +0530 Subject: [PATCH 083/275] Fix: Reuse PairedTransmitter outside loop to prevent repeated ZMQ socket creation (#268) --- 0mq/funcall.py | 40 +++++++++++++++++++++------------------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/0mq/funcall.py b/0mq/funcall.py index b316ff1b..a885ed1e 100644 --- a/0mq/funcall.py +++ b/0mq/funcall.py @@ -14,24 +14,26 @@ u = concore.initval(init_simtime_u) ym = concore2.initval(init_simtime_ym) -while(concore2.simtime> $GITHUB_OUTPUT - echo "base_sha=$(echo $PR_JSON | jq -r .base.sha)" >> $GITHUB_OUTPUT env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + REPO: ${{ github.repository }} + ISSUE_NUM: ${{ github.event.issue.number }} + #Use env vars for the API call to prevent injection + #Use quotes around variables to prevent word splitting + run: | + PR_JSON=$(gh api "repos/$REPO/pulls/$ISSUE_NUM") + echo "head_sha=$(echo "$PR_JSON" | jq -r .head.sha)" >> $GITHUB_OUTPUT + echo "base_sha=$(echo "$PR_JSON" | jq -r .base.sha)" >> $GITHUB_OUTPUT - uses: truongnh1992/gemini-ai-code-reviewer@main with: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} GEMINI_MODEL: gemini-2.5-flash - EXCLUDE: "*.md,*.txt,package-lock.json" - \ No newline at end of file + EXCLUDE: "*.md,*.txt,package-lock.json" \ No newline at end of file From 86f2e154e08a40619ba0a5a451cbfe1fce4c9c65 Mon Sep 17 00:00:00 2001 From: Avinash Kumar Deepak Date: Fri, 13 Feb 2026 22:06:46 +0530 Subject: [PATCH 085/275] fix bare except blocks - fixes #299 --- testsou/ccpymat.dir/concore.py | 8 ++++---- testsou/concore.py | 8 ++++---- tools/cwrap.py | 30 +++++++++++++++--------------- 3 files changed, 23 insertions(+), 23 deletions(-) diff --git a/testsou/ccpymat.dir/concore.py b/testsou/ccpymat.dir/concore.py index eab7e448..3cbd8de3 100644 --- a/testsou/ccpymat.dir/concore.py +++ b/testsou/ccpymat.dir/concore.py @@ -13,11 +13,11 @@ try: iport = literal_eval(open("concore.iport").read()) -except: +except Exception: iport = dict() try: oport = literal_eval(open("concore.oport").read()) -except: +except Exception: oport = dict() @@ -44,7 +44,7 @@ def read(port, name, initstr): try: infile = open(inpath+str(port)+"/"+name); ins = infile.read() - except: + except (OSError, IOError): ins = initstr while len(ins)==0: time.sleep(delay) @@ -68,7 +68,7 @@ def write(port, name, val, delta=0): outfile.write(str([simtime+delta]+val)) else: outfile.write(val) - except: + except (OSError, IOError): print("skipping"+outpath+str(port)+"/"+name); def initval(simtime_val): diff --git a/testsou/concore.py b/testsou/concore.py index eab7e448..3cbd8de3 100644 --- a/testsou/concore.py +++ b/testsou/concore.py @@ -13,11 +13,11 @@ try: iport = literal_eval(open("concore.iport").read()) -except: +except Exception: iport = dict() try: oport = literal_eval(open("concore.oport").read()) -except: +except Exception: oport = dict() @@ -44,7 +44,7 @@ def read(port, name, initstr): try: infile = open(inpath+str(port)+"/"+name); ins = infile.read() - except: + except (OSError, IOError): ins = initstr while len(ins)==0: time.sleep(delay) @@ -68,7 +68,7 @@ def write(port, name, val, delta=0): outfile.write(str([simtime+delta]+val)) else: outfile.write(val) - except: + except (OSError, IOError): print("skipping"+outpath+str(port)+"/"+name); def initval(simtime_val): diff --git a/tools/cwrap.py b/tools/cwrap.py index d1fda305..912423e7 100644 --- a/tools/cwrap.py +++ b/tools/cwrap.py @@ -12,51 +12,51 @@ try: apikey=open(concore.inpath+'1/concore.apikey',newline=None).readline().rstrip() -except: +except (OSError, IOError): try: #perhaps this should be removed for security apikey=open('./concore.apikey',newline=None).readline().rstrip() - except: + except (OSError, IOError): apikey = '' try: yuyu=open(concore.inpath+'1/concore.yuyu',newline=None).readline().rstrip() -except: +except (OSError, IOError): try: yuyu=open('./concore.yuyu',newline=None).readline().rstrip() - except: + except (OSError, IOError): yuyu = 'yuyu' try: name1=open(concore.inpath+'1/concore.name1',newline=None).readline().rstrip() -except: +except (OSError, IOError): try: name1=open('./concore.name1',newline=None).readline().rstrip() - except: + except (OSError, IOError): name1 = 'u' try: name2=open(concore.inpath+'1/concore.name2',newline=None).readline().rstrip() -except: +except (OSError, IOError): try: name2=open('./concore.name2',newline=None).readline().rstrip() - except: + except (OSError, IOError): name2 = 'ym' try: init_simtime_u = open(concore.inpath+'1/concore.init1',newline=None).readline().rstrip() -except: +except (OSError, IOError): try: init_simtime_u = open('./concore.init1',newline=None).readline().rstrip() - except: + except (OSError, IOError): init_simtime_u = "[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]" try: init_simtime_ym = open(concore.inpath+'1/concore.init2',newline=None).readline().rstrip() -except: +except (OSError, IOError): try: init_simtime_ym = open('./concore.init2',newline=None).readline().rstrip() - except: + except (OSError, IOError): init_simtime_ym = "[0.0, 0.0, 0.0]" logging.debug(f"API Key: {apikey}") @@ -92,7 +92,7 @@ if len(r.text)!=0: try: t=literal_eval(r.text)[0] - except: + except Exception: logging.error(f"bad eval {r.text}") timeout_count = 0 t1 = time.perf_counter() @@ -104,7 +104,7 @@ f = {'file1': open(concore.inpath+'1/'+name1, 'rb')} try: r = requests.post('http://www.controlcore.org/pm/'+yuyu+apikey+'&fetch='+name2, files=f,timeout=timeout_max) - except: + except Exception: logging.error("CW: bad request") timeout_count += 1 if r.status_code!=200 or time.perf_counter()-t1 > 1.1*timeout_max: #timeout_count>100: @@ -113,7 +113,7 @@ if len(r.text)!=0: try: t=literal_eval(r.text)[0] - except: + except Exception: logging.error(f"bad eval {r.text}") oldt = t oldym = r.text From 5074f24a5c6945164e040aa61a84888c421c3d2d Mon Sep 17 00:00:00 2001 From: Titas-Ghosh Date: Sat, 14 Feb 2026 03:08:32 +0530 Subject: [PATCH 086/275] Make concore validate fail on errors and check files --- concore_cli/README.md | 3 +++ concore_cli/cli.py | 7 +++++-- concore_cli/commands/validate.py | 28 +++++++++++++++++++++------- tests/test_cli.py | 13 +++++++++++++ 4 files changed, 42 insertions(+), 9 deletions(-) diff --git a/concore_cli/README.md b/concore_cli/README.md index e29c7657..b0da5057 100644 --- a/concore_cli/README.md +++ b/concore_cli/README.md @@ -79,6 +79,9 @@ Checks: - File references and naming conventions - ZMQ vs file-based communication +**Options:** +- `-s, --source ` - Source directory (default: src) + **Example:** ```bash concore validate workflow.graphml diff --git a/concore_cli/cli.py b/concore_cli/cli.py index 6e9ed076..615cb7b9 100644 --- a/concore_cli/cli.py +++ b/concore_cli/cli.py @@ -45,10 +45,13 @@ def run(workflow_file, source, output, type, auto_build): @cli.command() @click.argument('workflow_file', type=click.Path(exists=True)) -def validate(workflow_file): +@click.option('--source', '-s', default='src', help='Source directory') +def validate(workflow_file, source): """Validate a workflow file""" try: - validate_workflow(workflow_file, console) + ok = validate_workflow(workflow_file, source, console) + if not ok: + sys.exit(1) except Exception as e: console.print(f"[red]Error:[/red] {str(e)}") sys.exit(1) diff --git a/concore_cli/commands/validate.py b/concore_cli/commands/validate.py index fa1ea184..cbe7148e 100644 --- a/concore_cli/commands/validate.py +++ b/concore_cli/commands/validate.py @@ -5,8 +5,9 @@ import re import xml.etree.ElementTree as ET -def validate_workflow(workflow_file, console): +def validate_workflow(workflow_file, source_dir, console): workflow_path = Path(workflow_file) + source_root = (workflow_path.parent / source_dir) console.print(f"[cyan]Validating:[/cyan] {workflow_path.name}") console.print() @@ -15,31 +16,35 @@ def validate_workflow(workflow_file, console): warnings = [] info = [] + def finalize(): + show_results(console, errors, warnings, info) + return len(errors) == 0 + try: with open(workflow_path, 'r') as f: content = f.read() if not content.strip(): errors.append("File is empty") - return show_results(console, errors, warnings, info) + return finalize() # strict XML syntax check try: ET.fromstring(content) except ET.ParseError as e: errors.append(f"Invalid XML: {str(e)}") - return show_results(console, errors, warnings, info) + return finalize() try: soup = BeautifulSoup(content, 'xml') except Exception as e: errors.append(f"Invalid XML: {str(e)}") - return show_results(console, errors, warnings, info) + return finalize() root = soup.find('graphml') if not root: errors.append("Not a valid GraphML file - missing root element") - return show_results(console, errors, warnings, info) + return finalize() # check the graph attributes graph = soup.find('graph') @@ -64,6 +69,9 @@ def validate_workflow(workflow_file, console): warnings.append("No edges found in workflow") else: info.append(f"Found {len(edges)} edge(s)") + + if not source_root.exists(): + errors.append(f"Source directory not found: {source_root}") node_labels = [] for node in nodes: @@ -96,6 +104,10 @@ def validate_workflow(workflow_file, console): errors.append(f"Node '{label}' has no filename") elif not any(filename.endswith(ext) for ext in ['.py', '.cpp', '.m', '.v', '.java']): warnings.append(f"Node '{label}' has unusual file extension") + elif source_root.exists(): + file_path = source_root / filename + if not file_path.exists(): + errors.append(f"Missing source file: {filename}") else: warnings.append(f"Node {node_id} has no label") except Exception as e: @@ -138,12 +150,14 @@ def validate_workflow(workflow_file, console): if file_edges > 0: info.append(f"File-based edges: {file_edges}") - show_results(console, errors, warnings, info) - + return finalize() + except FileNotFoundError: console.print(f"[red]Error:[/red] File not found: {workflow_path}") + return False except Exception as e: console.print(f"[red]Validation failed:[/red] {str(e)}") + return False def show_results(console, errors, warnings, info): if errors: diff --git a/tests/test_cli.py b/tests/test_cli.py index 6aa78109..8d5a3994 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -58,6 +58,19 @@ def test_validate_valid_file(self): result = self.runner.invoke(cli, ['validate', 'test-project/workflow.graphml']) self.assertEqual(result.exit_code, 0) self.assertIn('Validation passed', result.output) + + def test_validate_missing_node_file(self): + with self.runner.isolated_filesystem(temp_dir=self.temp_dir): + result = self.runner.invoke(cli, ['init', 'test-project']) + self.assertEqual(result.exit_code, 0) + + missing_file = Path('test-project/src/script.py') + if missing_file.exists(): + missing_file.unlink() + + result = self.runner.invoke(cli, ['validate', 'test-project/workflow.graphml']) + self.assertNotEqual(result.exit_code, 0) + self.assertIn('Missing source file', result.output) def test_status_command(self): result = self.runner.invoke(cli, ['status']) From 88138af9f7b2963b153b4874112de5cade7a0d3e Mon Sep 17 00:00:00 2001 From: Titas-Ghosh Date: Sat, 14 Feb 2026 03:16:07 +0530 Subject: [PATCH 087/275] Treat missing source dir as warning in validate --- concore_cli/commands/validate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/concore_cli/commands/validate.py b/concore_cli/commands/validate.py index cbe7148e..74cd7900 100644 --- a/concore_cli/commands/validate.py +++ b/concore_cli/commands/validate.py @@ -71,7 +71,7 @@ def finalize(): info.append(f"Found {len(edges)} edge(s)") if not source_root.exists(): - errors.append(f"Source directory not found: {source_root}") + warnings.append(f"Source directory not found: {source_root}") node_labels = [] for node in nodes: From 914dd06cdd3944d78fff21aff0b1e744fcde687a Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Sat, 14 Feb 2026 10:51:08 +0530 Subject: [PATCH 088/275] Cleanup: Remove outdated concore copies in testsou (#289) --- testsou/ccpymat.dir/concore.py | 79 ------------------ testsou/concore.py | 79 ------------------ testsou/concoredocker.py | 58 ------------- testsou/concoreold.py | 58 ------------- testsou/funbody.dir/concore2.py | 113 -------------------------- testsou/funcall.dir/concore2.py | 113 -------------------------- testsou/mix.dir/concore2.py | 106 ------------------------ testsou/pmcpymat.dir/concore.py | 79 ------------------ testsou/powermetermax.dir/concore2.py | 106 ------------------------ 9 files changed, 791 deletions(-) delete mode 100644 testsou/ccpymat.dir/concore.py delete mode 100644 testsou/concore.py delete mode 100644 testsou/concoredocker.py delete mode 100644 testsou/concoreold.py delete mode 100644 testsou/funbody.dir/concore2.py delete mode 100644 testsou/funcall.dir/concore2.py delete mode 100644 testsou/mix.dir/concore2.py delete mode 100644 testsou/pmcpymat.dir/concore.py delete mode 100644 testsou/powermetermax.dir/concore2.py diff --git a/testsou/ccpymat.dir/concore.py b/testsou/ccpymat.dir/concore.py deleted file mode 100644 index 3cbd8de3..00000000 --- a/testsou/ccpymat.dir/concore.py +++ /dev/null @@ -1,79 +0,0 @@ -import time -import os -from ast import literal_eval -import sys - -#if windows, create script to kill this process -# because batch files don't provide easy way to know pid of last command -# ignored for posix!=windows, because "concorepid" is handled by script -# ignored for docker (linux!=windows), because handled by docker stop -if hasattr(sys, 'getwindowsversion'): - with open("concorekill.bat","w") as fpid: - fpid.write("taskkill /F /PID "+str(os.getpid())+"\n") - -try: - iport = literal_eval(open("concore.iport").read()) -except Exception: - iport = dict() -try: - oport = literal_eval(open("concore.oport").read()) -except Exception: - oport = dict() - - - -s = '' -olds = '' -delay = 1 -retrycount = 0 -inpath = "./in" #must be rel path for local -outpath = "./out" - -def unchanged(): - global olds,s - if olds==s: - s = '' - return True - else: - olds = s - return False - -def read(port, name, initstr): - global s,simtime,retrycount - time.sleep(delay) - try: - infile = open(inpath+str(port)+"/"+name); - ins = infile.read() - except (OSError, IOError): - ins = initstr - while len(ins)==0: - time.sleep(delay) - ins = infile.read() - retrycount += 1 - s += ins - inval = literal_eval(ins) - simtime = max(simtime,inval[0]) - return inval[1:] - -def write(port, name, val, delta=0): - global outpath,simtime - if isinstance(val,str): - time.sleep(2*delay) - elif isinstance(val,list)==False: - print("mywrite must have list or str") - quit() - try: - with open(outpath+str(port)+"/"+name,"w") as outfile: - if isinstance(val,list): - outfile.write(str([simtime+delta]+val)) - else: - outfile.write(val) - except (OSError, IOError): - print("skipping"+outpath+str(port)+"/"+name); - -def initval(simtime_val): - global simtime - val = literal_eval(simtime_val) - simtime = val[0] - return val[1:] - diff --git a/testsou/concore.py b/testsou/concore.py deleted file mode 100644 index 3cbd8de3..00000000 --- a/testsou/concore.py +++ /dev/null @@ -1,79 +0,0 @@ -import time -import os -from ast import literal_eval -import sys - -#if windows, create script to kill this process -# because batch files don't provide easy way to know pid of last command -# ignored for posix!=windows, because "concorepid" is handled by script -# ignored for docker (linux!=windows), because handled by docker stop -if hasattr(sys, 'getwindowsversion'): - with open("concorekill.bat","w") as fpid: - fpid.write("taskkill /F /PID "+str(os.getpid())+"\n") - -try: - iport = literal_eval(open("concore.iport").read()) -except Exception: - iport = dict() -try: - oport = literal_eval(open("concore.oport").read()) -except Exception: - oport = dict() - - - -s = '' -olds = '' -delay = 1 -retrycount = 0 -inpath = "./in" #must be rel path for local -outpath = "./out" - -def unchanged(): - global olds,s - if olds==s: - s = '' - return True - else: - olds = s - return False - -def read(port, name, initstr): - global s,simtime,retrycount - time.sleep(delay) - try: - infile = open(inpath+str(port)+"/"+name); - ins = infile.read() - except (OSError, IOError): - ins = initstr - while len(ins)==0: - time.sleep(delay) - ins = infile.read() - retrycount += 1 - s += ins - inval = literal_eval(ins) - simtime = max(simtime,inval[0]) - return inval[1:] - -def write(port, name, val, delta=0): - global outpath,simtime - if isinstance(val,str): - time.sleep(2*delay) - elif isinstance(val,list)==False: - print("mywrite must have list or str") - quit() - try: - with open(outpath+str(port)+"/"+name,"w") as outfile: - if isinstance(val,list): - outfile.write(str([simtime+delta]+val)) - else: - outfile.write(val) - except (OSError, IOError): - print("skipping"+outpath+str(port)+"/"+name); - -def initval(simtime_val): - global simtime - val = literal_eval(simtime_val) - simtime = val[0] - return val[1:] - diff --git a/testsou/concoredocker.py b/testsou/concoredocker.py deleted file mode 100644 index 6fb582a2..00000000 --- a/testsou/concoredocker.py +++ /dev/null @@ -1,58 +0,0 @@ -import time -from ast import literal_eval - -s = '' -olds = '' -delay = 1 -retrycount = 0 -inpath = "/in" -outpath = "/out" - -def unchanged(): - global olds,s - if olds==s: - s = '' - return True - else: - olds = s - return False - -def read(port, name, initstr): - global s,simtime,retrycount - time.sleep(delay) - try: - infile = open(inpath+str(port)+"/"+name); - ins = infile.read() - except: - ins = initstr - while len(ins)==0: - time.sleep(delay) - ins = infile.read() - retrycount += 1 - s += ins - inval = literal_eval(ins) - simtime = max(simtime,inval[0]) - return inval[1:] - -def write(port, name, val, delta=0): - global outpath,simtime - if isinstance(val,str): - time.sleep(2*delay) - elif isinstance(val,list)==False: - print("mywrite must have list or str") - quit() - try: - with open(outpath+str(port)+"/"+name,"w") as outfile: - if isinstance(val,list): - outfile.write(str([simtime+delta]+val)) - else: - outfile.write(val) - except: - print("skipping"+outpath+str(port)+"/"+name); - -def initval(simtime_val): - global simtime - val = literal_eval(simtime_val) - simtime = val[0] - return val[1:] - diff --git a/testsou/concoreold.py b/testsou/concoreold.py deleted file mode 100644 index 310d2b53..00000000 --- a/testsou/concoreold.py +++ /dev/null @@ -1,58 +0,0 @@ -import time -from ast import literal_eval - -s = '' -olds = '' -delay = 1 -retrycount = 0 -inpath = "./in" -outpath = "./out" - -def unchanged(): - global olds,s - if olds==s: - s = '' - return True - else: - olds = s - return False - -def read(port, name, initstr): - global s,simtime,retrycount - time.sleep(delay) - try: - infile = open(inpath+str(port)+"/"+name); - ins = infile.read() - except: - ins = initstr - while len(ins)==0: - time.sleep(delay) - ins = infile.read() - retrycount += 1 - s += ins - inval = literal_eval(ins) - simtime = max(simtime,inval[0]) - return inval[1:] - -def write(port, name, val, delta=0): - global outpath,simtime - if isinstance(val,str): - time.sleep(2*delay) - elif isinstance(val,list)==False: - print("mywrite must have list or str") - quit() - try: - with open(outpath+str(port)+"/"+name,"w") as outfile: - if isinstance(val,list): - outfile.write(str([simtime+delta]+val)) - else: - outfile.write(val) - except: - print("skipping"+outpath+str(port)+"/"+name); - -def initval(simtime_val): - global simtime - val = literal_eval(simtime_val) - simtime = val[0] - return val[1:] - diff --git a/testsou/funbody.dir/concore2.py b/testsou/funbody.dir/concore2.py deleted file mode 100644 index e3ec817c..00000000 --- a/testsou/funbody.dir/concore2.py +++ /dev/null @@ -1,113 +0,0 @@ -import time -import os -from ast import literal_eval -import sys -import re - -#if windows, create script to kill this process -# because batch files don't provide easy way to know pid of last command -# ignored for posix!=windows, because "concorepid" is handled by script -# ignored for docker (linux!=windows), because handled by docker stop -if hasattr(sys, 'getwindowsversion'): - with open("concorekill.bat","w") as fpid: - fpid.write("taskkill /F /PID "+str(os.getpid())+"\n") - -try: - iport = literal_eval(open("concore.iport").read()) -except: - iport = dict() -try: - oport = literal_eval(open("concore.oport").read()) -except: - oport = dict() - - -s = '' -olds = '' -delay = 1 -retrycount = 0 -inpath = "./in" #must be rel path for local -outpath = "./out" - -#9/21/22 -try: - sparams = open(inpath+"1/concore.params").read() - if sparams[0] == '"': #windows keeps "" need to remove - sparams = sparams[1:] - sparams = sparams[0:sparams.find('"')] - if sparams != '{': - print("converting sparams: "+sparams) - sparams = "{'"+re.sub(';',",'",re.sub('=',"':",re.sub(' ','',sparams)))+"}" - print("converted sparams: " + sparams) - try: - params = literal_eval(sparams) - except: - print("bad params: "+sparams) -except: - params = dict() -#9/30/22 -def tryparam(n,i): - try: - return params[n] - except: - return i - - -#9/12/21 -def default_maxtime(default): - global maxtime - try: - maxtime = literal_eval(open(inpath+"1/concore.maxtime").read()) - except: - maxtime = default -default_maxtime(100) - -def unchanged(): - global olds,s - if olds==s: - s = '' - return True - else: - olds = s - return False - -def read(port, name, initstr): - global s,simtime,retrycount - time.sleep(delay) - try: - infile = open(inpath+str(port)+"/"+name); - ins = infile.read() - except: - ins = initstr - while len(ins)==0: - time.sleep(delay) - ins = infile.read() - retrycount += 1 - s += ins - inval = literal_eval(ins) - simtime = max(simtime,inval[0]) - return inval[1:] - -def write(port, name, val, delta=0): - global outpath,simtime - if isinstance(val,str): - time.sleep(2*delay) - elif isinstance(val,list)==False: - print("mywrite must have list or str") - quit() - try: - with open(outpath+str(port)+"/"+name,"w") as outfile: - if isinstance(val,list): - outfile.write(str([simtime+delta]+val)) - simtime += delta - else: - outfile.write(val) - except: - print("skipping"+outpath+str(port)+"/"+name); - -def initval(simtime_val): - global simtime - val = literal_eval(simtime_val) - simtime = val[0] - return val[1:] - diff --git a/testsou/funcall.dir/concore2.py b/testsou/funcall.dir/concore2.py deleted file mode 100644 index e3ec817c..00000000 --- a/testsou/funcall.dir/concore2.py +++ /dev/null @@ -1,113 +0,0 @@ -import time -import os -from ast import literal_eval -import sys -import re - -#if windows, create script to kill this process -# because batch files don't provide easy way to know pid of last command -# ignored for posix!=windows, because "concorepid" is handled by script -# ignored for docker (linux!=windows), because handled by docker stop -if hasattr(sys, 'getwindowsversion'): - with open("concorekill.bat","w") as fpid: - fpid.write("taskkill /F /PID "+str(os.getpid())+"\n") - -try: - iport = literal_eval(open("concore.iport").read()) -except: - iport = dict() -try: - oport = literal_eval(open("concore.oport").read()) -except: - oport = dict() - - -s = '' -olds = '' -delay = 1 -retrycount = 0 -inpath = "./in" #must be rel path for local -outpath = "./out" - -#9/21/22 -try: - sparams = open(inpath+"1/concore.params").read() - if sparams[0] == '"': #windows keeps "" need to remove - sparams = sparams[1:] - sparams = sparams[0:sparams.find('"')] - if sparams != '{': - print("converting sparams: "+sparams) - sparams = "{'"+re.sub(';',",'",re.sub('=',"':",re.sub(' ','',sparams)))+"}" - print("converted sparams: " + sparams) - try: - params = literal_eval(sparams) - except: - print("bad params: "+sparams) -except: - params = dict() -#9/30/22 -def tryparam(n,i): - try: - return params[n] - except: - return i - - -#9/12/21 -def default_maxtime(default): - global maxtime - try: - maxtime = literal_eval(open(inpath+"1/concore.maxtime").read()) - except: - maxtime = default -default_maxtime(100) - -def unchanged(): - global olds,s - if olds==s: - s = '' - return True - else: - olds = s - return False - -def read(port, name, initstr): - global s,simtime,retrycount - time.sleep(delay) - try: - infile = open(inpath+str(port)+"/"+name); - ins = infile.read() - except: - ins = initstr - while len(ins)==0: - time.sleep(delay) - ins = infile.read() - retrycount += 1 - s += ins - inval = literal_eval(ins) - simtime = max(simtime,inval[0]) - return inval[1:] - -def write(port, name, val, delta=0): - global outpath,simtime - if isinstance(val,str): - time.sleep(2*delay) - elif isinstance(val,list)==False: - print("mywrite must have list or str") - quit() - try: - with open(outpath+str(port)+"/"+name,"w") as outfile: - if isinstance(val,list): - outfile.write(str([simtime+delta]+val)) - simtime += delta - else: - outfile.write(val) - except: - print("skipping"+outpath+str(port)+"/"+name); - -def initval(simtime_val): - global simtime - val = literal_eval(simtime_val) - simtime = val[0] - return val[1:] - diff --git a/testsou/mix.dir/concore2.py b/testsou/mix.dir/concore2.py deleted file mode 100644 index 6471e0b7..00000000 --- a/testsou/mix.dir/concore2.py +++ /dev/null @@ -1,106 +0,0 @@ -import time -import os -from ast import literal_eval -import sys -import re - -#if windows, create script to kill this process -# because batch files don't provide easy way to know pid of last command -# ignored for posix!=windows, because "concorepid" is handled by script -# ignored for docker (linux!=windows), because handled by docker stop -if hasattr(sys, 'getwindowsversion'): - with open("concorekill.bat","w") as fpid: - fpid.write("taskkill /F /PID "+str(os.getpid())+"\n") - -try: - iport = literal_eval(open("concore.iport").read()) -except: - iport = dict() -try: - oport = literal_eval(open("concore.oport").read()) -except: - oport = dict() - - -s = '' -olds = '' -delay = 1 -retrycount = 0 -inpath = "./in" #must be rel path for local -outpath = "./out" - -#9/21/22 -try: - sparams = open(inpath+"1/concore.params").read() - if sparams[0] == '"': #windows keeps "" need to remove - sparams = sparams[1:] - sparams = sparams[0:sparams.find('"')] - if sparams != '{': - print("converting sparams: "+sparams) - sparams = "{'"+re.sub(',',",'",re.sub('=',"':",re.sub(' ','',sparams)))+"}" - print("converted sparams: " + sparams) - try: - params = literal_eval(sparams) - except: - print("bad params: "+sparams) -except: - params = dict() - -#9/12/21 -def default_maxtime(default): - global maxtime - try: - maxtime = literal_eval(open(inpath+"1/concore.maxtime").read()) - except: - maxtime = default -default_maxtime(100) - -def unchanged(): - global olds,s - if olds==s: - s = '' - return True - else: - olds = s - return False - -def read(port, name, initstr): - global s,simtime,retrycount - time.sleep(delay) - try: - infile = open(inpath+str(port)+"/"+name); - ins = infile.read() - except: - ins = initstr - while len(ins)==0: - time.sleep(delay) - ins = infile.read() - retrycount += 1 - s += ins - inval = literal_eval(ins) - simtime = max(simtime,inval[0]) - return inval[1:] - -def write(port, name, val, delta=0): - global outpath,simtime - if isinstance(val,str): - time.sleep(2*delay) - elif isinstance(val,list)==False: - print("mywrite must have list or str") - quit() - try: - with open(outpath+str(port)+"/"+name,"w") as outfile: - if isinstance(val,list): - outfile.write(str([simtime+delta]+val)) - simtime += delta - else: - outfile.write(val) - except: - print("skipping"+outpath+str(port)+"/"+name); - -def initval(simtime_val): - global simtime - val = literal_eval(simtime_val) - simtime = val[0] - return val[1:] - diff --git a/testsou/pmcpymat.dir/concore.py b/testsou/pmcpymat.dir/concore.py deleted file mode 100644 index eab7e448..00000000 --- a/testsou/pmcpymat.dir/concore.py +++ /dev/null @@ -1,79 +0,0 @@ -import time -import os -from ast import literal_eval -import sys - -#if windows, create script to kill this process -# because batch files don't provide easy way to know pid of last command -# ignored for posix!=windows, because "concorepid" is handled by script -# ignored for docker (linux!=windows), because handled by docker stop -if hasattr(sys, 'getwindowsversion'): - with open("concorekill.bat","w") as fpid: - fpid.write("taskkill /F /PID "+str(os.getpid())+"\n") - -try: - iport = literal_eval(open("concore.iport").read()) -except: - iport = dict() -try: - oport = literal_eval(open("concore.oport").read()) -except: - oport = dict() - - - -s = '' -olds = '' -delay = 1 -retrycount = 0 -inpath = "./in" #must be rel path for local -outpath = "./out" - -def unchanged(): - global olds,s - if olds==s: - s = '' - return True - else: - olds = s - return False - -def read(port, name, initstr): - global s,simtime,retrycount - time.sleep(delay) - try: - infile = open(inpath+str(port)+"/"+name); - ins = infile.read() - except: - ins = initstr - while len(ins)==0: - time.sleep(delay) - ins = infile.read() - retrycount += 1 - s += ins - inval = literal_eval(ins) - simtime = max(simtime,inval[0]) - return inval[1:] - -def write(port, name, val, delta=0): - global outpath,simtime - if isinstance(val,str): - time.sleep(2*delay) - elif isinstance(val,list)==False: - print("mywrite must have list or str") - quit() - try: - with open(outpath+str(port)+"/"+name,"w") as outfile: - if isinstance(val,list): - outfile.write(str([simtime+delta]+val)) - else: - outfile.write(val) - except: - print("skipping"+outpath+str(port)+"/"+name); - -def initval(simtime_val): - global simtime - val = literal_eval(simtime_val) - simtime = val[0] - return val[1:] - diff --git a/testsou/powermetermax.dir/concore2.py b/testsou/powermetermax.dir/concore2.py deleted file mode 100644 index 6471e0b7..00000000 --- a/testsou/powermetermax.dir/concore2.py +++ /dev/null @@ -1,106 +0,0 @@ -import time -import os -from ast import literal_eval -import sys -import re - -#if windows, create script to kill this process -# because batch files don't provide easy way to know pid of last command -# ignored for posix!=windows, because "concorepid" is handled by script -# ignored for docker (linux!=windows), because handled by docker stop -if hasattr(sys, 'getwindowsversion'): - with open("concorekill.bat","w") as fpid: - fpid.write("taskkill /F /PID "+str(os.getpid())+"\n") - -try: - iport = literal_eval(open("concore.iport").read()) -except: - iport = dict() -try: - oport = literal_eval(open("concore.oport").read()) -except: - oport = dict() - - -s = '' -olds = '' -delay = 1 -retrycount = 0 -inpath = "./in" #must be rel path for local -outpath = "./out" - -#9/21/22 -try: - sparams = open(inpath+"1/concore.params").read() - if sparams[0] == '"': #windows keeps "" need to remove - sparams = sparams[1:] - sparams = sparams[0:sparams.find('"')] - if sparams != '{': - print("converting sparams: "+sparams) - sparams = "{'"+re.sub(',',",'",re.sub('=',"':",re.sub(' ','',sparams)))+"}" - print("converted sparams: " + sparams) - try: - params = literal_eval(sparams) - except: - print("bad params: "+sparams) -except: - params = dict() - -#9/12/21 -def default_maxtime(default): - global maxtime - try: - maxtime = literal_eval(open(inpath+"1/concore.maxtime").read()) - except: - maxtime = default -default_maxtime(100) - -def unchanged(): - global olds,s - if olds==s: - s = '' - return True - else: - olds = s - return False - -def read(port, name, initstr): - global s,simtime,retrycount - time.sleep(delay) - try: - infile = open(inpath+str(port)+"/"+name); - ins = infile.read() - except: - ins = initstr - while len(ins)==0: - time.sleep(delay) - ins = infile.read() - retrycount += 1 - s += ins - inval = literal_eval(ins) - simtime = max(simtime,inval[0]) - return inval[1:] - -def write(port, name, val, delta=0): - global outpath,simtime - if isinstance(val,str): - time.sleep(2*delay) - elif isinstance(val,list)==False: - print("mywrite must have list or str") - quit() - try: - with open(outpath+str(port)+"/"+name,"w") as outfile: - if isinstance(val,list): - outfile.write(str([simtime+delta]+val)) - simtime += delta - else: - outfile.write(val) - except: - print("skipping"+outpath+str(port)+"/"+name); - -def initval(simtime_val): - global simtime - val = literal_eval(simtime_val) - simtime = val[0] - return val[1:] - From 490a619c5478cdcbf76e694d02f2c4f0e7eaf5cb Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Sat, 14 Feb 2026 11:20:31 +0530 Subject: [PATCH 089/275] fix: replace deprecated concore2 imports with concore (Issue #287) --- 0mq/comm_node.py | 23 +++++++++---------- 0mq/funbody.py | 25 ++++++++++----------- 0mq/funbody2.py | 25 ++++++++++----------- 0mq/funbody_distributed.py | 25 ++++++++++----------- 0mq/funbody_zmq.py | 25 ++++++++++----------- 0mq/funbody_zmq2.py | 25 ++++++++++----------- 0mq/funcall.py | 25 ++++++++++----------- 0mq/funcall2.py | 25 ++++++++++----------- 0mq/funcall_distributed.py | 23 +++++++++---------- 0mq/funcall_zmq.py | 23 +++++++++---------- 0mq/funcall_zmq2.py | 23 +++++++++---------- measurements/Latency/funbody_distributed.py | 25 ++++++++++----------- measurements/Latency/funcall_distributed.py | 23 +++++++++---------- measurements/comm_node_test.py | 21 +++++++++-------- nintan/powermetermax.py | 15 ++++++------- ratc/learn3.py | 15 ++++++------- testsou/funbody.py | 23 +++++++++---------- testsou/funcall.py | 23 +++++++++---------- testsou/mix.py | 13 +++++------ testsou/powermetermax.py | 15 ++++++------- 20 files changed, 210 insertions(+), 230 deletions(-) diff --git a/0mq/comm_node.py b/0mq/comm_node.py index edba31db..6a49d9d6 100644 --- a/0mq/comm_node.py +++ b/0mq/comm_node.py @@ -1,25 +1,24 @@ import concore -import concore2 concore.delay = 0.07 -concore2.delay = 0.07 -concore2.inpath = concore.inpath -concore2.outpath = concore.outpath -concore2.simtime = 0 +concore.delay = 0.07 +concore.inpath = concore.inpath +concore.outpath = concore.outpath +concore.simtime = 0 concore.default_maxtime(100) init_simtime_u = "[0.0, 0.0, 0.0]" init_simtime_ym = "[0.0, 0.0, 0.0]" u = concore.initval(init_simtime_u) -ym = concore2.initval(init_simtime_ym) -while(concore2.simtime 0): @@ -50,17 +49,17 @@ if 'U2' in concore.oport: concore.write(concore.oport['U2'], "u", u_data_values) - old_concore2_simtime = concore2.simtime - while concore2.unchanged() or concore2.simtime <= old_concore2_simtime: + old_concore_simtime = concore.simtime + while concore.unchanged() or concore.simtime <= old_concore_simtime: # Assuming concore.iport['Y2'] is a file port (e.g., from pmpymax.py) - ym_data_values = concore2.read(concore.iport['Y2'], "ym", init_simtime_ym_str) - # time.sleep(concore2.delay) # Optional delay + ym_data_values = concore.read(concore.iport['Y2'], "ym", init_simtime_ym_str) + # time.sleep(concore.delay) # Optional delay - ym_full_to_send = [concore2.simtime] + ym_data_values + ym_full_to_send = [concore.simtime] + ym_data_values concore.write(PORT_NAME_F2_OUT, "ym_signal", ym_full_to_send) - print(f"funbody u={u_data_values} ym={ym_data_values} time={concore2.simtime}") + print(f"funbody u={u_data_values} ym={ym_data_values} time={concore.simtime}") print("funbody retry=" + str(concore.retrycount)) diff --git a/0mq/funbody_zmq.py b/0mq/funbody_zmq.py index 6a6b353f..5d980956 100644 --- a/0mq/funbody_zmq.py +++ b/0mq/funbody_zmq.py @@ -1,7 +1,6 @@ # funbody2_zmq.py import time import concore -import concore2 print("funbody using ZMQ via concore") @@ -15,21 +14,21 @@ # Standard concore initializations concore.delay = 0.07 -concore2.delay = 0.07 -concore2.inpath = concore.inpath -concore2.outpath = concore.outpath -concore2.simtime = 0 +concore.delay = 0.07 +concore.inpath = concore.inpath +concore.outpath = concore.outpath +concore.simtime = 0 concore.default_maxtime(100) init_simtime_u_str = "[0.0, 0.0, 0.0]" init_simtime_ym_str = "[0.0, 0.0, 0.0]" u_data_values = concore.initval(init_simtime_u_str) -ym_data_values = concore2.initval(init_simtime_ym_str) +ym_data_values = concore.initval(init_simtime_ym_str) print(f"Initial u_data_values: {u_data_values}, ym_data_values: {ym_data_values}") print(f"Max time: {concore.maxtime}") -while concore2.simtime < concore.maxtime: +while concore.simtime < concore.maxtime: received_u_data = concore.read(PORT_NAME_F2_F1, "u_signal", init_simtime_u_str) if not (isinstance(received_u_data, list) and len(received_u_data) > 0): @@ -49,17 +48,17 @@ if 'U2' in concore.oport: concore.write(concore.oport['U2'], "u", u_data_values) - old_concore2_simtime = concore2.simtime - while concore2.unchanged() or concore2.simtime <= old_concore2_simtime: + old_concore_simtime = concore.simtime + while concore.unchanged() or concore.simtime <= old_concore_simtime: # Assuming concore.iport['Y2'] is a file port (e.g., from pmpymax.py) - ym_data_values = concore2.read(concore.iport['Y2'], "ym", init_simtime_ym_str) - # time.sleep(concore2.delay) # Optional delay + ym_data_values = concore.read(concore.iport['Y2'], "ym", init_simtime_ym_str) + # time.sleep(concore.delay) # Optional delay - ym_full_to_send = [concore2.simtime] + ym_data_values + ym_full_to_send = [concore.simtime] + ym_data_values concore.write(PORT_NAME_F2_F1, "ym_signal", ym_full_to_send) - print(f"funbody u={u_data_values} ym={ym_data_values} time={concore2.simtime}") + print(f"funbody u={u_data_values} ym={ym_data_values} time={concore.simtime}") print("funbody retry=" + str(concore.retrycount)) diff --git a/0mq/funbody_zmq2.py b/0mq/funbody_zmq2.py index 04d94873..76583063 100644 --- a/0mq/funbody_zmq2.py +++ b/0mq/funbody_zmq2.py @@ -1,7 +1,6 @@ # funbody2_zmq.py import time import concore -import concore2 print("funbody using ZMQ via concore") @@ -15,21 +14,21 @@ # Standard concore initializations concore.delay = 0.07 -concore2.delay = 0.07 -concore2.inpath = concore.inpath -concore2.outpath = concore.outpath -concore2.simtime = 0 +concore.delay = 0.07 +concore.inpath = concore.inpath +concore.outpath = concore.outpath +concore.simtime = 0 concore.default_maxtime(100) init_simtime_u_str = "[0.0, 0.0, 0.0]" init_simtime_ym_str = "[0.0, 0.0, 0.0]" u_data_values = concore.initval(init_simtime_u_str) -ym_data_values = concore2.initval(init_simtime_ym_str) +ym_data_values = concore.initval(init_simtime_ym_str) print(f"Initial u_data_values: {u_data_values}, ym_data_values: {ym_data_values}") print(f"Max time: {concore.maxtime}") -while concore2.simtime < concore.maxtime: +while concore.simtime < concore.maxtime: received_u_data = concore.read(PORT_NAME_F2_OUT, "u_signal", init_simtime_u_str) if not (isinstance(received_u_data, list) and len(received_u_data) > 0): @@ -49,17 +48,17 @@ if 'U2' in concore.oport: concore.write(concore.oport['U2'], "u", u_data_values) - old_concore2_simtime = concore2.simtime - while concore2.unchanged() or concore2.simtime <= old_concore2_simtime: + old_concore_simtime = concore.simtime + while concore.unchanged() or concore.simtime <= old_concore_simtime: # Assuming concore.iport['Y2'] is a file port (e.g., from pmpymax.py) - ym_data_values = concore2.read(concore.iport['Y2'], "ym", init_simtime_ym_str) - # time.sleep(concore2.delay) # Optional delay + ym_data_values = concore.read(concore.iport['Y2'], "ym", init_simtime_ym_str) + # time.sleep(concore.delay) # Optional delay - ym_full_to_send = [concore2.simtime] + ym_data_values + ym_full_to_send = [concore.simtime] + ym_data_values concore.write(PORT_NAME_F2_OUT, "ym_signal", ym_full_to_send) - print(f"funbody u={u_data_values} ym={ym_data_values} time={concore2.simtime}") + print(f"funbody u={u_data_values} ym={ym_data_values} time={concore.simtime}") print("funbody retry=" + str(concore.retrycount)) diff --git a/0mq/funcall.py b/0mq/funcall.py index a885ed1e..85203ba1 100644 --- a/0mq/funcall.py +++ b/0mq/funcall.py @@ -1,39 +1,38 @@ import concore -import concore2 from osparc_control import PairedTransmitter print("funcall 0mq") concore.delay = 0.07 -concore2.delay = 0.07 -concore2.inpath = concore.inpath -concore2.outpath = concore.outpath -concore2.simtime = 0 +concore.delay = 0.07 +concore.inpath = concore.inpath +concore.outpath = concore.outpath +concore.simtime = 0 concore.default_maxtime(100) init_simtime_u = "[0.0, 0.0, 0.0]" init_simtime_ym = "[0.0, 0.0, 0.0]" u = concore.initval(init_simtime_u) -ym = concore2.initval(init_simtime_ym) +ym = concore.initval(init_simtime_ym) paired_transmitter = PairedTransmitter( remote_host="localhost", exposed_commands=[], remote_port=2345, listen_port=2346,) paired_transmitter.start_background_sync() try: - while(concore2.simtime 0: response_time = received_ym_data[0] if isinstance(response_time, (int, float)): - concore2.simtime = response_time + concore.simtime = response_time ym = received_ym_data[1:] else: print(f"Warning: Received ZMQ data's first element is not time: {received_ym_data}. Using as is.") ym = received_ym_data else: print(f"Warning: Received unexpected ZMQ data format: {received_ym_data}. Using default ym.") - ym = concore2.initval(init_simtime_ym_str) + ym = concore.initval(init_simtime_ym_str) # Assuming concore.oport['Y'] is a file port (e.g., to cpymax.py) - concore2.write(concore.oport['Y'], "ym", ym) + concore.write(concore.oport['Y'], "ym", ym) - print(f"funcall ZMQ u={u} ym={ym} time={concore2.simtime}") + print(f"funcall ZMQ u={u} ym={ym} time={concore.simtime}") print("funcall retry=" + str(concore.retrycount)) diff --git a/0mq/funcall_zmq.py b/0mq/funcall_zmq.py index b2f991d2..5b484b1b 100644 --- a/0mq/funcall_zmq.py +++ b/0mq/funcall_zmq.py @@ -1,7 +1,6 @@ # funcall2_zmq.py import time import concore -import concore2 print("funcall using ZMQ via concore") @@ -15,21 +14,21 @@ # Standard concore initializations concore.delay = 0.07 -concore2.delay = 0.07 -concore2.inpath = concore.inpath -concore2.outpath = concore.outpath -concore2.simtime = 0 +concore.delay = 0.07 +concore.inpath = concore.inpath +concore.outpath = concore.outpath +concore.simtime = 0 concore.default_maxtime(100) init_simtime_u_str = "[0.0, 0.0, 0.0]" init_simtime_ym_str = "[0.0, 0.0, 0.0]" u = concore.initval(init_simtime_u_str) -ym = concore2.initval(init_simtime_ym_str) +ym = concore.initval(init_simtime_ym_str) -print(f"Initial u: {u}, ym: {ym}, concore.simtime: {concore.simtime}, concore2.simtime: {concore2.simtime}") +print(f"Initial u: {u}, ym: {ym}, concore.simtime: {concore.simtime}, concore.simtime: {concore.simtime}") print(f"Max time: {concore.maxtime}") -while concore2.simtime < concore.maxtime: +while concore.simtime < concore.maxtime: while concore.unchanged(): # Assuming concore.iport['U'] is a file port (e.g., from cpymax.py) u = concore.read(concore.iport['U'], "u", init_simtime_u_str) @@ -44,19 +43,19 @@ if isinstance(received_ym_data, list) and len(received_ym_data) > 0: response_time = received_ym_data[0] if isinstance(response_time, (int, float)): - concore2.simtime = response_time + concore.simtime = response_time ym = received_ym_data[1:] else: print(f"Warning: Received ZMQ data's first element is not time: {received_ym_data}. Using as is.") ym = received_ym_data else: print(f"Warning: Received unexpected ZMQ data format: {received_ym_data}. Using default ym.") - ym = concore2.initval(init_simtime_ym_str) + ym = concore.initval(init_simtime_ym_str) # Assuming concore.oport['Y'] is a file port (e.g., to cpymax.py) - concore2.write(concore.oport['Y'], "ym", ym) + concore.write(concore.oport['Y'], "ym", ym) - print(f"funcall ZMQ u={u} ym={ym} time={concore2.simtime}") + print(f"funcall ZMQ u={u} ym={ym} time={concore.simtime}") print("funcall retry=" + str(concore.retrycount)) diff --git a/0mq/funcall_zmq2.py b/0mq/funcall_zmq2.py index 13c65842..10736ce2 100644 --- a/0mq/funcall_zmq2.py +++ b/0mq/funcall_zmq2.py @@ -1,7 +1,6 @@ # funcall2_zmq.py import time import concore -import concore2 print("funcall using ZMQ via concore") @@ -15,21 +14,21 @@ # Standard concore initializations concore.delay = 0.07 -concore2.delay = 0.07 -concore2.inpath = concore.inpath -concore2.outpath = concore.outpath -concore2.simtime = 0 +concore.delay = 0.07 +concore.inpath = concore.inpath +concore.outpath = concore.outpath +concore.simtime = 0 concore.default_maxtime(100) init_simtime_u_str = "[0.0, 0.0, 0.0]" init_simtime_ym_str = "[0.0, 0.0, 0.0]" u = concore.initval(init_simtime_u_str) -ym = concore2.initval(init_simtime_ym_str) +ym = concore.initval(init_simtime_ym_str) -print(f"Initial u: {u}, ym: {ym}, concore.simtime: {concore.simtime}, concore2.simtime: {concore2.simtime}") +print(f"Initial u: {u}, ym: {ym}, concore.simtime: {concore.simtime}, concore.simtime: {concore.simtime}") print(f"Max time: {concore.maxtime}") -while concore2.simtime < concore.maxtime: +while concore.simtime < concore.maxtime: while concore.unchanged(): # Assuming concore.iport['U'] is a file port (e.g., from cpymax.py) u = concore.read(concore.iport['U'], "u", init_simtime_u_str) @@ -44,19 +43,19 @@ if isinstance(received_ym_data, list) and len(received_ym_data) > 0: response_time = received_ym_data[0] if isinstance(response_time, (int, float)): - concore2.simtime = response_time + concore.simtime = response_time ym = received_ym_data[1:] else: print(f"Warning: Received ZMQ data's first element is not time: {received_ym_data}. Using as is.") ym = received_ym_data else: print(f"Warning: Received unexpected ZMQ data format: {received_ym_data}. Using default ym.") - ym = concore2.initval(init_simtime_ym_str) + ym = concore.initval(init_simtime_ym_str) # Assuming concore.oport['Y'] is a file port (e.g., to cpymax.py) - concore2.write(concore.oport['Y'], "ym", ym) + concore.write(concore.oport['Y'], "ym", ym) - print(f"funcall ZMQ u={u} ym={ym} time={concore2.simtime}") + print(f"funcall ZMQ u={u} ym={ym} time={concore.simtime}") print("funcall retry=" + str(concore.retrycount)) diff --git a/measurements/Latency/funbody_distributed.py b/measurements/Latency/funbody_distributed.py index eaae295c..b7e56849 100644 --- a/measurements/Latency/funbody_distributed.py +++ b/measurements/Latency/funbody_distributed.py @@ -1,7 +1,6 @@ # funbody2_zmq.py import time import concore -import concore2 print("funbody using ZMQ via concore") @@ -16,21 +15,21 @@ # Standard concore initializations concore.delay = 0.07 -concore2.delay = 0.07 -concore2.inpath = concore.inpath -concore2.outpath = concore.outpath -concore2.simtime = 0 +concore.delay = 0.07 +concore.inpath = concore.inpath +concore.outpath = concore.outpath +concore.simtime = 0 concore.default_maxtime(100) init_simtime_u_str = "[0.0, 0.0, 0.0]" init_simtime_ym_str = "[0.0, 0.0, 0.0]" u_data_values = concore.initval(init_simtime_u_str) -ym_data_values = concore2.initval(init_simtime_ym_str) +ym_data_values = concore.initval(init_simtime_ym_str) print(f"Initial u_data_values: {u_data_values}, ym_data_values: {ym_data_values}") print(f"Max time: {concore.maxtime}") -while concore2.simtime < concore.maxtime: +while concore.simtime < concore.maxtime: received_u_data = concore.read(PORT_NAME_F2_OUT, "u_signal", init_simtime_u_str) if not (isinstance(received_u_data, list) and len(received_u_data) > 0): @@ -50,17 +49,17 @@ if 'U2' in concore.oport: concore.write(concore.oport['U2'], "u", u_data_values) - old_concore2_simtime = concore2.simtime - while concore2.unchanged() or concore2.simtime <= old_concore2_simtime: + old_concore_simtime = concore.simtime + while concore.unchanged() or concore.simtime <= old_concore_simtime: # Assuming concore.iport['Y2'] is a file port (e.g., from pmpymax.py) - ym_data_values = concore2.read(concore.iport['Y2'], "ym", init_simtime_ym_str) - # time.sleep(concore2.delay) # Optional delay + ym_data_values = concore.read(concore.iport['Y2'], "ym", init_simtime_ym_str) + # time.sleep(concore.delay) # Optional delay - ym_full_to_send = [concore2.simtime] + ym_data_values + ym_full_to_send = [concore.simtime] + ym_data_values concore.write(PORT_NAME_F2_OUT, "ym_signal", ym_full_to_send) - print(f"funbody u={u_data_values} ym={ym_data_values} time={concore2.simtime}") + print(f"funbody u={u_data_values} ym={ym_data_values} time={concore.simtime}") print("funbody retry=" + str(concore.retrycount)) diff --git a/measurements/Latency/funcall_distributed.py b/measurements/Latency/funcall_distributed.py index 779324fb..30e8df90 100644 --- a/measurements/Latency/funcall_distributed.py +++ b/measurements/Latency/funcall_distributed.py @@ -1,7 +1,6 @@ # funcall_distributed.py (MODIFIED FOR LATENCY MEASUREMENT) import time import concore -import concore2 import csv # <--- ADDED: Import CSV library print("funcall using ZMQ via concore") @@ -16,24 +15,24 @@ # Standard concore initializations concore.delay = 0.07 -concore2.delay = 0.07 -concore2.inpath = concore.inpath -concore2.outpath = concore.outpath -concore2.simtime = 0 +concore.delay = 0.07 +concore.inpath = concore.inpath +concore.outpath = concore.outpath +concore.simtime = 0 concore.default_maxtime(100) # Recommend increasing this for more data points, e.g., 1000 init_simtime_u_str = "[0.0, 0.0, 0.0]" init_simtime_ym_str = "[0.0, 0.0, 0.0]" u = concore.initval(init_simtime_u_str) -ym = concore2.initval(init_simtime_ym_str) +ym = concore.initval(init_simtime_ym_str) # --- ADDED: Initialize a list to store latency values --- zeromq_latencies = [] -print(f"Initial u: {u}, ym: {ym}, concore.simtime: {concore.simtime}, concore2.simtime: {concore2.simtime}") +print(f"Initial u: {u}, ym: {ym}, concore.simtime: {concore.simtime}, concore.simtime: {concore.simtime}") print(f"Max time: {concore.maxtime}") -while concore2.simtime < concore.maxtime: +while concore.simtime < concore.maxtime: while concore.unchanged(): u = concore.read(concore.iport['U'], "u", init_simtime_u_str) @@ -53,18 +52,18 @@ if isinstance(received_ym_data, list) and len(received_ym_data) > 0: response_time = received_ym_data[0] if isinstance(response_time, (int, float)): - concore2.simtime = response_time + concore.simtime = response_time ym = received_ym_data[1:] else: print(f"Warning: Received ZMQ data's first element is not time: {received_ym_data}. Using as is.") ym = received_ym_data else: print(f"Warning: Received unexpected ZMQ data format: {received_ym_data}. Using default ym.") - ym = concore2.initval(init_simtime_ym_str) + ym = concore.initval(init_simtime_ym_str) - concore2.write(concore.oport['Y'], "ym", ym) + concore.write(concore.oport['Y'], "ym", ym) - print(f"funcall ZMQ u={u} ym={ym} time={concore2.simtime} | ZMQ Latency: {latency_ms:.4f} ms") + print(f"funcall ZMQ u={u} ym={ym} time={concore.simtime} | ZMQ Latency: {latency_ms:.4f} ms") print("funcall retry=" + str(concore.retrycount)) diff --git a/measurements/comm_node_test.py b/measurements/comm_node_test.py index dac015ed..d0db63e3 100644 --- a/measurements/comm_node_test.py +++ b/measurements/comm_node_test.py @@ -1,14 +1,13 @@ import concore -import concore2 import time import sys # --- Script Configuration --- concore.delay = 0.07 -concore2.delay = 0.07 -concore2.inpath = concore.inpath -concore2.outpath = concore.outpath -concore2.simtime = 0 +concore.delay = 0.07 +concore.inpath = concore.inpath +concore.outpath = concore.outpath +concore.simtime = 0 concore.default_maxtime(100) # This will be ignored by the new logic init_simtime_u = "[0.0, 0.0, 0.0]" init_simtime_ym = "[0.0, 0.0, 0.0]" @@ -19,7 +18,7 @@ # --- Main Script Logic --- u = concore.initval(init_simtime_u) -ym = concore2.initval(init_simtime_ym) +ym = concore.initval(init_simtime_ym) curr = 0 max_value = 100 iteration = 0 @@ -39,16 +38,16 @@ # Break if the loop condition is met after the first read if curr >= max_value: # Forward a final message to ensure the next node also terminates - concore2.write(concore.oport['Y'], "ym", [curr]) + concore.write(concore.oport['Y'], "ym", [curr]) break # 3. Wait for a message from the 'Y1' channel - old2 = concore2.simtime - while concore2.unchanged() or concore2.simtime <= old2: - ym = concore2.read(concore.iport['Y1'], "ym", init_simtime_ym) + old2 = concore.simtime + while concore.unchanged() or concore.simtime <= old2: + ym = concore.read(concore.iport['Y1'], "ym", init_simtime_ym) # 4. Forward it to the 'Y' channel - concore2.write(concore.oport['Y'], "ym", ym) + concore.write(concore.oport['Y'], "ym", ym) curr = ym[0] print(f"comm_node: u={u[0]:.2f} | ym={ym[0]:.2f}") diff --git a/nintan/powermetermax.py b/nintan/powermetermax.py index 5d6828ef..40ae9a83 100644 --- a/nintan/powermetermax.py +++ b/nintan/powermetermax.py @@ -1,14 +1,13 @@ #CW import concore -import concore2 import time print("powermeter") concore.delay = 0.07 -concore2.delay = 0.07 -concore2.inpath = concore.inpath -concore2.outpath = concore.outpath -concore2.simtime = 0 +concore.delay = 0.07 +concore.inpath = concore.inpath +concore.outpath = concore.outpath +concore.simtime = 0 #Nsim = 100 concore.default_maxtime(100) init_simtime_u = "[0.0, 0.0, 0.0]" @@ -19,9 +18,9 @@ while(concore.simtime Date: Sat, 14 Feb 2026 11:31:19 +0530 Subject: [PATCH 093/275] Update testsou/powermetermax.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- testsou/powermetermax.py | 1 - 1 file changed, 1 deletion(-) diff --git a/testsou/powermetermax.py b/testsou/powermetermax.py index 40ae9a83..479d40b3 100644 --- a/testsou/powermetermax.py +++ b/testsou/powermetermax.py @@ -3,7 +3,6 @@ import time print("powermeter") -concore.delay = 0.07 concore.delay = 0.07 concore.inpath = concore.inpath concore.outpath = concore.outpath From 0563c1cc44f428a7f525d82e37f3143f009c76ab Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Sat, 14 Feb 2026 11:31:28 +0530 Subject: [PATCH 094/275] Update testsou/funcall.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- testsou/funcall.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/testsou/funcall.py b/testsou/funcall.py index cb0c16e4..f6cb98bf 100644 --- a/testsou/funcall.py +++ b/testsou/funcall.py @@ -3,8 +3,6 @@ concore.delay = 0.07 concore.delay = 0.07 -concore.inpath = concore.inpath -concore.outpath = concore.outpath concore.simtime = 0 concore.default_maxtime(100) init_simtime_u = "[0.0, 0.0, 0.0]" From ab214a052a2f571bf07d39f465d3186325e03522 Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Sat, 14 Feb 2026 11:31:43 +0530 Subject: [PATCH 095/275] Update ratc/learn3.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- ratc/learn3.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/ratc/learn3.py b/ratc/learn3.py index 64336fde..a9625d50 100644 --- a/ratc/learn3.py +++ b/ratc/learn3.py @@ -5,8 +5,6 @@ GENERATE_PLOT = 0 concore.delay = 0.002 concore.delay = 0.002 -concore.inpath = concore.inpath -concore.outpath = concore.outpath concore.simtime = 0 fout=open(concore.outpath+'1/history.txt','w') From d34f485495bc2d36fca75c6d191a04f2c8a5a01b Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Sat, 14 Feb 2026 11:31:53 +0530 Subject: [PATCH 096/275] Update 0mq/funbody.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- 0mq/funbody.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/0mq/funbody.py b/0mq/funbody.py index a3146706..3ad6c051 100644 --- a/0mq/funbody.py +++ b/0mq/funbody.py @@ -26,8 +26,6 @@ paired_transmitter.start_background_sync() concore.delay = 0.07 -concore.inpath = concore.inpath -concore.outpath = concore.outpath concore.simtime = 0 concore.default_maxtime(100) init_simtime_u = "[0.0, 0.0, 0.0]" From b1d1d231ad61207d0b153d1e35869ad293077784 Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Sat, 14 Feb 2026 11:32:02 +0530 Subject: [PATCH 097/275] Update testsou/powermetermax.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- testsou/powermetermax.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/testsou/powermetermax.py b/testsou/powermetermax.py index 479d40b3..5294ceb8 100644 --- a/testsou/powermetermax.py +++ b/testsou/powermetermax.py @@ -4,8 +4,6 @@ print("powermeter") concore.delay = 0.07 -concore.inpath = concore.inpath -concore.outpath = concore.outpath concore.simtime = 0 #Nsim = 100 concore.default_maxtime(100) From 6c0fce9bbfc60d74caa4d0a93f31610c9a984b5b Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Sat, 14 Feb 2026 11:32:10 +0530 Subject: [PATCH 098/275] Update ratc/learn3.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- ratc/learn3.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ratc/learn3.py b/ratc/learn3.py index a9625d50..bbbc5476 100644 --- a/ratc/learn3.py +++ b/ratc/learn3.py @@ -4,7 +4,6 @@ import time GENERATE_PLOT = 0 concore.delay = 0.002 -concore.delay = 0.002 concore.simtime = 0 fout=open(concore.outpath+'1/history.txt','w') From fa1aa2623b4da83d38e9cda41bc1f59b78575a49 Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Sat, 14 Feb 2026 11:32:19 +0530 Subject: [PATCH 099/275] Update 0mq/funcall2.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- 0mq/funcall2.py | 1 - 1 file changed, 1 deletion(-) diff --git a/0mq/funcall2.py b/0mq/funcall2.py index 84dffe6b..ffc72421 100644 --- a/0mq/funcall2.py +++ b/0mq/funcall2.py @@ -2,7 +2,6 @@ from osparc_control import PairedTransmitter print("funcall 0mq") -concore.delay = 0.07 concore.delay = 0.07 concore.inpath = concore.inpath concore.outpath = concore.outpath From 36a0c9e9aa2b298b51c20fa3e358bdd7df8b80ec Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Sat, 14 Feb 2026 11:32:26 +0530 Subject: [PATCH 100/275] Update 0mq/funcall_distributed.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- 0mq/funcall_distributed.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/0mq/funcall_distributed.py b/0mq/funcall_distributed.py index 5aa5c47a..0bf6ce79 100644 --- a/0mq/funcall_distributed.py +++ b/0mq/funcall_distributed.py @@ -15,8 +15,6 @@ # Standard concore initializations concore.delay = 0.07 concore.delay = 0.07 -concore.inpath = concore.inpath -concore.outpath = concore.outpath concore.simtime = 0 concore.default_maxtime(100) init_simtime_u_str = "[0.0, 0.0, 0.0]" From efee0e8aa1e289af7cec0bc14fd8805217850e58 Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Sat, 14 Feb 2026 11:33:04 +0530 Subject: [PATCH 101/275] Update 0mq/funcall_zmq2.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- 0mq/funcall_zmq2.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/0mq/funcall_zmq2.py b/0mq/funcall_zmq2.py index 4864f57e..168fc929 100644 --- a/0mq/funcall_zmq2.py +++ b/0mq/funcall_zmq2.py @@ -14,8 +14,6 @@ # Standard concore initializations concore.delay = 0.07 -concore.inpath = concore.inpath -concore.outpath = concore.outpath concore.simtime = 0 concore.default_maxtime(100) init_simtime_u_str = "[0.0, 0.0, 0.0]" From 33820fddbf3109f82d971124e2503aacb317a74b Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Sat, 14 Feb 2026 11:33:28 +0530 Subject: [PATCH 102/275] Update measurements/comm_node_test.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- measurements/comm_node_test.py | 1 - 1 file changed, 1 deletion(-) diff --git a/measurements/comm_node_test.py b/measurements/comm_node_test.py index d0db63e3..b9cfcde4 100644 --- a/measurements/comm_node_test.py +++ b/measurements/comm_node_test.py @@ -4,7 +4,6 @@ # --- Script Configuration --- concore.delay = 0.07 -concore.delay = 0.07 concore.inpath = concore.inpath concore.outpath = concore.outpath concore.simtime = 0 From 6847bc15c7f53292ce0aabd7a9b3f9f8901dd075 Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Sat, 14 Feb 2026 11:33:42 +0530 Subject: [PATCH 103/275] Update 0mq/funbody_zmq.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- 0mq/funbody_zmq.py | 1 - 1 file changed, 1 deletion(-) diff --git a/0mq/funbody_zmq.py b/0mq/funbody_zmq.py index 5d980956..254088f9 100644 --- a/0mq/funbody_zmq.py +++ b/0mq/funbody_zmq.py @@ -16,7 +16,6 @@ concore.delay = 0.07 concore.delay = 0.07 concore.inpath = concore.inpath -concore.outpath = concore.outpath concore.simtime = 0 concore.default_maxtime(100) init_simtime_u_str = "[0.0, 0.0, 0.0]" From d682891232c30806f02d809b639550e2214277d3 Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Sat, 14 Feb 2026 11:33:56 +0530 Subject: [PATCH 104/275] Update 0mq/funbody_zmq.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- 0mq/funbody_zmq.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/0mq/funbody_zmq.py b/0mq/funbody_zmq.py index 254088f9..14e5f8ef 100644 --- a/0mq/funbody_zmq.py +++ b/0mq/funbody_zmq.py @@ -47,7 +47,9 @@ if 'U2' in concore.oport: concore.write(concore.oport['U2'], "u", u_data_values) - old_concore_simtime = concore.simtime + # Take a numeric snapshot of the current simulation time to avoid + # inadvertently sharing a reference with concore.simtime. + old_concore_simtime = float(concore.simtime) while concore.unchanged() or concore.simtime <= old_concore_simtime: # Assuming concore.iport['Y2'] is a file port (e.g., from pmpymax.py) ym_data_values = concore.read(concore.iport['Y2'], "ym", init_simtime_ym_str) From ab956e04f49f73475bd479efe18101e310335ab1 Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Sat, 14 Feb 2026 11:34:12 +0530 Subject: [PATCH 105/275] Update 0mq/funbody_distributed.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- 0mq/funbody_distributed.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/0mq/funbody_distributed.py b/0mq/funbody_distributed.py index b7e56849..e7e95e6d 100644 --- a/0mq/funbody_distributed.py +++ b/0mq/funbody_distributed.py @@ -16,8 +16,6 @@ # Standard concore initializations concore.delay = 0.07 concore.delay = 0.07 -concore.inpath = concore.inpath -concore.outpath = concore.outpath concore.simtime = 0 concore.default_maxtime(100) init_simtime_u_str = "[0.0, 0.0, 0.0]" From b6f87cd6809f39dacfae3adf15981d0f9bc7902c Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Sat, 14 Feb 2026 11:34:28 +0530 Subject: [PATCH 106/275] Update 0mq/funbody2.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- 0mq/funbody2.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/0mq/funbody2.py b/0mq/funbody2.py index 71d5f97d..b249428e 100644 --- a/0mq/funbody2.py +++ b/0mq/funbody2.py @@ -27,8 +27,6 @@ concore.delay = 0.07 concore.delay = 0.07 -concore.inpath = concore.inpath -concore.outpath = concore.outpath concore.simtime = 0 concore.default_maxtime(100) init_simtime_u = "[0.0, 0.0, 0.0]" From 3e6c9585114181306e7ec46879056c6bf14e7620 Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Sat, 14 Feb 2026 11:34:44 +0530 Subject: [PATCH 107/275] Update testsou/funcall.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- testsou/funcall.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testsou/funcall.py b/testsou/funcall.py index f6cb98bf..d53e7b9c 100644 --- a/testsou/funcall.py +++ b/testsou/funcall.py @@ -16,7 +16,7 @@ concore.write(concore.oport['U1'],"u",u) print(u) old2 = concore.simtime - while concore.unchanged() or concore.simtime <= old2: + while concore.unchanged() and concore.simtime <= old2: ym = concore.read(concore.iport['Y1'],"ym",init_simtime_ym) concore.write(concore.oport['Y'],"ym",ym) print("funbody u="+str(u)+" ym="+str(ym)+" time="+str(concore.simtime)) From aaebd5df1796afc71deddfcfaa1756fc35e05eab Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Sat, 14 Feb 2026 11:35:04 +0530 Subject: [PATCH 108/275] Update 0mq/funcall2.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- 0mq/funcall2.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/0mq/funcall2.py b/0mq/funcall2.py index ffc72421..78ca032e 100644 --- a/0mq/funcall2.py +++ b/0mq/funcall2.py @@ -3,8 +3,6 @@ print("funcall 0mq") concore.delay = 0.07 -concore.inpath = concore.inpath -concore.outpath = concore.outpath concore.simtime = 0 concore.default_maxtime(100) init_simtime_u = "[0.0, 0.0, 0.0]" From 9b886d0750b5711ac81088a5f1b957af5075a0a3 Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Sat, 14 Feb 2026 11:35:29 +0530 Subject: [PATCH 109/275] Update measurements/Latency/funcall_distributed.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- measurements/Latency/funcall_distributed.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/measurements/Latency/funcall_distributed.py b/measurements/Latency/funcall_distributed.py index 30e8df90..4332c9e3 100644 --- a/measurements/Latency/funcall_distributed.py +++ b/measurements/Latency/funcall_distributed.py @@ -16,8 +16,6 @@ # Standard concore initializations concore.delay = 0.07 concore.delay = 0.07 -concore.inpath = concore.inpath -concore.outpath = concore.outpath concore.simtime = 0 concore.default_maxtime(100) # Recommend increasing this for more data points, e.g., 1000 init_simtime_u_str = "[0.0, 0.0, 0.0]" From bda776f7eef9fe7adf73734249f14c076d54d887 Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Sat, 14 Feb 2026 11:36:02 +0530 Subject: [PATCH 110/275] Update 0mq/funbody_zmq2.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- 0mq/funbody_zmq2.py | 1 - 1 file changed, 1 deletion(-) diff --git a/0mq/funbody_zmq2.py b/0mq/funbody_zmq2.py index 76583063..96af0100 100644 --- a/0mq/funbody_zmq2.py +++ b/0mq/funbody_zmq2.py @@ -14,7 +14,6 @@ # Standard concore initializations concore.delay = 0.07 -concore.delay = 0.07 concore.inpath = concore.inpath concore.outpath = concore.outpath concore.simtime = 0 From e980a8165831941ed2587ab70c31f853e17cd472 Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Sat, 14 Feb 2026 11:36:24 +0530 Subject: [PATCH 111/275] Update 0mq/funbody2.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- 0mq/funbody2.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/0mq/funbody2.py b/0mq/funbody2.py index b249428e..ca4f3bbd 100644 --- a/0mq/funbody2.py +++ b/0mq/funbody2.py @@ -50,8 +50,8 @@ u = u[1:] concore.write(concore.oport['U2'],"u",u) print(u) - old2 = concore.simtime - while concore.unchanged() or concore.simtime <= old2: + old2 = float(concore.simtime) + while concore.simtime <= old2: ym = concore.read(concore.iport['Y2'],"ym",init_simtime_ym) ym = [concore.simtime]+ym print(f"Replying to {command.action} with {ym}") From 4f1382fb38c1078e3eb12cd9a83b6f895e6e604e Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Sat, 14 Feb 2026 11:36:52 +0530 Subject: [PATCH 112/275] Update 0mq/comm_node.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- 0mq/comm_node.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/0mq/comm_node.py b/0mq/comm_node.py index f78c88cf..13d29748 100644 --- a/0mq/comm_node.py +++ b/0mq/comm_node.py @@ -14,7 +14,7 @@ u = concore.read(concore.iport['U'],"u",init_simtime_u) concore.write(concore.oport['U1'],"u",u) print(u) - old2 = concore.simtime + old2 = float(concore.simtime) while concore.unchanged() or concore.simtime <= old2: ym = concore.read(concore.iport['Y1'],"ym",init_simtime_ym) concore.write(concore.oport['Y'],"ym",ym) From f8869e9d80493fff84c5664ae57408bf691a2a3c Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Sat, 14 Feb 2026 11:37:16 +0530 Subject: [PATCH 113/275] Update testsou/funbody.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- testsou/funbody.py | 1 - 1 file changed, 1 deletion(-) diff --git a/testsou/funbody.py b/testsou/funbody.py index 1032e478..900a064d 100644 --- a/testsou/funbody.py +++ b/testsou/funbody.py @@ -1,7 +1,6 @@ import concore print("funbody") -concore.delay = 0.07 concore.delay = 0.07 concore.inpath = concore.inpath concore.outpath = concore.outpath From dac0ec5845d2b34d296ab7adbf98b48f9538852e Mon Sep 17 00:00:00 2001 From: Avinash Kumar Deepak Date: Fri, 13 Feb 2026 21:38:37 +0530 Subject: [PATCH 114/275] use context managers for file I/O - fixes #298 --- complete_rebase.sh | 5 ++ concore.egg-info/PKG-INFO | 106 ++++++++++++++++++++++++++ concore.egg-info/SOURCES.txt | 24 ++++++ concore.egg-info/dependency_links.txt | 1 + concore.egg-info/entry_points.txt | 2 + concore.egg-info/requires.txt | 9 +++ concore.egg-info/top_level.txt | 2 + setup_branch.ps1 | 8 ++ 8 files changed, 157 insertions(+) create mode 100644 complete_rebase.sh create mode 100644 concore.egg-info/PKG-INFO create mode 100644 concore.egg-info/SOURCES.txt create mode 100644 concore.egg-info/dependency_links.txt create mode 100644 concore.egg-info/entry_points.txt create mode 100644 concore.egg-info/requires.txt create mode 100644 concore.egg-info/top_level.txt create mode 100644 setup_branch.ps1 diff --git a/complete_rebase.sh b/complete_rebase.sh new file mode 100644 index 00000000..139801d8 --- /dev/null +++ b/complete_rebase.sh @@ -0,0 +1,5 @@ +#!/bin/bash +cd "D:\gsoc org\concore-project\concore" +git add requirements-dev.txt +git rebase --continue +git push origin add-concoredocker-tests --force diff --git a/concore.egg-info/PKG-INFO b/concore.egg-info/PKG-INFO new file mode 100644 index 00000000..7924d4eb --- /dev/null +++ b/concore.egg-info/PKG-INFO @@ -0,0 +1,106 @@ +Metadata-Version: 2.4 +Name: concore +Version: 1.0.0 +Summary: Concore workflow management CLI +Home-page: https://github.com/ControlCore-Project/concore +Author: ControlCore Project +License: MIT +Requires-Python: >=3.9 +Description-Content-Type: text/markdown +License-File: LICENSE +Requires-Dist: click>=8.0.0 +Requires-Dist: rich>=10.0.0 +Requires-Dist: beautifulsoup4>=4.9.0 +Requires-Dist: lxml>=4.6.0 +Requires-Dist: psutil>=5.8.0 +Provides-Extra: dev +Requires-Dist: pytest>=6.0.0; extra == "dev" +Requires-Dist: pytest-cov>=2.10.0; extra == "dev" +Dynamic: author +Dynamic: home-page +Dynamic: license-file +Dynamic: requires-python + +# CONTROL-CORE: Integrated Development Environment for Closed-loop Neuromodulation Control Systems. + +[CONTROL-CORE](https://github.com/ControlCore-Project/) is a design and simulation framework, functioning as a visual Integrated Development Environment (IDE) for Closed-loop Neuromodulation Control Systems. At its center is _concore_, a lightweight protocol to simulate neuromodulation control systems. This repository consists of the implementation of _concore_ protocol and sample (demo and neuromodulation control systems) studies. In addition to its default standard Python implementation, _concore_ also supports developing studies in Matlab/Octave, Verilog, and C++. _concore_ also aims to support more language programs in the future. + +# The CONTROL-CORE Framework + +The CONTROL-CORE framework consists of the below projects. + +* _concore_: The CONTROL-CORE protocol, known as _concore_, allows modular simulation of controller and PM nodes to run on different operating systems, computing platforms, and programming languages. [This repository](https://github.com/ControlCore-Project/concore/) consists of _concore_ source code. The _concore_ documentation can be found [here](https://control-core.readthedocs.io/en/latest/index.html). A _concore_ study can be developed from programs written in different languages. That means, _concore_ facilitates a seamless communication across codes developed in different languages that it supports, through its simple file-based data sharing between the programs. + +* _concore_ Editor: This is the front-end for CONTROL-CORE. We forked [DHGWorkflow](https://github.com/ControlCore-Project/DHGWorkflow), a sibling project we developed, and extend it as the _concore_ Editor. + +* _Mediator_: The [Mediator](https://github.com/ControlCore-Project/mediator) allows the CONTROL-CORE studies to be distributed and run, rather than having all the programs that construct a study to be run just from a centralized location. + +* _concore-lite_: The [_concore-lite_](https://github.com/ControlCore-Project/concore-lite) repository consists of a simple example version of a _concore_ study. Please check out and run this, if you like to learn the _concore_ protocol without having to clone this large _concore_ repository. + +* documentation: The [source code repository](https://github.com/ControlCore-Project/documentation) of the ReadTheDocs documentation of CONTROL-CORE. + + +# The _concore_ Protocol + +_concore_ enables composing studies from programs developed in different languages. Currently supported languages are, Python, Matlab/Octave, Verilog, and C++. The studies are designed through the visual _concore_ Editor (DHGWorkflow) and interpreted into _concore_ through its parser. Neural control systems consist of loops (dicycles). Therefore, they cannot be represented by classic workflow standards (such as CWL or WDL). Therefore, _concore_ addresses a significant research gap to model closed-loop neuromodulation control systems. The _concore_ protocol shares data between the programs through file sharing, with no centralized entity (a broker or an orchestrator) to arbitrate communications between the programs. (In the distributed executions, the CONTROL-CORE Mediator enables connecting the disjoint pieces of the study through REST APIs). + + +# Installation and Getting Started Guide + +Please follow the [ReadTheDocs](https://control-core.readthedocs.io/en/latest/index.html) documentation and the [_concore-lite_](https://github.com/ControlCore-Project/concore-lite) repository to get started quick. + +Installation instructions for concore can be found [here](https://control-core.readthedocs.io/en/latest/installation.html). Usage instructions can be found [here](https://control-core.readthedocs.io/en/latest/usage.html). + +## Command-Line Interface (CLI) + +_concore_ now includes a command-line interface for easier workflow management. Install it with: + +```bash +pip install -e . +``` + +Quick start with the CLI: + +```bash +# Create a new project +concore init my-project + +# Validate your workflow +concore validate workflow.graphml + +# Run your workflow +concore run workflow.graphml --auto-build + +# Monitor running processes +concore status + +# Stop all processes +concore stop +``` + +For detailed CLI documentation, see [concore_cli/README.md](concore_cli/README.md). + +For a detailed and more scientific documentation, please read our extensive [open-access research paper on CONTROL-CORE](https://doi.org/10.1109/ACCESS.2022.3161471). This paper has a complete discussion on the CONTROL-CORE architecture and deployment, together with the commands to execute the studies in different programming languages and programming environments (Ubuntu, Windows, MacOS, Docker, and distributed execution). + + +# The _concore_ Repository + +_concore_ contains programs (such as physiological models or more commonly called "PMs" and controllers) and studies (i.e., graphml files that represents the studies as workflows). The _wrappers_ enable seamlessly extending a study into a distributed one with the CONTROL-CORE Mediator. + +_concore_ repository consists of several scripts at its root level. The demo folder consists of several sample programs and studies, mostly toy examples to learn the protocol. The ratc folder consists of the programs and studies of the rat cardiac experiments we developed with _concore_. + +If you have a bug to report in one of the CONTROL-CORE projects, please report it through relevant Issue Tracker. Similarly, please feel free to contribute your studies and code enhancements using pull requests. Questions and discussions can be made through the relevant Discussions forum. + +The _concore_ Issues can be reported [here](https://github.com/ControlCore-Project/concore/issues). + +The _concore_ discussion forum can be found [here](https://github.com/ControlCore-Project/concore/discussions). + +Please make sure to send your _concore_ pull requests to the [dev branch](https://github.com/ControlCore-Project/concore/tree/dev). + + +# Citing _concore_ + +If you use _concore_ in your research, please cite the below papers: + +* Kathiravelu, P., Arnold, M., Vijay, S., Jagwani, R., Goyal, P., Goel, A.K., Li, N., Horn, C., Pan, T., Kothare, M. V., and Mahmoudi, B. **Distributed Executions with CONTROL-CORE Integrated Development Environment (IDE) for Closed-loop Neuromodulation Control Systems.** In Cluster Computing – The Journal of Networks Software Tools and Applications (CLUSTER). May 2025. Accepted. Springer. +* Kathiravelu, P., Arnold, M., Fleischer, J., Yao, Y., Awasthi, S., Goel, A. K., Branen, A., Sarikhani, P., Kumar, G., Kothare, M. V., and Mahmoudi, B. **CONTROL-CORE: A Framework for Simulation and Design of Closed-Loop Peripheral Neuromodulation Control Systems**. In IEEE Access. March 2022. https://doi.org/10.1109/ACCESS.2022.3161471 diff --git a/concore.egg-info/SOURCES.txt b/concore.egg-info/SOURCES.txt new file mode 100644 index 00000000..99020ca8 --- /dev/null +++ b/concore.egg-info/SOURCES.txt @@ -0,0 +1,24 @@ +LICENSE +README.md +mkconcore.py +pyproject.toml +setup.py +concore.egg-info/PKG-INFO +concore.egg-info/SOURCES.txt +concore.egg-info/dependency_links.txt +concore.egg-info/entry_points.txt +concore.egg-info/requires.txt +concore.egg-info/top_level.txt +concore_cli/__init__.py +concore_cli/cli.py +concore_cli/commands/__init__.py +concore_cli/commands/init.py +concore_cli/commands/inspect.py +concore_cli/commands/run.py +concore_cli/commands/status.py +concore_cli/commands/stop.py +concore_cli/commands/validate.py +tests/test_cli.py +tests/test_concore.py +tests/test_concoredocker.py +tests/test_graph.py \ No newline at end of file diff --git a/concore.egg-info/dependency_links.txt b/concore.egg-info/dependency_links.txt new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/concore.egg-info/dependency_links.txt @@ -0,0 +1 @@ + diff --git a/concore.egg-info/entry_points.txt b/concore.egg-info/entry_points.txt new file mode 100644 index 00000000..a71a8db6 --- /dev/null +++ b/concore.egg-info/entry_points.txt @@ -0,0 +1,2 @@ +[console_scripts] +concore = concore_cli.cli:cli diff --git a/concore.egg-info/requires.txt b/concore.egg-info/requires.txt new file mode 100644 index 00000000..3895f3f9 --- /dev/null +++ b/concore.egg-info/requires.txt @@ -0,0 +1,9 @@ +click>=8.0.0 +rich>=10.0.0 +beautifulsoup4>=4.9.0 +lxml>=4.6.0 +psutil>=5.8.0 + +[dev] +pytest>=6.0.0 +pytest-cov>=2.10.0 diff --git a/concore.egg-info/top_level.txt b/concore.egg-info/top_level.txt new file mode 100644 index 00000000..3c4fe8d8 --- /dev/null +++ b/concore.egg-info/top_level.txt @@ -0,0 +1,2 @@ +concore_cli +mkconcore diff --git a/setup_branch.ps1 b/setup_branch.ps1 new file mode 100644 index 00000000..05b36bfc --- /dev/null +++ b/setup_branch.ps1 @@ -0,0 +1,8 @@ +cd "D:\gsoc org\concore-project\concore" +git config core.editor "notepad" +git fetch upstream +git checkout dev +git reset --hard upstream/dev +git branch -D refactor-file-io-v2 2>$null +git checkout -b refactor-file-io-v2 +Write-Host "Branch ready!" From 58915e36f0fc17299ba0053d845ed79bcc4b270b Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Sat, 14 Feb 2026 12:03:31 +0530 Subject: [PATCH 115/275] refactor: address remaining Copilot review suggestions - Remove duplicate concore.delay assignments (12 files) - Remove self-assignment no-ops: concore.inpath/outpath = concore.inpath/outpath (9 files) - Use float(concore.simtime) snapshot for old2/old_concore_simtime variables (7 files) --- 0mq/comm_node.py | 1 - 0mq/funbody.py | 2 +- 0mq/funbody2.py | 1 - 0mq/funbody_distributed.py | 3 +-- 0mq/funbody_zmq.py | 2 -- 0mq/funbody_zmq2.py | 4 +--- 0mq/funcall.py | 3 --- 0mq/funcall_distributed.py | 1 - 0mq/funcall_zmq.py | 3 --- measurements/Latency/funbody_distributed.py | 5 +---- measurements/Latency/funcall_distributed.py | 1 - measurements/comm_node_test.py | 4 +--- nintan/powermetermax.py | 3 --- testsou/funbody.py | 4 +--- testsou/funcall.py | 3 +-- testsou/mix.py | 3 --- 16 files changed, 7 insertions(+), 36 deletions(-) diff --git a/0mq/comm_node.py b/0mq/comm_node.py index 13d29748..2602cec7 100644 --- a/0mq/comm_node.py +++ b/0mq/comm_node.py @@ -1,6 +1,5 @@ import concore -concore.delay = 0.07 concore.delay = 0.07 concore.simtime = 0 concore.default_maxtime(100) diff --git a/0mq/funbody.py b/0mq/funbody.py index 3ad6c051..20064592 100644 --- a/0mq/funbody.py +++ b/0mq/funbody.py @@ -49,7 +49,7 @@ u = u[1:] concore.write(concore.oport['U2'],"u",u) print(u) - old2 = concore.simtime + old2 = float(concore.simtime) while concore.unchanged() or concore.simtime <= old2: ym = concore.read(concore.iport['Y2'],"ym",init_simtime_ym) ym = [concore.simtime]+ym diff --git a/0mq/funbody2.py b/0mq/funbody2.py index ca4f3bbd..491b14a1 100644 --- a/0mq/funbody2.py +++ b/0mq/funbody2.py @@ -25,7 +25,6 @@ paired_transmitter.start_background_sync() -concore.delay = 0.07 concore.delay = 0.07 concore.simtime = 0 concore.default_maxtime(100) diff --git a/0mq/funbody_distributed.py b/0mq/funbody_distributed.py index e7e95e6d..33da80b2 100644 --- a/0mq/funbody_distributed.py +++ b/0mq/funbody_distributed.py @@ -15,7 +15,6 @@ # Standard concore initializations concore.delay = 0.07 -concore.delay = 0.07 concore.simtime = 0 concore.default_maxtime(100) init_simtime_u_str = "[0.0, 0.0, 0.0]" @@ -47,7 +46,7 @@ if 'U2' in concore.oport: concore.write(concore.oport['U2'], "u", u_data_values) - old_concore_simtime = concore.simtime + old_concore_simtime = float(concore.simtime) while concore.unchanged() or concore.simtime <= old_concore_simtime: # Assuming concore.iport['Y2'] is a file port (e.g., from pmpymax.py) ym_data_values = concore.read(concore.iport['Y2'], "ym", init_simtime_ym_str) diff --git a/0mq/funbody_zmq.py b/0mq/funbody_zmq.py index 14e5f8ef..103e4e66 100644 --- a/0mq/funbody_zmq.py +++ b/0mq/funbody_zmq.py @@ -14,8 +14,6 @@ # Standard concore initializations concore.delay = 0.07 -concore.delay = 0.07 -concore.inpath = concore.inpath concore.simtime = 0 concore.default_maxtime(100) init_simtime_u_str = "[0.0, 0.0, 0.0]" diff --git a/0mq/funbody_zmq2.py b/0mq/funbody_zmq2.py index 96af0100..956af549 100644 --- a/0mq/funbody_zmq2.py +++ b/0mq/funbody_zmq2.py @@ -14,8 +14,6 @@ # Standard concore initializations concore.delay = 0.07 -concore.inpath = concore.inpath -concore.outpath = concore.outpath concore.simtime = 0 concore.default_maxtime(100) init_simtime_u_str = "[0.0, 0.0, 0.0]" @@ -47,7 +45,7 @@ if 'U2' in concore.oport: concore.write(concore.oport['U2'], "u", u_data_values) - old_concore_simtime = concore.simtime + old_concore_simtime = float(concore.simtime) while concore.unchanged() or concore.simtime <= old_concore_simtime: # Assuming concore.iport['Y2'] is a file port (e.g., from pmpymax.py) ym_data_values = concore.read(concore.iport['Y2'], "ym", init_simtime_ym_str) diff --git a/0mq/funcall.py b/0mq/funcall.py index 85203ba1..c5c63093 100644 --- a/0mq/funcall.py +++ b/0mq/funcall.py @@ -3,9 +3,6 @@ print("funcall 0mq") concore.delay = 0.07 -concore.delay = 0.07 -concore.inpath = concore.inpath -concore.outpath = concore.outpath concore.simtime = 0 concore.default_maxtime(100) init_simtime_u = "[0.0, 0.0, 0.0]" diff --git a/0mq/funcall_distributed.py b/0mq/funcall_distributed.py index 0bf6ce79..1092e263 100644 --- a/0mq/funcall_distributed.py +++ b/0mq/funcall_distributed.py @@ -14,7 +14,6 @@ # Standard concore initializations concore.delay = 0.07 -concore.delay = 0.07 concore.simtime = 0 concore.default_maxtime(100) init_simtime_u_str = "[0.0, 0.0, 0.0]" diff --git a/0mq/funcall_zmq.py b/0mq/funcall_zmq.py index 5b484b1b..ca12a784 100644 --- a/0mq/funcall_zmq.py +++ b/0mq/funcall_zmq.py @@ -14,9 +14,6 @@ # Standard concore initializations concore.delay = 0.07 -concore.delay = 0.07 -concore.inpath = concore.inpath -concore.outpath = concore.outpath concore.simtime = 0 concore.default_maxtime(100) init_simtime_u_str = "[0.0, 0.0, 0.0]" diff --git a/measurements/Latency/funbody_distributed.py b/measurements/Latency/funbody_distributed.py index b7e56849..33da80b2 100644 --- a/measurements/Latency/funbody_distributed.py +++ b/measurements/Latency/funbody_distributed.py @@ -15,9 +15,6 @@ # Standard concore initializations concore.delay = 0.07 -concore.delay = 0.07 -concore.inpath = concore.inpath -concore.outpath = concore.outpath concore.simtime = 0 concore.default_maxtime(100) init_simtime_u_str = "[0.0, 0.0, 0.0]" @@ -49,7 +46,7 @@ if 'U2' in concore.oport: concore.write(concore.oport['U2'], "u", u_data_values) - old_concore_simtime = concore.simtime + old_concore_simtime = float(concore.simtime) while concore.unchanged() or concore.simtime <= old_concore_simtime: # Assuming concore.iport['Y2'] is a file port (e.g., from pmpymax.py) ym_data_values = concore.read(concore.iport['Y2'], "ym", init_simtime_ym_str) diff --git a/measurements/Latency/funcall_distributed.py b/measurements/Latency/funcall_distributed.py index 4332c9e3..8e8e7590 100644 --- a/measurements/Latency/funcall_distributed.py +++ b/measurements/Latency/funcall_distributed.py @@ -15,7 +15,6 @@ # Standard concore initializations concore.delay = 0.07 -concore.delay = 0.07 concore.simtime = 0 concore.default_maxtime(100) # Recommend increasing this for more data points, e.g., 1000 init_simtime_u_str = "[0.0, 0.0, 0.0]" diff --git a/measurements/comm_node_test.py b/measurements/comm_node_test.py index b9cfcde4..465344d0 100644 --- a/measurements/comm_node_test.py +++ b/measurements/comm_node_test.py @@ -4,8 +4,6 @@ # --- Script Configuration --- concore.delay = 0.07 -concore.inpath = concore.inpath -concore.outpath = concore.outpath concore.simtime = 0 concore.default_maxtime(100) # This will be ignored by the new logic init_simtime_u = "[0.0, 0.0, 0.0]" @@ -41,7 +39,7 @@ break # 3. Wait for a message from the 'Y1' channel - old2 = concore.simtime + old2 = float(concore.simtime) while concore.unchanged() or concore.simtime <= old2: ym = concore.read(concore.iport['Y1'], "ym", init_simtime_ym) diff --git a/nintan/powermetermax.py b/nintan/powermetermax.py index 40ae9a83..5294ceb8 100644 --- a/nintan/powermetermax.py +++ b/nintan/powermetermax.py @@ -4,9 +4,6 @@ print("powermeter") concore.delay = 0.07 -concore.delay = 0.07 -concore.inpath = concore.inpath -concore.outpath = concore.outpath concore.simtime = 0 #Nsim = 100 concore.default_maxtime(100) diff --git a/testsou/funbody.py b/testsou/funbody.py index 900a064d..214604dc 100644 --- a/testsou/funbody.py +++ b/testsou/funbody.py @@ -2,8 +2,6 @@ print("funbody") concore.delay = 0.07 -concore.inpath = concore.inpath -concore.outpath = concore.outpath concore.simtime = 0 concore.default_maxtime(100) init_simtime_u = "[0.0, 0.0, 0.0]" @@ -16,7 +14,7 @@ u = concore.read(concore.iport['U1'],"u",init_simtime_u) concore.write(concore.oport['U2'],"u",u) print(u) - old2 = concore.simtime + old2 = float(concore.simtime) while concore.unchanged() or concore.simtime <= old2: ym = concore.read(concore.iport['Y2'],"ym",init_simtime_ym) concore.write(concore.oport['Y1'],"ym",ym) diff --git a/testsou/funcall.py b/testsou/funcall.py index d53e7b9c..d97619e9 100644 --- a/testsou/funcall.py +++ b/testsou/funcall.py @@ -1,7 +1,6 @@ import concore print("funcall") -concore.delay = 0.07 concore.delay = 0.07 concore.simtime = 0 concore.default_maxtime(100) @@ -15,7 +14,7 @@ u = concore.read(concore.iport['U'],"u",init_simtime_u) concore.write(concore.oport['U1'],"u",u) print(u) - old2 = concore.simtime + old2 = float(concore.simtime) while concore.unchanged() and concore.simtime <= old2: ym = concore.read(concore.iport['Y1'],"ym",init_simtime_ym) concore.write(concore.oport['Y'],"ym",ym) diff --git a/testsou/mix.py b/testsou/mix.py index bbbb4385..eb248eec 100644 --- a/testsou/mix.py +++ b/testsou/mix.py @@ -3,9 +3,6 @@ import time concore.delay = 0.005 -concore.delay = 0.005 -concore.inpath = concore.inpath -concore.outpath = concore.outpath concore.simtime = 0 concore.default_maxtime(150) init_simtime_u = "[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]" From e653cd8b4f48146547ff110685c770294dc696f7 Mon Sep 17 00:00:00 2001 From: Avinash Kumar Deepak Date: Fri, 13 Feb 2026 21:41:33 +0530 Subject: [PATCH 116/275] use context managers for file I/O --- complete_rebase.sh | 5 -- concore.egg-info/PKG-INFO | 106 -------------------------- concore.egg-info/SOURCES.txt | 24 ------ concore.egg-info/dependency_links.txt | 1 - concore.egg-info/entry_points.txt | 2 - concore.egg-info/requires.txt | 9 --- concore.egg-info/top_level.txt | 2 - setup_branch.ps1 | 8 -- tools/cwrap.py | 56 +++++++++----- 9 files changed, 35 insertions(+), 178 deletions(-) delete mode 100644 complete_rebase.sh delete mode 100644 concore.egg-info/PKG-INFO delete mode 100644 concore.egg-info/SOURCES.txt delete mode 100644 concore.egg-info/dependency_links.txt delete mode 100644 concore.egg-info/entry_points.txt delete mode 100644 concore.egg-info/requires.txt delete mode 100644 concore.egg-info/top_level.txt delete mode 100644 setup_branch.ps1 diff --git a/complete_rebase.sh b/complete_rebase.sh deleted file mode 100644 index 139801d8..00000000 --- a/complete_rebase.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash -cd "D:\gsoc org\concore-project\concore" -git add requirements-dev.txt -git rebase --continue -git push origin add-concoredocker-tests --force diff --git a/concore.egg-info/PKG-INFO b/concore.egg-info/PKG-INFO deleted file mode 100644 index 7924d4eb..00000000 --- a/concore.egg-info/PKG-INFO +++ /dev/null @@ -1,106 +0,0 @@ -Metadata-Version: 2.4 -Name: concore -Version: 1.0.0 -Summary: Concore workflow management CLI -Home-page: https://github.com/ControlCore-Project/concore -Author: ControlCore Project -License: MIT -Requires-Python: >=3.9 -Description-Content-Type: text/markdown -License-File: LICENSE -Requires-Dist: click>=8.0.0 -Requires-Dist: rich>=10.0.0 -Requires-Dist: beautifulsoup4>=4.9.0 -Requires-Dist: lxml>=4.6.0 -Requires-Dist: psutil>=5.8.0 -Provides-Extra: dev -Requires-Dist: pytest>=6.0.0; extra == "dev" -Requires-Dist: pytest-cov>=2.10.0; extra == "dev" -Dynamic: author -Dynamic: home-page -Dynamic: license-file -Dynamic: requires-python - -# CONTROL-CORE: Integrated Development Environment for Closed-loop Neuromodulation Control Systems. - -[CONTROL-CORE](https://github.com/ControlCore-Project/) is a design and simulation framework, functioning as a visual Integrated Development Environment (IDE) for Closed-loop Neuromodulation Control Systems. At its center is _concore_, a lightweight protocol to simulate neuromodulation control systems. This repository consists of the implementation of _concore_ protocol and sample (demo and neuromodulation control systems) studies. In addition to its default standard Python implementation, _concore_ also supports developing studies in Matlab/Octave, Verilog, and C++. _concore_ also aims to support more language programs in the future. - -# The CONTROL-CORE Framework - -The CONTROL-CORE framework consists of the below projects. - -* _concore_: The CONTROL-CORE protocol, known as _concore_, allows modular simulation of controller and PM nodes to run on different operating systems, computing platforms, and programming languages. [This repository](https://github.com/ControlCore-Project/concore/) consists of _concore_ source code. The _concore_ documentation can be found [here](https://control-core.readthedocs.io/en/latest/index.html). A _concore_ study can be developed from programs written in different languages. That means, _concore_ facilitates a seamless communication across codes developed in different languages that it supports, through its simple file-based data sharing between the programs. - -* _concore_ Editor: This is the front-end for CONTROL-CORE. We forked [DHGWorkflow](https://github.com/ControlCore-Project/DHGWorkflow), a sibling project we developed, and extend it as the _concore_ Editor. - -* _Mediator_: The [Mediator](https://github.com/ControlCore-Project/mediator) allows the CONTROL-CORE studies to be distributed and run, rather than having all the programs that construct a study to be run just from a centralized location. - -* _concore-lite_: The [_concore-lite_](https://github.com/ControlCore-Project/concore-lite) repository consists of a simple example version of a _concore_ study. Please check out and run this, if you like to learn the _concore_ protocol without having to clone this large _concore_ repository. - -* documentation: The [source code repository](https://github.com/ControlCore-Project/documentation) of the ReadTheDocs documentation of CONTROL-CORE. - - -# The _concore_ Protocol - -_concore_ enables composing studies from programs developed in different languages. Currently supported languages are, Python, Matlab/Octave, Verilog, and C++. The studies are designed through the visual _concore_ Editor (DHGWorkflow) and interpreted into _concore_ through its parser. Neural control systems consist of loops (dicycles). Therefore, they cannot be represented by classic workflow standards (such as CWL or WDL). Therefore, _concore_ addresses a significant research gap to model closed-loop neuromodulation control systems. The _concore_ protocol shares data between the programs through file sharing, with no centralized entity (a broker or an orchestrator) to arbitrate communications between the programs. (In the distributed executions, the CONTROL-CORE Mediator enables connecting the disjoint pieces of the study through REST APIs). - - -# Installation and Getting Started Guide - -Please follow the [ReadTheDocs](https://control-core.readthedocs.io/en/latest/index.html) documentation and the [_concore-lite_](https://github.com/ControlCore-Project/concore-lite) repository to get started quick. - -Installation instructions for concore can be found [here](https://control-core.readthedocs.io/en/latest/installation.html). Usage instructions can be found [here](https://control-core.readthedocs.io/en/latest/usage.html). - -## Command-Line Interface (CLI) - -_concore_ now includes a command-line interface for easier workflow management. Install it with: - -```bash -pip install -e . -``` - -Quick start with the CLI: - -```bash -# Create a new project -concore init my-project - -# Validate your workflow -concore validate workflow.graphml - -# Run your workflow -concore run workflow.graphml --auto-build - -# Monitor running processes -concore status - -# Stop all processes -concore stop -``` - -For detailed CLI documentation, see [concore_cli/README.md](concore_cli/README.md). - -For a detailed and more scientific documentation, please read our extensive [open-access research paper on CONTROL-CORE](https://doi.org/10.1109/ACCESS.2022.3161471). This paper has a complete discussion on the CONTROL-CORE architecture and deployment, together with the commands to execute the studies in different programming languages and programming environments (Ubuntu, Windows, MacOS, Docker, and distributed execution). - - -# The _concore_ Repository - -_concore_ contains programs (such as physiological models or more commonly called "PMs" and controllers) and studies (i.e., graphml files that represents the studies as workflows). The _wrappers_ enable seamlessly extending a study into a distributed one with the CONTROL-CORE Mediator. - -_concore_ repository consists of several scripts at its root level. The demo folder consists of several sample programs and studies, mostly toy examples to learn the protocol. The ratc folder consists of the programs and studies of the rat cardiac experiments we developed with _concore_. - -If you have a bug to report in one of the CONTROL-CORE projects, please report it through relevant Issue Tracker. Similarly, please feel free to contribute your studies and code enhancements using pull requests. Questions and discussions can be made through the relevant Discussions forum. - -The _concore_ Issues can be reported [here](https://github.com/ControlCore-Project/concore/issues). - -The _concore_ discussion forum can be found [here](https://github.com/ControlCore-Project/concore/discussions). - -Please make sure to send your _concore_ pull requests to the [dev branch](https://github.com/ControlCore-Project/concore/tree/dev). - - -# Citing _concore_ - -If you use _concore_ in your research, please cite the below papers: - -* Kathiravelu, P., Arnold, M., Vijay, S., Jagwani, R., Goyal, P., Goel, A.K., Li, N., Horn, C., Pan, T., Kothare, M. V., and Mahmoudi, B. **Distributed Executions with CONTROL-CORE Integrated Development Environment (IDE) for Closed-loop Neuromodulation Control Systems.** In Cluster Computing – The Journal of Networks Software Tools and Applications (CLUSTER). May 2025. Accepted. Springer. -* Kathiravelu, P., Arnold, M., Fleischer, J., Yao, Y., Awasthi, S., Goel, A. K., Branen, A., Sarikhani, P., Kumar, G., Kothare, M. V., and Mahmoudi, B. **CONTROL-CORE: A Framework for Simulation and Design of Closed-Loop Peripheral Neuromodulation Control Systems**. In IEEE Access. March 2022. https://doi.org/10.1109/ACCESS.2022.3161471 diff --git a/concore.egg-info/SOURCES.txt b/concore.egg-info/SOURCES.txt deleted file mode 100644 index 99020ca8..00000000 --- a/concore.egg-info/SOURCES.txt +++ /dev/null @@ -1,24 +0,0 @@ -LICENSE -README.md -mkconcore.py -pyproject.toml -setup.py -concore.egg-info/PKG-INFO -concore.egg-info/SOURCES.txt -concore.egg-info/dependency_links.txt -concore.egg-info/entry_points.txt -concore.egg-info/requires.txt -concore.egg-info/top_level.txt -concore_cli/__init__.py -concore_cli/cli.py -concore_cli/commands/__init__.py -concore_cli/commands/init.py -concore_cli/commands/inspect.py -concore_cli/commands/run.py -concore_cli/commands/status.py -concore_cli/commands/stop.py -concore_cli/commands/validate.py -tests/test_cli.py -tests/test_concore.py -tests/test_concoredocker.py -tests/test_graph.py \ No newline at end of file diff --git a/concore.egg-info/dependency_links.txt b/concore.egg-info/dependency_links.txt deleted file mode 100644 index 8b137891..00000000 --- a/concore.egg-info/dependency_links.txt +++ /dev/null @@ -1 +0,0 @@ - diff --git a/concore.egg-info/entry_points.txt b/concore.egg-info/entry_points.txt deleted file mode 100644 index a71a8db6..00000000 --- a/concore.egg-info/entry_points.txt +++ /dev/null @@ -1,2 +0,0 @@ -[console_scripts] -concore = concore_cli.cli:cli diff --git a/concore.egg-info/requires.txt b/concore.egg-info/requires.txt deleted file mode 100644 index 3895f3f9..00000000 --- a/concore.egg-info/requires.txt +++ /dev/null @@ -1,9 +0,0 @@ -click>=8.0.0 -rich>=10.0.0 -beautifulsoup4>=4.9.0 -lxml>=4.6.0 -psutil>=5.8.0 - -[dev] -pytest>=6.0.0 -pytest-cov>=2.10.0 diff --git a/concore.egg-info/top_level.txt b/concore.egg-info/top_level.txt deleted file mode 100644 index 3c4fe8d8..00000000 --- a/concore.egg-info/top_level.txt +++ /dev/null @@ -1,2 +0,0 @@ -concore_cli -mkconcore diff --git a/setup_branch.ps1 b/setup_branch.ps1 deleted file mode 100644 index 05b36bfc..00000000 --- a/setup_branch.ps1 +++ /dev/null @@ -1,8 +0,0 @@ -cd "D:\gsoc org\concore-project\concore" -git config core.editor "notepad" -git fetch upstream -git checkout dev -git reset --hard upstream/dev -git branch -D refactor-file-io-v2 2>$null -git checkout -b refactor-file-io-v2 -Write-Host "Branch ready!" diff --git a/tools/cwrap.py b/tools/cwrap.py index 912423e7..d19caee3 100644 --- a/tools/cwrap.py +++ b/tools/cwrap.py @@ -11,51 +11,63 @@ concore.delay = 0.02 try: - apikey=open(concore.inpath+'1/concore.apikey',newline=None).readline().rstrip() + with open(concore.inpath+'1/concore.apikey',newline=None) as f: + apikey=f.readline().rstrip() except (OSError, IOError): try: #perhaps this should be removed for security - apikey=open('./concore.apikey',newline=None).readline().rstrip() + with open('./concore.apikey',newline=None) as f: + apikey=f.readline().rstrip() except (OSError, IOError): apikey = '' try: - yuyu=open(concore.inpath+'1/concore.yuyu',newline=None).readline().rstrip() + with open(concore.inpath+'1/concore.yuyu',newline=None) as f: + yuyu=f.readline().rstrip() except (OSError, IOError): try: - yuyu=open('./concore.yuyu',newline=None).readline().rstrip() + with open('./concore.yuyu',newline=None) as f: + yuyu=f.readline().rstrip() except (OSError, IOError): yuyu = 'yuyu' try: - name1=open(concore.inpath+'1/concore.name1',newline=None).readline().rstrip() + with open(concore.inpath+'1/concore.name1',newline=None) as f: + name1=f.readline().rstrip() except (OSError, IOError): try: - name1=open('./concore.name1',newline=None).readline().rstrip() + with open('./concore.name1',newline=None) as f: + name1=f.readline().rstrip() except (OSError, IOError): name1 = 'u' try: - name2=open(concore.inpath+'1/concore.name2',newline=None).readline().rstrip() + with open(concore.inpath+'1/concore.name2',newline=None) as f: + name2=f.readline().rstrip() except (OSError, IOError): try: - name2=open('./concore.name2',newline=None).readline().rstrip() + with open('./concore.name2',newline=None) as f: + name2=f.readline().rstrip() except (OSError, IOError): name2 = 'ym' try: - init_simtime_u = open(concore.inpath+'1/concore.init1',newline=None).readline().rstrip() + with open(concore.inpath+'1/concore.init1',newline=None) as f: + init_simtime_u = f.readline().rstrip() except (OSError, IOError): try: - init_simtime_u = open('./concore.init1',newline=None).readline().rstrip() + with open('./concore.init1',newline=None) as f: + init_simtime_u = f.readline().rstrip() except (OSError, IOError): init_simtime_u = "[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]" try: - init_simtime_ym = open(concore.inpath+'1/concore.init2',newline=None).readline().rstrip() + with open(concore.inpath+'1/concore.init2',newline=None) as f: + init_simtime_ym = f.readline().rstrip() except (OSError, IOError): try: - init_simtime_ym = open('./concore.init2',newline=None).readline().rstrip() + with open('./concore.init2',newline=None) as f: + init_simtime_ym = f.readline().rstrip() except (OSError, IOError): init_simtime_ym = "[0.0, 0.0, 0.0]" @@ -82,10 +94,11 @@ logging.debug("CW outer loop") while concore.unchanged(): u = concore.read(1,name1,init_simtime_u) - f = {'file1': open(concore.inpath+'1/'+name1, 'rb')} - logging.debug(f"CW: before post u={u}") - logging.debug(f'http://www.controlcore.org/pm/{yuyu}{apikey}&fetch={name2}') - r = requests.post('http://www.controlcore.org/pm/'+yuyu+apikey+'&fetch='+name2, files=f,timeout=timeout_max) + with open(concore.inpath+'1/'+name1, 'rb') as f1: + f = {'file1': f1} + logging.debug(f"CW: before post u={u}") + logging.debug(f'http://www.controlcore.org/pm/{yuyu}{apikey}&fetch={name2}') + r = requests.post('http://www.controlcore.org/pm/'+yuyu+apikey+'&fetch='+name2, files=f,timeout=timeout_max) if r.status_code!=200: logging.error(f"bad POST request {r.status_code}") quit() @@ -101,11 +114,12 @@ while oldt==t or len(r.content)==0: time.sleep(concore.delay) logging.debug(f"CW waiting status={r.status_code} content={r.content.decode('utf-8')} t={t}") - f = {'file1': open(concore.inpath+'1/'+name1, 'rb')} - try: - r = requests.post('http://www.controlcore.org/pm/'+yuyu+apikey+'&fetch='+name2, files=f,timeout=timeout_max) - except Exception: - logging.error("CW: bad request") + with open(concore.inpath+'1/'+name1, 'rb') as f1: + f = {'file1': f1} + try: + r = requests.post('http://www.controlcore.org/pm/'+yuyu+apikey+'&fetch='+name2, files=f,timeout=timeout_max) + except Exception: + logging.error("CW: bad request") timeout_count += 1 if r.status_code!=200 or time.perf_counter()-t1 > 1.1*timeout_max: #timeout_count>100: logging.error(f"timeout or bad POST request {r.status_code}") From 62f8b28de86e53ff6dfbb5440b251ff24ef32cdd Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Sat, 14 Feb 2026 12:16:50 +0530 Subject: [PATCH 117/275] security: remove hardcoded Flask secret, disable debug mode, add SHA256 verification (Issue #272) --- Dockerfile.sh | 5 ++++- fri/server/main.py | 8 ++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/Dockerfile.sh b/Dockerfile.sh index 0e17693b..b8773daf 100644 --- a/Dockerfile.sh +++ b/Dockerfile.sh @@ -9,7 +9,10 @@ RUN mkdir /mcr-install \ WORKDIR /mcr-install -RUN wget https://ssd.mathworks.com/supportfiles/downloads/R2021a/Release/1/deployment_files/installer/complete/glnxa64/MATLAB_Runtime_R2021a_Update_1_glnxa64.zip +ENV MATLAB_RUNTIME_SHA256="b821022690804e498d2e5ad814dccb64aab17c5e4bc10a1e2a12498ef5364e0d" + +RUN wget https://ssd.mathworks.com/supportfiles/downloads/R2021a/Release/1/deployment_files/installer/complete/glnxa64/MATLAB_Runtime_R2021a_Update_1_glnxa64.zip \ + && echo "${MATLAB_RUNTIME_SHA256} MATLAB_Runtime_R2021a_Update_1_glnxa64.zip" | sha256sum -c - RUN unzip MATLAB_Runtime_R2021a_Update_1_glnxa64.zip \ && ./install -destinationFolder /opt/mcr -agreeToLicense yes -mode silent \ diff --git a/fri/server/main.py b/fri/server/main.py index f94bc663..6919e1ac 100644 --- a/fri/server/main.py +++ b/fri/server/main.py @@ -14,7 +14,9 @@ app = Flask(__name__) -app.secret_key = "secret key" +app.secret_key = os.environ.get("FLASK_SECRET_KEY") +if not app.secret_key: + raise RuntimeError("FLASK_SECRET_KEY environment variable not set") cors = CORS(app) app.config['CORS_HEADERS'] = 'Content-Type' @@ -407,4 +409,6 @@ def openJupyter(): if __name__ == "__main__": - app.run(host="0.0.0.0", port=5000) + # In production, use: + # gunicorn -w 4 -b 0.0.0.0:5000 main:app + app.run(host="0.0.0.0", port=5000, debug=False) From 37cbd17ec8d0411c406bbd967cc9a5c3f321437c Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Sat, 14 Feb 2026 12:28:56 +0530 Subject: [PATCH 118/275] security: address PR review - graceful dev fallback, fix gunicorn path, use ARG for SHA256 override --- Dockerfile.sh | 3 ++- fri/server/main.py | 14 ++++++++++---- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/Dockerfile.sh b/Dockerfile.sh index b8773daf..eec9f62b 100644 --- a/Dockerfile.sh +++ b/Dockerfile.sh @@ -9,7 +9,8 @@ RUN mkdir /mcr-install \ WORKDIR /mcr-install -ENV MATLAB_RUNTIME_SHA256="b821022690804e498d2e5ad814dccb64aab17c5e4bc10a1e2a12498ef5364e0d" +ARG MATLAB_RUNTIME_SHA256="b821022690804e498d2e5ad814dccb64aab17c5e4bc10a1e2a12498ef5364e0d" +ENV MATLAB_RUNTIME_SHA256=${MATLAB_RUNTIME_SHA256} RUN wget https://ssd.mathworks.com/supportfiles/downloads/R2021a/Release/1/deployment_files/installer/complete/glnxa64/MATLAB_Runtime_R2021a_Update_1_glnxa64.zip \ && echo "${MATLAB_RUNTIME_SHA256} MATLAB_Runtime_R2021a_Update_1_glnxa64.zip" | sha256sum -c - diff --git a/fri/server/main.py b/fri/server/main.py index 6919e1ac..a92f16df 100644 --- a/fri/server/main.py +++ b/fri/server/main.py @@ -14,9 +14,15 @@ app = Flask(__name__) -app.secret_key = os.environ.get("FLASK_SECRET_KEY") -if not app.secret_key: - raise RuntimeError("FLASK_SECRET_KEY environment variable not set") +secret_key = os.environ.get("FLASK_SECRET_KEY") +if not secret_key: + # In production, require an explicit FLASK_SECRET_KEY to be set. + # For local development and tests, fall back to a per-process random key + # so that importing this module does not fail hard. + if os.environ.get("FLASK_ENV") == "production": + raise RuntimeError("FLASK_SECRET_KEY environment variable not set in production") + secret_key = os.urandom(32) +app.secret_key = secret_key cors = CORS(app) app.config['CORS_HEADERS'] = 'Content-Type' @@ -410,5 +416,5 @@ def openJupyter(): if __name__ == "__main__": # In production, use: - # gunicorn -w 4 -b 0.0.0.0:5000 main:app + # gunicorn -w 4 -b 0.0.0.0:5000 fri.server.main:app app.run(host="0.0.0.0", port=5000, debug=False) From accc2b469f3c86f10562c032780c619418bcf3eb Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Sat, 14 Feb 2026 13:06:27 +0530 Subject: [PATCH 119/275] chore: upgrade FRI Dockerfile from Python 3.6 to 3.10 (fixes #270) --- fri/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fri/Dockerfile b/fri/Dockerfile index 450274f2..6a639285 100644 --- a/fri/Dockerfile +++ b/fri/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.6-slim +FROM python:3.10-slim RUN apt-get update && apt-get install -y ca-certificates \ && update-ca-certificates \ From 0fc83d0990588559cf71f77cc29b4285bebb3114 Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Sat, 14 Feb 2026 13:31:33 +0530 Subject: [PATCH 120/275] security: prevent path traversal in FRI /download endpoint (fixes #262) --- fri/server/main.py | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/fri/server/main.py b/fri/server/main.py index f94bc663..bf81ce33 100644 --- a/fri/server/main.py +++ b/fri/server/main.py @@ -1,4 +1,4 @@ -from flask import Flask, request, jsonify, send_file, send_from_directory +from flask import Flask, request, jsonify, send_file, send_from_directory, abort from werkzeug.utils import secure_filename import xml.etree.ElementTree as ET import os @@ -330,14 +330,35 @@ def contribute(): def download(dir): download_file = request.args.get('fetch') sub_folder = request.args.get('fetchDir') + + if not download_file: + abort(400, description="Missing file parameter") + + # Normalize the requested file path + safe_path = os.path.normpath(download_file) + + # Prevent absolute paths + if os.path.isabs(safe_path): + abort(400, description="Invalid file path") + + # Prevent directory traversal + if ".." in safe_path.split(os.sep): + abort(400, description="Directory traversal attempt detected") + dirname = secure_filename(dir) + "/" + secure_filename(sub_folder) directory_name = os.path.abspath(os.path.join(concore_path, dirname)) if not os.path.exists(directory_name): resp = jsonify({'message': 'Directory not found'}) resp.status_code = 400 return resp + + # Ensure final resolved path is within the intended directory + full_path = os.path.abspath(os.path.join(directory_name, safe_path)) + if not full_path.startswith(os.path.abspath(directory_name) + os.sep): + abort(403, description="Access denied") + try: - return send_from_directory(directory_name, download_file, as_attachment=True) + return send_from_directory(directory_name, safe_path, as_attachment=True) except: resp = jsonify({'message': 'file not found'}) resp.status_code = 400 From de233e87a62c5fab7c681b0e2d8f97a708f451fd Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Sat, 14 Feb 2026 13:40:43 +0530 Subject: [PATCH 121/275] security: use realpath to prevent symlink-based traversal bypass (fixes #262) --- fri/server/main.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/fri/server/main.py b/fri/server/main.py index bf81ce33..82b94474 100644 --- a/fri/server/main.py +++ b/fri/server/main.py @@ -346,15 +346,18 @@ def download(dir): abort(400, description="Directory traversal attempt detected") dirname = secure_filename(dir) + "/" + secure_filename(sub_folder) - directory_name = os.path.abspath(os.path.join(concore_path, dirname)) + concore_real = os.path.realpath(concore_path) + directory_name = os.path.realpath(os.path.join(concore_real, dirname)) + if not directory_name.startswith(concore_real + os.sep): + abort(403, description="Access denied") if not os.path.exists(directory_name): resp = jsonify({'message': 'Directory not found'}) resp.status_code = 400 return resp - # Ensure final resolved path is within the intended directory - full_path = os.path.abspath(os.path.join(directory_name, safe_path)) - if not full_path.startswith(os.path.abspath(directory_name) + os.sep): + # Ensure final resolved path is within the intended directory, resolving symlinks + full_path = os.path.realpath(os.path.join(directory_name, safe_path)) + if not full_path.startswith(directory_name + os.sep): abort(403, description="Access denied") try: From 7ad1c2552b8af1af006c3eee0fc3dd8336e8dc1e Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Sat, 14 Feb 2026 13:42:00 +0530 Subject: [PATCH 122/275] style: replace bare except with except Exception --- fri/server/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fri/server/main.py b/fri/server/main.py index 82b94474..20862861 100644 --- a/fri/server/main.py +++ b/fri/server/main.py @@ -362,7 +362,7 @@ def download(dir): try: return send_from_directory(directory_name, safe_path, as_attachment=True) - except: + except Exception: resp = jsonify({'message': 'file not found'}) resp.status_code = 400 return resp From e71ab64ecfbfe85d508ee27b7a09baf01ee4c44d Mon Sep 17 00:00:00 2001 From: GREENRAT-K405 Date: Sat, 14 Feb 2026 18:38:58 +0530 Subject: [PATCH 123/275] update docker file for python version --- Dockerfile.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Dockerfile.py b/Dockerfile.py index fb13fb56..3c3836f6 100644 --- a/Dockerfile.py +++ b/Dockerfile.py @@ -1,10 +1,9 @@ FROM jupyter/base-notebook USER root -RUN apt-get update -RUN apt-get install -y build-essential g++ libgl1-mesa-glx libx11-6 -RUN conda install matplotlib scipy -RUN pip install cvxopt +RUN apt-get update && apt-get install -y build-essential g++ libgl1-mesa-glx libx11-6 +COPY requirements.txt . +RUN pip install -r requirements.txt COPY . /src WORKDIR /src From f8c0c5eb57d024ad6bdbd2bef60c90a1c62cabbf Mon Sep 17 00:00:00 2001 From: Avinash Kumar Deepak Date: Sat, 14 Feb 2026 22:58:49 +0530 Subject: [PATCH 124/275] Use HTTPS for API calls --- tools/cwrap.py | 6 +++--- tools/pwrap.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tools/cwrap.py b/tools/cwrap.py index d19caee3..d60db4cb 100644 --- a/tools/cwrap.py +++ b/tools/cwrap.py @@ -97,8 +97,8 @@ with open(concore.inpath+'1/'+name1, 'rb') as f1: f = {'file1': f1} logging.debug(f"CW: before post u={u}") - logging.debug(f'http://www.controlcore.org/pm/{yuyu}{apikey}&fetch={name2}') - r = requests.post('http://www.controlcore.org/pm/'+yuyu+apikey+'&fetch='+name2, files=f,timeout=timeout_max) + logging.debug(f'https://www.controlcore.org/pm/{yuyu}{apikey}&fetch={name2}') + r = requests.post('https://www.controlcore.org/pm/'+yuyu+apikey+'&fetch='+name2, files=f,timeout=timeout_max) if r.status_code!=200: logging.error(f"bad POST request {r.status_code}") quit() @@ -117,7 +117,7 @@ with open(concore.inpath+'1/'+name1, 'rb') as f1: f = {'file1': f1} try: - r = requests.post('http://www.controlcore.org/pm/'+yuyu+apikey+'&fetch='+name2, files=f,timeout=timeout_max) + r = requests.post('https://www.controlcore.org/pm/'+yuyu+apikey+'&fetch='+name2, files=f,timeout=timeout_max) except Exception: logging.error("CW: bad request") timeout_count += 1 diff --git a/tools/pwrap.py b/tools/pwrap.py index 82aea7da..61698069 100644 --- a/tools/pwrap.py +++ b/tools/pwrap.py @@ -83,7 +83,7 @@ #initfiles = {'file1': open('./u', 'rb'), 'file2': open(concore.inpath+'1/ym', 'rb')} initfiles = {'file1': open('./'+name1, 'rb'), 'file2': open(concore.inpath+'1/'+name2, 'rb')} # POST Request to /init with u as file1 and ym as file2 -r = requests.post('http://www.controlcore.org/init/'+yuyu+apikey, files=initfiles) +r = requests.post('https://www.controlcore.org/init/'+yuyu+apikey, files=initfiles) while(concore.simtime Date: Sat, 14 Feb 2026 23:18:36 +0530 Subject: [PATCH 125/275] fix hardcoded java dockerfile, use wildcard in source path of copy --- Dockerfile.java => Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename Dockerfile.java => Dockerfile (87%) diff --git a/Dockerfile.java b/Dockerfile similarity index 87% rename from Dockerfile.java rename to Dockerfile index 1b32be00..324d0b40 100644 --- a/Dockerfile.java +++ b/Dockerfile @@ -3,7 +3,7 @@ WORKDIR /app # Only copy the JAR if it exists -COPY ./target/concore-0.0.1-SNAPSHOT.jar /app/concore.jar +COPY ./target/concore-*.jar /app/concore.jar # Ensure the JAR file is executable if present RUN [ -f /app/concore.jar ] && chmod +x /app/concore.jar || true From aac87c98ed7681055f11e4e20ada99984e1c4047 Mon Sep 17 00:00:00 2001 From: Avinash Kumar Deepak Date: Sat, 14 Feb 2026 23:22:15 +0530 Subject: [PATCH 126/275] allow tool path overrides via concore.tools config and env vars fixes #247 --- README.md | 20 +++++++++++ mkconcore.py | 45 +++++++++++++++++------ tests/test_tool_config.py | 76 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 131 insertions(+), 10 deletions(-) create mode 100644 tests/test_tool_config.py diff --git a/README.md b/README.md index 103ecb2e..37305503 100644 --- a/README.md +++ b/README.md @@ -57,6 +57,26 @@ concore stop For detailed CLI documentation, see [concore_cli/README.md](concore_cli/README.md). +## Configuration + +_concore_ supports customization through configuration files in the `CONCOREPATH` directory (defaults to the _concore_ installation directory): + +- **concore.tools** - Override tool paths (one per line, `KEY=value` format): + ``` + CPPEXE=/usr/local/bin/g++-12 + PYTHONEXE=/usr/bin/python3.11 + VEXE=/opt/iverilog/bin/iverilog + OCTAVEEXE=/snap/bin/octave + ``` + Supported keys: `CPPWIN`, `CPPEXE`, `VWIN`, `VEXE`, `PYTHONEXE`, `PYTHONWIN`, `MATLABEXE`, `MATLABWIN`, `OCTAVEEXE`, `OCTAVEWIN` + +- **concore.octave** - Treat `.m` files as Octave instead of MATLAB (presence = enabled) +- **concore.mcr** - MATLAB Compiler Runtime path (single line) +- **concore.sudo** - Docker command override (e.g., `docker` instead of `sudo docker`) +- **concore.repo** - Docker repository override + +Tool paths can also be set via environment variables (e.g., `CONCORE_CPPEXE=/usr/bin/g++`). Priority: config file > env var > defaults. + For a detailed and more scientific documentation, please read our extensive [open-access research paper on CONTROL-CORE](https://doi.org/10.1109/ACCESS.2022.3161471). This paper has a complete discussion on the CONTROL-CORE architecture and deployment, together with the commands to execute the studies in different programming languages and programming environments (Ubuntu, Windows, MacOS, Docker, and distributed execution). diff --git a/mkconcore.py b/mkconcore.py index 3b20f8e4..14abf34d 100644 --- a/mkconcore.py +++ b/mkconcore.py @@ -97,6 +97,19 @@ def safe_name(value, context, allow_path=False): SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +def _load_tool_config(filepath): + tools = {} + with open(filepath, "r") as f: + for line in f: + line = line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + k, v = line.split("=", 1) + k, v = k.strip(), v.strip() + if v: + tools[k] = v + return tools + def _resolve_concore_path(): script_concore = os.path.join(SCRIPT_DIR, "concore.py") if os.path.exists(script_concore): @@ -109,16 +122,16 @@ def _resolve_concore_path(): GRAPHML_FILE = sys.argv[1] TRIMMED_LOGS = True CONCOREPATH = _resolve_concore_path() -CPPWIN = "g++" #Windows C++ 6/22/21 -CPPEXE = "g++" #Ubuntu/macOS C++ 6/22/21 -VWIN = "iverilog" #Windows verilog 6/25/21 -VEXE = "iverilog" #Ubuntu/macOS verilog 6/25/21 -PYTHONEXE = "python3" #Ubuntu/macOS python3 -PYTHONWIN = "python" #Windows python3 -MATLABEXE = "matlab" #Ubuntu/macOS matlab -MATLABWIN = "matlab" #Windows matlab -OCTAVEEXE = "octave" #Ubuntu/macOS octave -OCTAVEWIN = "octave" #Windows octave +CPPWIN = os.environ.get("CONCORE_CPPWIN", "g++") #Windows C++ 6/22/21 +CPPEXE = os.environ.get("CONCORE_CPPEXE", "g++") #Ubuntu/macOS C++ 6/22/21 +VWIN = os.environ.get("CONCORE_VWIN", "iverilog") #Windows verilog 6/25/21 +VEXE = os.environ.get("CONCORE_VEXE", "iverilog") #Ubuntu/macOS verilog 6/25/21 +PYTHONEXE = os.environ.get("CONCORE_PYTHONEXE", "python3") #Ubuntu/macOS python3 +PYTHONWIN = os.environ.get("CONCORE_PYTHONWIN", "python") #Windows python3 +MATLABEXE = os.environ.get("CONCORE_MATLABEXE", "matlab") #Ubuntu/macOS matlab +MATLABWIN = os.environ.get("CONCORE_MATLABWIN", "matlab") #Windows matlab +OCTAVEEXE = os.environ.get("CONCORE_OCTAVEEXE", "octave") #Ubuntu/macOS octave +OCTAVEWIN = os.environ.get("CONCORE_OCTAVEWIN", "octave") #Windows octave M_IS_OCTAVE = False #treat .m as octave MCRPATH = "~/MATLAB/R2021a" #path to local Ubunta Matlab Compiler Runtime DOCKEREXE = "sudo docker"#assume simple docker install @@ -147,6 +160,18 @@ def _resolve_concore_path(): with open(CONCOREPATH+"/concore.repo", "r") as f: DOCKEREPO = f.readline().strip() #docker id for repo +if os.path.exists(CONCOREPATH+"/concore.tools"): + _tools = _load_tool_config(CONCOREPATH+"/concore.tools") + CPPWIN = _tools.get("CPPWIN", CPPWIN) + CPPEXE = _tools.get("CPPEXE", CPPEXE) + VWIN = _tools.get("VWIN", VWIN) + VEXE = _tools.get("VEXE", VEXE) + PYTHONEXE = _tools.get("PYTHONEXE", PYTHONEXE) + PYTHONWIN = _tools.get("PYTHONWIN", PYTHONWIN) + MATLABEXE = _tools.get("MATLABEXE", MATLABEXE) + MATLABWIN = _tools.get("MATLABWIN", MATLABWIN) + OCTAVEEXE = _tools.get("OCTAVEEXE", OCTAVEEXE) + OCTAVEWIN = _tools.get("OCTAVEWIN", OCTAVEWIN) prefixedgenode = "" sourcedir = sys.argv[2] diff --git a/tests/test_tool_config.py b/tests/test_tool_config.py new file mode 100644 index 00000000..58adc903 --- /dev/null +++ b/tests/test_tool_config.py @@ -0,0 +1,76 @@ +import pytest +import os + +# can't import mkconcore directly (sys.argv at module level), so we duplicate the parser +def _load_tool_config(filepath): + tools = {} + with open(filepath, "r") as f: + for line in f: + line = line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + k, v = line.split("=", 1) + k, v = k.strip(), v.strip() + if v: + tools[k] = v + return tools + + +class TestLoadToolConfig: + + def test_basic_overrides(self, temp_dir): + cfg = os.path.join(temp_dir, "concore.tools") + with open(cfg, "w") as f: + f.write("CPPEXE=/usr/local/bin/g++-12\n") + f.write("PYTHONEXE=/usr/bin/python3.11\n") + + tools = _load_tool_config(cfg) + assert tools["CPPEXE"] == "/usr/local/bin/g++-12" + assert tools["PYTHONEXE"] == "/usr/bin/python3.11" + assert "VEXE" not in tools + + def test_comments_and_blanks_ignored(self, temp_dir): + cfg = os.path.join(temp_dir, "concore.tools") + with open(cfg, "w") as f: + f.write("# custom tool paths\n") + f.write("\n") + f.write("OCTAVEEXE = /snap/bin/octave\n") + f.write("# MATLABEXE = /opt/matlab/bin/matlab\n") + + tools = _load_tool_config(cfg) + assert tools["OCTAVEEXE"] == "/snap/bin/octave" + assert "MATLABEXE" not in tools + + def test_empty_value_skipped(self, temp_dir): + cfg = os.path.join(temp_dir, "concore.tools") + with open(cfg, "w") as f: + f.write("CPPWIN=\n") + f.write("VEXE = \n") + + tools = _load_tool_config(cfg) + assert "CPPWIN" not in tools + assert "VEXE" not in tools + + def test_value_with_equals_sign(self, temp_dir): + cfg = os.path.join(temp_dir, "concore.tools") + with open(cfg, "w") as f: + f.write("CPPEXE=C:\\Program Files\\g++=fast\n") + + tools = _load_tool_config(cfg) + assert tools["CPPEXE"] == "C:\\Program Files\\g++=fast" + + def test_whitespace_around_key_value(self, temp_dir): + cfg = os.path.join(temp_dir, "concore.tools") + with open(cfg, "w") as f: + f.write(" VWIN = C:\\iverilog\\bin\\iverilog.exe \n") + + tools = _load_tool_config(cfg) + assert tools["VWIN"] == "C:\\iverilog\\bin\\iverilog.exe" + + def test_empty_file(self, temp_dir): + cfg = os.path.join(temp_dir, "concore.tools") + with open(cfg, "w") as f: + pass + + tools = _load_tool_config(cfg) + assert tools == {} From f9455c566008e623168ed32cbd45fbbb2dadfddd Mon Sep 17 00:00:00 2001 From: GREENRAT-K405 Date: Sat, 14 Feb 2026 23:51:59 +0530 Subject: [PATCH 127/275] update java docker file to two stage build --- Dockerfile | 14 -------------- Dockerfile.java | 17 +++++++++++++++++ 2 files changed, 17 insertions(+), 14 deletions(-) delete mode 100644 Dockerfile create mode 100644 Dockerfile.java diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 324d0b40..00000000 --- a/Dockerfile +++ /dev/null @@ -1,14 +0,0 @@ -FROM eclipse-temurin:17-jdk-alpine - -WORKDIR /app - -# Only copy the JAR if it exists -COPY ./target/concore-*.jar /app/concore.jar - -# Ensure the JAR file is executable if present -RUN [ -f /app/concore.jar ] && chmod +x /app/concore.jar || true - -EXPOSE 3000 - -# Run Java app only if the JAR exists, otherwise do nothing -CMD ["/bin/sh", "-c", "if [ -f /app/concore.jar ]; then java -jar /app/concore.jar; else echo 'No Java application found, exiting'; fi"] diff --git a/Dockerfile.java b/Dockerfile.java new file mode 100644 index 00000000..a3eb3ad0 --- /dev/null +++ b/Dockerfile.java @@ -0,0 +1,17 @@ +#build stage +FROM maven:3.9-eclipse-temurin-17-alpine AS builder +WORKDIR /build +COPY pom.xml . +RUN mvn dependency:go-offline -B +COPY src ./src +RUN mvn clean package -DskipTests + +#runtime stage +FROM eclipse-temurin:17-jdk-alpine + +WORKDIR /app + +# Copy the JAR from the build stage +COPY --from=builder /build/target/concore-*.jar /app/concore.jar +EXPOSE 3000 +CMD ["java", "-jar", "/app/concore.jar"] \ No newline at end of file From 302a94043ac3259e28d5ce9c2dff81fa632fcb82 Mon Sep 17 00:00:00 2001 From: GREENRAT-K405 Date: Sun, 15 Feb 2026 01:51:14 +0530 Subject: [PATCH 128/275] fix unnecessary matlab image bloating --- Dockerfile.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Dockerfile.sh b/Dockerfile.sh index eec9f62b..ccddea30 100644 --- a/Dockerfile.sh +++ b/Dockerfile.sh @@ -13,9 +13,8 @@ ARG MATLAB_RUNTIME_SHA256="b821022690804e498d2e5ad814dccb64aab17c5e4bc10a1e2a124 ENV MATLAB_RUNTIME_SHA256=${MATLAB_RUNTIME_SHA256} RUN wget https://ssd.mathworks.com/supportfiles/downloads/R2021a/Release/1/deployment_files/installer/complete/glnxa64/MATLAB_Runtime_R2021a_Update_1_glnxa64.zip \ - && echo "${MATLAB_RUNTIME_SHA256} MATLAB_Runtime_R2021a_Update_1_glnxa64.zip" | sha256sum -c - - -RUN unzip MATLAB_Runtime_R2021a_Update_1_glnxa64.zip \ + && echo "${MATLAB_RUNTIME_SHA256} MATLAB_Runtime_R2021a_Update_1_glnxa64.zip" | sha256sum -c - \ + && unzip MATLAB_Runtime_R2021a_Update_1_glnxa64.zip \ && ./install -destinationFolder /opt/mcr -agreeToLicense yes -mode silent \ && cd / \ && rm -rf mcr-install From 2e10f4800b5c30a5db359bf59af3e4d6e8a1097d Mon Sep 17 00:00:00 2001 From: Titas-Ghosh Date: Sun, 15 Feb 2026 02:14:40 +0530 Subject: [PATCH 129/275] Allow relative subdirectory paths in node labels --- mkconcore.py | 123 ++++++++++++++++++++++++++++++---------------- tests/test_cli.py | 24 +++++++++ 2 files changed, 104 insertions(+), 43 deletions(-) diff --git a/mkconcore.py b/mkconcore.py index 14abf34d..9457d74e 100644 --- a/mkconcore.py +++ b/mkconcore.py @@ -92,6 +92,24 @@ def safe_name(value, context, allow_path=False): if re.search(pattern, value): raise ValueError(f"Unsafe {context}: '{value}' contains illegal characters.") return value + +def safe_relpath(value, context): + """ + Allow relative subpaths while blocking traversal and absolute/drive paths. + """ + if not value: + raise ValueError(f"{context} cannot be empty") + normalized = value.replace("\\", "/") + safe_name(normalized, context, allow_path=True) + if normalized.startswith("/") or normalized.startswith("~"): + raise ValueError(f"Unsafe {context}: absolute paths are not allowed.") + if re.match(r"^[A-Za-z]:", normalized): + raise ValueError(f"Unsafe {context}: drive paths are not allowed.") + if ":" in normalized: + raise ValueError(f"Unsafe {context}: ':' is not allowed in relative paths.") + if any(part in ("", "..") for part in normalized.split("/")): + raise ValueError(f"Unsafe {context}: invalid path segment.") + return normalized MKCONCORE_VER = "22-09-18" @@ -273,14 +291,15 @@ def cleanup_script_files(): node_label = re.sub(r'(\s+|\n)', ' ', node_label) #Validate node labels - if ':' in node_label: - container_part, source_part = node_label.split(':', 1) - safe_name(container_part, f"Node container name '{container_part}'") - safe_name(source_part, f"Node source file '{source_part}'") - else: - safe_name(node_label, f"Node label '{node_label}'") - # Explicitly reject incorrect format to prevent later crashes and ambiguity - raise ValueError(f"Invalid node label '{node_label}': expected format 'container:source' with a ':' separator.") + if ':' in node_label: + container_part, source_part = node_label.split(':', 1) + safe_name(container_part, f"Node container name '{container_part}'") + source_part = safe_relpath(source_part, f"Node source file '{source_part}'") + node_label = f"{container_part}:{source_part}" + else: + safe_name(node_label, f"Node label '{node_label}'") + # Explicitly reject incorrect format to prevent later crashes and ambiguity + raise ValueError(f"Invalid node label '{node_label}': expected format 'container:source' with a ':' separator.") nodes_dict[node['id']] = node_label node_id_to_label_map[node['id']] = node_label.split(':')[0] @@ -466,12 +485,15 @@ def cleanup_script_files(): if not sourcecode: continue - if "." in sourcecode: - dockername, langext = os.path.splitext(sourcecode) - else: - dockername, langext = sourcecode, "" - - script_target_path = os.path.join(outdir, "src", sourcecode) + if "." in sourcecode: + dockername, langext = os.path.splitext(sourcecode) + else: + dockername, langext = sourcecode, "" + + script_target_path = os.path.join(outdir, "src", sourcecode) + script_target_parent = os.path.dirname(script_target_path) + if script_target_parent: + os.makedirs(script_target_parent, exist_ok=True) # If the script was specialized, it's already in outdir/src. If not, copy from sourcedir. if node_id_key not in node_edge_params: @@ -661,27 +683,33 @@ def cleanup_script_files(): # 4. Write final iport/oport files logging.info("Writing .iport and .oport files...") -for node_label, ports in node_port_mappings.items(): +for node_label, ports in node_port_mappings.items(): try: containername, sourcecode = node_label.split(':', 1) - if not sourcecode or "." not in sourcecode: continue - dockername = os.path.splitext(sourcecode)[0] - with open(os.path.join(outdir, "src", f"{dockername}.iport"), "w") as fport: - fport.write(str(ports['iport']).replace("'" + prefixedgenode, "'")) - with open(os.path.join(outdir, "src", f"{dockername}.oport"), "w") as fport: - fport.write(str(ports['oport']).replace("'" + prefixedgenode, "'")) + if not sourcecode or "." not in sourcecode: continue + dockername = os.path.splitext(sourcecode)[0] + iport_path = os.path.join(outdir, "src", f"{dockername}.iport") + oport_path = os.path.join(outdir, "src", f"{dockername}.oport") + iport_parent = os.path.dirname(iport_path) + if iport_parent: + os.makedirs(iport_parent, exist_ok=True) + with open(iport_path, "w") as fport: + fport.write(str(ports['iport']).replace("'" + prefixedgenode, "'")) + with open(oport_path, "w") as fport: + fport.write(str(ports['oport']).replace("'" + prefixedgenode, "'")) except ValueError: continue #if docker, make docker-dirs, generate build, run, stop, clear scripts and quit -if (concoretype=="docker"): - for node in nodes_dict: - containername,sourcecode = nodes_dict[node].split(':') - if len(sourcecode)!=0 and sourcecode.find(".")!=-1: #3/28/21 - dockername,langext = sourcecode.split(".") - if not os.path.exists(outdir+"/src/Dockerfile."+dockername): # 3/30/21 - try: +if (concoretype=="docker"): + for node in nodes_dict: + containername,sourcecode = nodes_dict[node].split(':') + if len(sourcecode)!=0 and sourcecode.find(".")!=-1: #3/28/21 + dockername,langext = sourcecode.split(".") + dockerfile_path = os.path.join(outdir, "src", f"Dockerfile.{dockername}") + if not os.path.exists(dockerfile_path): # 3/30/21 + try: if langext=="py": src_path = CONCOREPATH+"/Dockerfile.py" logging.info("assuming .py extension for Dockerfile") @@ -699,11 +727,14 @@ def cleanup_script_files(): logging.info("assuming .m extension for Dockerfile") with open(src_path) as fsource: source_content = fsource.read() - except: - logging.error(f"{CONCOREPATH} is not correct path to concore") - quit() - with open(outdir+"/src/Dockerfile."+dockername,"w") as fcopy: - fcopy.write(source_content) + except: + logging.error(f"{CONCOREPATH} is not correct path to concore") + quit() + dockerfile_parent = os.path.dirname(dockerfile_path) + if dockerfile_parent: + os.makedirs(dockerfile_parent, exist_ok=True) + with open(dockerfile_path,"w") as fcopy: + fcopy.write(source_content) if langext=="py": fcopy.write('CMD ["python", "-i", "'+sourcecode+'"]\n') if langext=="m": @@ -947,16 +978,22 @@ def cleanup_script_files(): if concoretype=="posix": fbuild.write('#!/bin/bash' + "\n") -for node in nodes_dict: - containername,sourcecode = nodes_dict[node].split(':') - if len(sourcecode)!=0: - if sourcecode.find(".")==-1: - logging.error("cannot pull container "+sourcecode+" with control core type "+concoretype) #3/28/21 - quit() - dockername,langext = sourcecode.split(".") - fbuild.write('mkdir '+containername+"\n") - if concoretype == "windows": - fbuild.write("copy .\\src\\"+sourcecode+" .\\"+containername+"\\"+sourcecode+"\n") +for node in nodes_dict: + containername,sourcecode = nodes_dict[node].split(':') + if len(sourcecode)!=0: + if sourcecode.find(".")==-1: + logging.error("cannot pull container "+sourcecode+" with control core type "+concoretype) #3/28/21 + quit() + dockername,langext = sourcecode.split(".") + fbuild.write('mkdir '+containername+"\n") + source_subdir = os.path.dirname(sourcecode).replace("\\", "/") + if source_subdir: + if concoretype == "windows": + fbuild.write("mkdir .\\"+containername+"\\"+source_subdir.replace("/", "\\")+"\n") + else: + fbuild.write("mkdir -p ./"+containername+"/"+source_subdir+"\n") + if concoretype == "windows": + fbuild.write("copy .\\src\\"+sourcecode+" .\\"+containername+"\\"+sourcecode+"\n") if langext == "py": fbuild.write("copy .\\src\\concore.py .\\" + containername + "\\concore.py\n") elif langext == "cpp": diff --git a/tests/test_cli.py b/tests/test_cli.py index 8d5a3994..b1853b49 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -114,6 +114,30 @@ def test_run_command_default_type(self): else: self.assertTrue(Path('out/build').exists()) + def test_run_command_subdir_source(self): + with self.runner.isolated_filesystem(temp_dir=self.temp_dir): + result = self.runner.invoke(cli, ['init', 'test-project']) + self.assertEqual(result.exit_code, 0) + + subdir = Path('test-project/src/subdir') + subdir.mkdir(parents=True, exist_ok=True) + shutil.move('test-project/src/script.py', subdir / 'script.py') + + workflow_path = Path('test-project/workflow.graphml') + content = workflow_path.read_text() + content = content.replace('N1:script.py', 'N1:subdir/script.py') + workflow_path.write_text(content) + + result = self.runner.invoke(cli, [ + 'run', + 'test-project/workflow.graphml', + '--source', 'test-project/src', + '--output', 'out', + '--type', 'posix' + ]) + self.assertEqual(result.exit_code, 0) + self.assertTrue(Path('out/src/subdir/script.py').exists()) + def test_run_command_existing_output(self): with self.runner.isolated_filesystem(temp_dir=self.temp_dir): result = self.runner.invoke(cli, ['init', 'test-project']) From 49eed7cbaf7b3f861e75c6b20b77d930134359de Mon Sep 17 00:00:00 2001 From: GREENRAT-K405 Date: Sun, 15 Feb 2026 02:30:38 +0530 Subject: [PATCH 130/275] update dockerfile.m to include essential packages --- Dockerfile.m | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Dockerfile.m b/Dockerfile.m index 54881fcb..02eaeff4 100644 --- a/Dockerfile.m +++ b/Dockerfile.m @@ -1,3 +1,8 @@ FROM mtmiller/octave + +USER root +RUN apt-get update && apt-get install -y --no-install-recommends octave-control octave-signal && rm -rf /var/lib/apt/lists/* +RUN echo "pkg load signal;" >> /etc/octave.conf && echo "pkg load control;" >> /etc/octave.conf + COPY . /src WORKDIR /src From 2a8e0acd6aebff150a9d9cddbf1861bc19b2d100 Mon Sep 17 00:00:00 2001 From: GREENRAT-K405 Date: Sun, 15 Feb 2026 03:07:21 +0530 Subject: [PATCH 131/275] clear apt cache to optimize verilog dockerfile --- Dockerfile.v | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile.v b/Dockerfile.v index e00aae73..ebe581a9 100644 --- a/Dockerfile.v +++ b/Dockerfile.v @@ -1,7 +1,7 @@ FROM ubuntu:20.04 RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ - iverilog + iverilog && rm -rf /var/lib/apt/lists/* COPY . /src WORKDIR /src From 6423a4990881a9eafa1c50f6026e6451b9591e01 Mon Sep 17 00:00:00 2001 From: GREENRAT-K405 Date: Sun, 15 Feb 2026 03:53:39 +0530 Subject: [PATCH 132/275] add .dockerignore to concore --- .dockerignore | 67 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..21476c34 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,67 @@ +# Git +.git +.gitignore + +# Python +__pycache__/ +*.py[cod] +*$py.class +.Python +env/ +venv/ +ENV/ +.venv/ +Pipfile.lock + +# Python Build / Distribution +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# C/C++ Build Artifacts +target/ +*.o +*.out +*.so +*.dylib +*.dll +CMakeCache.txt +CMakeFiles/ + +# Testing and Coverage +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# IDE and OS files +.vscode/ +.idea/ +*.swp +*.swo +.DS_Store +Thumbs.db + +Dockerfile* +docker-compose.yml +.dockerignore \ No newline at end of file From ceccbac0a2ba9dd60f269a246187094a303ea041 Mon Sep 17 00:00:00 2001 From: Sahil Lenka Date: Mon, 9 Feb 2026 16:17:06 +0530 Subject: [PATCH 133/275] Enhance workflow validation with source file and port checks - Add optional --source flag to validate command for file existence checking - Implement cycle detection to identify control loops in workflows - Add ZMQ port conflict detection and validation - Warn about reserved ports (< 1024) and invalid port ranges - Expand test coverage with 6 new comprehensive test cases - Update CLI documentation with new validation features This improves the user experience by catching common configuration errors before workflow execution, reducing runtime failures. --- concore_cli/README.md | 9 ++- concore_cli/cli.py | 5 +- concore_cli/commands/validate.py | 99 ++++++++++++++++++++++++++- tests/test_graph.py | 113 +++++++++++++++++++++++++++++++ 4 files changed, 222 insertions(+), 4 deletions(-) diff --git a/concore_cli/README.md b/concore_cli/README.md index e29c7657..e3b55b68 100644 --- a/concore_cli/README.md +++ b/concore_cli/README.md @@ -72,16 +72,23 @@ concore run workflow.graphml --source ./src --output ./build --auto-build Validates a GraphML workflow file before running. +**Options:** +- `-s, --source ` - Source directory to verify file references exist + Checks: - Valid XML structure - GraphML format compliance - Node and edge definitions - File references and naming conventions -- ZMQ vs file-based communication +- Source file existence (when --source provided) +- ZMQ port conflicts and reserved ports +- Circular dependencies (warns for control loops) +- Edge connectivity **Example:** ```bash concore validate workflow.graphml +concore validate workflow.graphml --source ./src ``` ### `concore status` diff --git a/concore_cli/cli.py b/concore_cli/cli.py index f7144533..4db6068f 100644 --- a/concore_cli/cli.py +++ b/concore_cli/cli.py @@ -47,10 +47,11 @@ def run(workflow_file, source, output, type, auto_build): @cli.command() @click.argument('workflow_file', type=click.Path(exists=True)) -def validate(workflow_file): +@click.option('--source', '-s', type=click.Path(exists=True), help='Source directory to check file references') +def validate(workflow_file, source): """Validate a workflow file""" try: - validate_workflow(workflow_file, console) + validate_workflow(workflow_file, console, source) except Exception as e: console.print(f"[red]Error:[/red] {str(e)}") sys.exit(1) diff --git a/concore_cli/commands/validate.py b/concore_cli/commands/validate.py index fa1ea184..7b34d722 100644 --- a/concore_cli/commands/validate.py +++ b/concore_cli/commands/validate.py @@ -5,7 +5,7 @@ import re import xml.etree.ElementTree as ET -def validate_workflow(workflow_file, console): +def validate_workflow(workflow_file, console, source_dir=None): workflow_path = Path(workflow_file) console.print(f"[cyan]Validating:[/cyan] {workflow_path.name}") @@ -138,6 +138,12 @@ def validate_workflow(workflow_file, console): if file_edges > 0: info.append(f"File-based edges: {file_edges}") + if source_dir: + _check_source_files(soup, Path(source_dir), errors, warnings) + + _check_cycles(soup, errors, warnings) + _check_zmq_ports(soup, errors, warnings) + show_results(console, errors, warnings, info) except FileNotFoundError: @@ -145,6 +151,97 @@ def validate_workflow(workflow_file, console): except Exception as e: console.print(f"[red]Validation failed:[/red] {str(e)}") +def _check_source_files(soup, source_path, errors, warnings): + nodes = soup.find_all('node') + + for node in nodes: + label_tag = node.find('y:NodeLabel') or node.find('NodeLabel') + if not label_tag or not label_tag.text: + continue + + label = label_tag.text.strip() + if ':' not in label: + continue + + parts = label.split(':') + if len(parts) != 2: + continue + + _, filename = parts + if not filename: + continue + + file_path = source_path / filename + if not file_path.exists(): + errors.append(f"Source file not found: {filename}") + +def _check_cycles(soup, errors, warnings): + nodes = soup.find_all('node') + edges = soup.find_all('edge') + + node_ids = [node.get('id') for node in nodes if node.get('id')] + if not node_ids: + return + + graph = {nid: [] for nid in node_ids} + for edge in edges: + source = edge.get('source') + target = edge.get('target') + if source and target and source in graph: + graph[source].append(target) + + def has_cycle_from(start, visited, rec_stack): + visited.add(start) + rec_stack.add(start) + + for neighbor in graph.get(start, []): + if neighbor not in visited: + if has_cycle_from(neighbor, visited, rec_stack): + return True + elif neighbor in rec_stack: + return True + + rec_stack.remove(start) + return False + + visited = set() + for node_id in node_ids: + if node_id not in visited: + if has_cycle_from(node_id, visited, set()): + warnings.append("Workflow contains cycles (expected for control loops)") + return + +def _check_zmq_ports(soup, errors, warnings): + edges = soup.find_all('edge') + port_pattern = re.compile(r"0x([a-fA-F0-9]+)_(\S+)") + + ports_used = {} + + for edge in edges: + label_tag = edge.find('y:EdgeLabel') or edge.find('EdgeLabel') + if not label_tag or not label_tag.text: + continue + + match = port_pattern.match(label_tag.text.strip()) + if not match: + continue + + port_hex = match.group(1) + port_name = match.group(2) + port_num = int(port_hex, 16) + + if port_num in ports_used: + existing_name = ports_used[port_num] + if existing_name != port_name: + errors.append(f"Port conflict: 0x{port_hex} used for both '{existing_name}' and '{port_name}'") + else: + ports_used[port_num] = port_name + + if port_num < 1024: + warnings.append(f"Port {port_num} (0x{port_hex}) is in reserved range (< 1024)") + elif port_num > 65535: + errors.append(f"Invalid port number: {port_num} (0x{port_hex}) exceeds maximum (65535)") + def show_results(console, errors, warnings, info): if errors: console.print("[red]✗ Validation failed[/red]\n") diff --git a/tests/test_graph.py b/tests/test_graph.py index 97102dce..7489e112 100644 --- a/tests/test_graph.py +++ b/tests/test_graph.py @@ -128,6 +128,119 @@ def test_validate_valid_graph(self): self.assertIn('Validation passed', result.output) self.assertIn('Workflow is valid', result.output) + + def test_validate_missing_source_file(self): + content = ''' + + + + n0:missing.py + + + + ''' + filepath = self.create_graph_file('workflow.graphml', content) + source_dir = Path(self.temp_dir) / 'src' + source_dir.mkdir() + + result = self.runner.invoke(cli, ['validate', filepath, '--source', str(source_dir)]) + + self.assertIn('Validation failed', result.output) + self.assertIn('Source file not found: missing.py', result.output) + + def test_validate_with_existing_source_file(self): + content = ''' + + + + n0:exists.py + + + + ''' + filepath = self.create_graph_file('workflow.graphml', content) + source_dir = Path(self.temp_dir) / 'src' + source_dir.mkdir() + (source_dir / 'exists.py').write_text('print("hello")') + + result = self.runner.invoke(cli, ['validate', filepath, '--source', str(source_dir)]) + + self.assertIn('Validation passed', result.output) + + def test_validate_zmq_port_conflict(self): + content = ''' + + + + n0:script1.py + + + n1:script2.py + + + 0x1234_portA + + + 0x1234_portB + + + + ''' + filepath = self.create_graph_file('conflict.graphml', content) + + result = self.runner.invoke(cli, ['validate', filepath]) + + self.assertIn('Validation failed', result.output) + self.assertIn('Port conflict', result.output) + + def test_validate_reserved_port(self): + content = ''' + + + + n0:script1.py + + + n1:script2.py + + + 0x50_data + + + + ''' + filepath = self.create_graph_file('reserved.graphml', content) + + result = self.runner.invoke(cli, ['validate', filepath]) + + self.assertIn('Port 80', result.output) + self.assertIn('reserved range', result.output) + + def test_validate_cycle_detection(self): + content = ''' + + + + n0:controller.py + + + n1:plant.py + + + control_signal + + + sensor_data + + + + ''' + filepath = self.create_graph_file('cycle.graphml', content) + + result = self.runner.invoke(cli, ['validate', filepath]) + + self.assertIn('cycles', result.output) + self.assertIn('control loops', result.output) if __name__ == '__main__': unittest.main() \ No newline at end of file From a39e30a4b0fa93394aa993ad6b8e614f3c5d6688 Mon Sep 17 00:00:00 2001 From: Sahil Lenka Date: Mon, 9 Feb 2026 16:28:02 +0530 Subject: [PATCH 134/275] Address Copilot review feedback - Add explicit validation for port 0 (invalid) - Enforce directory-only input for --source option - Use warnings parameter in _check_source_files for clarity - Add test coverage for port 0 and port > 65535 edge cases - Reorder port validation to check range before conflicts --- concore_cli/cli.py | 7 ++++- concore_cli/commands/validate.py | 11 ++++++-- tests/test_graph.py | 46 ++++++++++++++++++++++++++++++++ 3 files changed, 61 insertions(+), 3 deletions(-) diff --git a/concore_cli/cli.py b/concore_cli/cli.py index 4db6068f..f9e35f0c 100644 --- a/concore_cli/cli.py +++ b/concore_cli/cli.py @@ -47,7 +47,12 @@ def run(workflow_file, source, output, type, auto_build): @cli.command() @click.argument('workflow_file', type=click.Path(exists=True)) -@click.option('--source', '-s', type=click.Path(exists=True), help='Source directory to check file references') +@click.option( + '--source', + '-s', + type=click.Path(exists=True, file_okay=False, dir_okay=True, path_type=Path), + help='Source directory to check file references', +) def validate(workflow_file, source): """Validate a workflow file""" try: diff --git a/concore_cli/commands/validate.py b/concore_cli/commands/validate.py index 7b34d722..18456329 100644 --- a/concore_cli/commands/validate.py +++ b/concore_cli/commands/validate.py @@ -161,10 +161,12 @@ def _check_source_files(soup, source_path, errors, warnings): label = label_tag.text.strip() if ':' not in label: + warnings.append(f"Skipping node with invalid label format (expected 'ID:filename')") continue parts = label.split(':') if len(parts) != 2: + warnings.append(f"Skipping node '{label}' with invalid format") continue _, filename = parts @@ -230,6 +232,13 @@ def _check_zmq_ports(soup, errors, warnings): port_name = match.group(2) port_num = int(port_hex, 16) + if port_num < 1: + errors.append(f"Invalid port number: {port_num} (0x{port_hex}) must be at least 1") + continue + elif port_num > 65535: + errors.append(f"Invalid port number: {port_num} (0x{port_hex}) exceeds maximum (65535)") + continue + if port_num in ports_used: existing_name = ports_used[port_num] if existing_name != port_name: @@ -239,8 +248,6 @@ def _check_zmq_ports(soup, errors, warnings): if port_num < 1024: warnings.append(f"Port {port_num} (0x{port_hex}) is in reserved range (< 1024)") - elif port_num > 65535: - errors.append(f"Invalid port number: {port_num} (0x{port_hex}) exceeds maximum (65535)") def show_results(console, errors, warnings, info): if errors: diff --git a/tests/test_graph.py b/tests/test_graph.py index 7489e112..483fd06f 100644 --- a/tests/test_graph.py +++ b/tests/test_graph.py @@ -241,6 +241,52 @@ def test_validate_cycle_detection(self): self.assertIn('cycles', result.output) self.assertIn('control loops', result.output) + + def test_validate_port_zero(self): + content = ''' + + + + n0:script1.py + + + n1:script2.py + + + 0x0_invalid + + + + ''' + filepath = self.create_graph_file('port_zero.graphml', content) + + result = self.runner.invoke(cli, ['validate', filepath]) + + self.assertIn('Validation failed', result.output) + self.assertIn('must be at least 1', result.output) + + def test_validate_port_exceeds_maximum(self): + content = ''' + + + + n0:script1.py + + + n1:script2.py + + + 0x10000_toobig + + + + ''' + filepath = self.create_graph_file('port_max.graphml', content) + + result = self.runner.invoke(cli, ['validate', filepath]) + + self.assertIn('Validation failed', result.output) + self.assertIn('exceeds maximum (65535)', result.output) if __name__ == '__main__': unittest.main() \ No newline at end of file From ad0f393c45ad3498c23fbf1e6b30a68a2ba79b89 Mon Sep 17 00:00:00 2001 From: Sahil Lenka Date: Sun, 15 Feb 2026 04:00:48 +0530 Subject: [PATCH 135/275] Fix --- mkconcore.py | 54 ++++++++++++++++++++++++++++++++++------------------ 1 file changed, 35 insertions(+), 19 deletions(-) diff --git a/mkconcore.py b/mkconcore.py index 400475d6..0f519266 100644 --- a/mkconcore.py +++ b/mkconcore.py @@ -77,7 +77,8 @@ def safe_name(value, context): """ Validates that the input string does not contain characters dangerous - for filesystem paths or shell command injection. + for simple names (labels, filenames without paths). + Use safe_path() for validating directory/file paths. """ if not value: raise ValueError(f"{context} cannot be empty") @@ -86,22 +87,36 @@ def safe_name(value, context): raise ValueError(f"Unsafe {context}: '{value}' contains illegal characters.") return value -MKCONCORE_VER = "22-09-18" - -SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) - -def _resolve_concore_path(): - script_concore = os.path.join(SCRIPT_DIR, "concore.py") - if os.path.exists(script_concore): - return SCRIPT_DIR - cwd_concore = os.path.join(os.getcwd(), "concore.py") - if os.path.exists(cwd_concore): - return os.getcwd() - return SCRIPT_DIR - -GRAPHML_FILE = sys.argv[1] -TRIMMED_LOGS = True -CONCOREPATH = _resolve_concore_path() +def safe_path(value, context): + """ + Validates that a path string does not contain characters dangerous for shell command injection. + Unlike safe_name(), this allows path separators (/ and \) but still blocks dangerous shell metacharacters. + """ + if not value: + raise ValueError(f"{context} cannot be empty") + # Allow path separators but block control characters and shell metacharacters + # Blocks: control chars, *, ?, <, >, |, ;, &, $, `, ', ", (, ) + # Allows: /, \, -, _, ., alphanumeric, spaces, : + if re.search(r'[\x00-\x1F\x7F*?"<>|;&`$\'()]', value): + raise ValueError(f"Unsafe {context}: '{value}' contains illegal characters.") + return value + +MKCONCORE_VER = "22-09-18" + +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) + +def _resolve_concore_path(): + script_concore = os.path.join(SCRIPT_DIR, "concore.py") + if os.path.exists(script_concore): + return SCRIPT_DIR + cwd_concore = os.path.join(os.getcwd(), "concore.py") + if os.path.exists(cwd_concore): + return os.getcwd() + return SCRIPT_DIR + +GRAPHML_FILE = sys.argv[1] +TRIMMED_LOGS = True +CONCOREPATH = _resolve_concore_path() CPPWIN = "g++" #Windows C++ 6/22/21 CPPEXE = "g++" #Ubuntu/macOS C++ 6/22/21 VWIN = "iverilog" #Windows verilog 6/25/21 @@ -142,8 +157,9 @@ def _resolve_concore_path(): sourcedir = sys.argv[2] outdir = sys.argv[3] -# Validate outdir argument -safe_name(outdir, "Output directory argument") +# Validate path arguments +safe_path(outdir, "Output directory argument") +safe_path(sourcedir, "Source directory argument") if not os.path.isdir(sourcedir): logging.error(f"{sourcedir} does not exist") From 59c76b6b942f3f3f1eb9257a8111c9d8f0257084 Mon Sep 17 00:00:00 2001 From: Sahil Lenka Date: Sun, 15 Feb 2026 04:58:02 +0530 Subject: [PATCH 136/275] Merge upstream/dev --- abort_rebase.py | 12 +++++++ complete_push.py | 61 ++++++++++++++++++++++++++++++++ concore_cli/cli.py | 11 +++--- concore_cli/commands/validate.py | 60 +++++++++++++------------------ fix_and_push.bat | 8 +++++ mkconcore.py | 25 ++++++++++++- tests/test_graph.py | 2 +- 7 files changed, 135 insertions(+), 44 deletions(-) create mode 100644 abort_rebase.py create mode 100644 complete_push.py create mode 100644 fix_and_push.bat diff --git a/abort_rebase.py b/abort_rebase.py new file mode 100644 index 00000000..88dcb2fb --- /dev/null +++ b/abort_rebase.py @@ -0,0 +1,12 @@ +import subprocess +import os + +os.chdir(r'C:\Users\Sahil\concore') + +# Abort rebase +subprocess.run(['git', 'rebase', '--abort'], capture_output=True) + +# Check status +result = subprocess.run(['git', 'status'], capture_output=True, text=True) +print(result.stdout) +print(result.stderr) diff --git a/complete_push.py b/complete_push.py new file mode 100644 index 00000000..16c03613 --- /dev/null +++ b/complete_push.py @@ -0,0 +1,61 @@ +import subprocess +import sys +import os + +os.chdir(r'C:\Users\Sahil\concore') + +print("Aborting rebase and recovering branch state...") + +# Method 1: Direct HEAD manipulation +with open('.git/HEAD', 'w') as f: + f.write('ref: refs/heads/feature/enhanced-workflow-validation\n') + +# Remove rebase state +import shutil +try: + shutil.rmtree('.git/rebase-merge') + print("✓ Removed rebase-merge directory") +except: + pass + +try: + os.remove('.git/REBASE_HEAD') + print("✓ Removed REBASE_HEAD") +except: + pass + +# Reset to original HEAD +result = subprocess.run(['git', 'reset', '--hard', 'ad0f393'], capture_output=True, text=True) +print(result.stdout) +if result.stderr: + print(result.stderr) + +# Check status +result = subprocess.run(['git', 'status', '--short'], capture_output=True, text=True) +print("\nCurrent status:") +print(result.stdout) + +# Fetch upstream +print("\nFetching upstream...") +result = subprocess.run(['git', 'fetch', 'upstream'], capture_output=True, text=True) +if result.returncode != 0: + print(result.stderr) + +# Merge upstream/dev +print("\nMerging upstream/dev...") +result = subprocess.run(['git', 'merge', 'upstream/dev', '-m', 'Merge upstream/dev'], capture_output=True, text=True) +print(result.stdout) +if result.returncode != 0: + print("Merge conflicts or error:") + print(result.stderr) + sys.exit(1) + +# Push +print("\nPushing to origin...") +result = subprocess.run(['git', 'push', 'origin', 'feature/enhanced-workflow-validation', '--force-with-lease'], capture_output=True, text=True) +print(result.stdout) +if result.returncode != 0: + print(result.stderr) + sys.exit(1) + +print("\n✓ Successfully pushed!") diff --git a/concore_cli/cli.py b/concore_cli/cli.py index f9e35f0c..e6fc061d 100644 --- a/concore_cli/cli.py +++ b/concore_cli/cli.py @@ -47,16 +47,13 @@ def run(workflow_file, source, output, type, auto_build): @cli.command() @click.argument('workflow_file', type=click.Path(exists=True)) -@click.option( - '--source', - '-s', - type=click.Path(exists=True, file_okay=False, dir_okay=True, path_type=Path), - help='Source directory to check file references', -) +@click.option('--source', '-s', default='src', help='Source directory') def validate(workflow_file, source): """Validate a workflow file""" try: - validate_workflow(workflow_file, console, source) + ok = validate_workflow(workflow_file, source, console) + if not ok: + sys.exit(1) except Exception as e: console.print(f"[red]Error:[/red] {str(e)}") sys.exit(1) diff --git a/concore_cli/commands/validate.py b/concore_cli/commands/validate.py index 18456329..7cacab99 100644 --- a/concore_cli/commands/validate.py +++ b/concore_cli/commands/validate.py @@ -5,8 +5,9 @@ import re import xml.etree.ElementTree as ET -def validate_workflow(workflow_file, console, source_dir=None): +def validate_workflow(workflow_file, source_dir, console): workflow_path = Path(workflow_file) + source_root = (workflow_path.parent / source_dir) console.print(f"[cyan]Validating:[/cyan] {workflow_path.name}") console.print() @@ -15,31 +16,35 @@ def validate_workflow(workflow_file, console, source_dir=None): warnings = [] info = [] + def finalize(): + show_results(console, errors, warnings, info) + return len(errors) == 0 + try: with open(workflow_path, 'r') as f: content = f.read() if not content.strip(): errors.append("File is empty") - return show_results(console, errors, warnings, info) + return finalize() # strict XML syntax check try: ET.fromstring(content) except ET.ParseError as e: errors.append(f"Invalid XML: {str(e)}") - return show_results(console, errors, warnings, info) + return finalize() try: soup = BeautifulSoup(content, 'xml') except Exception as e: errors.append(f"Invalid XML: {str(e)}") - return show_results(console, errors, warnings, info) + return finalize() root = soup.find('graphml') if not root: errors.append("Not a valid GraphML file - missing root element") - return show_results(console, errors, warnings, info) + return finalize() # check the graph attributes graph = soup.find('graph') @@ -65,6 +70,9 @@ def validate_workflow(workflow_file, console, source_dir=None): else: info.append(f"Found {len(edges)} edge(s)") + if not source_root.exists(): + warnings.append(f"Source directory not found: {source_root}") + node_labels = [] for node in nodes: #check the node id @@ -84,6 +92,11 @@ def validate_workflow(workflow_file, console, source_dir=None): label = label_tag.text.strip() node_labels.append(label) + # reject shell metacharacters to prevent command injection (#251) + if re.search(r'[;&|`$\'"()\\]', label): + errors.append(f"Node '{label}' contains unsafe shell characters") + continue + if ':' not in label: warnings.append(f"Node '{label}' missing format 'ID:filename'") else: @@ -96,6 +109,10 @@ def validate_workflow(workflow_file, console, source_dir=None): errors.append(f"Node '{label}' has no filename") elif not any(filename.endswith(ext) for ext in ['.py', '.cpp', '.m', '.v', '.java']): warnings.append(f"Node '{label}' has unusual file extension") + elif source_root.exists(): + file_path = source_root / filename + if not file_path.exists(): + errors.append(f"Missing source file: {filename}") else: warnings.append(f"Node {node_id} has no label") except Exception as e: @@ -138,44 +155,17 @@ def validate_workflow(workflow_file, console, source_dir=None): if file_edges > 0: info.append(f"File-based edges: {file_edges}") - if source_dir: - _check_source_files(soup, Path(source_dir), errors, warnings) - _check_cycles(soup, errors, warnings) _check_zmq_ports(soup, errors, warnings) - show_results(console, errors, warnings, info) + return finalize() except FileNotFoundError: console.print(f"[red]Error:[/red] File not found: {workflow_path}") + return False except Exception as e: console.print(f"[red]Validation failed:[/red] {str(e)}") - -def _check_source_files(soup, source_path, errors, warnings): - nodes = soup.find_all('node') - - for node in nodes: - label_tag = node.find('y:NodeLabel') or node.find('NodeLabel') - if not label_tag or not label_tag.text: - continue - - label = label_tag.text.strip() - if ':' not in label: - warnings.append(f"Skipping node with invalid label format (expected 'ID:filename')") - continue - - parts = label.split(':') - if len(parts) != 2: - warnings.append(f"Skipping node '{label}' with invalid format") - continue - - _, filename = parts - if not filename: - continue - - file_path = source_path / filename - if not file_path.exists(): - errors.append(f"Source file not found: {filename}") + return False def _check_cycles(soup, errors, warnings): nodes = soup.find_all('node') diff --git a/fix_and_push.bat b/fix_and_push.bat new file mode 100644 index 00000000..f1a74e3c --- /dev/null +++ b/fix_and_push.bat @@ -0,0 +1,8 @@ +@echo off +cd C:\Users\Sahil\concore +rmdir /s /q .git\rebase-merge 2>nul +del /f .git\REBASE_HEAD 2>nul +git reset --hard ad0f393 +git fetch upstream +git merge upstream/dev -m "Merge upstream/dev" +git push origin feature/enhanced-workflow-validation --force-with-lease diff --git a/mkconcore.py b/mkconcore.py index 0f519266..45a4cc62 100644 --- a/mkconcore.py +++ b/mkconcore.py @@ -101,6 +101,25 @@ def safe_path(value, context): raise ValueError(f"Unsafe {context}: '{value}' contains illegal characters.") return value +def safe_relpath(value, context): + """ + Allow relative subpaths while blocking traversal and absolute/drive paths. + Used for node source file paths that may contain subdirectories. + """ + if not value: + raise ValueError(f"{context} cannot be empty") + normalized = value.replace("\\", "/") + safe_path(normalized, context) + if normalized.startswith("/") or normalized.startswith("~"): + raise ValueError(f"Unsafe {context}: absolute paths are not allowed.") + if re.match(r"^[A-Za-z]:", normalized): + raise ValueError(f"Unsafe {context}: drive paths are not allowed.") + if ":" in normalized: + raise ValueError(f"Unsafe {context}: ':' is not allowed in relative paths.") + if any(part in ("", "..") for part in normalized.split("/")): + raise ValueError(f"Unsafe {context}: invalid path segment.") + return normalized + MKCONCORE_VER = "22-09-18" SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) @@ -251,7 +270,8 @@ def _resolve_concore_path(): if ':' in node_label: container_part, source_part = node_label.split(':', 1) safe_name(container_part, f"Node container name '{container_part}'") - safe_name(source_part, f"Node source file '{source_part}'") + source_part = safe_relpath(source_part, f"Node source file '{source_part}'") + node_label = f"{container_part}:{source_part}" else: safe_name(node_label, f"Node label '{node_label}'") # Explicitly reject incorrect format to prevent later crashes and ambiguity @@ -447,6 +467,9 @@ def _resolve_concore_path(): dockername, langext = sourcecode, "" script_target_path = os.path.join(outdir, "src", sourcecode) + script_target_parent = os.path.dirname(script_target_path) + if script_target_parent: + os.makedirs(script_target_parent, exist_ok=True) # If the script was specialized, it's already in outdir/src. If not, copy from sourcedir. if node_id_key not in node_edge_params: diff --git a/tests/test_graph.py b/tests/test_graph.py index 11fcbd99..6aef0d3c 100644 --- a/tests/test_graph.py +++ b/tests/test_graph.py @@ -163,7 +163,7 @@ def test_validate_missing_source_file(self): result = self.runner.invoke(cli, ['validate', filepath, '--source', str(source_dir)]) self.assertIn('Validation failed', result.output) - self.assertIn('Source file not found: missing.py', result.output) + self.assertIn('Missing source file', result.output) def test_validate_with_existing_source_file(self): content = ''' From 8cc3d50d3403d0aafe8e5fbe7035719a46ad8cdd Mon Sep 17 00:00:00 2001 From: Sahil Lenka Date: Sun, 15 Feb 2026 04:58:30 +0530 Subject: [PATCH 137/275] Remove temporary helper scripts --- abort_rebase.py | 12 ---------- complete_push.py | 61 ------------------------------------------------ fix_and_push.bat | 8 ------- 3 files changed, 81 deletions(-) delete mode 100644 abort_rebase.py delete mode 100644 complete_push.py delete mode 100644 fix_and_push.bat diff --git a/abort_rebase.py b/abort_rebase.py deleted file mode 100644 index 88dcb2fb..00000000 --- a/abort_rebase.py +++ /dev/null @@ -1,12 +0,0 @@ -import subprocess -import os - -os.chdir(r'C:\Users\Sahil\concore') - -# Abort rebase -subprocess.run(['git', 'rebase', '--abort'], capture_output=True) - -# Check status -result = subprocess.run(['git', 'status'], capture_output=True, text=True) -print(result.stdout) -print(result.stderr) diff --git a/complete_push.py b/complete_push.py deleted file mode 100644 index 16c03613..00000000 --- a/complete_push.py +++ /dev/null @@ -1,61 +0,0 @@ -import subprocess -import sys -import os - -os.chdir(r'C:\Users\Sahil\concore') - -print("Aborting rebase and recovering branch state...") - -# Method 1: Direct HEAD manipulation -with open('.git/HEAD', 'w') as f: - f.write('ref: refs/heads/feature/enhanced-workflow-validation\n') - -# Remove rebase state -import shutil -try: - shutil.rmtree('.git/rebase-merge') - print("✓ Removed rebase-merge directory") -except: - pass - -try: - os.remove('.git/REBASE_HEAD') - print("✓ Removed REBASE_HEAD") -except: - pass - -# Reset to original HEAD -result = subprocess.run(['git', 'reset', '--hard', 'ad0f393'], capture_output=True, text=True) -print(result.stdout) -if result.stderr: - print(result.stderr) - -# Check status -result = subprocess.run(['git', 'status', '--short'], capture_output=True, text=True) -print("\nCurrent status:") -print(result.stdout) - -# Fetch upstream -print("\nFetching upstream...") -result = subprocess.run(['git', 'fetch', 'upstream'], capture_output=True, text=True) -if result.returncode != 0: - print(result.stderr) - -# Merge upstream/dev -print("\nMerging upstream/dev...") -result = subprocess.run(['git', 'merge', 'upstream/dev', '-m', 'Merge upstream/dev'], capture_output=True, text=True) -print(result.stdout) -if result.returncode != 0: - print("Merge conflicts or error:") - print(result.stderr) - sys.exit(1) - -# Push -print("\nPushing to origin...") -result = subprocess.run(['git', 'push', 'origin', 'feature/enhanced-workflow-validation', '--force-with-lease'], capture_output=True, text=True) -print(result.stdout) -if result.returncode != 0: - print(result.stderr) - sys.exit(1) - -print("\n✓ Successfully pushed!") diff --git a/fix_and_push.bat b/fix_and_push.bat deleted file mode 100644 index f1a74e3c..00000000 --- a/fix_and_push.bat +++ /dev/null @@ -1,8 +0,0 @@ -@echo off -cd C:\Users\Sahil\concore -rmdir /s /q .git\rebase-merge 2>nul -del /f .git\REBASE_HEAD 2>nul -git reset --hard ad0f393 -git fetch upstream -git merge upstream/dev -m "Merge upstream/dev" -git push origin feature/enhanced-workflow-validation --force-with-lease From ab15e7abe264c79ea8a671c6c5338cd9f4037f4c Mon Sep 17 00:00:00 2001 From: Sahil Lenka Date: Sun, 15 Feb 2026 05:05:39 +0530 Subject: [PATCH 138/275] Fix syntax warning: escape backslash in docstring --- mkconcore.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mkconcore.py b/mkconcore.py index 45a4cc62..5e830f7a 100644 --- a/mkconcore.py +++ b/mkconcore.py @@ -90,7 +90,7 @@ def safe_name(value, context): def safe_path(value, context): """ Validates that a path string does not contain characters dangerous for shell command injection. - Unlike safe_name(), this allows path separators (/ and \) but still blocks dangerous shell metacharacters. + Unlike safe_name(), this allows path separators (/ and \\) but still blocks dangerous shell metacharacters. """ if not value: raise ValueError(f"{context} cannot be empty") From 5ffd847e590fb748c7b91b661491475133663adf Mon Sep 17 00:00:00 2001 From: Sahil Lenka Date: Sun, 15 Feb 2026 05:28:19 +0530 Subject: [PATCH 139/275] Properly merge upstream/dev --- concore_cli/cli.py | 22 ++- mkconcore.py | 398 +++++++++++++++++++++++++-------------------- 2 files changed, 235 insertions(+), 185 deletions(-) diff --git a/concore_cli/cli.py b/concore_cli/cli.py index e6fc061d..615cb7b9 100644 --- a/concore_cli/cli.py +++ b/concore_cli/cli.py @@ -1,19 +1,17 @@ import click from rich.console import Console -from rich.table import Table -from rich.panel import Panel -from rich import print as rprint -import sys import os -from pathlib import Path +import sys from .commands.init import init_project from .commands.run import run_workflow from .commands.validate import validate_workflow from .commands.status import show_status from .commands.stop import stop_all +from .commands.inspect import inspect_workflow console = Console() +DEFAULT_EXEC_TYPE = 'windows' if os.name == 'nt' else 'posix' @click.group() @click.version_option(version='1.0.0', prog_name='concore') @@ -35,7 +33,7 @@ def init(name, template): @click.argument('workflow_file', type=click.Path(exists=True)) @click.option('--source', '-s', default='src', help='Source directory') @click.option('--output', '-o', default='out', help='Output directory') -@click.option('--type', '-t', default='windows', type=click.Choice(['windows', 'posix', 'docker']), help='Execution type') +@click.option('--type', '-t', default=DEFAULT_EXEC_TYPE, type=click.Choice(['windows', 'posix', 'docker']), help='Execution type') @click.option('--auto-build', is_flag=True, help='Automatically run build after generation') def run(workflow_file, source, output, type, auto_build): """Run a concore workflow""" @@ -58,6 +56,18 @@ def validate(workflow_file, source): console.print(f"[red]Error:[/red] {str(e)}") sys.exit(1) +@cli.command() +@click.argument('workflow_file', type=click.Path(exists=True)) +@click.option('--source', '-s', default='src', help='Source directory') +@click.option('--json', 'output_json', is_flag=True, help='Output in JSON format') +def inspect(workflow_file, source, output_json): + """Inspect a workflow file and show its structure""" + try: + inspect_workflow(workflow_file, source, output_json, console) + except Exception as e: + console.print(f"[red]Error:[/red] {str(e)}") + sys.exit(1) + @cli.command() def status(): """Show running concore processes""" diff --git a/mkconcore.py b/mkconcore.py index 5e830f7a..9457d74e 100644 --- a/mkconcore.py +++ b/mkconcore.py @@ -63,6 +63,7 @@ # - Sets the executable permission (`stat.S_IRWXU`) for the generated scripts on POSIX systems. from bs4 import BeautifulSoup +import atexit import logging import re import sys @@ -74,78 +75,81 @@ import shlex # Added for POSIX shell escaping # input validation helper -def safe_name(value, context): - """ - Validates that the input string does not contain characters dangerous - for simple names (labels, filenames without paths). - Use safe_path() for validating directory/file paths. - """ - if not value: - raise ValueError(f"{context} cannot be empty") - # blocks path traversal (/, \), control characters, and shell metacharacters (*, ?, <, >, |, ;, &, $, `, ', ", (, )) - if re.search(r'[\x00-\x1F\x7F\\/:*?"<>|;&`$\'()]', value): - raise ValueError(f"Unsafe {context}: '{value}' contains illegal characters.") - return value - -def safe_path(value, context): - """ - Validates that a path string does not contain characters dangerous for shell command injection. - Unlike safe_name(), this allows path separators (/ and \\) but still blocks dangerous shell metacharacters. - """ - if not value: - raise ValueError(f"{context} cannot be empty") - # Allow path separators but block control characters and shell metacharacters - # Blocks: control chars, *, ?, <, >, |, ;, &, $, `, ', ", (, ) - # Allows: /, \, -, _, ., alphanumeric, spaces, : - if re.search(r'[\x00-\x1F\x7F*?"<>|;&`$\'()]', value): - raise ValueError(f"Unsafe {context}: '{value}' contains illegal characters.") - return value - -def safe_relpath(value, context): - """ - Allow relative subpaths while blocking traversal and absolute/drive paths. - Used for node source file paths that may contain subdirectories. - """ - if not value: - raise ValueError(f"{context} cannot be empty") - normalized = value.replace("\\", "/") - safe_path(normalized, context) - if normalized.startswith("/") or normalized.startswith("~"): - raise ValueError(f"Unsafe {context}: absolute paths are not allowed.") - if re.match(r"^[A-Za-z]:", normalized): - raise ValueError(f"Unsafe {context}: drive paths are not allowed.") - if ":" in normalized: - raise ValueError(f"Unsafe {context}: ':' is not allowed in relative paths.") - if any(part in ("", "..") for part in normalized.split("/")): - raise ValueError(f"Unsafe {context}: invalid path segment.") - return normalized - -MKCONCORE_VER = "22-09-18" - -SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) - -def _resolve_concore_path(): - script_concore = os.path.join(SCRIPT_DIR, "concore.py") - if os.path.exists(script_concore): - return SCRIPT_DIR - cwd_concore = os.path.join(os.getcwd(), "concore.py") - if os.path.exists(cwd_concore): - return os.getcwd() - return SCRIPT_DIR - -GRAPHML_FILE = sys.argv[1] -TRIMMED_LOGS = True -CONCOREPATH = _resolve_concore_path() -CPPWIN = "g++" #Windows C++ 6/22/21 -CPPEXE = "g++" #Ubuntu/macOS C++ 6/22/21 -VWIN = "iverilog" #Windows verilog 6/25/21 -VEXE = "iverilog" #Ubuntu/macOS verilog 6/25/21 -PYTHONEXE = "python3" #Ubuntu/macOS python3 -PYTHONWIN = "python" #Windows python3 -MATLABEXE = "matlab" #Ubuntu/macOS matlab -MATLABWIN = "matlab" #Windows matlab -OCTAVEEXE = "octave" #Ubuntu/macOS octave -OCTAVEWIN = "octave" #Windows octave +def safe_name(value, context, allow_path=False): + """ + Validates that the input string does not contain characters dangerous + for filesystem paths or shell command injection. + """ + if not value: + raise ValueError(f"{context} cannot be empty") + # blocks control characters and shell metacharacters + # allow path separators and drive colons for full paths when needed + if allow_path: + pattern = r'[\x00-\x1F\x7F*?"<>|;&`$\'()]' + else: + # blocks path traversal (/, \, :) in addition to shell metacharacters + pattern = r'[\x00-\x1F\x7F\\/:*?"<>|;&`$\'()]' + if re.search(pattern, value): + raise ValueError(f"Unsafe {context}: '{value}' contains illegal characters.") + return value + +def safe_relpath(value, context): + """ + Allow relative subpaths while blocking traversal and absolute/drive paths. + """ + if not value: + raise ValueError(f"{context} cannot be empty") + normalized = value.replace("\\", "/") + safe_name(normalized, context, allow_path=True) + if normalized.startswith("/") or normalized.startswith("~"): + raise ValueError(f"Unsafe {context}: absolute paths are not allowed.") + if re.match(r"^[A-Za-z]:", normalized): + raise ValueError(f"Unsafe {context}: drive paths are not allowed.") + if ":" in normalized: + raise ValueError(f"Unsafe {context}: ':' is not allowed in relative paths.") + if any(part in ("", "..") for part in normalized.split("/")): + raise ValueError(f"Unsafe {context}: invalid path segment.") + return normalized + +MKCONCORE_VER = "22-09-18" + +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) + +def _load_tool_config(filepath): + tools = {} + with open(filepath, "r") as f: + for line in f: + line = line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + k, v = line.split("=", 1) + k, v = k.strip(), v.strip() + if v: + tools[k] = v + return tools + +def _resolve_concore_path(): + script_concore = os.path.join(SCRIPT_DIR, "concore.py") + if os.path.exists(script_concore): + return SCRIPT_DIR + cwd_concore = os.path.join(os.getcwd(), "concore.py") + if os.path.exists(cwd_concore): + return os.getcwd() + return SCRIPT_DIR + +GRAPHML_FILE = sys.argv[1] +TRIMMED_LOGS = True +CONCOREPATH = _resolve_concore_path() +CPPWIN = os.environ.get("CONCORE_CPPWIN", "g++") #Windows C++ 6/22/21 +CPPEXE = os.environ.get("CONCORE_CPPEXE", "g++") #Ubuntu/macOS C++ 6/22/21 +VWIN = os.environ.get("CONCORE_VWIN", "iverilog") #Windows verilog 6/25/21 +VEXE = os.environ.get("CONCORE_VEXE", "iverilog") #Ubuntu/macOS verilog 6/25/21 +PYTHONEXE = os.environ.get("CONCORE_PYTHONEXE", "python3") #Ubuntu/macOS python3 +PYTHONWIN = os.environ.get("CONCORE_PYTHONWIN", "python") #Windows python3 +MATLABEXE = os.environ.get("CONCORE_MATLABEXE", "matlab") #Ubuntu/macOS matlab +MATLABWIN = os.environ.get("CONCORE_MATLABWIN", "matlab") #Windows matlab +OCTAVEEXE = os.environ.get("CONCORE_OCTAVEEXE", "octave") #Ubuntu/macOS octave +OCTAVEWIN = os.environ.get("CONCORE_OCTAVEWIN", "octave") #Windows octave M_IS_OCTAVE = False #treat .m as octave MCRPATH = "~/MATLAB/R2021a" #path to local Ubunta Matlab Compiler Runtime DOCKEREXE = "sudo docker"#assume simple docker install @@ -163,22 +167,36 @@ def _resolve_concore_path(): M_IS_OCTAVE = True #treat .m as octave 9/27/21 if os.path.exists(CONCOREPATH+"/concore.mcr"): # 11/12/21 - MCRPATH = open(CONCOREPATH+"/concore.mcr", "r").readline().strip() #path to local Ubunta Matlab Compiler Runtime + with open(CONCOREPATH+"/concore.mcr", "r") as f: + MCRPATH = f.readline().strip() #path to local Ubunta Matlab Compiler Runtime if os.path.exists(CONCOREPATH+"/concore.sudo"): # 12/04/21 - DOCKEREXE = open(CONCOREPATH+"/concore.sudo", "r").readline().strip() #to omit sudo in docker + with open(CONCOREPATH+"/concore.sudo", "r") as f: + DOCKEREXE = f.readline().strip() #to omit sudo in docker if os.path.exists(CONCOREPATH+"/concore.repo"): # 12/04/21 - DOCKEREPO = open(CONCOREPATH+"/concore.repo", "r").readline().strip() #docker id for repo - + with open(CONCOREPATH+"/concore.repo", "r") as f: + DOCKEREPO = f.readline().strip() #docker id for repo + +if os.path.exists(CONCOREPATH+"/concore.tools"): + _tools = _load_tool_config(CONCOREPATH+"/concore.tools") + CPPWIN = _tools.get("CPPWIN", CPPWIN) + CPPEXE = _tools.get("CPPEXE", CPPEXE) + VWIN = _tools.get("VWIN", VWIN) + VEXE = _tools.get("VEXE", VEXE) + PYTHONEXE = _tools.get("PYTHONEXE", PYTHONEXE) + PYTHONWIN = _tools.get("PYTHONWIN", PYTHONWIN) + MATLABEXE = _tools.get("MATLABEXE", MATLABEXE) + MATLABWIN = _tools.get("MATLABWIN", MATLABWIN) + OCTAVEEXE = _tools.get("OCTAVEEXE", OCTAVEEXE) + OCTAVEWIN = _tools.get("OCTAVEWIN", OCTAVEWIN) prefixedgenode = "" sourcedir = sys.argv[2] outdir = sys.argv[3] -# Validate path arguments -safe_path(outdir, "Output directory argument") -safe_path(sourcedir, "Source directory argument") +# Validate outdir argument (allow full paths) +safe_name(outdir, "Output directory argument", allow_path=True) if not os.path.isdir(sourcedir): logging.error(f"{sourcedir} does not exist") @@ -230,6 +248,12 @@ def _resolve_concore_path(): funlock = open("unlock", "w") # 12/4/21 fparams = open("params", "w") # 9/18/22 +def cleanup_script_files(): + for fh in [fbuild, frun, fdebug, fstop, fclear, fmaxtime, funlock, fparams]: + if not fh.closed: + fh.close() +atexit.register(cleanup_script_files) + os.mkdir("src") os.chdir("..") @@ -243,8 +267,8 @@ def _resolve_concore_path(): logging.info(f"MCR path: {MCRPATH}") logging.info(f"Docker repository: {DOCKEREPO}") -f = open(GRAPHML_FILE, "r") -text_str = f.read() +with open(GRAPHML_FILE, "r") as f: + text_str = f.read() soup = BeautifulSoup(text_str, 'xml') @@ -267,15 +291,15 @@ def _resolve_concore_path(): node_label = re.sub(r'(\s+|\n)', ' ', node_label) #Validate node labels - if ':' in node_label: - container_part, source_part = node_label.split(':', 1) - safe_name(container_part, f"Node container name '{container_part}'") - source_part = safe_relpath(source_part, f"Node source file '{source_part}'") - node_label = f"{container_part}:{source_part}" - else: - safe_name(node_label, f"Node label '{node_label}'") - # Explicitly reject incorrect format to prevent later crashes and ambiguity - raise ValueError(f"Invalid node label '{node_label}': expected format 'container:source' with a ':' separator.") + if ':' in node_label: + container_part, source_part = node_label.split(':', 1) + safe_name(container_part, f"Node container name '{container_part}'") + source_part = safe_relpath(source_part, f"Node source file '{source_part}'") + node_label = f"{container_part}:{source_part}" + else: + safe_name(node_label, f"Node label '{node_label}'") + # Explicitly reject incorrect format to prevent later crashes and ambiguity + raise ValueError(f"Invalid node label '{node_label}': expected format 'container:source' with a ':' separator.") nodes_dict[node['id']] = node_label node_id_to_label_map[node['id']] = node_label.split(':')[0] @@ -461,15 +485,15 @@ def _resolve_concore_path(): if not sourcecode: continue - if "." in sourcecode: - dockername, langext = os.path.splitext(sourcecode) - else: - dockername, langext = sourcecode, "" - - script_target_path = os.path.join(outdir, "src", sourcecode) - script_target_parent = os.path.dirname(script_target_path) - if script_target_parent: - os.makedirs(script_target_parent, exist_ok=True) + if "." in sourcecode: + dockername, langext = os.path.splitext(sourcecode) + else: + dockername, langext = sourcecode, "" + + script_target_path = os.path.join(outdir, "src", sourcecode) + script_target_parent = os.path.dirname(script_target_path) + if script_target_parent: + os.makedirs(script_target_parent, exist_ok=True) # If the script was specialized, it's already in outdir/src. If not, copy from sourcedir. if node_id_key not in node_edge_params: @@ -494,121 +518,125 @@ def _resolve_concore_path(): #copy proper concore.py into /src try: if concoretype=="docker": - fsource = open(CONCOREPATH+"/concoredocker.py") + with open(CONCOREPATH+"/concoredocker.py") as fsource: + source_content = fsource.read() else: - fsource = open(CONCOREPATH+"/concore.py") + with open(CONCOREPATH+"/concore.py") as fsource: + source_content = fsource.read() except (FileNotFoundError, IOError) as e: logging.error(f"{CONCOREPATH} is not correct path to concore: {e}") quit() with open(outdir+"/src/concore.py","w") as fcopy: - fcopy.write(fsource.read()) -fsource.close() + fcopy.write(source_content) #copy proper concore.hpp into /src 6/22/21 try: if concoretype=="docker": - fsource = open(CONCOREPATH+"/concoredocker.hpp") + with open(CONCOREPATH+"/concoredocker.hpp") as fsource: + source_content = fsource.read() else: - fsource = open(CONCOREPATH+"/concore.hpp") + with open(CONCOREPATH+"/concore.hpp") as fsource: + source_content = fsource.read() except (FileNotFoundError, IOError) as e: logging.error(f"{CONCOREPATH} is not correct path to concore: {e}") quit() with open(outdir+"/src/concore.hpp","w") as fcopy: - fcopy.write(fsource.read()) -fsource.close() + fcopy.write(source_content) #copy proper concore.v into /src 6/25/21 try: if concoretype=="docker": - fsource = open(CONCOREPATH+"/concoredocker.v") + with open(CONCOREPATH+"/concoredocker.v") as fsource: + source_content = fsource.read() else: - fsource = open(CONCOREPATH+"/concore.v") + with open(CONCOREPATH+"/concore.v") as fsource: + source_content = fsource.read() except (FileNotFoundError, IOError) as e: logging.error(f"{CONCOREPATH} is not correct path to concore: {e}") quit() with open(outdir+"/src/concore.v","w") as fcopy: - fcopy.write(fsource.read()) -fsource.close() + fcopy.write(source_content) #copy mkcompile into /src 5/27/21 try: - fsource = open(CONCOREPATH+"/mkcompile") + with open(CONCOREPATH+"/mkcompile") as fsource: + source_content = fsource.read() except (FileNotFoundError, IOError) as e: logging.error(f"{CONCOREPATH} is not correct path to concore: {e}") quit() with open(outdir+"/src/mkcompile","w") as fcopy: - fcopy.write(fsource.read()) -fsource.close() + fcopy.write(source_content) os.chmod(outdir+"/src/mkcompile",stat.S_IRWXU) #copy concore*.m into /src 4/2/21 try: #maxtime in matlab 11/22/21 - fsource = open(CONCOREPATH+"/concore_default_maxtime.m") + with open(CONCOREPATH+"/concore_default_maxtime.m") as fsource: + source_content = fsource.read() except (FileNotFoundError, IOError) as e: logging.error(f"{CONCOREPATH} is not correct path to concore: {e}") quit() with open(outdir+"/src/concore_default_maxtime.m","w") as fcopy: - fcopy.write(fsource.read()) -fsource.close() + fcopy.write(source_content) try: - fsource = open(CONCOREPATH+"/concore_unchanged.m") + with open(CONCOREPATH+"/concore_unchanged.m") as fsource: + source_content = fsource.read() except (FileNotFoundError, IOError) as e: logging.error(f"{CONCOREPATH} is not correct path to concore: {e}") quit() with open(outdir+"/src/concore_unchanged.m","w") as fcopy: - fcopy.write(fsource.read()) -fsource.close() + fcopy.write(source_content) try: - fsource = open(CONCOREPATH+"/concore_read.m") + with open(CONCOREPATH+"/concore_read.m") as fsource: + source_content = fsource.read() except (FileNotFoundError, IOError) as e: logging.error(f"{CONCOREPATH} is not correct path to concore: {e}") quit() with open(outdir+"/src/concore_read.m","w") as fcopy: - fcopy.write(fsource.read()) -fsource.close() + fcopy.write(source_content) try: - fsource = open(CONCOREPATH+"/concore_write.m") + with open(CONCOREPATH+"/concore_write.m") as fsource: + source_content = fsource.read() except (FileNotFoundError, IOError) as e: logging.error(f"{CONCOREPATH} is not correct path to concore: {e}") quit() with open(outdir+"/src/concore_write.m","w") as fcopy: - fcopy.write(fsource.read()) -fsource.close() + fcopy.write(source_content) try: #4/9/21 - fsource = open(CONCOREPATH+"/concore_initval.m") + with open(CONCOREPATH+"/concore_initval.m") as fsource: + source_content = fsource.read() except (FileNotFoundError, IOError) as e: logging.error(f"{CONCOREPATH} is not correct path to concore: {e}") quit() with open(outdir+"/src/concore_initval.m","w") as fcopy: - fcopy.write(fsource.read()) -fsource.close() + fcopy.write(source_content) try: #11/19/21 - fsource = open(CONCOREPATH+"/concore_iport.m") + with open(CONCOREPATH+"/concore_iport.m") as fsource: + source_content = fsource.read() except (FileNotFoundError, IOError) as e: logging.error(f"{CONCOREPATH} is not correct path to concore: {e}") quit() with open(outdir+"/src/concore_iport.m","w") as fcopy: - fcopy.write(fsource.read()) -fsource.close() + fcopy.write(source_content) try: #11/19/21 - fsource = open(CONCOREPATH+"/concore_oport.m") + with open(CONCOREPATH+"/concore_oport.m") as fsource: + source_content = fsource.read() except (FileNotFoundError, IOError) as e: logging.error(f"{CONCOREPATH} is not correct path to concore: {e}") quit() with open(outdir+"/src/concore_oport.m","w") as fcopy: - fcopy.write(fsource.read()) -fsource.close() + fcopy.write(source_content) try: # 4/4/21 if concoretype=="docker": - fsource = open(CONCOREPATH+"/import_concoredocker.m") + with open(CONCOREPATH+"/import_concoredocker.m") as fsource: + source_content = fsource.read() else: - fsource = open(CONCOREPATH+"/import_concore.m") + with open(CONCOREPATH+"/import_concore.m") as fsource: + source_content = fsource.read() except (FileNotFoundError, IOError) as e: logging.error(f"{CONCOREPATH} is not correct path to concore: {e}") quit() with open(outdir+"/src/import_concore.m","w") as fcopy: - fcopy.write(fsource.read()) -fsource.close() + fcopy.write(source_content) # --- Generate iport and oport mappings --- logging.info("Generating iport/oport mappings...") @@ -655,47 +683,58 @@ def _resolve_concore_path(): # 4. Write final iport/oport files logging.info("Writing .iport and .oport files...") -for node_label, ports in node_port_mappings.items(): +for node_label, ports in node_port_mappings.items(): try: containername, sourcecode = node_label.split(':', 1) - if not sourcecode or "." not in sourcecode: continue - dockername = os.path.splitext(sourcecode)[0] - with open(os.path.join(outdir, "src", f"{dockername}.iport"), "w") as fport: - fport.write(str(ports['iport']).replace("'" + prefixedgenode, "'")) - with open(os.path.join(outdir, "src", f"{dockername}.oport"), "w") as fport: - fport.write(str(ports['oport']).replace("'" + prefixedgenode, "'")) + if not sourcecode or "." not in sourcecode: continue + dockername = os.path.splitext(sourcecode)[0] + iport_path = os.path.join(outdir, "src", f"{dockername}.iport") + oport_path = os.path.join(outdir, "src", f"{dockername}.oport") + iport_parent = os.path.dirname(iport_path) + if iport_parent: + os.makedirs(iport_parent, exist_ok=True) + with open(iport_path, "w") as fport: + fport.write(str(ports['iport']).replace("'" + prefixedgenode, "'")) + with open(oport_path, "w") as fport: + fport.write(str(ports['oport']).replace("'" + prefixedgenode, "'")) except ValueError: continue #if docker, make docker-dirs, generate build, run, stop, clear scripts and quit -if (concoretype=="docker"): - for node in nodes_dict: - containername,sourcecode = nodes_dict[node].split(':') - if len(sourcecode)!=0 and sourcecode.find(".")!=-1: #3/28/21 - dockername,langext = sourcecode.split(".") - if not os.path.exists(outdir+"/src/Dockerfile."+dockername): # 3/30/21 - try: +if (concoretype=="docker"): + for node in nodes_dict: + containername,sourcecode = nodes_dict[node].split(':') + if len(sourcecode)!=0 and sourcecode.find(".")!=-1: #3/28/21 + dockername,langext = sourcecode.split(".") + dockerfile_path = os.path.join(outdir, "src", f"Dockerfile.{dockername}") + if not os.path.exists(dockerfile_path): # 3/30/21 + try: if langext=="py": - fsource = open(CONCOREPATH+"/Dockerfile.py") + src_path = CONCOREPATH+"/Dockerfile.py" logging.info("assuming .py extension for Dockerfile") elif langext == "cpp": # 6/22/21 - fsource = open(CONCOREPATH+"/Dockerfile.cpp") + src_path = CONCOREPATH+"/Dockerfile.cpp" logging.info("assuming .cpp extension for Dockerfile") elif langext == "v": # 6/26/21 - fsource = open(CONCOREPATH+"/Dockerfile.v") + src_path = CONCOREPATH+"/Dockerfile.v" logging.info("assuming .v extension for Dockerfile") elif langext == "sh": # 5/19/21 - fsource = open(CONCOREPATH+"/Dockerfile.sh") + src_path = CONCOREPATH+"/Dockerfile.sh" logging.info("assuming .sh extension for Dockerfile") else: - fsource = open(CONCOREPATH+"/Dockerfile.m") + src_path = CONCOREPATH+"/Dockerfile.m" logging.info("assuming .m extension for Dockerfile") - except: - logging.error(f"{CONCOREPATH} is not correct path to concore") - quit() - with open(outdir+"/src/Dockerfile."+dockername,"w") as fcopy: - fcopy.write(fsource.read()) + with open(src_path) as fsource: + source_content = fsource.read() + except: + logging.error(f"{CONCOREPATH} is not correct path to concore") + quit() + dockerfile_parent = os.path.dirname(dockerfile_path) + if dockerfile_parent: + os.makedirs(dockerfile_parent, exist_ok=True) + with open(dockerfile_path,"w") as fcopy: + fcopy.write(source_content) if langext=="py": fcopy.write('CMD ["python", "-i", "'+sourcecode+'"]\n') if langext=="m": @@ -706,7 +745,6 @@ def _resolve_concore_path(): if langext=="v": fcopy.write('RUN iverilog ./'+sourcecode+'\n') # 7/02/21 fcopy.write('CMD ["./a.out"]\n') # 7/02/21 - fsource.close() fbuild.write('#!/bin/bash' + "\n") for node in nodes_dict: @@ -940,16 +978,22 @@ def _resolve_concore_path(): if concoretype=="posix": fbuild.write('#!/bin/bash' + "\n") -for node in nodes_dict: - containername,sourcecode = nodes_dict[node].split(':') - if len(sourcecode)!=0: - if sourcecode.find(".")==-1: - logging.error("cannot pull container "+sourcecode+" with control core type "+concoretype) #3/28/21 - quit() - dockername,langext = sourcecode.split(".") - fbuild.write('mkdir '+containername+"\n") - if concoretype == "windows": - fbuild.write("copy .\\src\\"+sourcecode+" .\\"+containername+"\\"+sourcecode+"\n") +for node in nodes_dict: + containername,sourcecode = nodes_dict[node].split(':') + if len(sourcecode)!=0: + if sourcecode.find(".")==-1: + logging.error("cannot pull container "+sourcecode+" with control core type "+concoretype) #3/28/21 + quit() + dockername,langext = sourcecode.split(".") + fbuild.write('mkdir '+containername+"\n") + source_subdir = os.path.dirname(sourcecode).replace("\\", "/") + if source_subdir: + if concoretype == "windows": + fbuild.write("mkdir .\\"+containername+"\\"+source_subdir.replace("/", "\\")+"\n") + else: + fbuild.write("mkdir -p ./"+containername+"/"+source_subdir+"\n") + if concoretype == "windows": + fbuild.write("copy .\\src\\"+sourcecode+" .\\"+containername+"\\"+sourcecode+"\n") if langext == "py": fbuild.write("copy .\\src\\concore.py .\\" + containername + "\\concore.py\n") elif langext == "cpp": @@ -1237,10 +1281,6 @@ def _resolve_concore_path(): frun.close() fbuild.close() fdebug.close() -fstop.close() -fclear.close() -fmaxtime.close() -fparams.close() if concoretype != "windows": os.chmod(outdir+"/build",stat.S_IRWXU) os.chmod(outdir+"/run",stat.S_IRWXU) @@ -1249,4 +1289,4 @@ def _resolve_concore_path(): os.chmod(outdir+"/clear",stat.S_IRWXU) os.chmod(outdir+"/maxtime",stat.S_IRWXU) os.chmod(outdir+"/params",stat.S_IRWXU) - os.chmod(outdir+"/unlock",stat.S_IRWXU) \ No newline at end of file + os.chmod(outdir+"/unlock",stat.S_IRWXU) From 1b737522b4891d778edd1b0d322b5daa73090c2e Mon Sep 17 00:00:00 2001 From: Sahil Lenka Date: Sun, 15 Feb 2026 05:39:00 +0530 Subject: [PATCH 140/275] Add test coverage for inspect command The inspect command had no tests, which left a gap in our test coverage. Added 4 new tests covering: - Basic workflow inspection - JSON output format - Missing file error handling - Detection of missing source files All tests pass. --- tests/test_cli.py | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/tests/test_cli.py b/tests/test_cli.py index b1853b49..f852d5c1 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -150,6 +150,47 @@ def test_run_command_existing_output(self): '--output', 'output' ]) self.assertIn('already exists', result.output.lower()) + + def test_inspect_command_basic(self): + with self.runner.isolated_filesystem(temp_dir=self.temp_dir): + result = self.runner.invoke(cli, ['init', 'test-project']) + self.assertEqual(result.exit_code, 0) + + result = self.runner.invoke(cli, ['inspect', 'test-project/workflow.graphml']) + self.assertEqual(result.exit_code, 0) + self.assertIn('Workflow Overview', result.output) + self.assertIn('Nodes:', result.output) + self.assertIn('Edges:', result.output) + + def test_inspect_missing_file(self): + result = self.runner.invoke(cli, ['inspect', 'nonexistent.graphml']) + self.assertNotEqual(result.exit_code, 0) + + def test_inspect_json_output(self): + with self.runner.isolated_filesystem(temp_dir=self.temp_dir): + result = self.runner.invoke(cli, ['init', 'test-project']) + self.assertEqual(result.exit_code, 0) + + result = self.runner.invoke(cli, ['inspect', 'test-project/workflow.graphml', '--json']) + self.assertEqual(result.exit_code, 0) + + import json + output_data = json.loads(result.output) + self.assertIn('workflow', output_data) + self.assertIn('nodes', output_data) + self.assertIn('edges', output_data) + self.assertEqual(output_data['workflow'], 'workflow.graphml') + + def test_inspect_missing_source_file(self): + with self.runner.isolated_filesystem(temp_dir=self.temp_dir): + result = self.runner.invoke(cli, ['init', 'test-project']) + self.assertEqual(result.exit_code, 0) + + Path('test-project/src/script.py').unlink() + + result = self.runner.invoke(cli, ['inspect', 'test-project/workflow.graphml', '--source', 'src']) + self.assertEqual(result.exit_code, 0) + self.assertIn('Missing files', result.output) if __name__ == '__main__': unittest.main() From 8690784ef6764244409e142131254b0716204362 Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Sun, 15 Feb 2026 10:33:26 +0530 Subject: [PATCH 141/275] fix: validate arguments before usage in mkconcore.py (Issue #267) --- mkconcore.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/mkconcore.py b/mkconcore.py index 9457d74e..18a1ecb3 100644 --- a/mkconcore.py +++ b/mkconcore.py @@ -137,6 +137,11 @@ def _resolve_concore_path(): return os.getcwd() return SCRIPT_DIR +if len(sys.argv) < 4: + logging.error("usage: py mkconcore.py file.graphml sourcedir outdir [type]") + logging.error(" type must be posix (macos or ubuntu), windows, or docker") + quit() + GRAPHML_FILE = sys.argv[1] TRIMMED_LOGS = True CONCOREPATH = _resolve_concore_path() @@ -202,11 +207,7 @@ def _resolve_concore_path(): logging.error(f"{sourcedir} does not exist") quit() -if len(sys.argv) < 4: - logging.error("usage: py mkconcore.py file.graphml sourcedir outdir [type]") - logging.error(" type must be posix (macos or ubuntu), windows, or docker") - quit() -elif len(sys.argv) == 4: +if len(sys.argv) == 4: prefixedgenode = outdir+"_" #nodes and edges prefixed with outdir_ only in case no type specified 3/24/21 concoretype = "docker" else: From 9e8e756b7360347d244716312e497e241f2dc5df Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Sun, 15 Feb 2026 10:46:21 +0530 Subject: [PATCH 142/275] fix: use print+sys.exit for early arg check before logging is configured --- mkconcore.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mkconcore.py b/mkconcore.py index 18a1ecb3..4e01dde4 100644 --- a/mkconcore.py +++ b/mkconcore.py @@ -138,9 +138,9 @@ def _resolve_concore_path(): return SCRIPT_DIR if len(sys.argv) < 4: - logging.error("usage: py mkconcore.py file.graphml sourcedir outdir [type]") - logging.error(" type must be posix (macos or ubuntu), windows, or docker") - quit() + print("usage: py mkconcore.py file.graphml sourcedir outdir [type]") + print(" type must be posix (macos or ubuntu), windows, or docker") + sys.exit(1) GRAPHML_FILE = sys.argv[1] TRIMMED_LOGS = True From 7803521fce2b1199ab358500de00e0ca6163a1ef Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Sat, 14 Feb 2026 13:21:33 +0530 Subject: [PATCH 143/275] security: remove API key logging from wrapper scripts (fixes #263) --- demo/cwrap.py | 3 +-- demo/pwrap.py | 1 - ratc/cwrap.py | 3 +-- ratc/pwrap.py | 1 - 4 files changed, 2 insertions(+), 6 deletions(-) diff --git a/demo/cwrap.py b/demo/cwrap.py index b8cfb4b6..f9f33e59 100644 --- a/demo/cwrap.py +++ b/demo/cwrap.py @@ -58,7 +58,6 @@ except: init_simtime_ym = "[0.0, 0.0, 0.0]" -print(apikey) print(yuyu) print(name1+'='+init_simtime_u) print(name2+'='+init_simtime_ym) @@ -83,7 +82,7 @@ u = concore.read(1,name1,init_simtime_u) f = {'file1': open(concore.inpath+'1/'+name1, 'rb')} print("CW: before post u="+str(u)) - print('http://www.controlcore.org/pm/'+yuyu+apikey+'&fetch='+name2) + print('http://www.controlcore.org/pm/'+yuyu+''+'&fetch='+name2) r = requests.post('http://www.controlcore.org/pm/'+yuyu+apikey+'&fetch='+name2, files=f,timeout=timeout_max) if r.status_code!=200: print("bad POST request "+str(r.status_code)) diff --git a/demo/pwrap.py b/demo/pwrap.py index beb90116..283ca0ce 100644 --- a/demo/pwrap.py +++ b/demo/pwrap.py @@ -61,7 +61,6 @@ except: init_simtime_ym = "[0.0, 0.0, 0.0]" -print(apikey) print(yuyu) print(name1+'='+init_simtime_u) print(name2+'='+init_simtime_ym) diff --git a/ratc/cwrap.py b/ratc/cwrap.py index b8cfb4b6..f9f33e59 100644 --- a/ratc/cwrap.py +++ b/ratc/cwrap.py @@ -58,7 +58,6 @@ except: init_simtime_ym = "[0.0, 0.0, 0.0]" -print(apikey) print(yuyu) print(name1+'='+init_simtime_u) print(name2+'='+init_simtime_ym) @@ -83,7 +82,7 @@ u = concore.read(1,name1,init_simtime_u) f = {'file1': open(concore.inpath+'1/'+name1, 'rb')} print("CW: before post u="+str(u)) - print('http://www.controlcore.org/pm/'+yuyu+apikey+'&fetch='+name2) + print('http://www.controlcore.org/pm/'+yuyu+''+'&fetch='+name2) r = requests.post('http://www.controlcore.org/pm/'+yuyu+apikey+'&fetch='+name2, files=f,timeout=timeout_max) if r.status_code!=200: print("bad POST request "+str(r.status_code)) diff --git a/ratc/pwrap.py b/ratc/pwrap.py index beb90116..283ca0ce 100644 --- a/ratc/pwrap.py +++ b/ratc/pwrap.py @@ -61,7 +61,6 @@ except: init_simtime_ym = "[0.0, 0.0, 0.0]" -print(apikey) print(yuyu) print(name1+'='+init_simtime_u) print(name2+'='+init_simtime_ym) From df5bbee3ef73b519c6a3e764526c93806b01cacf Mon Sep 17 00:00:00 2001 From: Avinash Kumar Deepak Date: Sun, 15 Feb 2026 16:18:01 +0530 Subject: [PATCH 144/275] Fix filename parsing for files with multiple dots --- mkconcore.py | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/mkconcore.py b/mkconcore.py index 9457d74e..03728d49 100644 --- a/mkconcore.py +++ b/mkconcore.py @@ -706,7 +706,7 @@ def cleanup_script_files(): for node in nodes_dict: containername,sourcecode = nodes_dict[node].split(':') if len(sourcecode)!=0 and sourcecode.find(".")!=-1: #3/28/21 - dockername,langext = sourcecode.split(".") + dockername,langext = sourcecode.rsplit(".", 1) dockerfile_path = os.path.join(outdir, "src", f"Dockerfile.{dockername}") if not os.path.exists(dockerfile_path): # 3/30/21 try: @@ -750,7 +750,7 @@ def cleanup_script_files(): for node in nodes_dict: containername,sourcecode = nodes_dict[node].split(':') if len(sourcecode)!=0 and sourcecode.find(".")!=-1: #3/28/21 - dockername,langext = sourcecode.split(".") + dockername,langext = sourcecode.rsplit(".", 1) fbuild.write("mkdir docker-"+dockername+"\n") fbuild.write("cd docker-"+dockername+"\n") fbuild.write("cp ../src/Dockerfile."+dockername+" Dockerfile\n") @@ -788,7 +788,7 @@ def cleanup_script_files(): # Use safe_container frun.write(DOCKEREXE+' run --name='+safe_container+volswr[i]+volsro[i]+" "+DOCKEREPO+"/docker-"+shlex.quote(sourcecode)+"&\n") else: - dockername,langext = sourcecode.split(".") + dockername,langext = sourcecode.rsplit(".", 1) logging.debug(f"Generating Docker run command for {dockername}: {DOCKEREXE} run --name={containername+volswr[i]+volsro[i]} docker-{dockername}") # Use safe_container frun.write(DOCKEREXE+' run --name='+safe_container+volswr[i]+volsro[i]+" docker-"+shlex.quote(dockername)+"&\n") @@ -800,8 +800,8 @@ def cleanup_script_files(): for node in nodes_dict: containername,sourcecode = nodes_dict[node].split(':') if len(sourcecode)!=0: - #dockername,langext = sourcecode.split(".") - dockername = sourcecode.split(".")[0] # 3/28/21 + #dockername,langext = sourcecode.rsplit(".", 1) + dockername = sourcecode.rsplit(".", 1)[0] # 3/28/21 safe_container = shlex.quote(containername) fstop.write(DOCKEREXE+' stop '+safe_container+"\n") fstop.write(DOCKEREXE+' rm '+safe_container+"\n") @@ -813,7 +813,7 @@ def cleanup_script_files(): for node in nodes_dict: containername,sourcecode = nodes_dict[node].split(':') if len(sourcecode)!=0: - dockername = sourcecode.split(".")[0] #3/28/21 + dockername = sourcecode.rsplit(".", 1)[0] #3/28/21 writeedges = volswr[i] while writeedges.find(":") != -1: #scape volume path using shlex.quote for Docker commands (defense-in-depth) @@ -833,7 +833,7 @@ def cleanup_script_files(): for node in nodes_dict: containername,sourcecode = nodes_dict[node].split(':') if len(sourcecode)!=0: - dockername = sourcecode.split(".")[0] #3/28/21 + dockername = sourcecode.rsplit(".", 1)[0] #3/28/21 writeedges = volswr[i] while writeedges.find(":") != -1: fmaxtime.write(' -v ') @@ -850,7 +850,7 @@ def cleanup_script_files(): for node in nodes_dict: containername,sourcecode = nodes_dict[node].split(':') if len(sourcecode)!=0: - dockername = sourcecode.split(".")[0] #3/28/21 + dockername = sourcecode.rsplit(".", 1)[0] #3/28/21 writeedges = volswr[i] while writeedges.find(":") != -1: fmaxtime.write('sudo docker cp concore.maxtime concore:/') @@ -876,7 +876,7 @@ def cleanup_script_files(): for node in nodes_dict: containername,sourcecode = nodes_dict[node].split(':') if len(sourcecode)!=0: - dockername = sourcecode.split(".")[0] #3/28/21 + dockername = sourcecode.rsplit(".", 1)[0] #3/28/21 writeedges = volswr[i] while writeedges.find(":") != -1: fparams.write(' -v ') @@ -893,7 +893,7 @@ def cleanup_script_files(): for node in nodes_dict: containername,sourcecode = nodes_dict[node].split(':') if len(sourcecode)!=0: - dockername = sourcecode.split(".")[0] #3/28/21 + dockername = sourcecode.rsplit(".", 1)[0] #3/28/21 writeedges = volswr[i] while writeedges.find(":") != -1: fparams.write('sudo docker cp concore.params concore:/') @@ -918,7 +918,7 @@ def cleanup_script_files(): for node in nodes_dict: containername,sourcecode = nodes_dict[node].split(':') if len(sourcecode)!=0: - dockername = sourcecode.split(".")[0] #3/28/21 + dockername = sourcecode.rsplit(".", 1)[0] #3/28/21 writeedges = volswr[i] while writeedges.find(":") != -1: funlock.write(' -v ') @@ -935,7 +935,7 @@ def cleanup_script_files(): for node in nodes_dict: containername,sourcecode = nodes_dict[node].split(':') if len(sourcecode)!=0: - dockername = sourcecode.split(".")[0] #3/28/21 + dockername = sourcecode.rsplit(".", 1)[0] #3/28/21 writeedges = volswr[i] while writeedges.find(":") != -1: funlock.write('sudo docker cp ~/concore.apikey concore:/') @@ -955,7 +955,7 @@ def cleanup_script_files(): for node in nodes_dict: containername,sourcecode = nodes_dict[node].split(':') if len(sourcecode)!=0 and sourcecode.find(".")!=-1: #3/28/21 - dockername,langext = sourcecode.split(".") + dockername,langext = sourcecode.rsplit(".", 1) # safe_container added to debug line (POSIX) safe_container = shlex.quote(containername) safe_image = shlex.quote("docker-" + dockername) # escape docker image name @@ -984,7 +984,7 @@ def cleanup_script_files(): if sourcecode.find(".")==-1: logging.error("cannot pull container "+sourcecode+" with control core type "+concoretype) #3/28/21 quit() - dockername,langext = sourcecode.split(".") + dockername,langext = sourcecode.rsplit(".", 1) fbuild.write('mkdir '+containername+"\n") source_subdir = os.path.dirname(sourcecode).replace("\\", "/") if source_subdir: @@ -1041,7 +1041,7 @@ def cleanup_script_files(): containername,sourcecode = edges_dict[edges][0].split(':') outcount[nodes_num[edges_dict[edges][0]]] += 1 if len(sourcecode)!=0: - dockername,langext = sourcecode.split(".") + dockername,langext = sourcecode.rsplit(".", 1) fbuild.write('cd '+containername+"\n") if concoretype=="windows": fbuild.write("mklink /J out"+str(outcount[nodes_num[edges_dict[edges][0]]])+" ..\\"+str(edges)+"\n") @@ -1054,7 +1054,7 @@ def cleanup_script_files(): for node in nodes_dict: containername,sourcecode = nodes_dict[node].split(':') if len(sourcecode)!=0: - dockername,langext = sourcecode.split(".") + dockername,langext = sourcecode.rsplit(".", 1) fbuild.write('cd '+containername+"\n") for pair in indir[i]: volname,dirname = pair.split(':/') @@ -1075,7 +1075,7 @@ def cleanup_script_files(): for node in nodes_dict: containername,sourcecode = nodes_dict[node].split(':') if len(sourcecode)!=0: - dockername,langext = sourcecode.split(".") + dockername,langext = sourcecode.rsplit(".", 1) if not (langext in ["py","m","sh","cpp","v"]): # 6/22/21 logging.error(f"Extension .{langext} is unsupported") quit() @@ -1191,7 +1191,7 @@ def cleanup_script_files(): for node in nodes_dict: containername,sourcecode = nodes_dict[node].split(':') if len(sourcecode)!=0: - dockername = sourcecode.split(".")[0] # 3/28/21 + dockername = sourcecode.rsplit(".", 1)[0] # 3/28/21 if concoretype=="windows": q_container = f'"{containername}"' fstop.write('cmd /C '+q_container+"\\concorekill\n") @@ -1209,7 +1209,7 @@ def cleanup_script_files(): for node in nodes_dict: containername,sourcecode = nodes_dict[node].split(':') if len(sourcecode)!=0: - dockername = sourcecode.split(".")[0] #3/28/21 + dockername = sourcecode.rsplit(".", 1)[0] #3/28/21 writeedges = volswr[i] while writeedges.find(":") != -1: path_part = writeedges.split(":")[0].split("-v")[1].strip() @@ -1229,7 +1229,7 @@ def cleanup_script_files(): for node in nodes_dict: containername,sourcecode = nodes_dict[node].split(':') if len(sourcecode)!=0: - dockername = sourcecode.split(".")[0] #3/28/21 + dockername = sourcecode.rsplit(".", 1)[0] #3/28/21 writeedges = volswr[i] while writeedges.find(":") != -1: path_part = writeedges.split(":")[0].split("-v")[1].strip() @@ -1247,7 +1247,7 @@ def cleanup_script_files(): for node in nodes_dict: containername,sourcecode = nodes_dict[node].split(':') if len(sourcecode)!=0: - dockername = sourcecode.split(".")[0] #3/28/21 + dockername = sourcecode.rsplit(".", 1)[0] #3/28/21 writeedges = volswr[i] while writeedges.find(":") != -1: path_part = writeedges.split(":")[0].split("-v")[1].strip() @@ -1265,7 +1265,7 @@ def cleanup_script_files(): for node in nodes_dict: containername,sourcecode = nodes_dict[node].split(':') if len(sourcecode)!=0: - dockername = sourcecode.split(".")[0] #3/28/21 + dockername = sourcecode.rsplit(".", 1)[0] #3/28/21 writeedges = volswr[i] while writeedges.find(":") != -1: path_part = writeedges.split(":")[0].split("-v")[1].strip() From 270656ebaae5b181c99c14a294165d564da3ccbc Mon Sep 17 00:00:00 2001 From: Avinash Kumar Deepak Date: Sun, 15 Feb 2026 17:22:14 +0530 Subject: [PATCH 145/275] advance simtime after write in C++ and Verilog --- concore.hpp | 2 ++ concore.v | 1 + 2 files changed, 3 insertions(+) diff --git a/concore.hpp b/concore.hpp index 65109afe..69f38b16 100644 --- a/concore.hpp +++ b/concore.hpp @@ -494,6 +494,7 @@ class Concore{ outfile< Date: Sun, 15 Feb 2026 19:16:40 +0530 Subject: [PATCH 146/275] optimize python dockerfile to be much lighter --- Dockerfile.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/Dockerfile.py b/Dockerfile.py index 3c3836f6..1a2a5169 100644 --- a/Dockerfile.py +++ b/Dockerfile.py @@ -1,9 +1,8 @@ -FROM jupyter/base-notebook +FROM python:3.10-slim -USER root -RUN apt-get update && apt-get install -y build-essential g++ libgl1-mesa-glx libx11-6 -COPY requirements.txt . -RUN pip install -r requirements.txt -COPY . /src WORKDIR /src +RUN apt-get update && apt-get install -y build-essential g++ libgl1-mesa-glx libx11-6 && rm -rf /var/lib/apt/lists/* +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt +COPY . . From 20812998c5c089d440236617263905ca2cc39722 Mon Sep 17 00:00:00 2001 From: Avinash Kumar Deepak Date: Sun, 15 Feb 2026 22:27:13 +0530 Subject: [PATCH 147/275] fix: add missing return in concoredocker.py write() after ZMQ send --- concoredocker.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/concoredocker.py b/concoredocker.py index c81e7303..61a6a45e 100644 --- a/concoredocker.py +++ b/concoredocker.py @@ -276,12 +276,20 @@ def write(port_identifier, name, val, delta=0): if isinstance(port_identifier, str) and port_identifier in zmq_ports: zmq_p = zmq_ports[port_identifier] try: - zmq_p.send_json_with_retry(val) + zmq_val = convert_numpy_to_python(val) + if isinstance(zmq_val, list): + # Prepend simtime to match file-based write behavior + payload = [simtime + delta] + zmq_val + zmq_p.send_json_with_retry(payload) + simtime += delta + else: + zmq_p.send_json_with_retry(zmq_val) except zmq.error.ZMQError as e: logging.error(f"ZMQ write error on port {port_identifier} (name: {name}): {e}") except Exception as e: logging.error(f"Unexpected error during ZMQ write on port {port_identifier} (name: {name}): {e}") - + return + try: file_port_num = int(port_identifier) file_path = os.path.join(outpath, str(file_port_num), name) From 143d6e96010f8f876ac78116d709635d6c412b80 Mon Sep 17 00:00:00 2001 From: Sahil Lenka Date: Mon, 16 Feb 2026 05:58:30 +0530 Subject: [PATCH 148/275] fix simtime update in write_FM and write_SM functions --- concore.hpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/concore.hpp b/concore.hpp index b74ddd7e..f56bf755 100644 --- a/concore.hpp +++ b/concore.hpp @@ -451,6 +451,7 @@ class Concore{ outfile< Date: Mon, 16 Feb 2026 06:34:21 +0530 Subject: [PATCH 149/275] fix simtime not advancing in MATLAB write function --- concore_write.m | 1 + 1 file changed, 1 insertion(+) diff --git a/concore_write.m b/concore_write.m index 20bce14e..b4bf42ca 100644 --- a/concore_write.m +++ b/concore_write.m @@ -5,6 +5,7 @@ function concore_write(port, name, val, delta) outstr = cat(2,"[",num2str(concore.simtime+delta),num2str(val,",%e"),"]"); fprintf(output1,'%s',outstr); fclose(output1); + concore.simtime = concore.simtime + delta; catch exc disp(['skipping ' concore.outpath num2str(port) '/' name]); end From b607b8e53f0fbb13715e27b42a658f071c049bcf Mon Sep 17 00:00:00 2001 From: Avinash Kumar Deepak Date: Mon, 16 Feb 2026 15:03:11 +0530 Subject: [PATCH 150/275] fix java params to use semicolon separator --- concoredocker.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/concoredocker.java b/concoredocker.java index dde521c8..a991178a 100644 --- a/concoredocker.java +++ b/concoredocker.java @@ -36,7 +36,7 @@ public static void main(String[] args) { } if (!sparams.equals("{")) { System.out.println("converting sparams: " + sparams); - sparams = "{'" + sparams.replaceAll(",", ",'").replaceAll("=", "':").replaceAll(" ", "") + "}"; + sparams = "{'" + sparams.replaceAll(";", ",'").replaceAll("=", "':").replaceAll(" ", "") + "}"; System.out.println("converted sparams: " + sparams); } try { From 926e5ee1da873ae9edeebf9beb7290a72f3ce41b Mon Sep 17 00:00:00 2001 From: Avinash Kumar Deepak Date: Mon, 16 Feb 2026 15:24:55 +0530 Subject: [PATCH 151/275] add missing deps to pyproject.toml fixes #354 --- pyproject.toml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 63a8f6b7..2c12b5bc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ name = "concore" version = "1.0.0" description = "Concore workflow management CLI" readme = "README.md" -requires-python = ">=3.8" +requires-python = ">=3.9" license = {text = "MIT"} dependencies = [ "click>=8.0.0", @@ -15,6 +15,10 @@ dependencies = [ "beautifulsoup4>=4.9.0", "lxml>=4.6.0", "psutil>=5.8.0", + "numpy>=1.19.0", + "pyzmq>=22.0.0", + "scipy>=1.5.0", + "matplotlib>=3.3.0", ] [project.optional-dependencies] From 82cea619a46963508e280ff360ad8e78fe4e0a48 Mon Sep 17 00:00:00 2001 From: Avinash Kumar Deepak Date: Mon, 16 Feb 2026 23:20:28 +0530 Subject: [PATCH 152/275] fix: guard against empty string crash in parser() and bounds check in read_FM/read_SM --- concore.hpp | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/concore.hpp b/concore.hpp index b74ddd7e..28dacac8 100644 --- a/concore.hpp +++ b/concore.hpp @@ -240,6 +240,7 @@ class Concore{ */ vector parser(string f){ vector temp; + if(f.empty()) return temp; string value = ""; //Changing last bracket to comma to use comma as a delimiter @@ -330,6 +331,10 @@ class Concore{ s += ins; vector inval = parser(ins); + if(inval.empty()) + inval = parser(initstr); + if(inval.empty()) + return inval; simtime = simtime > inval[0] ? simtime : inval[0]; //returning a string with data excluding simtime @@ -389,6 +394,10 @@ class Concore{ s += ins; vector inval = parser(ins); + if(inval.empty()) + inval = parser(initstr); + if(inval.empty()) + return inval; simtime = simtime > inval[0] ? simtime : inval[0]; //returning a string with data excluding simtime From 454acd8ee102eb2f09c120bd7457f64fb52b5965 Mon Sep 17 00:00:00 2001 From: Avinash Kumar Deepak Date: Tue, 17 Feb 2026 13:40:40 +0530 Subject: [PATCH 153/275] fix: align concoredocker.py ZMQ read/write with simtime framing (#382) --- concoredocker.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/concoredocker.py b/concoredocker.py index c81e7303..5582ba9a 100644 --- a/concoredocker.py +++ b/concoredocker.py @@ -211,6 +211,11 @@ def read(port_identifier, name, initstr_val): zmq_p = zmq_ports[port_identifier] try: message = zmq_p.recv_json_with_retry() + if isinstance(message, list) and len(message) > 0: + first_element = message[0] + if isinstance(first_element, (int, float)): + simtime = max(simtime, first_element) + return message[1:] return message except zmq.error.ZMQError as e: logging.error(f"ZMQ read error on port {port_identifier} (name: {name}): {e}. Returning default.") @@ -276,7 +281,13 @@ def write(port_identifier, name, val, delta=0): if isinstance(port_identifier, str) and port_identifier in zmq_ports: zmq_p = zmq_ports[port_identifier] try: - zmq_p.send_json_with_retry(val) + zmq_val = convert_numpy_to_python(val) + if isinstance(zmq_val, list): + payload = [simtime + delta] + zmq_val + zmq_p.send_json_with_retry(payload) + simtime += delta + else: + zmq_p.send_json_with_retry(zmq_val) except zmq.error.ZMQError as e: logging.error(f"ZMQ write error on port {port_identifier} (name: {name}): {e}") except Exception as e: From 4dd4d7de891bcc6dc6a6bb4ffae3b5d5f8d7c34d Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Wed, 18 Feb 2026 11:58:37 +0530 Subject: [PATCH 154/275] refactor: remove duplicate Docker import logic, prevent Windows batch generation in Docker, and fix file descriptor leaks (fixes #285) --- import_concore.m | 105 ++++++++++++++++++++++++----------------- import_concoredocker.m | 87 +++++++++++++++++----------------- 2 files changed, 108 insertions(+), 84 deletions(-) diff --git a/import_concore.m b/import_concore.m index 018c10e5..ad95889c 100644 --- a/import_concore.m +++ b/import_concore.m @@ -1,42 +1,63 @@ -function import_concore - global concore; - - try - pid = getpid(); - catch exc - pid = feature('getpid'); - end - outputpid = fopen('concorekill.bat','w'); - fprintf(outputpid,'%s',['taskkill /F /PID ',num2str(pid)]); - fclose(outputpid); - - - try - iportfile = fopen('concore.iport'); - concore.iports = fscanf(iportfile,'%c'); - catch exc - iportfile = ''; - end - - try - oportfile = fopen('concore.oport'); - concore.oports = fscanf(oportfile,'%c'); - catch exc - oportfile = ''; - end - - concore.s = ''; - concore.olds = ''; - concore.delay = 1; - concore.retrycount = 0; - if exist('/in1','dir')==7 % 5/20/21 work for docker or local - concore.inpath = '/in'; - concore.outpath = '/out'; - else - concore.inpath = 'in'; - concore.outpath = 'out'; - end - concore.simtime = 0; - - concore_default_maxtime(100); -end +function import_concore + global concore; + + if ispc + try + pid = getpid(); + catch exc + pid = feature('getpid'); + end + outputpid = fopen('concorekill.bat','w'); + if outputpid ~= -1 + fprintf(outputpid,'%s',['taskkill /F /PID ',num2str(pid)]); + fclose(outputpid); + else + warning('import_concore:ConcoreKillFileOpenFailed', ... + 'Could not create concorekill.bat. Continuing without kill script.'); + end + end + + + try + iportfile = fopen('concore.iport'); + concore.iports = fscanf(iportfile,'%c'); + if isnumeric(iportfile) && iportfile ~= -1 + fclose(iportfile); + end + iportfile = -1; + catch exc + if exist('iportfile', 'var') && isnumeric(iportfile) && iportfile ~= -1 + fclose(iportfile); + end + iportfile = -1; + end + + try + oportfile = fopen('concore.oport'); + concore.oports = fscanf(oportfile,'%c'); + if isnumeric(oportfile) && oportfile ~= -1 + fclose(oportfile); + end + oportfile = -1; + catch exc + if exist('oportfile', 'var') && isnumeric(oportfile) && oportfile ~= -1 + fclose(oportfile); + end + oportfile = -1; + end + + concore.s = ''; + concore.olds = ''; + concore.delay = 1; + concore.retrycount = 0; + if exist('/in1','dir')==7 % 5/20/21 work for docker or local + concore.inpath = '/in'; + concore.outpath = '/out'; + else + concore.inpath = 'in'; + concore.outpath = 'out'; + end + concore.simtime = 0; + + concore_default_maxtime(100); +end diff --git a/import_concoredocker.m b/import_concoredocker.m index 018c10e5..dd385e16 100644 --- a/import_concoredocker.m +++ b/import_concoredocker.m @@ -1,42 +1,45 @@ -function import_concore - global concore; - - try - pid = getpid(); - catch exc - pid = feature('getpid'); - end - outputpid = fopen('concorekill.bat','w'); - fprintf(outputpid,'%s',['taskkill /F /PID ',num2str(pid)]); - fclose(outputpid); - - - try - iportfile = fopen('concore.iport'); - concore.iports = fscanf(iportfile,'%c'); - catch exc - iportfile = ''; - end - - try - oportfile = fopen('concore.oport'); - concore.oports = fscanf(oportfile,'%c'); - catch exc - oportfile = ''; - end - - concore.s = ''; - concore.olds = ''; - concore.delay = 1; - concore.retrycount = 0; - if exist('/in1','dir')==7 % 5/20/21 work for docker or local - concore.inpath = '/in'; - concore.outpath = '/out'; - else - concore.inpath = 'in'; - concore.outpath = 'out'; - end - concore.simtime = 0; - - concore_default_maxtime(100); -end +function import_concore + global concore; + + % Docker/Linux environment — no Windows batch file generation + + + iportfile = fopen('concore.iport'); + if iportfile ~= -1 + try + concore.iports = fscanf(iportfile,'%c'); + catch exc + concore.iports = ''; + end + fclose(iportfile); + else + concore.iports = ''; + end + + oportfile = fopen('concore.oport'); + if oportfile ~= -1 + try + concore.oports = fscanf(oportfile,'%c'); + catch exc + concore.oports = ''; + end + fclose(oportfile); + else + concore.oports = ''; + end + + concore.s = ''; + concore.olds = ''; + concore.delay = 1; + concore.retrycount = 0; + if exist('/in1','dir')==7 % 5/20/21 work for docker or local + concore.inpath = '/in'; + concore.outpath = '/out'; + else + concore.inpath = 'in'; + concore.outpath = 'out'; + end + concore.simtime = 0; + + concore_default_maxtime(100); +end From fb5b10e44b980b501d9c8cc1e5ccc1aa91a1a40f Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Wed, 18 Feb 2026 12:33:53 +0530 Subject: [PATCH 155/275] security: harden contribute.py with token validation, safe error handling, and retry logic (fixes #288) --- contribute.py | 374 ++++++++++++++++++++++++++++---------------------- 1 file changed, 213 insertions(+), 161 deletions(-) diff --git a/contribute.py b/contribute.py index ef5d8ceb..a5d3456d 100644 --- a/contribute.py +++ b/contribute.py @@ -1,162 +1,214 @@ -import github -from github import Github -import os,sys,platform,base64,time - -# Intializing the Variables -BOT_TOKEN = os.environ.get('CONCORE_BOT_TOKEN', '') -BOT_ACCOUNT = 'concore-bot' #bot account name -REPO_NAME = 'concore-studies' #study repo name -UPSTREAM_ACCOUNT = 'ControlCore-Project' #upstream account name -STUDY_NAME = sys.argv[1] -STUDY_NAME_PATH = sys.argv[2] -AUTHOR_NAME = sys.argv[3] -BRANCH_NAME = sys.argv[4] -PR_TITLE = sys.argv[5] -PR_BODY = sys.argv[6] - -# Defining Functions -def checkInputValidity(): - if not AUTHOR_NAME or not STUDY_NAME or not STUDY_NAME_PATH: - print("Please Provide necessary Inputs") - exit(1) - if not os.path.isdir(STUDY_NAME_PATH): - print("Directory doesnot Exists.Invalid Path") - exit(1) - -def printPR(pr): - print(f'Check your example here https://github.com/{UPSTREAM_ACCOUNT}/{REPO_NAME}/pulls/{pr.number}',end="") - -def anyOpenPR(upstream_repo): - try: - prs = upstream_repo.get_pulls(state='open', head=f'{BOT_ACCOUNT}:{BRANCH_NAME}') - return prs[0] if prs.totalCount > 0 else None - except Exception: - print("Unable to fetch PR status. Try again later.") - exit(1) - -def commitAndUpdateRef(repo,tree_content,commit,branch): - try: - new_tree = repo.create_git_tree(tree=tree_content,base_tree=commit.commit.tree) - new_commit = repo.create_git_commit(f"Committing Study Named {STUDY_NAME}",new_tree,[commit.commit]) - if len(repo.compare(base=commit.commit.sha,head=new_commit.sha).files) == 0: - print("Your don't have any new changes.May be your example is already accepted.If this is not the case try with different fields.") - exit(1) - ref = repo.get_git_ref("heads/"+branch.name) - ref.edit(new_commit.sha,True) - except Exception as e: - print("failed to Upload your example.Please try after some time.",end="") - exit(1) - - -def appendBlobInTree(repo,content,file_path,tree_content): - blob = repo.create_git_blob(content,'utf-8') - tree_content.append( github.InputGitTreeElement(path=file_path,mode="100644",type="blob",sha=blob.sha)) - - -def runWorkflow(repo,upstream_repo): - openPR = anyOpenPR(upstream_repo) - if not openPR: - try: - repo.get_workflow("pull_request.yml").create_dispatch( - ref=BRANCH_NAME, - inputs={'title': f"[BOT]: {PR_TITLE}", 'body': PR_BODY, 'upstreamRepo': UPSTREAM_ACCOUNT, 'botRepo': BOT_ACCOUNT, 'repo': REPO_NAME} - ) - printPRStatus(upstream_repo) - except Exception as e: - print(f"Error triggering workflow. Try again later.\n ERROR: {e}") - exit(1) - else: - print(f"Successfully uploaded. Waiting for approval: https://github.com/{UPSTREAM_ACCOUNT}/{REPO_NAME}/pull/{openPR.number}") - -def printPRStatus(upstream_repo): - attempts = 5 - delay = 2 - for i in range(attempts): - print(f"Attempt: {i}") - try: - latest_pr = upstream_repo.get_pulls(state='open', sort='created', direction='desc')[0] - print(f"Check your example here: https://github.com/{UPSTREAM_ACCOUNT}/{REPO_NAME}/pull/{latest_pr.number}") - return - except Exception: - time.sleep(delay) - delay *= 2 - print("Uploaded successfully, but unable to fetch status.") - - -def isImageFile(filename): - image_extensions = ['.jpeg', '.jpg', '.png','.gif'] - return any(filename.endswith(ext) for ext in image_extensions) - -def remove_prefix(text, prefix): - if text.startswith(prefix): - return text[len(prefix):] - return text - - -# Decode Github Token -def decode_token(encoded_token): - decoded_bytes = encoded_token.encode("ascii") - convertedbytes = base64.b64decode(decoded_bytes) - decoded_token = convertedbytes.decode("ascii") - return decoded_token - - -# check if directory path is Valid -checkInputValidity() - - -# Authenticating Github with Access token -try: - BRANCH_NAME = AUTHOR_NAME.replace(" ", "_") + "_" + STUDY_NAME if BRANCH_NAME == "#" else BRANCH_NAME.replace(" ", "_") - PR_TITLE = f"Contributing Study {STUDY_NAME} by {AUTHOR_NAME}" if PR_TITLE == "#" else PR_TITLE - PR_BODY = f"Study Name: {STUDY_NAME}\nAuthor Name: {AUTHOR_NAME}" if PR_BODY == "#" else PR_BODY - DIR_PATH = STUDY_NAME - DIR_PATH = DIR_PATH.replace(" ","_") - g = Github(BOT_TOKEN) - repo = g.get_user(BOT_ACCOUNT).get_repo(REPO_NAME) - upstream_repo = g.get_repo(f'{UPSTREAM_ACCOUNT}/{REPO_NAME}') #controlcore-Project/concore-studies - base_ref = upstream_repo.get_branch(repo.default_branch) - - try: - repo.get_branch(BRANCH_NAME) - is_present = True - except github.GithubException: - print(f"No Branch is available with the name {BRANCH_NAME}") - is_present = False -except Exception as e: - print("Authentication failed", end="") - exit(1) - - -try: - if not is_present: - repo.create_git_ref(f"refs/heads/{BRANCH_NAME}", base_ref.commit.sha) - branch = repo.get_branch(BRANCH_NAME) -except Exception: - print("Unable to create study. Try again later.") - exit(1) - - -tree_content = [] - -try: - for root, dirs, files in os.walk(STUDY_NAME_PATH): - files = [f for f in files if not f[0] == '.'] - for filename in files: - path = f"{root}/{filename}" - if isImageFile(filename): - with open(file=path, mode='rb') as file: - image = file.read() - content = base64.b64encode(image).decode('utf-8') - else: - with open(file=path, mode='r') as file: - content = file.read() - file_path = f'{DIR_PATH+remove_prefix(path,STUDY_NAME_PATH)}' - if(platform.uname()[0]=='Windows'): file_path=file_path.replace("\\","/") - appendBlobInTree(repo,content,file_path,tree_content) - commitAndUpdateRef(repo,tree_content,base_ref.commit,branch) - runWorkflow(repo,upstream_repo) -except Exception as e: - print(e) - print("Some error Occured.Please try again after some time.",end="") +import github +from github import Github +import os,sys,platform,base64,time,re +import requests + +# Intializing the Variables +BOT_TOKEN = os.environ.get('CONCORE_BOT_TOKEN', '') + +# Fix 1: Fail fast if token is missing +if not BOT_TOKEN: + print("Error: CONCORE_BOT_TOKEN environment variable is not set.") + sys.exit(1) + +# Fix 2: Token format validation +token_pattern = r"^(ghp_|github_pat_)[A-Za-z0-9_]{20,}$" +if not re.match(token_pattern, BOT_TOKEN): + print("Error: Invalid GitHub token format.") + sys.exit(1) +BOT_ACCOUNT = 'concore-bot' #bot account name +REPO_NAME = 'concore-studies' #study repo name +UPSTREAM_ACCOUNT = 'ControlCore-Project' #upstream account name +STUDY_NAME = sys.argv[1] +STUDY_NAME_PATH = sys.argv[2] +AUTHOR_NAME = sys.argv[3] +BRANCH_NAME = sys.argv[4] +PR_TITLE = sys.argv[5] +PR_BODY = sys.argv[6] + +# Defining Functions +def checkInputValidity(): + if not AUTHOR_NAME or not STUDY_NAME or not STUDY_NAME_PATH: + print("Please Provide necessary Inputs") + exit(1) + if not os.path.isdir(STUDY_NAME_PATH): + print("Directory doesnot Exists.Invalid Path") + exit(1) + +# Fix 5: Retry + backoff wrapper for GitHub API requests +def github_request(method, url, headers=None, json=None, retries=3): + for attempt in range(retries): + try: + response = requests.request(method, url, headers=headers, json=json, timeout=30) + if response.status_code == 429 or response.status_code >= 500: + wait_time = 2 ** attempt + time.sleep(wait_time) + continue + return response + except requests.exceptions.ConnectionError: + print("Network error while contacting GitHub API.") + sys.exit(1) + except requests.exceptions.Timeout: + print("GitHub API request timed out.") + sys.exit(1) + print("Error: GitHub API request failed after retries.") + sys.exit(1) + +# Fix 4: Correct PR URL (singular 'pull' not 'pulls') +def printPR(pr): + print(f'Check your example here https://github.com/{UPSTREAM_ACCOUNT}/{REPO_NAME}/pull/{pr.number}',end="") + +def anyOpenPR(upstream_repo): + try: + prs = upstream_repo.get_pulls(state='open', head=f'{BOT_ACCOUNT}:{BRANCH_NAME}') + return prs[0] if prs.totalCount > 0 else None + except requests.exceptions.ConnectionError: + print("Network error while fetching PR status.") + exit(1) + except requests.exceptions.Timeout: + print("Request timed out while fetching PR status.") + exit(1) + except Exception: + print("Unable to fetch PR status. Try again later.") + exit(1) + +def commitAndUpdateRef(repo,tree_content,commit,branch): + try: + new_tree = repo.create_git_tree(tree=tree_content,base_tree=commit.commit.tree) + new_commit = repo.create_git_commit(f"Committing Study Named {STUDY_NAME}",new_tree,[commit.commit]) + if len(repo.compare(base=commit.commit.sha,head=new_commit.sha).files) == 0: + print("Your don't have any new changes.May be your example is already accepted.If this is not the case try with different fields.") + exit(1) + ref = repo.get_git_ref("heads/"+branch.name) + ref.edit(new_commit.sha,True) + except requests.exceptions.HTTPError as e: + print(f"GitHub API error: {e.response.status_code}") + exit(1) + except Exception: + print("Failed to upload your example. Please try after some time.",end="") + exit(1) + + +def appendBlobInTree(repo,content,file_path,tree_content): + blob = repo.create_git_blob(content,'utf-8') + tree_content.append( github.InputGitTreeElement(path=file_path,mode="100644",type="blob",sha=blob.sha)) + + +def runWorkflow(repo,upstream_repo): + openPR = anyOpenPR(upstream_repo) + if not openPR: + try: + repo.get_workflow("pull_request.yml").create_dispatch( + ref=BRANCH_NAME, + inputs={'title': f"[BOT]: {PR_TITLE}", 'body': PR_BODY, 'upstreamRepo': UPSTREAM_ACCOUNT, 'botRepo': BOT_ACCOUNT, 'repo': REPO_NAME} + ) + printPRStatus(upstream_repo) + except requests.exceptions.HTTPError as e: + print(f"GitHub API error while triggering workflow: {e.response.status_code}") + exit(1) + except Exception: + print("Error triggering workflow. Try again later.") + exit(1) + else: + print(f"Successfully uploaded. Waiting for approval: https://github.com/{UPSTREAM_ACCOUNT}/{REPO_NAME}/pull/{openPR.number}") + +def printPRStatus(upstream_repo): + attempts = 5 + delay = 2 + for i in range(attempts): + print(f"Attempt: {i}") + try: + latest_pr = upstream_repo.get_pulls(state='open', sort='created', direction='desc')[0] + print(f"Check your example here: https://github.com/{UPSTREAM_ACCOUNT}/{REPO_NAME}/pull/{latest_pr.number}") + return + except Exception: + time.sleep(delay) + delay *= 2 + print("Uploaded successfully, but unable to fetch status.") + + +def isImageFile(filename): + image_extensions = ['.jpeg', '.jpg', '.png','.gif'] + return any(filename.endswith(ext) for ext in image_extensions) + +def remove_prefix(text, prefix): + if text.startswith(prefix): + return text[len(prefix):] + return text + + +# Fix 9: Removed unused decode_token() function + +# check if directory path is Valid +checkInputValidity() + + +# Authenticating Github with Access token +try: + BRANCH_NAME = AUTHOR_NAME.replace(" ", "_") + "_" + STUDY_NAME if BRANCH_NAME == "#" else BRANCH_NAME.replace(" ", "_") + PR_TITLE = f"Contributing Study {STUDY_NAME} by {AUTHOR_NAME}" if PR_TITLE == "#" else PR_TITLE + PR_BODY = f"Study Name: {STUDY_NAME}\nAuthor Name: {AUTHOR_NAME}" if PR_BODY == "#" else PR_BODY + DIR_PATH = STUDY_NAME + DIR_PATH = DIR_PATH.replace(" ","_") + g = Github(BOT_TOKEN) + repo = g.get_user(BOT_ACCOUNT).get_repo(REPO_NAME) + upstream_repo = g.get_repo(f'{UPSTREAM_ACCOUNT}/{REPO_NAME}') #controlcore-Project/concore-studies + base_ref = upstream_repo.get_branch(repo.default_branch) + + try: + repo.get_branch(BRANCH_NAME) + is_present = True + except github.GithubException: + print(f"No Branch is available with the name {BRANCH_NAME}") + is_present = False +except requests.exceptions.ConnectionError: + print("Network error during GitHub authentication.") + exit(1) +except requests.exceptions.Timeout: + print("GitHub authentication request timed out.") + exit(1) +except Exception: + print("Authentication failed", end="") + exit(1) + + +try: + if not is_present: + repo.create_git_ref(f"refs/heads/{BRANCH_NAME}", base_ref.commit.sha) + branch = repo.get_branch(BRANCH_NAME) +except Exception: + print("Unable to create study. Try again later.") + exit(1) + + +tree_content = [] + +try: + for root, dirs, files in os.walk(STUDY_NAME_PATH): + files = [f for f in files if not f[0] == '.'] + for filename in files: + path = f"{root}/{filename}" + if isImageFile(filename): + with open(file=path, mode='rb') as file: + image = file.read() + content = base64.b64encode(image).decode('utf-8') + else: + with open(file=path, mode='r') as file: + content = file.read() + file_path = f'{DIR_PATH+remove_prefix(path,STUDY_NAME_PATH)}' + if(platform.uname()[0]=='Windows'): file_path=file_path.replace("\\","/") + appendBlobInTree(repo,content,file_path,tree_content) + commitAndUpdateRef(repo,tree_content,base_ref.commit,branch) + runWorkflow(repo,upstream_repo) +except requests.exceptions.HTTPError as e: + print(f"GitHub API error: {e.response.status_code}") + exit(1) +except requests.exceptions.ConnectionError: + print("Network error while uploading study.") + exit(1) +except requests.exceptions.Timeout: + print("Request timed out while uploading study.") + exit(1) +except Exception: + print("Some error occurred. Please try again after some time.",end="") exit(1) \ No newline at end of file From 9ace7557ee671cabc406e392b84eb49104890c44 Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Wed, 18 Feb 2026 12:50:39 +0530 Subject: [PATCH 156/275] security: address PR review - fix exception types to github.GithubException, expand token regex, remove unused requests import, fix spelling (fixes #288) --- contribute.py | 60 ++++++++++++++++++++------------------------------- 1 file changed, 23 insertions(+), 37 deletions(-) diff --git a/contribute.py b/contribute.py index a5d3456d..900a9fc4 100644 --- a/contribute.py +++ b/contribute.py @@ -1,9 +1,8 @@ import github from github import Github import os,sys,platform,base64,time,re -import requests -# Intializing the Variables +# Initializing the Variables BOT_TOKEN = os.environ.get('CONCORE_BOT_TOKEN', '') # Fix 1: Fail fast if token is missing @@ -12,7 +11,7 @@ sys.exit(1) # Fix 2: Token format validation -token_pattern = r"^(ghp_|github_pat_)[A-Za-z0-9_]{20,}$" +token_pattern = r"^((ghp_|github_pat_|ghs_)[A-Za-z0-9_]{20,}|[0-9a-fA-F]{40})$" if not re.match(token_pattern, BOT_TOKEN): print("Error: Invalid GitHub token format.") sys.exit(1) @@ -32,25 +31,21 @@ def checkInputValidity(): print("Please Provide necessary Inputs") exit(1) if not os.path.isdir(STUDY_NAME_PATH): - print("Directory doesnot Exists.Invalid Path") + print("Directory does not Exists.Invalid Path") exit(1) -# Fix 5: Retry + backoff wrapper for GitHub API requests -def github_request(method, url, headers=None, json=None, retries=3): +# Fix 5: Retry + backoff wrapper for PyGithub operations +def with_retry(operation, retries=3): + """Retry wrapper for PyGithub operations with exponential backoff.""" for attempt in range(retries): try: - response = requests.request(method, url, headers=headers, json=json, timeout=30) - if response.status_code == 429 or response.status_code >= 500: + return operation() + except github.GithubException as e: + if (e.status == 429 or e.status >= 500) and attempt < retries - 1: wait_time = 2 ** attempt time.sleep(wait_time) continue - return response - except requests.exceptions.ConnectionError: - print("Network error while contacting GitHub API.") - sys.exit(1) - except requests.exceptions.Timeout: - print("GitHub API request timed out.") - sys.exit(1) + raise print("Error: GitHub API request failed after retries.") sys.exit(1) @@ -62,11 +57,11 @@ def anyOpenPR(upstream_repo): try: prs = upstream_repo.get_pulls(state='open', head=f'{BOT_ACCOUNT}:{BRANCH_NAME}') return prs[0] if prs.totalCount > 0 else None - except requests.exceptions.ConnectionError: - print("Network error while fetching PR status.") - exit(1) - except requests.exceptions.Timeout: - print("Request timed out while fetching PR status.") + except github.GithubException as e: + if e.status == 429 or e.status >= 500: + print("GitHub API rate limit or server error while fetching PR status.") + else: + print("Unable to fetch PR status. Try again later.") exit(1) except Exception: print("Unable to fetch PR status. Try again later.") @@ -81,8 +76,8 @@ def commitAndUpdateRef(repo,tree_content,commit,branch): exit(1) ref = repo.get_git_ref("heads/"+branch.name) ref.edit(new_commit.sha,True) - except requests.exceptions.HTTPError as e: - print(f"GitHub API error: {e.response.status_code}") + except github.GithubException as e: + print(f"GitHub API error: {e.status}") exit(1) except Exception: print("Failed to upload your example. Please try after some time.",end="") @@ -103,8 +98,8 @@ def runWorkflow(repo,upstream_repo): inputs={'title': f"[BOT]: {PR_TITLE}", 'body': PR_BODY, 'upstreamRepo': UPSTREAM_ACCOUNT, 'botRepo': BOT_ACCOUNT, 'repo': REPO_NAME} ) printPRStatus(upstream_repo) - except requests.exceptions.HTTPError as e: - print(f"GitHub API error while triggering workflow: {e.response.status_code}") + except github.GithubException as e: + print(f"GitHub API error while triggering workflow: {e.status}") exit(1) except Exception: print("Error triggering workflow. Try again later.") @@ -161,11 +156,8 @@ def remove_prefix(text, prefix): except github.GithubException: print(f"No Branch is available with the name {BRANCH_NAME}") is_present = False -except requests.exceptions.ConnectionError: - print("Network error during GitHub authentication.") - exit(1) -except requests.exceptions.Timeout: - print("GitHub authentication request timed out.") +except github.GithubException as e: + print(f"GitHub API error during authentication: {e.status}") exit(1) except Exception: print("Authentication failed", end="") @@ -200,14 +192,8 @@ def remove_prefix(text, prefix): appendBlobInTree(repo,content,file_path,tree_content) commitAndUpdateRef(repo,tree_content,base_ref.commit,branch) runWorkflow(repo,upstream_repo) -except requests.exceptions.HTTPError as e: - print(f"GitHub API error: {e.response.status_code}") - exit(1) -except requests.exceptions.ConnectionError: - print("Network error while uploading study.") - exit(1) -except requests.exceptions.Timeout: - print("Request timed out while uploading study.") +except github.GithubException as e: + print(f"GitHub API error: {e.status}") exit(1) except Exception: print("Some error occurred. Please try again after some time.",end="") From d55ace74f09210f2dc9d603c706ac43b19a2472c Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Wed, 18 Feb 2026 13:11:46 +0530 Subject: [PATCH 157/275] fix: make Windows destroy.bat call stop and clear before deletion (fixes #342) --- destroy.bat | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/destroy.bat b/destroy.bat index 561ec023..68b31a18 100644 --- a/destroy.bat +++ b/destroy.bat @@ -1,3 +1,25 @@ -@echo off -if exist "%1\stop.bat" (rmdir /s/q %1) else (echo "%1 is not a concore study") - +@echo off + +if not exist "%1" ( + echo "%1 does not exist" + exit /b 1 +) + +if not exist "%1\stop.bat" ( + echo "%1 is not a concore study" + exit /b 1 +) + +echo Stopping study... +call "%1\stop.bat" + +if exist "%1\clear.bat" ( + echo Clearing study... + call "%1\clear.bat" +) + +echo Removing study directory... +rmdir /s /q "%1" + +echo Done. + From 8cd332646752e89275e33cdebd87393efb0bd906 Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Wed, 18 Feb 2026 15:23:47 +0530 Subject: [PATCH 158/275] maintenance: update oldeditgraph DHGWorkflow link to archived repository location (fixes #346) --- oldeditgraph | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/oldeditgraph b/oldeditgraph index 2d2553aa..ae8b5912 100755 --- a/oldeditgraph +++ b/oldeditgraph @@ -2,12 +2,12 @@ which open if [ $? == 0 ] then - open https://pradeeban.github.io/DHGWorkflow/ + open https://kathiravelulab.github.io/DHGWorkflow/ else which xdg-open if [ $? == 0 ] then - xdg-open https://pradeeban.github.io/DHGWorkflow/ + xdg-open https://kathiravelulab.github.io/DHGWorkflow/ else echo "unable to open browser for DHGWorkflow" fi From 83666ad3b9bc554f584b61676fcbd1ac4d688d8b Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Wed, 18 Feb 2026 15:39:06 +0530 Subject: [PATCH 159/275] security: sanitize fetch parameter with secure_filename in /download endpoint (fixes #358) --- fri/server/main.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/fri/server/main.py b/fri/server/main.py index c2e1e659..90dfd7a0 100644 --- a/fri/server/main.py +++ b/fri/server/main.py @@ -433,6 +433,11 @@ def download(dir): if not download_file: abort(400, description="Missing file parameter") + download_file = secure_filename(download_file) + + if download_file == "": + abort(400, description="Invalid filename") + # Normalize the requested file path safe_path = os.path.normpath(download_file) From 6a40d0f53e7b84b1a97f120bdb7e5cfdb750d66c Mon Sep 17 00:00:00 2001 From: Avinash Kumar Deepak Date: Wed, 18 Feb 2026 15:43:07 +0530 Subject: [PATCH 160/275] fix: use double for simtime/maxtime in concoredocker.hpp --- concoredocker.hpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/concoredocker.hpp b/concoredocker.hpp index 2471e3ec..c142592a 100644 --- a/concoredocker.hpp +++ b/concoredocker.hpp @@ -23,8 +23,8 @@ class Concore { int retrycount = 0; std::string inpath = "/in"; std::string outpath = "/out"; - int simtime = 0; - int maxtime = 100; + double simtime = 0; + double maxtime = 100; std::unordered_map params; std::string stripstr(const std::string& str) { @@ -119,7 +119,7 @@ class Concore { return params.count(n) ? params[n] : i; } - void default_maxtime(int defaultValue) { + void default_maxtime(double defaultValue) { maxtime = defaultValue; std::ifstream file(inpath + "/1/concore.maxtime"); if (file) { @@ -166,7 +166,7 @@ class Concore { try { std::vector inval = parselist(ins); if (!inval.empty()) { - int file_simtime = (int)std::stod(inval[0]); + double file_simtime = std::stod(inval[0]); simtime = std::max(simtime, file_simtime); return std::vector(inval.begin() + 1, inval.end()); } @@ -195,7 +195,7 @@ class Concore { try { std::vector val = parselist(simtime_val); if (!val.empty()) { - simtime = (int)std::stod(val[0]); + simtime = std::stod(val[0]); return std::vector(val.begin() + 1, val.end()); } } catch (...) {} From fefe4571502615a0693a91fe4a88c256ab123ef9 Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Wed, 18 Feb 2026 15:56:52 +0530 Subject: [PATCH 161/275] fix: correct subprocess handling and HTTP status codes in /library endpoint (fixes #359) --- fri/server/main.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/fri/server/main.py b/fri/server/main.py index c2e1e659..42d88561 100644 --- a/fri/server/main.py +++ b/fri/server/main.py @@ -507,18 +507,18 @@ def library(dir): result = subprocess.run(["cmd.exe", "/c", r"..\library.bat", library_path, filename], cwd=dir_path, check=True, capture_output=True, text=True) proc = result.stdout else: - proc = subprocess.check_output([r"../library", library_path, filename], cwd=dir_path) + proc = subprocess.check_output([r"../library", library_path, filename], cwd=dir_path, stderr=subprocess.STDOUT) proc = proc.decode("utf-8") resp = jsonify({'message': proc}) - resp.status_code = 201 + resp.status_code = 200 return resp except subprocess.CalledProcessError as e: error_output = get_error_output(e) resp = jsonify({'message': f'Command execution failed: {error_output}'}) - resp.status_code = 500 + resp.status_code = 400 return resp except Exception as e: - resp = jsonify({'message': 'There is an Error'}) + resp = jsonify({'message': 'Internal server error'}) resp.status_code = 500 return resp From e2c9853bbe057d6b1cfc31c53e8b55e2925e7f6a Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Wed, 18 Feb 2026 16:22:10 +0530 Subject: [PATCH 162/275] security: remove shell=True from /contribute endpoint to prevent command injection (fixes #360) --- fri/server/main.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/fri/server/main.py b/fri/server/main.py index c2e1e659..ddc33c4f 100644 --- a/fri/server/main.py +++ b/fri/server/main.py @@ -396,16 +396,18 @@ def contribute(): validate_text_field(PR_TITLE, 'title', max_length=512) validate_text_field(PR_BODY, 'desc', max_length=8192) + # Build base command depending on platform if(platform.uname()[0]=='Windows'): - # Use cmd.exe /c to invoke contribute.bat on Windows - proc = subprocess.run(["cmd.exe", "/c", "contribute.bat", STUDY_NAME, STUDY_NAME_PATH, AUTHOR_NAME, BRANCH_NAME, PR_TITLE, PR_BODY], cwd=concore_path, check=True, capture_output=True, text=True) - output_string = proc.stdout + cmd = ["cmd.exe", "/c", "contribute.bat", STUDY_NAME, STUDY_NAME_PATH, AUTHOR_NAME] else: - if len(BRANCH_NAME)==0: - proc = check_output([r"./contribute",STUDY_NAME,STUDY_NAME_PATH,AUTHOR_NAME],cwd=concore_path) - else: - proc = check_output([r"./contribute",STUDY_NAME,STUDY_NAME_PATH,AUTHOR_NAME,BRANCH_NAME,PR_TITLE,PR_BODY],cwd=concore_path) - output_string = proc.decode() + cmd = [r"./contribute", STUDY_NAME, STUDY_NAME_PATH, AUTHOR_NAME] + + # Append optional branch/PR args only when BRANCH_NAME is provided + if len(BRANCH_NAME) > 0: + cmd.extend([BRANCH_NAME, PR_TITLE, PR_BODY]) + + proc = subprocess.run(cmd, cwd=concore_path, check=True, capture_output=True, text=True) + output_string = proc.stdout status=200 if output_string.find("/pulls/")!=-1: status=200 From 0047de57c0aed50b272fd2e0bcce1dc87902a6d8 Mon Sep 17 00:00:00 2001 From: Avinash Kumar Deepak Date: Wed, 18 Feb 2026 17:01:01 +0530 Subject: [PATCH 163/275] test: add ZMQ simtime framing tests for concoredocker.py --- tests/test_concoredocker.py | 82 +++++++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) diff --git a/tests/test_concoredocker.py b/tests/test_concoredocker.py index 7677f0ae..2b9a260d 100644 --- a/tests/test_concoredocker.py +++ b/tests/test_concoredocker.py @@ -1,4 +1,5 @@ import os +import pytest class TestSafeLiteralEval: @@ -151,3 +152,84 @@ def test_returns_default_when_file_missing(self, temp_dir): assert result == [5, 5] concoredocker.inpath = old_inpath concoredocker.delay = old_delay + + +class TestZMQ: + @pytest.fixture(autouse=True) + def reset_zmq_ports(self): + import concoredocker + original_ports = concoredocker.zmq_ports.copy() + yield + concoredocker.zmq_ports.clear() + concoredocker.zmq_ports.update(original_ports) + + def test_write_prepends_simtime(self): + import concoredocker + + class DummyPort: + def __init__(self): + self.sent = None + + def send_json_with_retry(self, message): + self.sent = message + + dummy = DummyPort() + concoredocker.zmq_ports["test_zmq"] = dummy + concoredocker.simtime = 3.0 + + concoredocker.write("test_zmq", "data", [1.0, 2.0], delta=2) + + assert dummy.sent == [5.0, 1.0, 2.0] + assert concoredocker.simtime == 5.0 + + def test_read_strips_simtime(self): + import concoredocker + + class DummyPort: + def recv_json_with_retry(self): + return [10.0, 4.0, 5.0] + + concoredocker.zmq_ports["test_zmq"] = DummyPort() + concoredocker.simtime = 0 + + result = concoredocker.read("test_zmq", "data", "[]") + + assert result == [4.0, 5.0] + assert concoredocker.simtime == 10.0 + + def test_read_updates_simtime_monotonically(self): + import concoredocker + + class DummyPort: + def recv_json_with_retry(self): + return [2.0, 99.0] + + concoredocker.zmq_ports["test_zmq"] = DummyPort() + concoredocker.simtime = 5.0 + + concoredocker.read("test_zmq", "data", "[]") + + assert concoredocker.simtime == 5.0 + + def test_write_read_roundtrip(self): + import concoredocker + + class DummyPort: + def __init__(self): + self.buffer = None + + def send_json_with_retry(self, message): + self.buffer = message + + def recv_json_with_retry(self): + return self.buffer + + dummy = DummyPort() + concoredocker.zmq_ports["roundtrip"] = dummy + concoredocker.simtime = 0 + + original = [1.5, 2.5, 3.5] + concoredocker.write("roundtrip", "data", original) + result = concoredocker.read("roundtrip", "data", "[]") + + assert result == original From 9fe3608a34259bc95b99f18d1bcdfebccdfd1410 Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Wed, 18 Feb 2026 17:27:22 +0530 Subject: [PATCH 164/275] security: protect /openJupyter endpoint with API key and process control (fixes #361) --- fri/server/main.py | 56 +++++++++-- tests/test_openjupyter_security.py | 152 +++++++++++++++++++++++++++++ 2 files changed, 199 insertions(+), 9 deletions(-) create mode 100644 tests/test_openjupyter_security.py diff --git a/fri/server/main.py b/fri/server/main.py index c2e1e659..3f15a3fd 100644 --- a/fri/server/main.py +++ b/fri/server/main.py @@ -84,6 +84,20 @@ def get_error_output(e): cur_path = os.path.dirname(os.path.abspath(__file__)) concore_path = os.path.abspath(os.path.join(cur_path, '../../')) +# API key authentication for sensitive endpoints +API_KEY = os.environ.get("CONCORE_API_KEY") + +def require_api_key(): + """Require a valid API key via X-API-KEY header.""" + if not API_KEY: + abort(500, description="Server not configured with API key") + provided = request.headers.get("X-API-KEY") + if provided != API_KEY: + abort(403, description="Unauthorized") + +# Track single Jupyter process to prevent multiple concurrent launches +jupyter_process = None + app = Flask(__name__) secret_key = os.environ.get("FLASK_SECRET_KEY") @@ -536,15 +550,39 @@ def getFilesList(dir): @app.route('/openJupyter/', methods=['POST']) def openJupyter(): - proc = subprocess.Popen(['jupyter', 'lab'], shell=False, stdout=subprocess.PIPE, cwd=concore_path) - if proc.poll() is None: - resp = jsonify({'message': 'Successfuly opened Jupyter'}) - resp.status_code = 308 - return resp - else: - resp = jsonify({'message': 'There is an Error'}) - resp.status_code = 500 - return resp + global jupyter_process + + require_api_key() + + if jupyter_process and jupyter_process.poll() is None: + return jsonify({"message": "Jupyter already running"}), 409 + + try: + jupyter_process = subprocess.Popen( + ['jupyter', 'lab', '--no-browser'], + shell=False, + cwd=concore_path, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL + ) + return jsonify({"message": "Jupyter Lab started"}), 200 + except Exception: + return jsonify({"error": "Failed to start Jupyter"}), 500 + + +@app.route('/stopJupyter/', methods=['POST']) +def stopJupyter(): + global jupyter_process + + require_api_key() + + if not jupyter_process or jupyter_process.poll() is not None: + return jsonify({"message": "No running Jupyter instance"}), 404 + + jupyter_process.terminate() + jupyter_process = None + + return jsonify({"message": "Jupyter stopped"}), 200 if __name__ == "__main__": diff --git a/tests/test_openjupyter_security.py b/tests/test_openjupyter_security.py new file mode 100644 index 00000000..204f5b98 --- /dev/null +++ b/tests/test_openjupyter_security.py @@ -0,0 +1,152 @@ +"""Tests for the secured /openJupyter/ and /stopJupyter/ endpoints.""" +import os +import sys +import pytest +from unittest.mock import patch, MagicMock + +# Ensure the project root is on the path +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +# Set a test API key before importing the app module +TEST_API_KEY = "test-secret-key-12345" + + +@pytest.fixture(autouse=True) +def reset_jupyter_process(): + """Reset the module-level jupyter_process before each test.""" + import fri.server.main as mod + mod.jupyter_process = None + yield + mod.jupyter_process = None + + +@pytest.fixture +def client(): + """Create a Flask test client with the API key configured.""" + with patch.dict(os.environ, {"CONCORE_API_KEY": TEST_API_KEY}): + # Re-read env var after patching + import fri.server.main as mod + mod.API_KEY = TEST_API_KEY + mod.app.config["TESTING"] = True + with mod.app.test_client() as c: + yield c + + +@pytest.fixture +def client_no_key(): + """Create a Flask test client without API key configured.""" + import fri.server.main as mod + mod.API_KEY = None + mod.app.config["TESTING"] = True + with mod.app.test_client() as c: + yield c + + +class TestOpenJupyterAuth: + """Test authentication on /openJupyter/ endpoint.""" + + def test_missing_api_key_header_returns_403(self, client): + """Request without X-API-KEY header should be rejected.""" + resp = client.post("/openJupyter/") + assert resp.status_code == 403 + + def test_wrong_api_key_returns_403(self, client): + """Request with wrong key should be rejected.""" + resp = client.post("/openJupyter/", headers={"X-API-KEY": "wrong-key"}) + assert resp.status_code == 403 + + def test_server_without_api_key_configured_returns_500(self, client_no_key): + """If CONCORE_API_KEY is not set on server, return 500.""" + resp = client_no_key.post( + "/openJupyter/", headers={"X-API-KEY": "anything"} + ) + assert resp.status_code == 500 + + +class TestOpenJupyterProcess: + """Test process control on /openJupyter/ endpoint.""" + + @patch("fri.server.main.subprocess.Popen") + def test_authorized_request_starts_jupyter(self, mock_popen, client): + """Valid API key should start Jupyter Lab.""" + mock_proc = MagicMock() + mock_proc.poll.return_value = None # process running + mock_popen.return_value = mock_proc + + resp = client.post( + "/openJupyter/", headers={"X-API-KEY": TEST_API_KEY} + ) + assert resp.status_code == 200 + data = resp.get_json() + assert data["message"] == "Jupyter Lab started" + + # Verify Popen was called with --no-browser and DEVNULL + call_args = mock_popen.call_args + assert "--no-browser" in call_args[0][0] + assert call_args[1].get("shell") is False + + @patch("fri.server.main.subprocess.Popen") + def test_duplicate_launch_returns_409(self, mock_popen, client): + """Second launch while first is still running should return 409.""" + mock_proc = MagicMock() + mock_proc.poll.return_value = None # still running + mock_popen.return_value = mock_proc + + # First launch + resp1 = client.post( + "/openJupyter/", headers={"X-API-KEY": TEST_API_KEY} + ) + assert resp1.status_code == 200 + + # Second launch should be rejected + resp2 = client.post( + "/openJupyter/", headers={"X-API-KEY": TEST_API_KEY} + ) + assert resp2.status_code == 409 + data = resp2.get_json() + assert data["message"] == "Jupyter already running" + + @patch("fri.server.main.subprocess.Popen", side_effect=Exception("fail")) + def test_popen_failure_returns_500(self, mock_popen, client): + """If Popen raises, return 500.""" + resp = client.post( + "/openJupyter/", headers={"X-API-KEY": TEST_API_KEY} + ) + assert resp.status_code == 500 + data = resp.get_json() + assert "error" in data + + +class TestStopJupyter: + """Test /stopJupyter/ endpoint.""" + + def test_stop_without_auth_returns_403(self, client): + """Request without API key should be rejected.""" + resp = client.post("/stopJupyter/") + assert resp.status_code == 403 + + def test_stop_when_no_process_returns_404(self, client): + """Stop with no running process returns 404.""" + resp = client.post( + "/stopJupyter/", headers={"X-API-KEY": TEST_API_KEY} + ) + assert resp.status_code == 404 + + @patch("fri.server.main.subprocess.Popen") + def test_stop_running_process_returns_200(self, mock_popen, client): + """Stop a running Jupyter instance returns 200.""" + mock_proc = MagicMock() + mock_proc.poll.return_value = None # running + mock_popen.return_value = mock_proc + + # Start first + client.post("/openJupyter/", headers={"X-API-KEY": TEST_API_KEY}) + + # Stop + resp = client.post( + "/stopJupyter/", headers={"X-API-KEY": TEST_API_KEY} + ) + assert resp.status_code == 200 + data = resp.get_json() + assert data["message"] == "Jupyter stopped" + mock_proc.terminate.assert_called_once() From 5bf69a253ded4de80949e8bcca880f72539a66fc Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Wed, 18 Feb 2026 17:40:23 +0530 Subject: [PATCH 165/275] test: skip openJupyter security tests when flask is not installed in CI --- tests/test_openjupyter_security.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_openjupyter_security.py b/tests/test_openjupyter_security.py index 204f5b98..d3a1023c 100644 --- a/tests/test_openjupyter_security.py +++ b/tests/test_openjupyter_security.py @@ -7,6 +7,9 @@ # Ensure the project root is on the path sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +# Skip entire module if flask is not installed (e.g. in CI with minimal deps) +pytest.importorskip("flask", reason="flask not installed — skipping server endpoint tests") + # Set a test API key before importing the app module TEST_API_KEY = "test-secret-key-12345" From 51e70762212d1a2a7109fc525880b0d3886033eb Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Wed, 18 Feb 2026 17:48:47 +0530 Subject: [PATCH 166/275] security: address PR review - timing-safe comparison, thread lock, specific exceptions, graceful termination --- fri/server/main.py | 59 +++++++++++++++++++----------- tests/test_openjupyter_security.py | 3 +- 2 files changed, 40 insertions(+), 22 deletions(-) diff --git a/fri/server/main.py b/fri/server/main.py index 3f15a3fd..2f0fdb2b 100644 --- a/fri/server/main.py +++ b/fri/server/main.py @@ -6,10 +6,15 @@ from subprocess import call,check_output from pathlib import Path import json +import logging import platform import re +import secrets +import threading from flask_cors import CORS, cross_origin +logger = logging.getLogger(__name__) + # Input validation pattern for safe names (alphanumeric, dash, underscore, slash, dot, space) SAFE_INPUT_PATTERN = re.compile(r'^[a-zA-Z0-9_\-/. ]+$') # Pattern for filenames - no path separators or .. allowed @@ -91,12 +96,13 @@ def require_api_key(): """Require a valid API key via X-API-KEY header.""" if not API_KEY: abort(500, description="Server not configured with API key") - provided = request.headers.get("X-API-KEY") - if provided != API_KEY: + provided = request.headers.get("X-API-KEY") or "" + if not secrets.compare_digest(provided, API_KEY): abort(403, description="Unauthorized") # Track single Jupyter process to prevent multiple concurrent launches jupyter_process = None +jupyter_lock = threading.Lock() app = Flask(__name__) @@ -554,20 +560,22 @@ def openJupyter(): require_api_key() - if jupyter_process and jupyter_process.poll() is None: - return jsonify({"message": "Jupyter already running"}), 409 + with jupyter_lock: + if jupyter_process and jupyter_process.poll() is None: + return jsonify({"message": "Jupyter already running"}), 409 - try: - jupyter_process = subprocess.Popen( - ['jupyter', 'lab', '--no-browser'], - shell=False, - cwd=concore_path, - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL - ) - return jsonify({"message": "Jupyter Lab started"}), 200 - except Exception: - return jsonify({"error": "Failed to start Jupyter"}), 500 + try: + jupyter_process = subprocess.Popen( + ['jupyter', 'lab', '--no-browser'], + shell=False, + cwd=concore_path, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL + ) + return jsonify({"message": "Jupyter Lab started"}), 200 + except (FileNotFoundError, PermissionError, OSError) as e: + logger.error("Failed to start Jupyter: %s", e) + return jsonify({"error": "Failed to start Jupyter"}), 500 @app.route('/stopJupyter/', methods=['POST']) @@ -576,13 +584,22 @@ def stopJupyter(): require_api_key() - if not jupyter_process or jupyter_process.poll() is not None: - return jsonify({"message": "No running Jupyter instance"}), 404 - - jupyter_process.terminate() - jupyter_process = None + with jupyter_lock: + if not jupyter_process or jupyter_process.poll() is not None: + return jsonify({"message": "No running Jupyter instance"}), 404 - return jsonify({"message": "Jupyter stopped"}), 200 + try: + jupyter_process.terminate() + # Wait for Jupyter to terminate gracefully + jupyter_process.wait(timeout=5) + except subprocess.TimeoutExpired: + # Force kill if it did not exit in time + jupyter_process.kill() + jupyter_process.wait(timeout=5) + finally: + jupyter_process = None + + return jsonify({"message": "Jupyter stopped"}), 200 if __name__ == "__main__": diff --git a/tests/test_openjupyter_security.py b/tests/test_openjupyter_security.py index d3a1023c..230a4819 100644 --- a/tests/test_openjupyter_security.py +++ b/tests/test_openjupyter_security.py @@ -109,7 +109,7 @@ def test_duplicate_launch_returns_409(self, mock_popen, client): data = resp2.get_json() assert data["message"] == "Jupyter already running" - @patch("fri.server.main.subprocess.Popen", side_effect=Exception("fail")) + @patch("fri.server.main.subprocess.Popen", side_effect=OSError("fail")) def test_popen_failure_returns_500(self, mock_popen, client): """If Popen raises, return 500.""" resp = client.post( @@ -153,3 +153,4 @@ def test_stop_running_process_returns_200(self, mock_popen, client): data = resp.get_json() assert data["message"] == "Jupyter stopped" mock_proc.terminate.assert_called_once() + mock_proc.wait.assert_called() From 1bfd16c57d6e4c1fc3e7197c04909bcd01e1cbb3 Mon Sep 17 00:00:00 2001 From: Avinash Kumar Deepak Date: Wed, 18 Feb 2026 19:01:31 +0530 Subject: [PATCH 167/275] use named logger, drop force=True --- concore.py | 67 ++++++++++++++++++++++++++---------------------------- 1 file changed, 32 insertions(+), 35 deletions(-) diff --git a/concore.py b/concore.py index cb97ab61..3762bbff 100644 --- a/concore.py +++ b/concore.py @@ -9,11 +9,8 @@ import numpy as np import signal -logging.basicConfig( - level=logging.INFO, - format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', - force=True -) +logger = logging.getLogger('concore') +logger.addHandler(logging.NullHandler()) #these lines mute the noisy library logging.getLogger('matplotlib').setLevel(logging.WARNING) @@ -53,10 +50,10 @@ def __init__(self, port_type, address, zmq_socket_type): # Bind or connect if self.port_type == "bind": self.socket.bind(address) - logging.info(f"ZMQ Port bound to {address}") + logger.info(f"ZMQ Port bound to {address}") else: self.socket.connect(address) - logging.info(f"ZMQ Port connected to {address}") + logger.info(f"ZMQ Port connected to {address}") def send_json_with_retry(self, message): """Send JSON message with retries if timeout occurs.""" @@ -65,9 +62,9 @@ def send_json_with_retry(self, message): self.socket.send_json(message) return except zmq.Again: - logging.warning(f"Send timeout (attempt {attempt + 1}/5)") + logger.warning(f"Send timeout (attempt {attempt + 1}/5)") time.sleep(0.5) - logging.error("Failed to send after retries.") + logger.error("Failed to send after retries.") return def recv_json_with_retry(self): @@ -76,9 +73,9 @@ def recv_json_with_retry(self): try: return self.socket.recv_json() except zmq.Again: - logging.warning(f"Receive timeout (attempt {attempt + 1}/5)") + logger.warning(f"Receive timeout (attempt {attempt + 1}/5)") time.sleep(0.5) - logging.error("Failed to receive after retries.") + logger.error("Failed to receive after retries.") return None # Global ZeroMQ ports registry @@ -94,20 +91,20 @@ def init_zmq_port(port_name, port_type, address, socket_type_str): socket_type_str (str): String representation of ZMQ socket type (e.g., "REQ", "REP", "PUB", "SUB"). """ if port_name in zmq_ports: - logging.info(f"ZMQ Port {port_name} already initialized.") + logger.info(f"ZMQ Port {port_name} already initialized.") return # Avoid reinitialization try: # Map socket type string to actual ZMQ constant (e.g., zmq.REQ, zmq.REP) zmq_socket_type = getattr(zmq, socket_type_str.upper()) zmq_ports[port_name] = ZeroMQPort(port_type, address, zmq_socket_type) - logging.info(f"Initialized ZMQ port: {port_name} ({socket_type_str}) on {address}") + logger.info(f"Initialized ZMQ port: {port_name} ({socket_type_str}) on {address}") except AttributeError: - logging.error(f"Error: Invalid ZMQ socket type string '{socket_type_str}'.") + logger.error(f"Error: Invalid ZMQ socket type string '{socket_type_str}'.") except zmq.error.ZMQError as e: - logging.error(f"Error initializing ZMQ port {port_name} on {address}: {e}") + logger.error(f"Error initializing ZMQ port {port_name} on {address}: {e}") except Exception as e: - logging.error(f"An unexpected error occurred during ZMQ port initialization for {port_name}: {e}") + logger.error(f"An unexpected error occurred during ZMQ port initialization for {port_name}: {e}") def terminate_zmq(): """Clean up all ZMQ sockets and contexts before exit.""" @@ -127,7 +124,7 @@ def terminate_zmq(): port.context.term() print(f"Closed ZMQ port: {port_name}") except Exception as e: - logging.error(f"Error while terminating ZMQ port {port.address}: {e}") + logger.error(f"Error while terminating ZMQ port {port.address}: {e}") zmq_ports.clear() _cleanup_in_progress = False @@ -242,9 +239,9 @@ def parse_params(sparams: str) -> dict: sparams = sparams[1:-1] # Parse params using clean function instead of regex - logging.debug("parsing sparams: "+sparams) + logger.debug("parsing sparams: "+sparams) params = parse_params(sparams) - logging.debug("parsed params: " + str(params)) + logger.debug("parsed params: " + str(params)) else: params = dict() else: @@ -306,17 +303,17 @@ def read(port_identifier, name, initstr_val): return message[1:] return message except zmq.error.ZMQError as e: - logging.error(f"ZMQ read error on port {port_identifier} (name: {name}): {e}. Returning default.") + logger.error(f"ZMQ read error on port {port_identifier} (name: {name}): {e}. Returning default.") return default_return_val except Exception as e: - logging.error(f"Unexpected error during ZMQ read on port {port_identifier} (name: {name}): {e}. Returning default.") + logger.error(f"Unexpected error during ZMQ read on port {port_identifier} (name: {name}): {e}. Returning default.") return default_return_val # Case 2: File-based port try: file_port_num = int(port_identifier) except ValueError: - logging.error(f"Error: Invalid port identifier '{port_identifier}' for file operation. Must be integer or ZMQ name.") + logger.error(f"Error: Invalid port identifier '{port_identifier}' for file operation. Must be integer or ZMQ name.") return default_return_val time.sleep(delay) @@ -330,7 +327,7 @@ def read(port_identifier, name, initstr_val): ins = str(initstr_val) s += ins # Update s to break unchanged() loop except Exception as e: - logging.error(f"Error reading {file_path}: {e}. Using default value.") + logger.error(f"Error reading {file_path}: {e}. Using default value.") return default_return_val # Retry logic if file is empty @@ -342,12 +339,12 @@ def read(port_identifier, name, initstr_val): with open(file_path, "r") as infile: ins = infile.read() except Exception as e: - logging.warning(f"Retry {attempts + 1}: Error reading {file_path} - {e}") + logger.warning(f"Retry {attempts + 1}: Error reading {file_path} - {e}") attempts += 1 retrycount += 1 if len(ins) == 0: - logging.error(f"Max retries reached for {file_path}, using default value.") + logger.error(f"Max retries reached for {file_path}, using default value.") return default_return_val s += ins @@ -361,10 +358,10 @@ def read(port_identifier, name, initstr_val): simtime = max(simtime, current_simtime_from_file) return inval[1:] else: - logging.warning(f"Warning: Unexpected data format in {file_path}: {ins}. Returning raw content or default.") + logger.warning(f"Warning: Unexpected data format in {file_path}: {ins}. Returning raw content or default.") return inval except Exception as e: - logging.error(f"Error parsing content from {file_path} ('{ins}'): {e}. Returning default.") + logger.error(f"Error parsing content from {file_path} ('{ins}'): {e}. Returning default.") return default_return_val @@ -389,9 +386,9 @@ def write(port_identifier, name, val, delta=0): else: zmq_p.send_json_with_retry(zmq_val) except zmq.error.ZMQError as e: - logging.error(f"ZMQ write error on port {port_identifier} (name: {name}): {e}") + logger.error(f"ZMQ write error on port {port_identifier} (name: {name}): {e}") except Exception as e: - logging.error(f"Unexpected error during ZMQ write on port {port_identifier} (name: {name}): {e}") + logger.error(f"Unexpected error during ZMQ write on port {port_identifier} (name: {name}): {e}") return # Case 2: File-based port @@ -399,14 +396,14 @@ def write(port_identifier, name, val, delta=0): file_port_num = int(port_identifier) file_path = os.path.join(outpath + str(file_port_num), name) except ValueError: - logging.error(f"Error: Invalid port identifier '{port_identifier}' for file operation. Must be integer or ZMQ name.") + logger.error(f"Error: Invalid port identifier '{port_identifier}' for file operation. Must be integer or ZMQ name.") return # File writing rules if isinstance(val, str): time.sleep(2 * delay) # string writes wait longer elif not isinstance(val, list): - logging.error(f"File write to {file_path} must have list or str value, got {type(val)}") + logger.error(f"File write to {file_path} must have list or str value, got {type(val)}") return try: @@ -420,7 +417,7 @@ def write(port_identifier, name, val, delta=0): else: outfile.write(val) except Exception as e: - logging.error(f"Error writing to {file_path}: {e}") + logger.error(f"Error writing to {file_path}: {e}") def initval(simtime_val_str): """ @@ -436,12 +433,12 @@ def initval(simtime_val_str): simtime = first_element return val[1:] else: - logging.error(f"Error: First element in initval string '{simtime_val_str}' is not a number. Using data part as is or empty.") + logger.error(f"Error: First element in initval string '{simtime_val_str}' is not a number. Using data part as is or empty.") return val[1:] if len(val) > 1 else [] else: - logging.error(f"Error: initval string '{simtime_val_str}' is not a list or is empty. Returning empty list.") + logger.error(f"Error: initval string '{simtime_val_str}' is not a list or is empty. Returning empty list.") return [] except Exception as e: - logging.error(f"Error parsing simtime_val_str '{simtime_val_str}': {e}. Returning empty list.") + logger.error(f"Error parsing simtime_val_str '{simtime_val_str}': {e}. Returning empty list.") return [] From 586a7c2cc1dee147c2a360b711a72feb0da9d0b4 Mon Sep 17 00:00:00 2001 From: Avinash Kumar Deepak Date: Wed, 18 Feb 2026 21:51:47 +0530 Subject: [PATCH 168/275] fix maxtime as double in concoredocker.java, fixes --- concoredocker.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/concoredocker.java b/concoredocker.java index 053a8ff2..c987d278 100644 --- a/concoredocker.java +++ b/concoredocker.java @@ -26,7 +26,7 @@ public class concoredocker { private static Map params = new HashMap<>(); // simtime as double to preserve fractional values (e.g. "[0.0, ...]") private static double simtime = 0; - private static int maxtime; + private static double maxtime; public static void main(String[] args) { try { @@ -109,12 +109,12 @@ private static Map parseFile(String filename) throws IOException * Sets maxtime from concore.maxtime file, or uses defaultValue if file not found. * Catches both IOException and RuntimeException to match Python safe_literal_eval. */ - private static void defaultMaxTime(int defaultValue) { + private static void defaultMaxTime(double defaultValue) { try { String content = new String(Files.readAllBytes(Paths.get(inpath + "1/concore.maxtime"))); Object parsed = literalEval(content.trim()); if (parsed instanceof Number) { - maxtime = ((Number) parsed).intValue(); + maxtime = ((Number) parsed).doubleValue(); } else { maxtime = defaultValue; } From 5354b99cb16e780f3957c87841cfc72b642c84b2 Mon Sep 17 00:00:00 2001 From: Titas-Ghosh Date: Thu, 19 Feb 2026 01:20:59 +0530 Subject: [PATCH 169/275] Fix mkconcore path handling for nested output directories --- mkconcore.py | 28 +++++++++++++++------------- tests/test_cli.py | 15 +++++++++++++++ 2 files changed, 30 insertions(+), 13 deletions(-) diff --git a/mkconcore.py b/mkconcore.py index 0a119481..ed5f708a 100644 --- a/mkconcore.py +++ b/mkconcore.py @@ -142,7 +142,8 @@ def _resolve_concore_path(): print(" type must be posix (macos or ubuntu), windows, or docker") sys.exit(1) -GRAPHML_FILE = sys.argv[1] +ORIGINAL_CWD = os.getcwd() +GRAPHML_FILE = os.path.abspath(sys.argv[1]) TRIMMED_LOGS = True CONCOREPATH = _resolve_concore_path() CPPWIN = os.environ.get("CONCORE_CPPWIN", "g++") #Windows C++ 6/22/21 @@ -196,10 +197,10 @@ def _resolve_concore_path(): OCTAVEEXE = _tools.get("OCTAVEEXE", OCTAVEEXE) OCTAVEWIN = _tools.get("OCTAVEWIN", OCTAVEWIN) -prefixedgenode = "" -sourcedir = sys.argv[2] -outdir = sys.argv[3] - +prefixedgenode = "" +sourcedir = os.path.abspath(sys.argv[2]) +outdir = os.path.abspath(sys.argv[3]) + # Validate outdir argument (allow full paths) safe_name(outdir, "Output directory argument", allow_path=True) @@ -207,10 +208,11 @@ def _resolve_concore_path(): logging.error(f"{sourcedir} does not exist") quit() -if len(sys.argv) == 4: - prefixedgenode = outdir+"_" #nodes and edges prefixed with outdir_ only in case no type specified 3/24/21 - concoretype = "docker" -else: +if len(sys.argv) == 4: + # Use only the output directory name in generated prefixes. + prefixedgenode = os.path.basename(os.path.normpath(outdir)) + "_" + concoretype = "docker" +else: concoretype = sys.argv[4] if not (concoretype in ["posix","windows","docker","macos","ubuntu"]): logging.error(" type must be posix (macos or ubuntu), windows, or docker") @@ -227,8 +229,8 @@ def _resolve_concore_path(): logging.error(f"if intended, Remove/Rename {outdir} first") quit() -os.mkdir(outdir) -os.chdir(outdir) +os.makedirs(outdir) +os.chdir(outdir) if concoretype == "windows": fbuild = open("build.bat","w") frun = open("run.bat", "w") @@ -255,8 +257,8 @@ def cleanup_script_files(): fh.close() atexit.register(cleanup_script_files) -os.mkdir("src") -os.chdir("..") +os.mkdir("src") +os.chdir(ORIGINAL_CWD) logging.info(f"mkconcore {MKCONCORE_VER}") logging.info(f"Concore path: {CONCOREPATH}") diff --git a/tests/test_cli.py b/tests/test_cli.py index f852d5c1..19dae726 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -114,6 +114,21 @@ def test_run_command_default_type(self): else: self.assertTrue(Path('out/build').exists()) + def test_run_command_nested_output_path(self): + with self.runner.isolated_filesystem(temp_dir=self.temp_dir): + result = self.runner.invoke(cli, ['init', 'test-project']) + self.assertEqual(result.exit_code, 0) + + result = self.runner.invoke(cli, [ + 'run', + 'test-project/workflow.graphml', + '--source', 'test-project/src', + '--output', 'build/out', + '--type', 'posix' + ]) + self.assertEqual(result.exit_code, 0) + self.assertTrue(Path('build/out/src/concore.py').exists()) + def test_run_command_subdir_source(self): with self.runner.isolated_filesystem(temp_dir=self.temp_dir): result = self.runner.invoke(cli, ['init', 'test-project']) From 0487c24a17b504c864ef0452bc59a0df39a54eed Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Thu, 19 Feb 2026 01:59:52 +0530 Subject: [PATCH 170/275] security: remove hardcoded Flask secret key and load from environment (fixes #362) --- README.md | 10 ++++++++++ fri/server/main.py | 23 ++++++++++++++--------- 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 37305503..ba7e8064 100644 --- a/README.md +++ b/README.md @@ -77,6 +77,16 @@ _concore_ supports customization through configuration files in the `CONCOREPATH Tool paths can also be set via environment variables (e.g., `CONCORE_CPPEXE=/usr/bin/g++`). Priority: config file > env var > defaults. +### Security Configuration + +Set a secure secret key for the Flask server before running in production: + +```bash +export FLASK_SECRET_KEY=$(python -c "import secrets; print(secrets.token_hex(32))") +``` + +Do **NOT** commit your secret key to version control. If `FLASK_SECRET_KEY` is not set, a temporary random key will be generated automatically (suitable for local development only). + For a detailed and more scientific documentation, please read our extensive [open-access research paper on CONTROL-CORE](https://doi.org/10.1109/ACCESS.2022.3161471). This paper has a complete discussion on the CONTROL-CORE architecture and deployment, together with the commands to execute the studies in different programming languages and programming environments (Ubuntu, Windows, MacOS, Docker, and distributed execution). diff --git a/fri/server/main.py b/fri/server/main.py index c2e1e659..243d562f 100644 --- a/fri/server/main.py +++ b/fri/server/main.py @@ -2,6 +2,7 @@ from werkzeug.utils import secure_filename import xml.etree.ElementTree as ET import os +import secrets import subprocess from subprocess import call,check_output from pathlib import Path @@ -86,15 +87,19 @@ def get_error_output(e): app = Flask(__name__) -secret_key = os.environ.get("FLASK_SECRET_KEY") -if not secret_key: - # In production, require an explicit FLASK_SECRET_KEY to be set. - # For local development and tests, fall back to a per-process random key - # so that importing this module does not fail hard. - if os.environ.get("FLASK_ENV") == "production": - raise RuntimeError("FLASK_SECRET_KEY environment variable not set in production") - secret_key = os.urandom(32) -app.secret_key = secret_key +app.secret_key = os.getenv("FLASK_SECRET_KEY") + +if not app.secret_key: + # In production, require an explicit secret key to avoid session issues + flask_env = os.getenv("FLASK_ENV", "").lower() + if flask_env in ("development", "dev") or app.debug: + # Generate temporary key for development environments where a secret key + # has not been explicitly configured. + app.secret_key = secrets.token_hex(32) + else: + raise RuntimeError( + "FLASK_SECRET_KEY environment variable must be set in production." + ) cors = CORS(app) app.config['CORS_HEADERS'] = 'Content-Type' From edf0c3a37424a4e94d724dc88d7899f621a5a273 Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Thu, 19 Feb 2026 02:21:05 +0530 Subject: [PATCH 171/275] refactor: remove duplicated logic in build() endpoint and dynamically construct command (fixes #363) --- fri/server/main.py | 94 ++++++++++++---------------------------------- 1 file changed, 25 insertions(+), 69 deletions(-) diff --git a/fri/server/main.py b/fri/server/main.py index c2e1e659..aec9792f 100644 --- a/fri/server/main.py +++ b/fri/server/main.py @@ -190,76 +190,32 @@ def build(dir): if not os.path.exists(dir_path): - if(platform.uname()[0]=='Windows'): - if(out_dir == None or out_dir == ""): - if(docker == 'true'): - try: - output_bytes = subprocess.check_output(["makedocker", makestudy_dir], cwd=concore_path, shell=True) - output_str = output_bytes.decode("utf-8") - proc = 0 - except subprocess.CalledProcessError as e: - output_str = f"Docker study creation failed with return code {e.returncode} (check duplicate directory)" - proc = 1 - else: - try: - output_bytes = subprocess.check_output(["makestudy", makestudy_dir], cwd=concore_path, shell=True) - output_str = output_bytes.decode("utf-8") - proc = 0 - except subprocess.CalledProcessError as e: - output_str = f"Study creation failed with return code {e.returncode} (check duplicate directory)" - proc = 1 - else: - if(docker == 'true'): - try: - output_bytes = subprocess.check_output(["makedocker", makestudy_dir, out_dir], cwd=concore_path, shell=True) - output_str = output_bytes.decode("utf-8") - proc = 0 - except subprocess.CalledProcessError as e: - output_str = f"Docker study creation failed with return code {e.returncode} (check duplicate directory)" - proc = 1 - else: - try: - output_bytes = subprocess.check_output(["makestudy", makestudy_dir, out_dir], cwd=concore_path, shell=True) - output_str = output_bytes.decode("utf-8") - proc = 0 - except subprocess.CalledProcessError as e: - output_str = f"Study creation failed with return code {e.returncode} (check duplicate directory)" - proc = 1 + # Determine command name and error label based on docker flag + if docker == 'true': + cmd_name = "makedocker" + error_label = "Docker study" else: - if(out_dir == None or out_dir == ""): - if(docker == 'true'): - try: - output_bytes = subprocess.check_output([r"./makedocker", makestudy_dir], cwd=concore_path) - output_str = output_bytes.decode("utf-8") - proc = 0 - except subprocess.CalledProcessError as e: - output_str = f"Docker study creation failed with return code {e.returncode} (check duplicate directory)" - proc = 1 - else: - try: - output_bytes = subprocess.check_output([r"./makestudy", makestudy_dir], cwd=concore_path) - output_str = output_bytes.decode("utf-8") - proc = 0 - except subprocess.CalledProcessError as e: - output_str = f"Study creation failed with return code {e.returncode} (check duplicate directory)" - proc = 1 - else: - if(docker == 'true'): - try: - output_bytes = subprocess.check_output([r"./makedocker", makestudy_dir, out_dir], cwd=concore_path) - output_str = output_bytes.decode("utf-8") - proc = 0 - except subprocess.CalledProcessError as e: - output_str = f"Docker study creation failed with return code {e.returncode} (check duplicate directory)" - proc = 1 - else: - try: - output_bytes = subprocess.check_output([r"./makestudy", makestudy_dir, out_dir], cwd=concore_path) - output_str = output_bytes.decode("utf-8") - proc = 0 - except subprocess.CalledProcessError as e: - output_str = f"Study creation failed with return code {e.returncode} (check duplicate directory)" - proc = 1 + cmd_name = "makestudy" + error_label = "Study" + + # Build command list + cmd = [cmd_name, makestudy_dir] + if out_dir != None and out_dir != "": + cmd.append(out_dir) + + # OS-specific adjustments + is_windows = platform.uname()[0] == 'Windows' + if not is_windows: + cmd[0] = f"./{cmd[0]}" + + try: + output_bytes = subprocess.check_output(cmd, cwd=concore_path, shell=is_windows) + output_str = output_bytes.decode("utf-8") + proc = 0 + except subprocess.CalledProcessError as e: + output_str = f"{error_label} creation failed with return code {e.returncode} (check duplicate directory)" + proc = 1 + if(proc == 0): resp = jsonify({'message': 'Directory successfully created'}) resp.status_code = 201 From 84aeabab6e46d4054644b956d0fe11a8cba9ee54 Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Thu, 19 Feb 2026 02:35:16 +0530 Subject: [PATCH 172/275] refactor: replace verbose try/except param handling in pidsig.py with concore.tryparam() (fixes #365) --- tools/pidsig.py | 35 +++++++---------------------------- 1 file changed, 7 insertions(+), 28 deletions(-) diff --git a/tools/pidsig.py b/tools/pidsig.py index 42b3dda8..1a921af1 100644 --- a/tools/pidsig.py +++ b/tools/pidsig.py @@ -7,34 +7,13 @@ Prev_Error = 0 I = 0 -try: - sp = concore.params['sp'] -except: - sp = 67.5 -try: - Kp = concore.params['Kp'] -except: - Kp = 0.1 -try: - Ki = concore.params['Ki'] -except: - Ki = 0.01 -try: - Kd = concore.params['Kd'] -except: - Kd = 0.03 -try: - freq = concore.params['freq'] -except: - freq = 30 -try: - sigout = concore.params['sigout'] -except: - sigout = True -try: - cin = concore.params['cin'] -except: - cin = 'hr' +sp = concore.tryparam('sp', 67.5) +Kp = concore.tryparam('Kp', 0.1) +Ki = concore.tryparam('Ki', 0.01) +Kd = concore.tryparam('Kd', 0.03) +freq = concore.tryparam('freq', 30) +sigout = concore.tryparam('sigout', True) +cin = concore.tryparam('cin', 'hr') def pid_controller(ym): global Prev_Error, I, freq From 1d82977f1a51d99080836a3da4a54e6afee872bc Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Thu, 19 Feb 2026 02:46:46 +0530 Subject: [PATCH 173/275] fix: use computed PID frequency in pid2.py instead of hardcoded value (fixes #366) --- tools/pid2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/pid2.py b/tools/pid2.py index 1c720f85..225e87c0 100644 --- a/tools/pid2.py +++ b/tools/pid2.py @@ -44,7 +44,7 @@ def pid_controller(ym): if freq<10: freq = 10 Prev_ErrorF = ErrorF - ustar = np.array([amp,30]) + ustar = np.array([amp,freq]) return ustar From 176cc9411946279519be3e4152a11766282ed14a Mon Sep 17 00:00:00 2001 From: Titas-Ghosh Date: Thu, 19 Feb 2026 03:02:21 +0530 Subject: [PATCH 174/275] Fix ZMQ specialization collisions for shared source scripts --- copy_with_port_portname.py | 57 +++++++++++++++------- mkconcore.py | 97 +++++++++++++++++++++++++------------- tests/test_cli.py | 41 ++++++++++++++++ 3 files changed, 143 insertions(+), 52 deletions(-) diff --git a/copy_with_port_portname.py b/copy_with_port_portname.py index 1a0a033a..1398092b 100644 --- a/copy_with_port_portname.py +++ b/copy_with_port_portname.py @@ -5,21 +5,40 @@ import logging import json -def run_specialization_script(template_script_path, output_dir, edge_params_list, python_exe, copy_script_path): +def _normalize_output_relpath(template_script_path, output_relpath=None): + if output_relpath: + relpath = output_relpath.replace("\\", "/").lstrip("/") + else: + relpath = os.path.basename(template_script_path) + if not relpath: + raise ValueError("Output relative path cannot be empty.") + return relpath + + +def _join_output_path(output_dir, output_relpath): + return os.path.join(output_dir, *output_relpath.split("/")) + + +def run_specialization_script( + template_script_path, + output_dir, + edge_params_list, + python_exe, + copy_script_path, + output_relpath=None +): """ Calls the copy script to generate a specialized version of a node's script. Returns the basename of the generated script on success, None on failure. """ - # The new copy script generates a standardized filename, e.g., "original.py" base_template_name = os.path.basename(template_script_path) - template_root, template_ext = os.path.splitext(base_template_name) - output_filename = f"{template_root}{template_ext}" - expected_output_path = os.path.join(output_dir, output_filename) + output_relpath = _normalize_output_relpath(template_script_path, output_relpath) + expected_output_path = _join_output_path(output_dir, output_relpath) # If the specialized file already exists, we don't need to regenerate it. if os.path.exists(expected_output_path): logging.info(f"Specialized script '{expected_output_path}' already exists. Using existing.") - return output_filename + return output_relpath # Convert the list of parameters to a JSON string for command line argument edge_params_json_str = json.dumps(edge_params_list) @@ -31,13 +50,15 @@ def run_specialization_script(template_script_path, output_dir, edge_params_list output_dir, edge_params_json_str # Pass the JSON string as the last argument ] + if output_relpath: + cmd.append(output_relpath) logging.info(f"Running specialization for '{base_template_name}': {' '.join(cmd)}") try: result = subprocess.run(cmd, capture_output=True, text=True, check=True, encoding='utf-8') - logging.info(f"Successfully generated specialized script '{output_filename}'.") + logging.info(f"Successfully generated specialized script '{output_relpath}'.") if result.stdout: logging.debug(f"copy_with_port_portname.py stdout:\n{result.stdout.strip()}") if result.stderr: logging.warning(f"copy_with_port_portname.py stderr:\n{result.stderr.strip()}") - return output_filename + return output_relpath except subprocess.CalledProcessError as e: logging.error(f"Error calling specialization script for '{template_script_path}':") logging.error(f"Command: {' '.join(e.cmd)}") @@ -50,7 +71,7 @@ def run_specialization_script(template_script_path, output_dir, edge_params_list return None -def create_modified_script(template_script_path, output_dir, edge_params_json_str): +def create_modified_script(template_script_path, output_dir, edge_params_json_str, output_relpath=None): """ Creates a modified Python script by injecting ZMQ port and port name definitions from a JSON object. @@ -121,17 +142,16 @@ def create_modified_script(template_script_path, output_dir, edge_params_json_st modified_lines = lines[:insert_index] + definitions + lines[insert_index:] # --- Determine and create output file --- - base_template_name = os.path.basename(template_script_path) - template_root, template_ext = os.path.splitext(base_template_name) - - # Standardized output filename for a node with one or more specializations - output_filename = f"{template_root}{template_ext}" - output_script_path = os.path.join(output_dir, output_filename) + output_relpath = _normalize_output_relpath(template_script_path, output_relpath) + output_script_path = _join_output_path(output_dir, output_relpath) try: if not os.path.exists(output_dir): os.makedirs(output_dir) print(f"Created output directory: {output_dir}") + output_parent = os.path.dirname(output_script_path) + if output_parent and not os.path.exists(output_parent): + os.makedirs(output_parent, exist_ok=True) with open(output_script_path, 'w') as f: f.writelines(modified_lines) @@ -149,8 +169,8 @@ def create_modified_script(template_script_path, output_dir, edge_params_json_st datefmt='%Y-%m-%d %H:%M:%S' ) - if len(sys.argv) != 4: - print("\nUsage: python3 copy_with_port_portname.py ''\n") + if len(sys.argv) not in [4, 5]: + print("\nUsage: python3 copy_with_port_portname.py '' [OUTPUT_RELATIVE_PATH]\n") print("Example JSON: '[{\"port\": \"2355\", \"port_name\": \"FUNBODY_REP_1\", \"source_node_label\": \"nodeA\", \"target_node_label\": \"nodeB\"}]'") print("Note: The JSON string must be enclosed in single quotes in shell.\n") sys.exit(1) @@ -158,5 +178,6 @@ def create_modified_script(template_script_path, output_dir, edge_params_json_st template_script_path_arg = sys.argv[1] output_directory_arg = sys.argv[2] json_params_arg = sys.argv[3] + output_relpath_arg = sys.argv[4] if len(sys.argv) == 5 else None - create_modified_script(template_script_path_arg, output_directory_arg, json_params_arg) \ No newline at end of file + create_modified_script(template_script_path_arg, output_directory_arg, json_params_arg, output_relpath_arg) diff --git a/mkconcore.py b/mkconcore.py index 0a119481..000d2df8 100644 --- a/mkconcore.py +++ b/mkconcore.py @@ -418,40 +418,69 @@ def cleanup_script_files(): logging.warning(f"Error processing edge for parameter aggregation: {e}") # --- Now, run the specialization for each node that has aggregated parameters --- -if node_edge_params: - logging.info("Running script specialization process...") - specialized_scripts_output_dir = os.path.abspath(os.path.join(outdir, "src")) - os.makedirs(specialized_scripts_output_dir, exist_ok=True) - - for node_id, params_list in node_edge_params.items(): - current_node_full_label = nodes_dict[node_id] - try: - container_name, original_script = current_node_full_label.split(':', 1) - except ValueError: - continue # Skip if label format is wrong - - if not original_script or "." not in original_script: - continue # Skip if not a script file - - template_script_full_path = os.path.join(sourcedir, original_script) - if not os.path.exists(template_script_full_path): - logging.error(f"Cannot specialize: Original script '{template_script_full_path}' not found in '{sourcedir}'.") - continue - - new_script_basename = copy_with_port_portname.run_specialization_script( - template_script_full_path, - specialized_scripts_output_dir, - params_list, - python_executable, - copy_script_py_path - ) - - if new_script_basename: - # Update nodes_dict to point to the new comprehensive specialized script - nodes_dict[node_id] = f"{container_name}:{new_script_basename}" - logging.info(f"Node ID '{node_id}' ('{container_name}') updated to use specialized script '{new_script_basename}'.") - else: - logging.error(f"Failed to generate specialized script for node ID '{node_id}'. It will retain its original script.") +if node_edge_params: + logging.info("Running script specialization process...") + specialized_scripts_output_dir = os.path.abspath(os.path.join(outdir, "src")) + os.makedirs(specialized_scripts_output_dir, exist_ok=True) + + # Build one specialization plan per source script. This avoids collisions + # when multiple nodes reference the same script and need different ZMQ params. + script_edge_params = {} + script_nodes = {} + for node_id, params_list in node_edge_params.items(): + current_node_full_label = nodes_dict.get(node_id, "") + try: + container_name, original_script = current_node_full_label.split(':', 1) + except ValueError: + continue + + if not original_script or "." not in original_script: + continue + + script_nodes.setdefault(original_script, []).append((node_id, container_name)) + script_edge_params.setdefault(original_script, []) + seen_keys = { + ( + p.get("port"), + p.get("port_name"), + p.get("source_node_label"), + p.get("target_node_label") + ) + for p in script_edge_params[original_script] + } + for edge_param in params_list: + edge_key = ( + edge_param.get("port"), + edge_param.get("port_name"), + edge_param.get("source_node_label"), + edge_param.get("target_node_label") + ) + if edge_key not in seen_keys: + script_edge_params[original_script].append(edge_param) + seen_keys.add(edge_key) + + for original_script, merged_params in script_edge_params.items(): + template_script_full_path = os.path.join(sourcedir, original_script) + if not os.path.exists(template_script_full_path): + logging.error(f"Cannot specialize: Original script '{template_script_full_path}' not found in '{sourcedir}'.") + continue + + new_script_relpath = copy_with_port_portname.run_specialization_script( + template_script_full_path, + specialized_scripts_output_dir, + merged_params, + python_executable, + copy_script_py_path, + output_relpath=original_script + ) + + if not new_script_relpath: + logging.error(f"Failed to generate specialized script for source '{original_script}'.") + continue + + for node_id, container_name in script_nodes.get(original_script, []): + nodes_dict[node_id] = f"{container_name}:{new_script_relpath}" + logging.info(f"Node ID '{node_id}' ('{container_name}') updated to use specialized script '{new_script_relpath}'.") #not right for PM2_1_1 and PM2_1_2 volswr = len(nodes_dict)*[''] diff --git a/tests/test_cli.py b/tests/test_cli.py index f852d5c1..a48ea4fd 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -138,6 +138,47 @@ def test_run_command_subdir_source(self): self.assertEqual(result.exit_code, 0) self.assertTrue(Path('out/src/subdir/script.py').exists()) + def test_run_command_shared_source_specialization_merges_edge_params(self): + with self.runner.isolated_filesystem(temp_dir=self.temp_dir): + Path('src').mkdir() + Path('src/common.py').write_text( + "import concore\n\n" + "def step():\n" + " return None\n" + ) + + workflow = """ + + + + + A:common.py + B:common.py + C:common.py + 0x1000_AB + 0x1001_BC + + +""" + Path('workflow.graphml').write_text(workflow) + + result = self.runner.invoke(cli, [ + 'run', + 'workflow.graphml', + '--source', 'src', + '--output', 'out', + '--type', 'posix' + ]) + self.assertEqual(result.exit_code, 0) + + specialized_script = Path('out/src/common.py') + self.assertTrue(specialized_script.exists()) + content = specialized_script.read_text() + self.assertIn('PORT_NAME_A_B', content) + self.assertIn('PORT_A_B', content) + self.assertIn('PORT_NAME_B_C', content) + self.assertIn('PORT_B_C', content) + def test_run_command_existing_output(self): with self.runner.isolated_filesystem(temp_dir=self.temp_dir): result = self.runner.invoke(cli, ['init', 'test-project']) From 08bd61b0852edacf65d2e5c28311a2be1f8fbff8 Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Thu, 19 Feb 2026 11:29:23 +0530 Subject: [PATCH 175/275] refactor: remove Fix N counter prefixes from comments per review feedback --- contribute.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/contribute.py b/contribute.py index 900a9fc4..59fba5a9 100644 --- a/contribute.py +++ b/contribute.py @@ -5,12 +5,12 @@ # Initializing the Variables BOT_TOKEN = os.environ.get('CONCORE_BOT_TOKEN', '') -# Fix 1: Fail fast if token is missing +# Fail fast if token is missing if not BOT_TOKEN: print("Error: CONCORE_BOT_TOKEN environment variable is not set.") sys.exit(1) -# Fix 2: Token format validation +# Token format validation token_pattern = r"^((ghp_|github_pat_|ghs_)[A-Za-z0-9_]{20,}|[0-9a-fA-F]{40})$" if not re.match(token_pattern, BOT_TOKEN): print("Error: Invalid GitHub token format.") @@ -34,7 +34,7 @@ def checkInputValidity(): print("Directory does not Exists.Invalid Path") exit(1) -# Fix 5: Retry + backoff wrapper for PyGithub operations +# Retry + backoff wrapper for PyGithub operations def with_retry(operation, retries=3): """Retry wrapper for PyGithub operations with exponential backoff.""" for attempt in range(retries): @@ -49,7 +49,7 @@ def with_retry(operation, retries=3): print("Error: GitHub API request failed after retries.") sys.exit(1) -# Fix 4: Correct PR URL (singular 'pull' not 'pulls') +# Correct PR URL (singular 'pull' not 'pulls') def printPR(pr): print(f'Check your example here https://github.com/{UPSTREAM_ACCOUNT}/{REPO_NAME}/pull/{pr.number}',end="") @@ -132,8 +132,6 @@ def remove_prefix(text, prefix): return text -# Fix 9: Removed unused decode_token() function - # check if directory path is Valid checkInputValidity() From 379e86d5b5101652ad143b425c804d9a2dc713a0 Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Thu, 19 Feb 2026 11:45:14 +0530 Subject: [PATCH 176/275] fix: respect DOCKEREXE configuration and remove hardcoded sudo docker (fixes #343) --- mkconcore.py | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/mkconcore.py b/mkconcore.py index 974c9bbf..eff438f8 100644 --- a/mkconcore.py +++ b/mkconcore.py @@ -158,7 +158,7 @@ def _resolve_concore_path(): OCTAVEWIN = os.environ.get("CONCORE_OCTAVEWIN", "octave") #Windows octave M_IS_OCTAVE = False #treat .m as octave MCRPATH = "~/MATLAB/R2021a" #path to local Ubunta Matlab Compiler Runtime -DOCKEREXE = "sudo docker"#assume simple docker install +DOCKEREXE = os.environ.get("DOCKEREXE", "docker")#default to docker, allow env override DOCKEREPO = "markgarnold"#where pulls come from 3/28/21 INDIRNAME = ":/in" OUTDIRNAME = ":/out" @@ -858,8 +858,8 @@ def cleanup_script_files(): fmaxtime.write('#!/bin/bash' + "\n") fmaxtime.write('echo "$1" >concore.maxtime\n') fmaxtime.write('echo "FROM alpine:3.8" > Dockerfile\n') - fmaxtime.write('sudo docker build -t docker-concore .\n') - fmaxtime.write('sudo docker run --name=concore') + fmaxtime.write(f'{DOCKEREXE} build -t docker-concore .\n') + fmaxtime.write(f'{DOCKEREXE} run --name=concore') # -v VCZ:/VCZ -v VPZ:/VPZ i=0 # 9/12/21 for node in nodes_dict: @@ -885,15 +885,15 @@ def cleanup_script_files(): dockername = sourcecode.rsplit(".", 1)[0] #3/28/21 writeedges = volswr[i] while writeedges.find(":") != -1: - fmaxtime.write('sudo docker cp concore.maxtime concore:/') + fmaxtime.write(f'{DOCKEREXE} cp concore.maxtime concore:/') # escape destination path in docker cp vol_path = writeedges.split(":")[0].split("-v ")[1].strip() fmaxtime.write(shlex.quote(vol_path+"/concore.maxtime")+"\n") writeedges = writeedges[writeedges.find(":")+1:] i=i+1 - fmaxtime.write('sudo docker stop concore \n') - fmaxtime.write('sudo docker rm concore\n') - fmaxtime.write('sudo docker rmi docker-concore\n') + fmaxtime.write(f'{DOCKEREXE} stop concore \n') + fmaxtime.write(f'{DOCKEREXE} rm concore\n') + fmaxtime.write(f'{DOCKEREXE} rmi docker-concore\n') fmaxtime.write('rm Dockerfile\n') fmaxtime.write('rm concore.maxtime\n') fmaxtime.close() @@ -901,8 +901,8 @@ def cleanup_script_files(): fparams.write('#!/bin/bash' + "\n") fparams.write('echo "$1" >concore.params\n') fparams.write('echo "FROM alpine:3.8" > Dockerfile\n') - fparams.write('sudo docker build -t docker-concore .\n') - fparams.write('sudo docker run --name=concore') + fparams.write(f'{DOCKEREXE} build -t docker-concore .\n') + fparams.write(f'{DOCKEREXE} run --name=concore') # -v VCZ:/VCZ -v VPZ:/VPZ i=0 # 9/12/21 for node in nodes_dict: @@ -928,23 +928,23 @@ def cleanup_script_files(): dockername = sourcecode.rsplit(".", 1)[0] #3/28/21 writeedges = volswr[i] while writeedges.find(":") != -1: - fparams.write('sudo docker cp concore.params concore:/') + fparams.write(f'{DOCKEREXE} cp concore.params concore:/') # escape destination path vol_path = writeedges.split(":")[0].split("-v ")[1].strip() fparams.write(shlex.quote(vol_path+"/concore.params")+"\n") writeedges = writeedges[writeedges.find(":")+1:] i=i+1 - fparams.write('sudo docker stop concore \n') - fparams.write('sudo docker rm concore\n') - fparams.write('sudo docker rmi docker-concore\n') + fparams.write(f'{DOCKEREXE} stop concore \n') + fparams.write(f'{DOCKEREXE} rm concore\n') + fparams.write(f'{DOCKEREXE} rmi docker-concore\n') fparams.write('rm Dockerfile\n') fparams.write('rm concore.params\n') fparams.close() funlock.write('#!/bin/bash' + "\n") funlock.write('echo "FROM alpine:3.8" > Dockerfile\n') - funlock.write('sudo docker build -t docker-concore .\n') - funlock.write('sudo docker run --name=concore') + funlock.write(f'{DOCKEREXE} build -t docker-concore .\n') + funlock.write(f'{DOCKEREXE} run --name=concore') # -v VCZ:/VCZ -v VPZ:/VPZ i=0 # 9/12/21 for node in nodes_dict: @@ -970,15 +970,15 @@ def cleanup_script_files(): dockername = sourcecode.rsplit(".", 1)[0] #3/28/21 writeedges = volswr[i] while writeedges.find(":") != -1: - funlock.write('sudo docker cp ~/concore.apikey concore:/') + funlock.write(f'{DOCKEREXE} cp ~/concore.apikey concore:/') # escape destination path vol_path = writeedges.split(":")[0].split("-v ")[1].strip() funlock.write(shlex.quote(vol_path+"/concore.apikey")+"\n") writeedges = writeedges[writeedges.find(":")+1:] i=i+1 - funlock.write('sudo docker stop concore \n') - funlock.write('sudo docker rm concore\n') - funlock.write('sudo docker rmi docker-concore\n') + funlock.write(f'{DOCKEREXE} stop concore \n') + funlock.write(f'{DOCKEREXE} rm concore\n') + funlock.write(f'{DOCKEREXE} rmi docker-concore\n') funlock.write('rm Dockerfile\n') funlock.close() From ec5690a74e3b290793391666eacb82f41499eb06 Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Thu, 19 Feb 2026 14:18:58 +0530 Subject: [PATCH 177/275] fix: prevent NameError in measurement scripts by adding fallback port variable definitions (fixes #345) --- measurements/A.py | 17 +++++++++++++++++ measurements/B.py | 25 +++++++++++++++++++++++++ measurements/C.py | 17 +++++++++++++++++ measurements/readme.md | 4 +++- 4 files changed, 62 insertions(+), 1 deletion(-) diff --git a/measurements/A.py b/measurements/A.py index b7d0fb6e..0db49368 100644 --- a/measurements/A.py +++ b/measurements/A.py @@ -5,6 +5,23 @@ import psutil import sys +# --- Fallback Definitions for Direct Execution --- +# These values are normally injected by copy_with_port_portname.py during study generation. +# When running this script directly, safe defaults are used instead. +_STANDALONE_MODE = False + +if 'PORT_NAME_F1_F2' not in globals(): + PORT_NAME_F1_F2 = "F1_F2" + _STANDALONE_MODE = True + +if 'PORT_F1_F2' not in globals(): + PORT_F1_F2 = "5555" + _STANDALONE_MODE = True + +if _STANDALONE_MODE: + print("Warning: Port variables not injected. Running in standalone mode with default values.") + print(" For full study behavior, run via study generation (makestudy).") + # --- ZMQ Initialization --- # This REQ socket connects to Node B concore.init_zmq_port( diff --git a/measurements/B.py b/measurements/B.py index 8ab05d09..595f3d8b 100644 --- a/measurements/B.py +++ b/measurements/B.py @@ -2,6 +2,31 @@ import concore import time +# --- Fallback Definitions for Direct Execution --- +# These values are normally injected by copy_with_port_portname.py during study generation. +# When running this script directly, safe defaults are used instead. +_STANDALONE_MODE = False + +if 'PORT_NAME_F1_F2' not in globals(): + PORT_NAME_F1_F2 = "F1_F2" + _STANDALONE_MODE = True + +if 'PORT_F1_F2' not in globals(): + PORT_F1_F2 = "5555" + _STANDALONE_MODE = True + +if 'PORT_NAME_F2_F3' not in globals(): + PORT_NAME_F2_F3 = "F2_F3" + _STANDALONE_MODE = True + +if 'PORT_F2_F3' not in globals(): + PORT_F2_F3 = "5556" + _STANDALONE_MODE = True + +if _STANDALONE_MODE: + print("Warning: Port variables not injected. Running in standalone mode with default values.") + print(" For full study behavior, run via study generation (makestudy).") + # --- ZMQ Initialization --- # This REP socket binds and waits for requests from Node A concore.init_zmq_port( diff --git a/measurements/C.py b/measurements/C.py index 1448069d..03337657 100644 --- a/measurements/C.py +++ b/measurements/C.py @@ -5,6 +5,23 @@ import os import sys +# --- Fallback Definitions for Direct Execution --- +# These values are normally injected by copy_with_port_portname.py during study generation. +# When running this script directly, safe defaults are used instead. +_STANDALONE_MODE = False + +if 'PORT_NAME_F2_F3' not in globals(): + PORT_NAME_F2_F3 = "F2_F3" + _STANDALONE_MODE = True + +if 'PORT_F2_F3' not in globals(): + PORT_F2_F3 = "5556" + _STANDALONE_MODE = True + +if _STANDALONE_MODE: + print("Warning: Port variables not injected. Running in standalone mode with default values.") + print(" For full study behavior, run via study generation (makestudy).") + # --- ZMQ Initialization --- # This REP socket binds and waits for requests from Node B concore.init_zmq_port( diff --git a/measurements/readme.md b/measurements/readme.md index f22f9f46..41088cc5 100644 --- a/measurements/readme.md +++ b/measurements/readme.md @@ -32,4 +32,6 @@ This folder contains studies and source code specifically for measuring communic Here you will find studies and source code focused on measuring communication times using ZeroMQ within a single system setup. # Network Communication Measurements (Two Systems Required) -For the Latency, Throughput, and CPU Usage measurements, two different systems are required. These systems should be connected over the same network to ensure efficient and accurate communication measurements between them. This setup is crucial for evaluating network-dependent performance metrics effectively. \ No newline at end of file +For the Latency, Throughput, and CPU Usage measurements, two different systems are required. These systems should be connected over the same network to ensure efficient and accurate communication measurements between them. This setup is crucial for evaluating network-dependent performance metrics effectively. +# Important Note on Port Variables +The measurement benchmark scripts (A.py, B.py, C.py) expect port variables such as `PORT_NAME_F1_F2`, `PORT_F1_F2`, `PORT_NAME_F2_F3`, and `PORT_F2_F3` to be injected by `copy_with_port_portname.py` during study generation. Running these scripts directly will use safe fallback defaults and may not reflect full study behavior. For accurate results, always run measurements through the study generation workflow (e.g., `makestudy`). From b315c131c157106c309608560e1283cb9675694c Mon Sep 17 00:00:00 2001 From: Avinash Kumar Deepak Date: Thu, 19 Feb 2026 16:10:46 +0530 Subject: [PATCH 178/275] fix: path slash and retry loop in concoredocker.hpp) --- concoredocker.hpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/concoredocker.hpp b/concoredocker.hpp index cbce7184..b978f32b 100644 --- a/concoredocker.hpp +++ b/concoredocker.hpp @@ -81,7 +81,7 @@ class Concore { std::vector read(int port, const std::string& name, const std::string& initstr) { std::this_thread::sleep_for(std::chrono::seconds(delay)); - std::string file_path = inpath + std::to_string(port) + "/" + name; + std::string file_path = inpath + "/" + std::to_string(port) + "/" + name; std::ifstream infile(file_path); std::string ins; @@ -94,6 +94,7 @@ class Concore { int attempts = 0, max_retries = 5; while (ins.empty() && attempts < max_retries) { std::this_thread::sleep_for(std::chrono::seconds(delay)); + infile.close(); infile.open(file_path); if (infile) std::getline(infile, ins); attempts++; @@ -110,7 +111,7 @@ class Concore { } void write(int port, const std::string& name, const std::vector& val, int delta = 0) { - std::string file_path = outpath + std::to_string(port) + "/" + name; + std::string file_path = outpath + "/" + std::to_string(port) + "/" + name; std::ofstream outfile(file_path); if (!outfile) { std::cerr << "Error writing to " << file_path << "\n"; From 7a40f00c2eeb756895af55c340ac8e2f20475e66 Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Thu, 19 Feb 2026 18:06:27 +0530 Subject: [PATCH 179/275] fix: correct Flask-CORS package name in requirements.txt (fixes #368) --- fri/requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fri/requirements.txt b/fri/requirements.txt index 2c7516da..85974b38 100644 --- a/fri/requirements.txt +++ b/fri/requirements.txt @@ -1,5 +1,6 @@ Flask gunicorn==20.1.0 -FLASK_CORS +flask-cors +# Optional dependency for /openJupyter endpoint jupyterlab PyGithub \ No newline at end of file From 97b669b4802c86ee09719120591523b54bde0950 Mon Sep 17 00:00:00 2001 From: Avinash Kumar Deepak Date: Thu, 19 Feb 2026 19:48:51 +0530 Subject: [PATCH 180/275] fix: error on duplicate node labels in mkconcore and validate --- concore_cli/commands/validate.py | 9 ++++++++- mkconcore.py | 6 ++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/concore_cli/commands/validate.py b/concore_cli/commands/validate.py index 7cacab99..6adf5fc4 100644 --- a/concore_cli/commands/validate.py +++ b/concore_cli/commands/validate.py @@ -117,7 +117,14 @@ def finalize(): warnings.append(f"Node {node_id} has no label") except Exception as e: warnings.append(f"Error parsing node: {str(e)}") - + + # duplicate labels cause silent corruption in mkconcore.py (#384) + seen = set() + for label in node_labels: + if label in seen: + errors.append(f"Duplicate node label: '{label}'") + seen.add(label) + node_ids = {node.get('id') for node in nodes if node.get('id')} for edge in edges: source = edge.get('source') diff --git a/mkconcore.py b/mkconcore.py index 0a119481..da935f82 100644 --- a/mkconcore.py +++ b/mkconcore.py @@ -307,6 +307,12 @@ def cleanup_script_files(): except (IndexError, AttributeError): logging.debug('A node with no valid properties encountered and ignored') +label_values = list(nodes_dict.values()) +duplicates = {label for label in label_values if label_values.count(label) > 1} +if duplicates: + logging.error(f"Duplicate node labels found: {sorted(duplicates)}") + quit() + for edge in edges_text: try: data = edge.find('data', recursive=False) From 4e3a098301c52e9ca54f9426703013884548d94f Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Fri, 20 Feb 2026 10:32:32 +0530 Subject: [PATCH 181/275] fix(simtime): remove global simtime mutation from write() to ensure deterministic cross-language behavior (#385) --- concore.hpp | 4 +- concore.py | 6 +- concore.v | 2 +- concore_write.m | 2 +- tests/test_concore.py | 143 ++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 151 insertions(+), 6 deletions(-) diff --git a/concore.hpp b/concore.hpp index ceb63724..56d8dc7d 100644 --- a/concore.hpp +++ b/concore.hpp @@ -503,7 +503,7 @@ class Concore{ outfile< Date: Fri, 20 Feb 2026 10:59:08 +0530 Subject: [PATCH 182/275] cleanup: add comprehensive Python bytecode and cache ignore rules (#373) --- .gitignore | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 82494f13..0488ad98 100644 --- a/.gitignore +++ b/.gitignore @@ -1,12 +1,19 @@ -# Python +# Python bytecode/cache __pycache__/ +*.pyc +*.pyo +*.pyd *.py[cod] *.class *.so .Python + +# Virtual environments venv/ env/ ENV/ +.env/ +.venv/ # IDE .vscode/ From f66ce521535ef38f8152ecae5ff569bbd95ec3e6 Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Fri, 20 Feb 2026 11:08:29 +0530 Subject: [PATCH 183/275] fix: replace quit() with sys.exit(1) for reliable process termination (#375) --- 0mq/funbody.py | 3 ++- demo/cwrap.py | 5 +++-- demo/pwrap.py | 5 +++-- tools/pidmayuresh1.py | 3 ++- tools/pidmayuresh3.py | 3 ++- tools/pidsig.py | 3 ++- 6 files changed, 14 insertions(+), 8 deletions(-) diff --git a/0mq/funbody.py b/0mq/funbody.py index 20064592..4330b51d 100644 --- a/0mq/funbody.py +++ b/0mq/funbody.py @@ -1,5 +1,6 @@ import concore import time +import sys from osparc_control import CommandManifest from osparc_control import CommandParameter from osparc_control import CommandType @@ -58,7 +59,7 @@ request_id=command.request_id, payload=ym) else: print("undefined action"+str(command.action)) - quit() + sys.exit(1) #concore.write(concore.oport['Y1'],"ym",ym) print("funbody u="+str(u)+" ym="+str(ym)+" time="+str(concore.simtime)) paired_transmitter.stop_background_sync() diff --git a/demo/cwrap.py b/demo/cwrap.py index f9f33e59..360b8d8e 100644 --- a/demo/cwrap.py +++ b/demo/cwrap.py @@ -4,6 +4,7 @@ import time from ast import literal_eval import os +import sys #time.sleep(7) timeout_max = 20 @@ -86,7 +87,7 @@ r = requests.post('http://www.controlcore.org/pm/'+yuyu+apikey+'&fetch='+name2, files=f,timeout=timeout_max) if r.status_code!=200: print("bad POST request "+str(r.status_code)) - quit() + sys.exit(1) if len(r.text)!=0: try: t=literal_eval(r.text)[0] @@ -107,7 +108,7 @@ timeout_count += 1 if r.status_code!=200 or time.perf_counter()-t1 > 1.1*timeout_max: #timeout_count>100: print("timeout or bad POST request "+str(r.status_code)) - quit() + sys.exit(1) if len(r.text)!=0: try: t=literal_eval(r.text)[0] diff --git a/demo/pwrap.py b/demo/pwrap.py index 283ca0ce..2da6889b 100644 --- a/demo/pwrap.py +++ b/demo/pwrap.py @@ -4,6 +4,7 @@ import time from ast import literal_eval import os +import sys #time.sleep(7) timeout_max=20 @@ -93,7 +94,7 @@ r = requests.post('http://www.controlcore.org/ctl/'+yuyu+apikey+'&fetch='+name1, files=f,timeout=timeout_max) if r.status_code!=200: print("bad POST request "+str(r.status_code)) - quit() + sys.exit(1) if len(r.text)!=0: try: t=literal_eval(r.text)[0] @@ -114,7 +115,7 @@ timeout_count += 1 if r.status_code!=200 or time.perf_counter()-t1 > 1.1*timeout_max: #timeout_count>200: print("timeout or bad POST request "+str(r.status_code)) - quit() + sys.exit(1) if len(r.text)!=0: try: t=literal_eval(r.text)[0] diff --git a/tools/pidmayuresh1.py b/tools/pidmayuresh1.py index 178b3086..29774f56 100644 --- a/tools/pidmayuresh1.py +++ b/tools/pidmayuresh1.py @@ -1,6 +1,7 @@ import numpy as np import math import concore +import sys dT = 0.1 global Prev_Error, I, freq Prev_Error = 0 @@ -22,7 +23,7 @@ def pid_controller(ym): Error = sp - ym[0] else: print('invalid control input '+cin) - quit() + sys.exit(1) P = Error I = I + Error*dT D = (Error - Prev_Error )/dT diff --git a/tools/pidmayuresh3.py b/tools/pidmayuresh3.py index 900d32f5..446a6511 100644 --- a/tools/pidmayuresh3.py +++ b/tools/pidmayuresh3.py @@ -1,6 +1,7 @@ import numpy as np import math import concore +import sys dT = 0.1 sp = concore.tryparam('sp', 67.5) @@ -20,7 +21,7 @@ def pid_controller(state, ym, sp, Kp, Ki, Kd, sigout, cin, low, up): Error = sp - ym[0] else: print('invalid control input '+cin) - quit() + sys.exit(1) P = Error I = I + Error*dT D = (Error - Prev_Error )/dT diff --git a/tools/pidsig.py b/tools/pidsig.py index 1a921af1..da07ee26 100644 --- a/tools/pidsig.py +++ b/tools/pidsig.py @@ -2,6 +2,7 @@ import math import concore import logging +import sys dT = 0.1 global Prev_Error, I, freq Prev_Error = 0 @@ -23,7 +24,7 @@ def pid_controller(ym): Error = sp - ym[0] else: logging.error(f'invalid control input {cin}') - quit() + sys.exit(1) P = Error I = I + Error*dT D = (Error - Prev_Error )/dT From 93a8759d804a916f44d34dca3a7aa8dce3a0409a Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Fri, 20 Feb 2026 11:29:11 +0530 Subject: [PATCH 184/275] fix: remove mixed tabs/spaces indentation in bangbang.py (#376) --- humanc/bangbang.py | 5 ++--- tools/bangbang.py | 5 ++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/humanc/bangbang.py b/humanc/bangbang.py index eed72b83..b5020514 100644 --- a/humanc/bangbang.py +++ b/humanc/bangbang.py @@ -8,9 +8,8 @@ def bangbang_controller(ym): amp = 3 elif ym[1]<65: amp = 1 - - - ustar = np.array([amp,30]) + + ustar = np.array([amp,30]) return ustar diff --git a/tools/bangbang.py b/tools/bangbang.py index 3bf9e5a2..9b723822 100644 --- a/tools/bangbang.py +++ b/tools/bangbang.py @@ -9,9 +9,8 @@ def bangbang_controller(ym): amp = 3 elif ym[1]<65: amp = 1 - - - ustar = np.array([amp,30]) + + ustar = np.array([amp,30]) return ustar From 52b0c3b855a8b0b684d684da7639a47bf26d0178 Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Fri, 20 Feb 2026 11:46:46 +0530 Subject: [PATCH 185/275] refactor: consolidate duplicate PID controller (pidmayuresh3 wrapper) Replace the duplicate PID implementation in tools/pidmayuresh3.py with a thin backward-compatibility wrapper that re-exports from pidmayuresh.py. - pidmayuresh.py remains the canonical implementation (uses logging) - pidmayuresh3.py now imports from pidmayuresh.py with a DeprecationWarning - No PID logic, API, or default parameters changed - All existing imports of pidmayuresh3 continue to work Closes #378 --- tools/pidmayuresh3.py | 72 ++++++++++++++----------------------------- 1 file changed, 23 insertions(+), 49 deletions(-) diff --git a/tools/pidmayuresh3.py b/tools/pidmayuresh3.py index 900d32f5..af369fd9 100644 --- a/tools/pidmayuresh3.py +++ b/tools/pidmayuresh3.py @@ -1,52 +1,26 @@ -import numpy as np -import math -import concore -dT = 0.1 - -sp = concore.tryparam('sp', 67.5) -Kp = concore.tryparam('Kp', 0.075) -Ki = concore.tryparam('Ki', 0.02) -Kd = concore.tryparam('Kd', 0.005) -freq = concore.tryparam('freq',30) -sigout = concore.tryparam('sigout',True) -cin = concore.tryparam('cin', 'hr') - -def pid_controller(state, ym, sp, Kp, Ki, Kd, sigout, cin, low, up): - Prev_Error = state[0] - I = state[1] - if cin == 'hr': - Error = sp - ym[1] - elif cin == 'map': - Error = sp - ym[0] - else: - print('invalid control input '+cin) - quit() - P = Error - I = I + Error*dT - D = (Error - Prev_Error )/dT - amp = Kp*P + Ki*I + Kd*D - Prev_Error = Error - if sigout: - amp = (up-low)/(1.0 + math.exp(amp)) + low - state = [Prev_Error, I] - return (state, amp) - - -concore.default_maxtime(150) -concore.delay = 0.02 -init_simtime_ym = "[0.0, 70.0,91]" -ym = np.array(concore.initval(init_simtime_ym)) -state = [0.0, 0.0] -print("Mayuresh's PID controller: sp is "+str(sp)) -print(concore.params) -while(concore.simtime Date: Fri, 20 Feb 2026 11:47:06 +0530 Subject: [PATCH 186/275] remove issue ref from inline comment --- concore_cli/commands/validate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/concore_cli/commands/validate.py b/concore_cli/commands/validate.py index 6adf5fc4..d9a39dd4 100644 --- a/concore_cli/commands/validate.py +++ b/concore_cli/commands/validate.py @@ -118,7 +118,7 @@ def finalize(): except Exception as e: warnings.append(f"Error parsing node: {str(e)}") - # duplicate labels cause silent corruption in mkconcore.py (#384) + # duplicate labels cause silent corruption in mkconcore.py seen = set() for label in node_labels: if label in seen: From b27755a1b33d4033e818265c749bb108d8e09222 Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Fri, 20 Feb 2026 11:53:08 +0530 Subject: [PATCH 187/275] fix: use try/except for relative+absolute import in pidmayuresh3 wrapper MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Apply Copilot review suggestion — try relative import first (from .pidmayuresh) for package-style usage, with fallback to absolute import for direct script execution. --- tools/pidmayuresh3.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tools/pidmayuresh3.py b/tools/pidmayuresh3.py index af369fd9..1fd1f2d8 100644 --- a/tools/pidmayuresh3.py +++ b/tools/pidmayuresh3.py @@ -20,7 +20,12 @@ # Re-execute the canonical module so run-time behaviour is identical # when this file is invoked directly (e.g., via a study graph). -from pidmayuresh import * # noqa: F401,F403 +try: + # Prefer relative import when part of the tools package + from .pidmayuresh import * # type: ignore[attr-defined] # noqa: F401,F403 +except ImportError: + # Fallback for script-style execution (e.g., `python tools/pidmayuresh3.py`) + from pidmayuresh import * # noqa: F401,F403 From 01fed627f0b385df1d0d497d50ab255acd40a7e8 Mon Sep 17 00:00:00 2001 From: Avinash Kumar Deepak Date: Fri, 20 Feb 2026 23:13:43 +0530 Subject: [PATCH 188/275] fix: add include guard, default_maxtime, tryparam, and initval bounds check to concore.hpp --- concore.hpp | 104 +++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 103 insertions(+), 1 deletion(-) diff --git a/concore.hpp b/concore.hpp index ceb63724..d2a8b3e9 100644 --- a/concore.hpp +++ b/concore.hpp @@ -1,4 +1,7 @@ // concore.hpp -- this C++ include file will be the equivalent of concore.py +#ifndef CONCORE_HPP +#define CONCORE_HPP + #include #include #include //for setprecision @@ -19,6 +22,7 @@ #endif #include #include +#include using namespace std; @@ -47,8 +51,10 @@ class Concore{ double delay = 1; int retrycount = 0; double simtime; + int maxtime = 100; map iport; map oport; + map params; /** * @brief Constructor for Concore class. @@ -57,7 +63,9 @@ class Concore{ */ Concore(){ iport = mapParser("concore.iport"); - oport = mapParser("concore.oport"); + oport = mapParser("concore.oport"); + default_maxtime(100); + load_params(); int iport_number = -1; int oport_number = -1; @@ -592,6 +600,96 @@ class Concore{ } } + /** + * @brief Strips leading and trailing whitespace from a string. + * @param str The input string. + * @return The stripped string. + */ + string stripstr(string str){ + size_t start = str.find_first_not_of(" \t\n\r"); + if (start == string::npos) return ""; + size_t end = str.find_last_not_of(" \t\n\r"); + return str.substr(start, end - start + 1); + } + + /** + * @brief Strips surrounding single or double quotes from a string. + * @param str The input string. + * @return The unquoted string. + */ + string stripquotes(string str){ + if (str.size() >= 2 && ((str.front() == '\'' && str.back() == '\'') || (str.front() == '"' && str.back() == '"'))) + return str.substr(1, str.size() - 2); + return str; + } + + /** + * @brief Parses a dict-formatted string into a string-to-string map. + * @param str The input string in {key: val, ...} format. + * @return A map of key-value string pairs. + */ + map parsedict(string str){ + map result; + string trimmed = stripstr(str); + if (trimmed.size() < 2 || trimmed.front() != '{' || trimmed.back() != '}') + return result; + string inner = trimmed.substr(1, trimmed.size() - 2); + stringstream ss(inner); + string token; + while (getline(ss, token, ',')) { + size_t colon = token.find(':'); + if (colon == string::npos) continue; + string key = stripquotes(stripstr(token.substr(0, colon))); + string val = stripquotes(stripstr(token.substr(colon + 1))); + if (!key.empty()) result[key] = val; + } + return result; + } + + /** + * @brief Sets maxtime from the concore.maxtime file, falling back to defaultValue. + * @param defaultValue The fallback value if the file is missing. + */ + void default_maxtime(int defaultValue){ + maxtime = defaultValue; + ifstream file(inpath + "/1/concore.maxtime"); + if (file) { + file >> maxtime; + } + } + + /** + * @brief Loads simulation parameters from concore.params into the params map. + */ + void load_params(){ + ifstream file(inpath + "/1/concore.params"); + if (!file) return; + stringstream buffer; + buffer << file.rdbuf(); + string sparams = buffer.str(); + + if (!sparams.empty() && sparams[0] == '"') { + sparams = sparams.substr(1, sparams.find('"', 1) - 1); + } + + if (!sparams.empty() && sparams[0] != '{') { + sparams = "{\"" + regex_replace(regex_replace(regex_replace(sparams, regex(","), ",\""), regex("="), "\":"), regex(" "), "") + "}"; + } + try { + params = parsedict(sparams); + } catch (...) {} + } + + /** + * @brief Returns the value of a param by name, or a default if not found. + * @param n The parameter name. + * @param i The default value. + * @return The parameter value or the default. + */ + string tryparam(string n, string i){ + return params.count(n) ? params[n] : i; + } + /** * @brief Initializes the system with the given input values. * @param f The input string containing the values. @@ -601,6 +699,8 @@ class Concore{ //parsing vector val = parser(f); + if (val.empty()) return val; + //determining simtime simtime = val[0]; @@ -609,3 +709,5 @@ class Concore{ return val; } }; + +#endif // CONCORE_HPP From 82de8376b9a352beffaf7153fb58b39a156357dc Mon Sep 17 00:00:00 2001 From: Pradeeban Kathiravelu Date: Fri, 20 Feb 2026 08:53:26 -0900 Subject: [PATCH 189/275] Clarify network communication measurements section Reworded the explanation for network communication measurements to enhance clarity and conciseness. --- measurements/readme.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/measurements/readme.md b/measurements/readme.md index 41088cc5..621a296e 100644 --- a/measurements/readme.md +++ b/measurements/readme.md @@ -32,6 +32,7 @@ This folder contains studies and source code specifically for measuring communic Here you will find studies and source code focused on measuring communication times using ZeroMQ within a single system setup. # Network Communication Measurements (Two Systems Required) -For the Latency, Throughput, and CPU Usage measurements, two different systems are required. These systems should be connected over the same network to ensure efficient and accurate communication measurements between them. This setup is crucial for evaluating network-dependent performance metrics effectively. +For the Latency, Throughput, and CPU Usage measurements, two different systems are required. These systems should be connected to the same network to ensure efficient, accurate communication between them. This setup is crucial for effectively evaluating network-dependent performance metrics. + # Important Note on Port Variables The measurement benchmark scripts (A.py, B.py, C.py) expect port variables such as `PORT_NAME_F1_F2`, `PORT_F1_F2`, `PORT_NAME_F2_F3`, and `PORT_F2_F3` to be injected by `copy_with_port_portname.py` during study generation. Running these scripts directly will use safe fallback defaults and may not reflect full study behavior. For accurate results, always run measurements through the study generation workflow (e.g., `makestudy`). From 24ebe32f8fe9fa03b64273fae0f9c8b35220485c Mon Sep 17 00:00:00 2001 From: Titas-Ghosh Date: Fri, 20 Feb 2026 01:04:28 +0530 Subject: [PATCH 190/275] Fix docker build script paths for subdirectory node sources --- mkconcore.py | 58 +++++++++++++++++++++++++---------------------- tests/test_cli.py | 29 ++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 27 deletions(-) diff --git a/mkconcore.py b/mkconcore.py index f2fb6f77..090be3fb 100644 --- a/mkconcore.py +++ b/mkconcore.py @@ -785,33 +785,37 @@ def cleanup_script_files(): fcopy.write('CMD ["./a.out"]\n') # 7/02/21 fbuild.write('#!/bin/bash' + "\n") - for node in nodes_dict: - containername,sourcecode = nodes_dict[node].split(':') - if len(sourcecode)!=0 and sourcecode.find(".")!=-1: #3/28/21 - dockername,langext = sourcecode.rsplit(".", 1) - fbuild.write("mkdir docker-"+dockername+"\n") - fbuild.write("cd docker-"+dockername+"\n") - fbuild.write("cp ../src/Dockerfile."+dockername+" Dockerfile\n") - #copy sourcefiles from ./src into corresponding directories - fbuild.write("cp ../src/"+sourcecode+" .\n") - if langext == "py": #4/29/21 - fbuild.write("cp ../src/concore.py .\n") - elif langext == "cpp": #6/22/21 - fbuild.write("cp ../src/concore.hpp .\n") - elif langext == "v": #6/25/21 - fbuild.write("cp ../src/concore.v .\n") - if langext == "m": - fbuild.write("cp ../src/concore_*.m .\n") - fbuild.write("cp ../src/import_concore.m .\n") - if langext == "sh": #5/27/21 - fbuild.write("chmod u+x "+sourcecode+"\n") - fbuild.write("cp ../src/"+dockername+".iport concore.iport\n") - fbuild.write("cp ../src/"+dockername+".oport concore.oport\n") - #include data files in here if they exist - if os.path.isdir(sourcedir+"/"+dockername+".dir"): - fbuild.write("cp -r ../src/"+dockername+".dir/* .\n") - fbuild.write(DOCKEREXE+" build -t docker-"+dockername+" .\n") - fbuild.write("cd ..\n") + for node in nodes_dict: + containername,sourcecode = nodes_dict[node].split(':') + if len(sourcecode)!=0 and sourcecode.find(".")!=-1: #3/28/21 + dockername,langext = sourcecode.rsplit(".", 1) + # dockername can include subdirectories (e.g. subdir/script). + # Compute a stable prefix to reach study root from docker build dir. + docker_subpath_depth = dockername.count("/") + dockername.count("\\") + docker_rel_prefix = "../" * (docker_subpath_depth + 1) + fbuild.write("mkdir docker-"+dockername+"\n") + fbuild.write("cd docker-"+dockername+"\n") + fbuild.write("cp "+docker_rel_prefix+"src/Dockerfile."+dockername+" Dockerfile\n") + #copy sourcefiles from ./src into corresponding directories + fbuild.write("cp "+docker_rel_prefix+"src/"+sourcecode+" .\n") + if langext == "py": #4/29/21 + fbuild.write("cp "+docker_rel_prefix+"src/concore.py .\n") + elif langext == "cpp": #6/22/21 + fbuild.write("cp "+docker_rel_prefix+"src/concore.hpp .\n") + elif langext == "v": #6/25/21 + fbuild.write("cp "+docker_rel_prefix+"src/concore.v .\n") + if langext == "m": + fbuild.write("cp "+docker_rel_prefix+"src/concore_*.m .\n") + fbuild.write("cp "+docker_rel_prefix+"src/import_concore.m .\n") + if langext == "sh": #5/27/21 + fbuild.write("chmod u+x "+sourcecode+"\n") + fbuild.write("cp "+docker_rel_prefix+"src/"+dockername+".iport concore.iport\n") + fbuild.write("cp "+docker_rel_prefix+"src/"+dockername+".oport concore.oport\n") + #include data files in here if they exist + if os.path.isdir(sourcedir+"/"+dockername+".dir"): + fbuild.write("cp -r "+docker_rel_prefix+"src/"+dockername+".dir/* .\n") + fbuild.write(DOCKEREXE+" build -t docker-"+dockername+" .\n") + fbuild.write("cd "+docker_rel_prefix+"\n") fbuild.close() diff --git a/tests/test_cli.py b/tests/test_cli.py index 4723002b..0871fdef 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -153,6 +153,35 @@ def test_run_command_subdir_source(self): self.assertEqual(result.exit_code, 0) self.assertTrue(Path('out/src/subdir/script.py').exists()) + def test_run_command_docker_subdir_source_build_paths(self): + with self.runner.isolated_filesystem(temp_dir=self.temp_dir): + result = self.runner.invoke(cli, ['init', 'test-project']) + self.assertEqual(result.exit_code, 0) + + subdir = Path('test-project/src/subdir') + subdir.mkdir(parents=True, exist_ok=True) + shutil.move('test-project/src/script.py', subdir / 'script.py') + + workflow_path = Path('test-project/workflow.graphml') + content = workflow_path.read_text() + content = content.replace('N1:script.py', 'N1:subdir/script.py') + workflow_path.write_text(content) + + result = self.runner.invoke(cli, [ + 'run', + 'test-project/workflow.graphml', + '--source', 'test-project/src', + '--output', 'out', + '--type', 'docker' + ]) + self.assertEqual(result.exit_code, 0) + + build_script = Path('out/build').read_text() + self.assertIn('cp ../../src/Dockerfile.subdir/script Dockerfile', build_script) + self.assertIn('cp ../../src/subdir/script.py .', build_script) + self.assertIn('cp ../../src/subdir/script.iport concore.iport', build_script) + self.assertIn('cd ../../', build_script) + def test_run_command_shared_source_specialization_merges_edge_params(self): with self.runner.isolated_filesystem(temp_dir=self.temp_dir): Path('src').mkdir() From 0ace53aa350b60a6610964c8e1d79b1fa9312260 Mon Sep 17 00:00:00 2001 From: Titas-Ghosh Date: Fri, 20 Feb 2026 23:40:03 +0530 Subject: [PATCH 191/275] Refine docker subdir fix to minimal mkconcore line changes --- mkconcore.py | 31 ++++++++++++++----------------- tests/test_cli.py | 9 +++++---- 2 files changed, 19 insertions(+), 21 deletions(-) diff --git a/mkconcore.py b/mkconcore.py index 090be3fb..0387e0c1 100644 --- a/mkconcore.py +++ b/mkconcore.py @@ -789,33 +789,30 @@ def cleanup_script_files(): containername,sourcecode = nodes_dict[node].split(':') if len(sourcecode)!=0 and sourcecode.find(".")!=-1: #3/28/21 dockername,langext = sourcecode.rsplit(".", 1) - # dockername can include subdirectories (e.g. subdir/script). - # Compute a stable prefix to reach study root from docker build dir. - docker_subpath_depth = dockername.count("/") + dockername.count("\\") - docker_rel_prefix = "../" * (docker_subpath_depth + 1) - fbuild.write("mkdir docker-"+dockername+"\n") - fbuild.write("cd docker-"+dockername+"\n") - fbuild.write("cp "+docker_rel_prefix+"src/Dockerfile."+dockername+" Dockerfile\n") + dockerbuilddir = "docker-"+dockername.replace("/", "__").replace("\\", "__") + fbuild.write("mkdir "+dockerbuilddir+"\n") + fbuild.write("cd "+dockerbuilddir+"\n") + fbuild.write("cp ../src/Dockerfile."+dockername+" Dockerfile\n") #copy sourcefiles from ./src into corresponding directories - fbuild.write("cp "+docker_rel_prefix+"src/"+sourcecode+" .\n") + fbuild.write("cp ../src/"+sourcecode+" .\n") if langext == "py": #4/29/21 - fbuild.write("cp "+docker_rel_prefix+"src/concore.py .\n") + fbuild.write("cp ../src/concore.py .\n") elif langext == "cpp": #6/22/21 - fbuild.write("cp "+docker_rel_prefix+"src/concore.hpp .\n") + fbuild.write("cp ../src/concore.hpp .\n") elif langext == "v": #6/25/21 - fbuild.write("cp "+docker_rel_prefix+"src/concore.v .\n") + fbuild.write("cp ../src/concore.v .\n") if langext == "m": - fbuild.write("cp "+docker_rel_prefix+"src/concore_*.m .\n") - fbuild.write("cp "+docker_rel_prefix+"src/import_concore.m .\n") + fbuild.write("cp ../src/concore_*.m .\n") + fbuild.write("cp ../src/import_concore.m .\n") if langext == "sh": #5/27/21 fbuild.write("chmod u+x "+sourcecode+"\n") - fbuild.write("cp "+docker_rel_prefix+"src/"+dockername+".iport concore.iport\n") - fbuild.write("cp "+docker_rel_prefix+"src/"+dockername+".oport concore.oport\n") + fbuild.write("cp ../src/"+dockername+".iport concore.iport\n") + fbuild.write("cp ../src/"+dockername+".oport concore.oport\n") #include data files in here if they exist if os.path.isdir(sourcedir+"/"+dockername+".dir"): - fbuild.write("cp -r "+docker_rel_prefix+"src/"+dockername+".dir/* .\n") + fbuild.write("cp -r ../src/"+dockername+".dir/* .\n") fbuild.write(DOCKEREXE+" build -t docker-"+dockername+" .\n") - fbuild.write("cd "+docker_rel_prefix+"\n") + fbuild.write("cd ..\n") fbuild.close() diff --git a/tests/test_cli.py b/tests/test_cli.py index 0871fdef..33316a71 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -177,10 +177,11 @@ def test_run_command_docker_subdir_source_build_paths(self): self.assertEqual(result.exit_code, 0) build_script = Path('out/build').read_text() - self.assertIn('cp ../../src/Dockerfile.subdir/script Dockerfile', build_script) - self.assertIn('cp ../../src/subdir/script.py .', build_script) - self.assertIn('cp ../../src/subdir/script.iport concore.iport', build_script) - self.assertIn('cd ../../', build_script) + self.assertIn('mkdir docker-subdir__script', build_script) + self.assertIn('cp ../src/Dockerfile.subdir/script Dockerfile', build_script) + self.assertIn('cp ../src/subdir/script.py .', build_script) + self.assertIn('cp ../src/subdir/script.iport concore.iport', build_script) + self.assertIn('cd ..', build_script) def test_run_command_shared_source_specialization_merges_edge_params(self): with self.runner.isolated_filesystem(temp_dir=self.temp_dir): From d6f49c228208a765fd94af8c32364c77c57b42c7 Mon Sep 17 00:00:00 2001 From: Titas-Ghosh Date: Fri, 20 Feb 2026 23:41:24 +0530 Subject: [PATCH 192/275] Minimize mkconcore diff for docker subdir build path fix --- mkconcore.py | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/mkconcore.py b/mkconcore.py index 0387e0c1..af19ff50 100644 --- a/mkconcore.py +++ b/mkconcore.py @@ -792,27 +792,27 @@ def cleanup_script_files(): dockerbuilddir = "docker-"+dockername.replace("/", "__").replace("\\", "__") fbuild.write("mkdir "+dockerbuilddir+"\n") fbuild.write("cd "+dockerbuilddir+"\n") - fbuild.write("cp ../src/Dockerfile."+dockername+" Dockerfile\n") - #copy sourcefiles from ./src into corresponding directories - fbuild.write("cp ../src/"+sourcecode+" .\n") - if langext == "py": #4/29/21 - fbuild.write("cp ../src/concore.py .\n") - elif langext == "cpp": #6/22/21 - fbuild.write("cp ../src/concore.hpp .\n") - elif langext == "v": #6/25/21 - fbuild.write("cp ../src/concore.v .\n") - if langext == "m": - fbuild.write("cp ../src/concore_*.m .\n") - fbuild.write("cp ../src/import_concore.m .\n") - if langext == "sh": #5/27/21 - fbuild.write("chmod u+x "+sourcecode+"\n") - fbuild.write("cp ../src/"+dockername+".iport concore.iport\n") - fbuild.write("cp ../src/"+dockername+".oport concore.oport\n") - #include data files in here if they exist - if os.path.isdir(sourcedir+"/"+dockername+".dir"): - fbuild.write("cp -r ../src/"+dockername+".dir/* .\n") - fbuild.write(DOCKEREXE+" build -t docker-"+dockername+" .\n") - fbuild.write("cd ..\n") + fbuild.write("cp ../src/Dockerfile."+dockername+" Dockerfile\n") + #copy sourcefiles from ./src into corresponding directories + fbuild.write("cp ../src/"+sourcecode+" .\n") + if langext == "py": #4/29/21 + fbuild.write("cp ../src/concore.py .\n") + elif langext == "cpp": #6/22/21 + fbuild.write("cp ../src/concore.hpp .\n") + elif langext == "v": #6/25/21 + fbuild.write("cp ../src/concore.v .\n") + if langext == "m": + fbuild.write("cp ../src/concore_*.m .\n") + fbuild.write("cp ../src/import_concore.m .\n") + if langext == "sh": #5/27/21 + fbuild.write("chmod u+x "+sourcecode+"\n") + fbuild.write("cp ../src/"+dockername+".iport concore.iport\n") + fbuild.write("cp ../src/"+dockername+".oport concore.oport\n") + #include data files in here if they exist + if os.path.isdir(sourcedir+"/"+dockername+".dir"): + fbuild.write("cp -r ../src/"+dockername+".dir/* .\n") + fbuild.write(DOCKEREXE+" build -t docker-"+dockername+" .\n") + fbuild.write("cd ..\n") fbuild.close() From 6cdf4c92395009592228612ad11f35f140c801a1 Mon Sep 17 00:00:00 2001 From: Titas-Ghosh Date: Fri, 20 Feb 2026 23:42:59 +0530 Subject: [PATCH 193/275] Drop unrelated duplicate-label change from docker subdir PR --- mkconcore.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/mkconcore.py b/mkconcore.py index af19ff50..f5387215 100644 --- a/mkconcore.py +++ b/mkconcore.py @@ -309,13 +309,7 @@ def cleanup_script_files(): except (IndexError, AttributeError): logging.debug('A node with no valid properties encountered and ignored') -label_values = list(nodes_dict.values()) -duplicates = {label for label in label_values if label_values.count(label) > 1} -if duplicates: - logging.error(f"Duplicate node labels found: {sorted(duplicates)}") - quit() - -for edge in edges_text: +for edge in edges_text: try: data = edge.find('data', recursive=False) if data: From e15cf19bb7eec4eec87cfb3642926340e7e48ae6 Mon Sep 17 00:00:00 2001 From: GREENRAT-K405 Date: Fri, 20 Feb 2026 23:43:55 +0530 Subject: [PATCH 194/275] update mkconcore logic to select scripts to add to src --- mkconcore.py | 214 ++++++++++++++++++++++----------------------------- 1 file changed, 92 insertions(+), 122 deletions(-) diff --git a/mkconcore.py b/mkconcore.py index f2fb6f77..18a97aed 100644 --- a/mkconcore.py +++ b/mkconcore.py @@ -553,128 +553,98 @@ def cleanup_script_files(): shutil.copytree(os.path.join(sourcedir, dir_for_node), os.path.join(outdir, "src", dir_for_node), dirs_exist_ok=True) -#copy proper concore.py into /src -try: - if concoretype=="docker": - with open(CONCOREPATH+"/concoredocker.py") as fsource: - source_content = fsource.read() - else: - with open(CONCOREPATH+"/concore.py") as fsource: - source_content = fsource.read() -except (FileNotFoundError, IOError) as e: - logging.error(f"{CONCOREPATH} is not correct path to concore: {e}") - quit() -with open(outdir+"/src/concore.py","w") as fcopy: - fcopy.write(source_content) - -#copy proper concore.hpp into /src 6/22/21 -try: - if concoretype=="docker": - with open(CONCOREPATH+"/concoredocker.hpp") as fsource: - source_content = fsource.read() - else: - with open(CONCOREPATH+"/concore.hpp") as fsource: - source_content = fsource.read() -except (FileNotFoundError, IOError) as e: - logging.error(f"{CONCOREPATH} is not correct path to concore: {e}") - quit() -with open(outdir+"/src/concore.hpp","w") as fcopy: - fcopy.write(source_content) - -#copy proper concore.v into /src 6/25/21 -try: - if concoretype=="docker": - with open(CONCOREPATH+"/concoredocker.v") as fsource: - source_content = fsource.read() - else: - with open(CONCOREPATH+"/concore.v") as fsource: - source_content = fsource.read() -except (FileNotFoundError, IOError) as e: - logging.error(f"{CONCOREPATH} is not correct path to concore: {e}") - quit() -with open(outdir+"/src/concore.v","w") as fcopy: - fcopy.write(source_content) - -#copy mkcompile into /src 5/27/21 -try: - with open(CONCOREPATH+"/mkcompile") as fsource: - source_content = fsource.read() -except (FileNotFoundError, IOError) as e: - logging.error(f"{CONCOREPATH} is not correct path to concore: {e}") - quit() -with open(outdir+"/src/mkcompile","w") as fcopy: - fcopy.write(source_content) -os.chmod(outdir+"/src/mkcompile",stat.S_IRWXU) - -#copy concore*.m into /src 4/2/21 -try: #maxtime in matlab 11/22/21 - with open(CONCOREPATH+"/concore_default_maxtime.m") as fsource: - source_content = fsource.read() -except (FileNotFoundError, IOError) as e: - logging.error(f"{CONCOREPATH} is not correct path to concore: {e}") - quit() -with open(outdir+"/src/concore_default_maxtime.m","w") as fcopy: - fcopy.write(source_content) -try: - with open(CONCOREPATH+"/concore_unchanged.m") as fsource: - source_content = fsource.read() -except (FileNotFoundError, IOError) as e: - logging.error(f"{CONCOREPATH} is not correct path to concore: {e}") - quit() -with open(outdir+"/src/concore_unchanged.m","w") as fcopy: - fcopy.write(source_content) -try: - with open(CONCOREPATH+"/concore_read.m") as fsource: - source_content = fsource.read() -except (FileNotFoundError, IOError) as e: - logging.error(f"{CONCOREPATH} is not correct path to concore: {e}") - quit() -with open(outdir+"/src/concore_read.m","w") as fcopy: - fcopy.write(source_content) -try: - with open(CONCOREPATH+"/concore_write.m") as fsource: - source_content = fsource.read() -except (FileNotFoundError, IOError) as e: - logging.error(f"{CONCOREPATH} is not correct path to concore: {e}") - quit() -with open(outdir+"/src/concore_write.m","w") as fcopy: - fcopy.write(source_content) -try: #4/9/21 - with open(CONCOREPATH+"/concore_initval.m") as fsource: - source_content = fsource.read() -except (FileNotFoundError, IOError) as e: - logging.error(f"{CONCOREPATH} is not correct path to concore: {e}") - quit() -with open(outdir+"/src/concore_initval.m","w") as fcopy: - fcopy.write(source_content) -try: #11/19/21 - with open(CONCOREPATH+"/concore_iport.m") as fsource: - source_content = fsource.read() -except (FileNotFoundError, IOError) as e: - logging.error(f"{CONCOREPATH} is not correct path to concore: {e}") - quit() -with open(outdir+"/src/concore_iport.m","w") as fcopy: - fcopy.write(source_content) -try: #11/19/21 - with open(CONCOREPATH+"/concore_oport.m") as fsource: - source_content = fsource.read() -except (FileNotFoundError, IOError) as e: - logging.error(f"{CONCOREPATH} is not correct path to concore: {e}") - quit() -with open(outdir+"/src/concore_oport.m","w") as fcopy: - fcopy.write(source_content) -try: # 4/4/21 - if concoretype=="docker": - with open(CONCOREPATH+"/import_concoredocker.m") as fsource: - source_content = fsource.read() - else: - with open(CONCOREPATH+"/import_concore.m") as fsource: - source_content = fsource.read() -except (FileNotFoundError, IOError) as e: - logging.error(f"{CONCOREPATH} is not correct path to concore: {e}") - quit() -with open(outdir+"/src/import_concore.m","w") as fcopy: - fcopy.write(source_content) +#determine languages used in the graphml +required_langs = set() +for node in nodes_dict: + containername, sourcecode = nodes_dict[node].split(':') + if len(sourcecode) != 0 and "." in sourcecode: + langext = sourcecode.split(".")[-1] + required_langs.add(langext) +logging.info(f"Languages detected in graph: {required_langs}") + +if 'py' in required_langs: + try: + if concoretype=="docker": + fsource = open(CONCOREPATH+"/concoredocker.py") + else: + fsource = open(CONCOREPATH+"/concore.py") + except (FileNotFoundError, IOError): + print(CONCOREPATH+" is not correct path to concore (missing python files)") + quit() + with open(outdir+"/src/concore.py","w") as fcopy: + fcopy.write(fsource.read()) + fsource.close() + +if 'cpp' in required_langs: + try: + if concoretype=="docker": + fsource = open(CONCOREPATH+"/concoredocker.hpp") + else: + fsource = open(CONCOREPATH+"/concore.hpp") + except (FileNotFoundError, IOError): + print(CONCOREPATH+" is not correct path to concore (missing C++ files)") + quit() + with open(outdir+"/src/concore.hpp","w") as fcopy: + fcopy.write(fsource.read()) + fsource.close() + +if 'v' in required_langs: + try: + if concoretype=="docker": + fsource = open(CONCOREPATH+"/concoredocker.v") + else: + fsource = open(CONCOREPATH+"/concore.v") + except (FileNotFoundError, IOError): + print(CONCOREPATH+" is not correct path to concore (missing Verilog files)") + quit() + with open(outdir+"/src/concore.v","w") as fcopy: + fcopy.write(fsource.read()) + fsource.close() + +if 'm' in required_langs: + try: + fsource = open(CONCOREPATH+"/concore_default_maxtime.m") + with open(outdir+"/src/concore_default_maxtime.m","w") as fcopy: fcopy.write(fsource.read()) + fsource.close() + + fsource = open(CONCOREPATH+"/concore_unchanged.m") + with open(outdir+"/src/concore_unchanged.m","w") as fcopy: fcopy.write(fsource.read()) + fsource.close() + + fsource = open(CONCOREPATH+"/concore_read.m") + with open(outdir+"/src/concore_read.m","w") as fcopy: fcopy.write(fsource.read()) + fsource.close() + + fsource = open(CONCOREPATH+"/concore_write.m") + with open(outdir+"/src/concore_write.m","w") as fcopy: fcopy.write(fsource.read()) + fsource.close() + + fsource = open(CONCOREPATH+"/concore_initval.m") + with open(outdir+"/src/concore_initval.m","w") as fcopy: fcopy.write(fsource.read()) + fsource.close() + + fsource = open(CONCOREPATH+"/concore_iport.m") + with open(outdir+"/src/concore_iport.m","w") as fcopy: fcopy.write(fsource.read()) + fsource.close() + + fsource = open(CONCOREPATH+"/concore_oport.m") + with open(outdir+"/src/concore_oport.m","w") as fcopy: fcopy.write(fsource.read()) + fsource.close() + + if concoretype=="docker": + fsource = open(CONCOREPATH+"/import_concoredocker.m") + else: + fsource = open(CONCOREPATH+"/import_concore.m") + with open(outdir+"/src/import_concore.m","w") as fcopy: fcopy.write(fsource.read()) + fsource.close() + + fsource = open(CONCOREPATH+"/mkcompile") + with open(outdir+"/src/mkcompile","w") as fcopy: fcopy.write(fsource.read()) + fsource.close() + os.chmod(outdir+"/src/mkcompile",stat.S_IRWXU) + except Exception as e: + print(CONCOREPATH+" is not correct path to concore (missing MATLAB files):", e) + quit() # --- Generate iport and oport mappings --- logging.info("Generating iport/oport mappings...") From a5d58e8b0a49bf89e4b0c45d1308bb7e9f82095e Mon Sep 17 00:00:00 2001 From: Titas-Ghosh Date: Fri, 20 Feb 2026 23:44:59 +0530 Subject: [PATCH 195/275] Restore duplicate-label guard while keeping docker subdir fix scoped --- mkconcore.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/mkconcore.py b/mkconcore.py index f5387215..2caccca4 100644 --- a/mkconcore.py +++ b/mkconcore.py @@ -284,7 +284,7 @@ def cleanup_script_files(): nodes_dict = dict() node_id_to_label_map = dict() # Helper to get clean node labels from GraphML ID -for node in nodes_text: +for node in nodes_text: try: data = node.find('data', recursive=False) if data: @@ -306,9 +306,15 @@ def cleanup_script_files(): nodes_dict[node['id']] = node_label node_id_to_label_map[node['id']] = node_label.split(':')[0] - except (IndexError, AttributeError): - logging.debug('A node with no valid properties encountered and ignored') - + except (IndexError, AttributeError): + logging.debug('A node with no valid properties encountered and ignored') + +label_values = list(nodes_dict.values()) +duplicates = {label for label in label_values if label_values.count(label) > 1} +if duplicates: + logging.error(f"Duplicate node labels found: {sorted(duplicates)}") + quit() + for edge in edges_text: try: data = edge.find('data', recursive=False) From 7a1f0e2d8f3cf8dcf390acc1fd29ce0b854c75e6 Mon Sep 17 00:00:00 2001 From: Titas-Ghosh Date: Fri, 20 Feb 2026 23:46:19 +0530 Subject: [PATCH 196/275] Reduce mkconcore changes to only docker build-dir normalization --- mkconcore.py | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/mkconcore.py b/mkconcore.py index 2caccca4..5b0286a2 100644 --- a/mkconcore.py +++ b/mkconcore.py @@ -284,7 +284,7 @@ def cleanup_script_files(): nodes_dict = dict() node_id_to_label_map = dict() # Helper to get clean node labels from GraphML ID -for node in nodes_text: +for node in nodes_text: try: data = node.find('data', recursive=False) if data: @@ -306,16 +306,16 @@ def cleanup_script_files(): nodes_dict[node['id']] = node_label node_id_to_label_map[node['id']] = node_label.split(':')[0] - except (IndexError, AttributeError): - logging.debug('A node with no valid properties encountered and ignored') - -label_values = list(nodes_dict.values()) -duplicates = {label for label in label_values if label_values.count(label) > 1} -if duplicates: - logging.error(f"Duplicate node labels found: {sorted(duplicates)}") - quit() - -for edge in edges_text: + except (IndexError, AttributeError): + logging.debug('A node with no valid properties encountered and ignored') + +label_values = list(nodes_dict.values()) +duplicates = {label for label in label_values if label_values.count(label) > 1} +if duplicates: + logging.error(f"Duplicate node labels found: {sorted(duplicates)}") + quit() + +for edge in edges_text: try: data = edge.find('data', recursive=False) if data: @@ -785,13 +785,13 @@ def cleanup_script_files(): fcopy.write('CMD ["./a.out"]\n') # 7/02/21 fbuild.write('#!/bin/bash' + "\n") - for node in nodes_dict: - containername,sourcecode = nodes_dict[node].split(':') - if len(sourcecode)!=0 and sourcecode.find(".")!=-1: #3/28/21 - dockername,langext = sourcecode.rsplit(".", 1) - dockerbuilddir = "docker-"+dockername.replace("/", "__").replace("\\", "__") - fbuild.write("mkdir "+dockerbuilddir+"\n") - fbuild.write("cd "+dockerbuilddir+"\n") + for node in nodes_dict: + containername,sourcecode = nodes_dict[node].split(':') + if len(sourcecode)!=0 and sourcecode.find(".")!=-1: #3/28/21 + dockername,langext = sourcecode.rsplit(".", 1) + dockerbuilddir = "docker-"+dockername.replace("/", "__").replace("\\", "__") + fbuild.write("mkdir "+dockerbuilddir+"\n") + fbuild.write("cd "+dockerbuilddir+"\n") fbuild.write("cp ../src/Dockerfile."+dockername+" Dockerfile\n") #copy sourcefiles from ./src into corresponding directories fbuild.write("cp ../src/"+sourcecode+" .\n") From dc726cbe3966249868bcbdb1802dd948b2184c58 Mon Sep 17 00:00:00 2001 From: Avinash Kumar Deepak Date: Sat, 21 Feb 2026 12:58:50 +0530 Subject: [PATCH 197/275] fix: use concore.maxtime instead of hardcoded Nsim in C++ nodes --- testsou/cpymat.cpp | 2 +- testsou/pmpymat.cpp | 3 +-- testsou/powermeter.cpp | 3 +-- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/testsou/cpymat.cpp b/testsou/cpymat.cpp index 55a9d04c..aac38391 100644 --- a/testsou/cpymat.cpp +++ b/testsou/cpymat.cpp @@ -21,7 +21,7 @@ int main() auto wallclock1 = chrono::high_resolution_clock::now(); vector ym; - while(concore.simtime ym = concore.initval(init_simtime_ym); vector u; - while(concore.simtime ym = concore.initval(init_simtime_ym); vector u; - while(concore.simtime Date: Sat, 21 Feb 2026 20:17:09 +0530 Subject: [PATCH 198/275] Add phase-1 protocol conformance fixtures and runner --- tests/protocol_fixtures/README.md | 12 ++ .../python_phase1_cases.json | 105 ++++++++++++++++ tests/protocol_fixtures/schema.phase1.json | 62 ++++++++++ tests/test_protocol_conformance.py | 115 ++++++++++++++++++ 4 files changed, 294 insertions(+) create mode 100644 tests/protocol_fixtures/README.md create mode 100644 tests/protocol_fixtures/python_phase1_cases.json create mode 100644 tests/protocol_fixtures/schema.phase1.json create mode 100644 tests/test_protocol_conformance.py diff --git a/tests/protocol_fixtures/README.md b/tests/protocol_fixtures/README.md new file mode 100644 index 00000000..0b85c8bc --- /dev/null +++ b/tests/protocol_fixtures/README.md @@ -0,0 +1,12 @@ +# Protocol Conformance Fixtures (Phase 1) + +This directory contains the phase-1 protocol conformance baseline for Python. + +- `schema.phase1.json`: fixture document shape and supported case targets. +- `python_phase1_cases.json`: initial baseline cases (report-only mode metadata). + +Phase-1 scope: + +- No runtime behavior changes. +- Python-only execution through `tests/test_protocol_conformance.py`. +- Fixture format is language-neutral to enable future cross-binding runners. diff --git a/tests/protocol_fixtures/python_phase1_cases.json b/tests/protocol_fixtures/python_phase1_cases.json new file mode 100644 index 00000000..f8a72185 --- /dev/null +++ b/tests/protocol_fixtures/python_phase1_cases.json @@ -0,0 +1,105 @@ +{ + "schema_version": "1.0", + "runtime": "python", + "mode": "report_only", + "cases": [ + { + "id": "parse_params/simple_types_and_whitespace", + "target": "parse_params", + "description": "Semicolon-delimited params preserve string values and coerce literals.", + "input": { + "sparams": "delay=5; coeffs=[1,2,3]; label = hello world" + }, + "expected": { + "result": { + "delay": 5, + "coeffs": [ + 1, + 2, + 3 + ], + "label": "hello world" + } + } + }, + { + "id": "parse_params/embedded_equals_not_split", + "target": "parse_params", + "description": "Only the first '=' is used as key/value separator.", + "input": { + "sparams": "url=https://example.com?a=1&b=2" + }, + "expected": { + "result": { + "url": "https://example.com?a=1&b=2" + } + } + }, + { + "id": "initval/valid_list_sets_simtime", + "target": "initval", + "description": "initval sets simtime to first numeric entry and returns payload tail.", + "input": { + "initial_simtime": 0, + "simtime_val_str": "[12.5, \"a\", 3]" + }, + "expected": { + "result": [ + "a", + 3 + ], + "simtime_after": 12.5 + } + }, + { + "id": "initval/invalid_input_returns_empty_and_preserves_simtime", + "target": "initval", + "description": "Invalid non-list input returns [] and leaves simtime unchanged.", + "input": { + "initial_simtime": 7, + "simtime_val_str": "not_a_list" + }, + "expected": { + "result": [], + "simtime_after": 7 + } + }, + { + "id": "write_zmq/list_payload_prepends_timestamp_without_mutation", + "target": "write_zmq", + "description": "write() prepends simtime+delta for list payloads but does not mutate global simtime.", + "input": { + "initial_simtime": 10, + "delta": 2, + "name": "data", + "value": [ + 1.5, + 2.5 + ] + }, + "expected": { + "sent_payload": [ + 12, + 1.5, + 2.5 + ], + "simtime_after": 10 + } + }, + { + "id": "write_zmq/non_list_payload_forwarded_as_is", + "target": "write_zmq", + "description": "Non-list payloads are forwarded as-is and simtime remains unchanged.", + "input": { + "initial_simtime": 10, + "delta": 3, + "name": "status", + "value": "ok" + }, + "expected": { + "sent_payload": "ok", + "simtime_after": 10 + } + } + ] +} diff --git a/tests/protocol_fixtures/schema.phase1.json b/tests/protocol_fixtures/schema.phase1.json new file mode 100644 index 00000000..3b54e135 --- /dev/null +++ b/tests/protocol_fixtures/schema.phase1.json @@ -0,0 +1,62 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Concore Protocol Conformance Fixtures (Phase 1)", + "description": "Language-neutral fixture format. Phase 1 executes Python-only baseline checks.", + "type": "object", + "required": [ + "schema_version", + "runtime", + "mode", + "cases" + ], + "properties": { + "schema_version": { + "type": "string" + }, + "runtime": { + "type": "string" + }, + "mode": { + "type": "string", + "enum": [ + "report_only" + ] + }, + "cases": { + "type": "array", + "items": { + "type": "object", + "required": [ + "id", + "target", + "input", + "expected" + ], + "properties": { + "id": { + "type": "string" + }, + "target": { + "type": "string", + "enum": [ + "parse_params", + "initval", + "write_zmq" + ] + }, + "description": { + "type": "string" + }, + "input": { + "type": "object" + }, + "expected": { + "type": "object" + } + }, + "additionalProperties": true + } + } + }, + "additionalProperties": false +} diff --git a/tests/test_protocol_conformance.py b/tests/test_protocol_conformance.py new file mode 100644 index 00000000..e831165a --- /dev/null +++ b/tests/test_protocol_conformance.py @@ -0,0 +1,115 @@ +import json +from pathlib import Path + +import pytest + +import concore + + +FIXTURE_DIR = Path(__file__).parent / "protocol_fixtures" +SCHEMA_PATH = FIXTURE_DIR / "schema.phase1.json" +CASES_PATH = FIXTURE_DIR / "python_phase1_cases.json" +SUPPORTED_TARGETS = {"parse_params", "initval", "write_zmq"} + + +def _load_json(path): + with path.open("r", encoding="utf-8") as f: + return json.load(f) + + +def _validate_fixture_document_shape(doc): + required_top = {"schema_version", "runtime", "mode", "cases"} + missing = required_top - set(doc.keys()) + if missing: + raise AssertionError(f"Fixture document missing required top-level keys: {sorted(missing)}") + if doc["runtime"] != "python": + raise AssertionError(f"Phase-1 fixture runtime must be 'python', found: {doc['runtime']}") + if doc["mode"] != "report_only": + raise AssertionError(f"Phase-1 fixture mode must be 'report_only', found: {doc['mode']}") + if not isinstance(doc["cases"], list) or not doc["cases"]: + raise AssertionError("Fixture document must contain a non-empty 'cases' list") + + for idx, case in enumerate(doc["cases"]): + for key in ("id", "target", "input", "expected"): + if key not in case: + raise AssertionError(f"Case index {idx} missing required key '{key}'") + if case["target"] not in SUPPORTED_TARGETS: + raise AssertionError( + f"Case '{case['id']}' has unsupported target '{case['target']}'" + ) + + +def _run_parse_params_case(case): + result = concore.parse_params(case["input"]["sparams"]) + assert result == case["expected"]["result"] + + +def _run_initval_case(case): + old_simtime = concore.simtime + try: + concore.simtime = case["input"]["initial_simtime"] + result = concore.initval(case["input"]["simtime_val_str"]) + assert result == case["expected"]["result"] + assert concore.simtime == case["expected"]["simtime_after"] + finally: + concore.simtime = old_simtime + + +def _run_write_zmq_case(case): + class DummyPort: + def __init__(self): + self.sent_payload = None + + def send_json_with_retry(self, message): + self.sent_payload = message + + old_simtime = concore.simtime + port_name = f"fixture_{case['id'].replace('/', '_')}" + existing_port = concore.zmq_ports.get(port_name) + dummy_port = DummyPort() + + try: + concore.simtime = case["input"]["initial_simtime"] + concore.zmq_ports[port_name] = dummy_port + concore.write( + port_name, + case["input"]["name"], + case["input"]["value"], + delta=case["input"]["delta"], + ) + assert dummy_port.sent_payload == case["expected"]["sent_payload"] + assert concore.simtime == case["expected"]["simtime_after"] + finally: + concore.simtime = old_simtime + if existing_port is None: + concore.zmq_ports.pop(port_name, None) + else: + concore.zmq_ports[port_name] = existing_port + + +def _run_case(case): + if case["target"] == "parse_params": + _run_parse_params_case(case) + elif case["target"] == "initval": + _run_initval_case(case) + elif case["target"] == "write_zmq": + _run_write_zmq_case(case) + else: + raise AssertionError(f"Unsupported target: {case['target']}") + + +def _load_cases(): + doc = _load_json(CASES_PATH) + _validate_fixture_document_shape(doc) + return doc["cases"] + + +def test_phase1_schema_file_present_and_basic_shape(): + schema = _load_json(SCHEMA_PATH) + assert schema["title"] == "Concore Protocol Conformance Fixtures (Phase 1)" + assert "cases" in schema["properties"] + + +@pytest.mark.parametrize("case", _load_cases(), ids=lambda case: case["id"]) +def test_phase1_python_protocol_conformance(case): + _run_case(case) From cc3f74acae8306d10019ea1d36524ae0e3be6cf0 Mon Sep 17 00:00:00 2001 From: Titas-Ghosh Date: Sat, 21 Feb 2026 20:43:10 +0530 Subject: [PATCH 199/275] Rename protocol fixtures README to avoid repo-level README confusion --- tests/protocol_fixtures/{README.md => PROTOCOL_FIXTURES.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/protocol_fixtures/{README.md => PROTOCOL_FIXTURES.md} (100%) diff --git a/tests/protocol_fixtures/README.md b/tests/protocol_fixtures/PROTOCOL_FIXTURES.md similarity index 100% rename from tests/protocol_fixtures/README.md rename to tests/protocol_fixtures/PROTOCOL_FIXTURES.md From c254d046e68c4b4339870f0aff536f3c52f1d131 Mon Sep 17 00:00:00 2001 From: Avinash Kumar Deepak Date: Sat, 21 Feb 2026 21:14:38 +0530 Subject: [PATCH 200/275] fix: pull shared logic into concore_base, fix docker path/cleanup bugs --- concore.py | 400 ++++-------------------------------- concore_base.py | 364 ++++++++++++++++++++++++++++++++ concoredocker.py | 342 +++++------------------------- mkconcore.py | 14 +- tests/test_concoredocker.py | 6 +- 5 files changed, 467 insertions(+), 659 deletions(-) create mode 100644 concore_base.py diff --git a/concore.py b/concore.py index 21fe83d2..5bd112ba 100644 --- a/concore.py +++ b/concore.py @@ -9,6 +9,8 @@ import numpy as np import signal +import concore_base + logger = logging.getLogger('concore') logger.addHandler(logging.NullHandler()) @@ -27,117 +29,53 @@ with open("concorekill.bat","w") as fpid: fpid.write("taskkill /F /PID "+str(os.getpid())+"\n") -# =================================================================== -# ZeroMQ Communication Wrapper -# =================================================================== -class ZeroMQPort: - def __init__(self, port_type, address, zmq_socket_type): - """ - port_type: "bind" or "connect" - address: ZeroMQ address (e.g., "tcp://*:5555") - zmq_socket_type: zmq.REQ, zmq.REP, zmq.PUB, zmq.SUB etc. - """ - self.context = zmq.Context() - self.socket = self.context.socket(zmq_socket_type) - self.port_type = port_type # "bind" or "connect" - self.address = address +ZeroMQPort = concore_base.ZeroMQPort +convert_numpy_to_python = concore_base.convert_numpy_to_python +safe_literal_eval = concore_base.safe_literal_eval +parse_params = concore_base.parse_params - # Configure timeouts & immediate close on failure - self.socket.setsockopt(zmq.RCVTIMEO, 2000) # 2 sec receive timeout - self.socket.setsockopt(zmq.SNDTIMEO, 2000) # 2 sec send timeout - self.socket.setsockopt(zmq.LINGER, 0) # Drop pending messages on close +# Global variables +zmq_ports = {} +_cleanup_in_progress = False + +s = '' +olds = '' +delay = 1 +retrycount = 0 +inpath = "./in" #must be rel path for local +outpath = "./out" +simtime = 0 - # Bind or connect - if self.port_type == "bind": - self.socket.bind(address) - logger.info(f"ZMQ Port bound to {address}") - else: - self.socket.connect(address) - logger.info(f"ZMQ Port connected to {address}") - - def send_json_with_retry(self, message): - """Send JSON message with retries if timeout occurs.""" - for attempt in range(5): - try: - self.socket.send_json(message) - return - except zmq.Again: - logger.warning(f"Send timeout (attempt {attempt + 1}/5)") - time.sleep(0.5) - logger.error("Failed to send after retries.") - return +def _port_path(base, port_num): + return base + str(port_num) - def recv_json_with_retry(self): - """Receive JSON message with retries if timeout occurs.""" - for attempt in range(5): - try: - return self.socket.recv_json() - except zmq.Again: - logger.warning(f"Receive timeout (attempt {attempt + 1}/5)") - time.sleep(0.5) - logger.error("Failed to receive after retries.") - return None +concore_params_file = os.path.join(_port_path(inpath, 1), "concore.params") +concore_maxtime_file = os.path.join(_port_path(inpath, 1), "concore.maxtime") -# Global ZeroMQ ports registry -zmq_ports = {} -_cleanup_in_progress = False +# Load input/output ports if present +iport = safe_literal_eval("concore.iport", {}) +oport = safe_literal_eval("concore.oport", {}) + +_mod = sys.modules[__name__] +# =================================================================== +# ZeroMQ Communication Wrapper +# =================================================================== def init_zmq_port(port_name, port_type, address, socket_type_str): - """ - Initializes and registers a ZeroMQ port. - port_name (str): A unique name for this ZMQ port. - port_type (str): "bind" or "connect". - address (str): The ZMQ address (e.g., "tcp://*:5555", "tcp://localhost:5555"). - socket_type_str (str): String representation of ZMQ socket type (e.g., "REQ", "REP", "PUB", "SUB"). - """ - if port_name in zmq_ports: - logger.info(f"ZMQ Port {port_name} already initialized.") - return # Avoid reinitialization - - try: - # Map socket type string to actual ZMQ constant (e.g., zmq.REQ, zmq.REP) - zmq_socket_type = getattr(zmq, socket_type_str.upper()) - zmq_ports[port_name] = ZeroMQPort(port_type, address, zmq_socket_type) - logger.info(f"Initialized ZMQ port: {port_name} ({socket_type_str}) on {address}") - except AttributeError: - logger.error(f"Error: Invalid ZMQ socket type string '{socket_type_str}'.") - except zmq.error.ZMQError as e: - logger.error(f"Error initializing ZMQ port {port_name} on {address}: {e}") - except Exception as e: - logger.error(f"An unexpected error occurred during ZMQ port initialization for {port_name}: {e}") + concore_base.init_zmq_port(_mod, port_name, port_type, address, socket_type_str) def terminate_zmq(): """Clean up all ZMQ sockets and contexts before exit.""" - global _cleanup_in_progress - - if _cleanup_in_progress: - return # Already cleaning up, prevent reentrant calls - - if not zmq_ports: - return # No ports to clean up - - _cleanup_in_progress = True - print("\nCleaning up ZMQ resources...") - for port_name, port in zmq_ports.items(): - try: - port.socket.close() - port.context.term() - print(f"Closed ZMQ port: {port_name}") - except Exception as e: - logger.error(f"Error while terminating ZMQ port {port.address}: {e}") - zmq_ports.clear() - _cleanup_in_progress = False + concore_base.terminate_zmq(_mod) def signal_handler(sig, frame): """Handle interrupt signals gracefully.""" print(f"\nReceived signal {sig}, shutting down gracefully...") - # Prevent terminate_zmq from being called twice: once here and once via atexit try: atexit.unregister(terminate_zmq) except Exception: - # If unregister fails for any reason, proceed with explicit cleanup anyway pass - terminate_zmq() + concore_base.terminate_zmq(_mod) sys.exit(0) # Register cleanup handlers @@ -146,116 +84,13 @@ def signal_handler(sig, frame): if not hasattr(sys, 'getwindowsversion'): signal.signal(signal.SIGTERM, signal_handler) # Handle termination (Unix only) -# --- ZeroMQ Integration End --- - - -# NumPy Type Conversion Helper -def convert_numpy_to_python(obj): - #Recursively convert numpy types to native Python types. - #This is necessary because literal_eval cannot parse numpy representations - #like np.float64(1.0), but can parse native Python types like 1.0. - if isinstance(obj, np.generic): - # Convert numpy scalar types to Python native types - return obj.item() - elif isinstance(obj, list): - return [convert_numpy_to_python(item) for item in obj] - elif isinstance(obj, tuple): - return tuple(convert_numpy_to_python(item) for item in obj) - elif isinstance(obj, dict): - return {key: convert_numpy_to_python(value) for key, value in obj.items()} - else: - return obj - -# =================================================================== -# File & Parameter Handling -# =================================================================== -def safe_literal_eval(filename, defaultValue): - try: - with open(filename, "r") as file: - return literal_eval(file.read()) - except (FileNotFoundError, SyntaxError, ValueError, Exception) as e: - # Keep print for debugging, but can be made quieter - # print(f"Info: Error reading {filename} or file not found, using default: {e}") - return defaultValue - - -# Load input/output ports if present -iport = safe_literal_eval("concore.iport", {}) -oport = safe_literal_eval("concore.oport", {}) - -# Global variables -s = '' -olds = '' -delay = 1 -retrycount = 0 -inpath = "./in" #must be rel path for local -outpath = "./out" -simtime = 0 -concore_params_file = os.path.join(inpath + "1", "concore.params") -concore_maxtime_file = os.path.join(inpath + "1", "concore.maxtime") - -#9/21/22 -# =================================================================== -# Parameter Parsing -# =================================================================== -def parse_params(sparams: str) -> dict: - params = {} - if not sparams: - return params - - s = sparams.strip() - - #full dict literal - if s.startswith("{") and s.endswith("}"): - try: - val = literal_eval(s) - if isinstance(val, dict): - return val - except (ValueError, SyntaxError): - pass - - for item in s.split(";"): - if "=" in item: - key, value = item.split("=", 1) # split only once - key=key.strip() - value=value.strip() - #try to convert to python type (int, float, list, etc.) - # Use literal_eval to preserve backward compatibility (integers/lists) - # Fallback to string for unquoted values (paths, URLs) - try: - params[key] = literal_eval(value) - except (ValueError, SyntaxError): - params[key] = value - return params - -try: - sparams_path = concore_params_file - if os.path.exists(sparams_path): - with open(sparams_path, "r") as f: - sparams = f.read().strip() - if sparams: # Ensure sparams is not empty - # Windows sometimes keeps quotes - if sparams[0] == '"' and sparams[-1] == '"': #windows keeps "" need to remove - sparams = sparams[1:-1] - - # Parse params using clean function instead of regex - logger.debug("parsing sparams: "+sparams) - params = parse_params(sparams) - logger.debug("parsed params: " + str(params)) - else: - params = dict() - else: - params = dict() -except Exception as e: - # print(f"Info: concore.params not found or error reading, using empty dict: {e}") - params = dict() +params = concore_base.load_params(concore_params_file) #9/30/22 def tryparam(n, i): """Return parameter `n` from params dict, else default `i`.""" return params.get(n, i) - #9/12/21 # =================================================================== # Simulation Time Handling @@ -269,178 +104,17 @@ def default_maxtime(default): def unchanged(): """Check if global string `s` is unchanged since last call.""" - global olds, s - if olds == s: - s = '' - return True - olds = s - return False + return concore_base.unchanged(_mod) # =================================================================== # I/O Handling (File + ZMQ) # =================================================================== def read(port_identifier, name, initstr_val): - global s, simtime, retrycount - - # Default return - default_return_val = initstr_val - if isinstance(initstr_val, str): - try: - default_return_val = literal_eval(initstr_val) - except (SyntaxError, ValueError): - pass - - # Case 1: ZMQ port - if isinstance(port_identifier, str) and port_identifier in zmq_ports: - zmq_p = zmq_ports[port_identifier] - try: - message = zmq_p.recv_json_with_retry() - # Strip simtime prefix if present (mirroring file-based read behavior) - if isinstance(message, list) and len(message) > 0: - first_element = message[0] - if isinstance(first_element, (int, float)): - simtime = max(simtime, first_element) - return message[1:] - return message - except zmq.error.ZMQError as e: - logger.error(f"ZMQ read error on port {port_identifier} (name: {name}): {e}. Returning default.") - return default_return_val - except Exception as e: - logger.error(f"Unexpected error during ZMQ read on port {port_identifier} (name: {name}): {e}. Returning default.") - return default_return_val - - # Case 2: File-based port - try: - file_port_num = int(port_identifier) - except ValueError: - logger.error(f"Error: Invalid port identifier '{port_identifier}' for file operation. Must be integer or ZMQ name.") - return default_return_val - - time.sleep(delay) - file_path = os.path.join(inpath + str(file_port_num), name) - ins = "" - - try: - with open(file_path, "r") as infile: - ins = infile.read() - except FileNotFoundError: - ins = str(initstr_val) - s += ins # Update s to break unchanged() loop - except Exception as e: - logger.error(f"Error reading {file_path}: {e}. Using default value.") - return default_return_val - - # Retry logic if file is empty - attempts = 0 - max_retries = 5 - while len(ins) == 0 and attempts < max_retries: - time.sleep(delay) - try: - with open(file_path, "r") as infile: - ins = infile.read() - except Exception as e: - logger.warning(f"Retry {attempts + 1}: Error reading {file_path} - {e}") - attempts += 1 - retrycount += 1 - - if len(ins) == 0: - logger.error(f"Max retries reached for {file_path}, using default value.") - return default_return_val - - s += ins - - # Try parsing - try: - inval = literal_eval(ins) - if isinstance(inval, list) and len(inval) > 0: - current_simtime_from_file = inval[0] - if isinstance(current_simtime_from_file, (int, float)): - simtime = max(simtime, current_simtime_from_file) - return inval[1:] - else: - logger.warning(f"Warning: Unexpected data format in {file_path}: {ins}. Returning raw content or default.") - return inval - except Exception as e: - logger.error(f"Error parsing content from {file_path} ('{ins}'): {e}. Returning default.") - return default_return_val + return concore_base.read(_mod, port_identifier, name, initstr_val) def write(port_identifier, name, val, delta=0): - """ - Write data either to ZMQ port or file. - `val` is the data payload (list or string); write() prepends [simtime + delta] internally. - """ - global simtime - - # Case 1: ZMQ port - if isinstance(port_identifier, str) and port_identifier in zmq_ports: - zmq_p = zmq_ports[port_identifier] - try: - # Keep ZMQ payloads JSON-serializable by normalizing numpy types. - zmq_val = convert_numpy_to_python(val) - if isinstance(zmq_val, list): - # Prepend simtime to match file-based write behavior - payload = [simtime + delta] + zmq_val - zmq_p.send_json_with_retry(payload) - # simtime must not be mutated here. - # Mutation breaks cross-language determinism (see issue #385). - else: - zmq_p.send_json_with_retry(zmq_val) - except zmq.error.ZMQError as e: - logger.error(f"ZMQ write error on port {port_identifier} (name: {name}): {e}") - except Exception as e: - logger.error(f"Unexpected error during ZMQ write on port {port_identifier} (name: {name}): {e}") - return - - # Case 2: File-based port - try: - file_port_num = int(port_identifier) - file_path = os.path.join(outpath + str(file_port_num), name) - except ValueError: - logger.error(f"Error: Invalid port identifier '{port_identifier}' for file operation. Must be integer or ZMQ name.") - return - - # File writing rules - if isinstance(val, str): - time.sleep(2 * delay) # string writes wait longer - elif not isinstance(val, list): - logger.error(f"File write to {file_path} must have list or str value, got {type(val)}") - return - - try: - with open(file_path, "w") as outfile: - if isinstance(val, list): - # Convert numpy types to native Python types - val_converted = convert_numpy_to_python(val) - data_to_write = [simtime + delta] + val_converted - outfile.write(str(data_to_write)) - # simtime must not be mutated here. - # Mutation breaks cross-language determinism (see issue #385). - else: - outfile.write(val) - except Exception as e: - logger.error(f"Error writing to {file_path}: {e}") + concore_base.write(_mod, port_identifier, name, val, delta) def initval(simtime_val_str): - """ - Initialize simtime from string containing a list. - Example: "[10, 'foo', 'bar']" → simtime=10, returns ['foo','bar'] - """ - global simtime - try: - val = literal_eval(simtime_val_str) - if isinstance(val, list) and len(val) > 0: - first_element = val[0] - if isinstance(first_element, (int, float)): - simtime = first_element - return val[1:] - else: - logger.error(f"Error: First element in initval string '{simtime_val_str}' is not a number. Using data part as is or empty.") - return val[1:] if len(val) > 1 else [] - else: - logger.error(f"Error: initval string '{simtime_val_str}' is not a list or is empty. Returning empty list.") - return [] - - except Exception as e: - logger.error(f"Error parsing simtime_val_str '{simtime_val_str}': {e}. Returning empty list.") - return [] + return concore_base.initval(_mod, simtime_val_str) diff --git a/concore_base.py b/concore_base.py new file mode 100644 index 00000000..f6b3e1b9 --- /dev/null +++ b/concore_base.py @@ -0,0 +1,364 @@ +import time +import logging +import os +from ast import literal_eval +import zmq +import numpy as np + +logger = logging.getLogger('concore') +logger.addHandler(logging.NullHandler()) + +# =================================================================== +# ZeroMQ Communication Wrapper +# =================================================================== +class ZeroMQPort: + def __init__(self, port_type, address, zmq_socket_type): + """ + port_type: "bind" or "connect" + address: ZeroMQ address (e.g., "tcp://*:5555") + zmq_socket_type: zmq.REQ, zmq.REP, zmq.PUB, zmq.SUB etc. + """ + self.context = zmq.Context() + self.socket = self.context.socket(zmq_socket_type) + self.port_type = port_type # "bind" or "connect" + self.address = address + + # Configure timeouts & immediate close on failure + self.socket.setsockopt(zmq.RCVTIMEO, 2000) # 2 sec receive timeout + self.socket.setsockopt(zmq.SNDTIMEO, 2000) # 2 sec send timeout + self.socket.setsockopt(zmq.LINGER, 0) # Drop pending messages on close + + # Bind or connect + if self.port_type == "bind": + self.socket.bind(address) + logger.info(f"ZMQ Port bound to {address}") + else: + self.socket.connect(address) + logger.info(f"ZMQ Port connected to {address}") + + def send_json_with_retry(self, message): + """Send JSON message with retries if timeout occurs.""" + for attempt in range(5): + try: + self.socket.send_json(message) + return + except zmq.Again: + logger.warning(f"Send timeout (attempt {attempt + 1}/5)") + time.sleep(0.5) + logger.error("Failed to send after retries.") + return + + def recv_json_with_retry(self): + """Receive JSON message with retries if timeout occurs.""" + for attempt in range(5): + try: + return self.socket.recv_json() + except zmq.Again: + logger.warning(f"Receive timeout (attempt {attempt + 1}/5)") + time.sleep(0.5) + logger.error("Failed to receive after retries.") + return None + + +def init_zmq_port(mod, port_name, port_type, address, socket_type_str): + """ + Initializes and registers a ZeroMQ port. + mod: calling module (has zmq_ports dict) + port_name (str): A unique name for this ZMQ port. + port_type (str): "bind" or "connect". + address (str): The ZMQ address (e.g., "tcp://*:5555", "tcp://localhost:5555"). + socket_type_str (str): String representation of ZMQ socket type (e.g., "REQ", "REP", "PUB", "SUB"). + """ + if port_name in mod.zmq_ports: + logger.info(f"ZMQ Port {port_name} already initialized.") + return # Avoid reinitialization + + try: + # Map socket type string to actual ZMQ constant (e.g., zmq.REQ, zmq.REP) + zmq_socket_type = getattr(zmq, socket_type_str.upper()) + mod.zmq_ports[port_name] = ZeroMQPort(port_type, address, zmq_socket_type) + logger.info(f"Initialized ZMQ port: {port_name} ({socket_type_str}) on {address}") + except AttributeError: + logger.error(f"Error: Invalid ZMQ socket type string '{socket_type_str}'.") + except zmq.error.ZMQError as e: + logger.error(f"Error initializing ZMQ port {port_name} on {address}: {e}") + except Exception as e: + logger.error(f"An unexpected error occurred during ZMQ port initialization for {port_name}: {e}") + +def terminate_zmq(mod): + """Clean up all ZMQ sockets and contexts before exit.""" + if mod._cleanup_in_progress: + return # Already cleaning up, prevent reentrant calls + + if not mod.zmq_ports: + return # No ports to clean up + + mod._cleanup_in_progress = True + print("\nCleaning up ZMQ resources...") + for port_name, port in mod.zmq_ports.items(): + try: + port.socket.close() + port.context.term() + print(f"Closed ZMQ port: {port_name}") + except Exception as e: + logger.error(f"Error while terminating ZMQ port {port.address}: {e}") + mod.zmq_ports.clear() + mod._cleanup_in_progress = False + +# --- ZeroMQ Integration End --- + + +# NumPy Type Conversion Helper +def convert_numpy_to_python(obj): + #Recursively convert numpy types to native Python types. + #This is necessary because literal_eval cannot parse numpy representations + #like np.float64(1.0), but can parse native Python types like 1.0. + if isinstance(obj, np.generic): + # Convert numpy scalar types to Python native types + return obj.item() + elif isinstance(obj, list): + return [convert_numpy_to_python(item) for item in obj] + elif isinstance(obj, tuple): + return tuple(convert_numpy_to_python(item) for item in obj) + elif isinstance(obj, dict): + return {key: convert_numpy_to_python(value) for key, value in obj.items()} + else: + return obj + +# =================================================================== +# File & Parameter Handling +# =================================================================== +def safe_literal_eval(filename, defaultValue): + try: + with open(filename, "r") as file: + return literal_eval(file.read()) + except (FileNotFoundError, SyntaxError, ValueError, Exception) as e: + # print(f"Info: Error reading {filename} or file not found, using default: {e}") + return defaultValue + +#9/21/22 +# =================================================================== +# Parameter Parsing +# =================================================================== +def parse_params(sparams): + params = {} + if not sparams: + return params + + s = sparams.strip() + + #full dict literal + if s.startswith("{") and s.endswith("}"): + try: + val = literal_eval(s) + if isinstance(val, dict): + return val + except (ValueError, SyntaxError): + pass + + for item in s.split(";"): + if "=" in item: + key, value = item.split("=", 1) # split only once + key=key.strip() + value=value.strip() + #try to convert to python type (int, float, list, etc.) + # Use literal_eval to preserve backward compatibility (integers/lists) + # Fallback to string for unquoted values (paths, URLs) + try: + params[key] = literal_eval(value) + except (ValueError, SyntaxError): + params[key] = value + return params + +def load_params(params_file): + try: + if os.path.exists(params_file): + with open(params_file, "r") as f: + sparams = f.read().strip() + if sparams: + # Windows sometimes keeps quotes + if sparams[0] == '"' and sparams[-1] == '"': #windows keeps "" need to remove + sparams = sparams[1:-1] + logger.debug("parsing sparams: "+sparams) + p = parse_params(sparams) + logger.debug("parsed params: " + str(p)) + return p + return dict() + except Exception: + return dict() + +# =================================================================== +# I/O Handling (File + ZMQ) +# =================================================================== + +def unchanged(mod): + """Check if global string `s` is unchanged since last call.""" + if mod.olds == mod.s: + mod.s = '' + return True + mod.olds = mod.s + return False + + +def read(mod, port_identifier, name, initstr_val): + # Default return + default_return_val = initstr_val + if isinstance(initstr_val, str): + try: + default_return_val = literal_eval(initstr_val) + except (SyntaxError, ValueError): + pass + + # Case 1: ZMQ port + if isinstance(port_identifier, str) and port_identifier in mod.zmq_ports: + zmq_p = mod.zmq_ports[port_identifier] + try: + message = zmq_p.recv_json_with_retry() + # Strip simtime prefix if present (mirroring file-based read behavior) + if isinstance(message, list) and len(message) > 0: + first_element = message[0] + if isinstance(first_element, (int, float)): + mod.simtime = max(mod.simtime, first_element) + return message[1:] + return message + except zmq.error.ZMQError as e: + logger.error(f"ZMQ read error on port {port_identifier} (name: {name}): {e}. Returning default.") + return default_return_val + except Exception as e: + logger.error(f"Unexpected error during ZMQ read on port {port_identifier} (name: {name}): {e}. Returning default.") + return default_return_val + + # Case 2: File-based port + try: + file_port_num = int(port_identifier) + except ValueError: + logger.error(f"Error: Invalid port identifier '{port_identifier}' for file operation. Must be integer or ZMQ name.") + return default_return_val + + time.sleep(mod.delay) + port_dir = mod._port_path(mod.inpath, file_port_num) + file_path = os.path.join(port_dir, name) + ins = "" + + try: + with open(file_path, "r") as infile: + ins = infile.read() + except FileNotFoundError: + ins = str(initstr_val) + mod.s += ins # Update s to break unchanged() loop + except Exception as e: + logger.error(f"Error reading {file_path}: {e}. Using default value.") + return default_return_val + + # Retry logic if file is empty + attempts = 0 + max_retries = 5 + while len(ins) == 0 and attempts < max_retries: + time.sleep(mod.delay) + try: + with open(file_path, "r") as infile: + ins = infile.read() + except Exception as e: + logger.warning(f"Retry {attempts + 1}: Error reading {file_path} - {e}") + attempts += 1 + mod.retrycount += 1 + + if len(ins) == 0: + logger.error(f"Max retries reached for {file_path}, using default value.") + return default_return_val + + mod.s += ins + + # Try parsing + try: + inval = literal_eval(ins) + if isinstance(inval, list) and len(inval) > 0: + current_simtime_from_file = inval[0] + if isinstance(current_simtime_from_file, (int, float)): + mod.simtime = max(mod.simtime, current_simtime_from_file) + return inval[1:] + else: + logger.warning(f"Warning: Unexpected data format in {file_path}: {ins}. Returning raw content or default.") + return inval + except Exception as e: + logger.error(f"Error parsing content from {file_path} ('{ins}'): {e}. Returning default.") + return default_return_val + + +def write(mod, port_identifier, name, val, delta=0): + """ + Write data either to ZMQ port or file. + `val` is the data payload (list or string); write() prepends [simtime + delta] internally. + """ + # Case 1: ZMQ port + if isinstance(port_identifier, str) and port_identifier in mod.zmq_ports: + zmq_p = mod.zmq_ports[port_identifier] + try: + # Keep ZMQ payloads JSON-serializable by normalizing numpy types. + zmq_val = convert_numpy_to_python(val) + if isinstance(zmq_val, list): + # Prepend simtime to match file-based write behavior + payload = [mod.simtime + delta] + zmq_val + zmq_p.send_json_with_retry(payload) + # simtime must not be mutated here. + # Mutation breaks cross-language determinism (see issue #385). + else: + zmq_p.send_json_with_retry(zmq_val) + except zmq.error.ZMQError as e: + logger.error(f"ZMQ write error on port {port_identifier} (name: {name}): {e}") + except Exception as e: + logger.error(f"Unexpected error during ZMQ write on port {port_identifier} (name: {name}): {e}") + return + + # Case 2: File-based port + try: + file_port_num = int(port_identifier) + port_dir = mod._port_path(mod.outpath, file_port_num) + file_path = os.path.join(port_dir, name) + except ValueError: + logger.error(f"Error: Invalid port identifier '{port_identifier}' for file operation. Must be integer or ZMQ name.") + return + + # File writing rules + if isinstance(val, str): + time.sleep(2 * mod.delay) # string writes wait longer + elif not isinstance(val, list): + logger.error(f"File write to {file_path} must have list or str value, got {type(val)}") + return + + try: + with open(file_path, "w") as outfile: + if isinstance(val, list): + # Convert numpy types to native Python types + val_converted = convert_numpy_to_python(val) + data_to_write = [mod.simtime + delta] + val_converted + outfile.write(str(data_to_write)) + # simtime must not be mutated here. + # Mutation breaks cross-language determinism (see issue #385). + else: + outfile.write(val) + except Exception as e: + logger.error(f"Error writing to {file_path}: {e}") + +def initval(mod, simtime_val_str): + """ + Initialize simtime from string containing a list. + Example: "[10, 'foo', 'bar']" -> simtime=10, returns ['foo','bar'] + """ + try: + val = literal_eval(simtime_val_str) + if isinstance(val, list) and len(val) > 0: + first_element = val[0] + if isinstance(first_element, (int, float)): + mod.simtime = first_element + return val[1:] + else: + logger.error(f"Error: First element in initval string '{simtime_val_str}' is not a number. Using data part as is or empty.") + return val[1:] if len(val) > 1 else [] + else: + logger.error(f"Error: initval string '{simtime_val_str}' is not a list or is empty. Returning empty list.") + return [] + + except Exception as e: + logger.error(f"Error parsing simtime_val_str '{simtime_val_str}': {e}. Returning empty list.") + return [] diff --git a/concoredocker.py b/concoredocker.py index 0f1714e0..84d4caa5 100644 --- a/concoredocker.py +++ b/concoredocker.py @@ -3,115 +3,27 @@ import re import os import logging +import atexit +import sys import zmq import numpy as np +import signal + +import concore_base logging.basicConfig( level=logging.INFO, format='%(levelname)s - %(message)s' ) -class ZeroMQPort: - def __init__(self, port_type, address, zmq_socket_type): - self.context = zmq.Context() - self.socket = self.context.socket(zmq_socket_type) - self.port_type = port_type - self.address = address - - # Configure timeouts & immediate close on failure - self.socket.setsockopt(zmq.RCVTIMEO, 2000) # 2 sec receive timeout - self.socket.setsockopt(zmq.SNDTIMEO, 2000) # 2 sec send timeout - self.socket.setsockopt(zmq.LINGER, 0) # Drop pending messages on close - - # Bind or connect - if self.port_type == "bind": - self.socket.bind(address) - logging.info(f"ZMQ Port bound to {address}") - else: - self.socket.connect(address) - logging.info(f"ZMQ Port connected to {address}") - - def send_json_with_retry(self, message): - """Send JSON message with retries if timeout occurs.""" - for attempt in range(5): - try: - self.socket.send_json(message) - return - except zmq.Again: - logging.warning(f"Send timeout (attempt {attempt + 1}/5)") - time.sleep(0.5) - logging.error("Failed to send after retries.") - return +ZeroMQPort = concore_base.ZeroMQPort +convert_numpy_to_python = concore_base.convert_numpy_to_python +safe_literal_eval = concore_base.safe_literal_eval +parse_params = concore_base.parse_params - def recv_json_with_retry(self): - """Receive JSON message with retries if timeout occurs.""" - for attempt in range(5): - try: - return self.socket.recv_json() - except zmq.Again: - logging.warning(f"Receive timeout (attempt {attempt + 1}/5)") - time.sleep(0.5) - logging.error("Failed to receive after retries.") - return None - -# Global ZeroMQ ports registry +# Global variables zmq_ports = {} - -def init_zmq_port(port_name, port_type, address, socket_type_str): - """ - Initializes and registers a ZeroMQ port. - port_name (str): A unique name for this ZMQ port. - port_type (str): "bind" or "connect". - address (str): The ZMQ address (e.g., "tcp://*:5555", "tcp://localhost:5555"). - socket_type_str (str): String representation of ZMQ socket type (e.g., "REQ", "REP", "PUB", "SUB"). - """ - if port_name in zmq_ports: - logging.info(f"ZMQ Port {port_name} already initialized.") - return#avoid reinstallation - try: - # Map socket type string to actual ZMQ constant (e.g., zmq.REQ, zmq.REP) - zmq_socket_type = getattr(zmq, socket_type_str.upper()) - zmq_ports[port_name] = ZeroMQPort(port_type, address, zmq_socket_type) - logging.info(f"Initialized ZMQ port: {port_name} ({socket_type_str}) on {address}") - except AttributeError: - logging.error(f"Error: Invalid ZMQ socket type string '{socket_type_str}'.") - except zmq.error.ZMQError as e: - logging.error(f"Error initializing ZMQ port {port_name} on {address}: {e}") - except Exception as e: - logging.error(f"An unexpected error occurred during ZMQ port initialization for {port_name}: {e}") - -def terminate_zmq(): - for port in zmq_ports.values(): - try: - port.socket.close() - port.context.term() - except Exception as e: - logging.error(f"Error while terminating ZMQ port {port.address}: {e}") -# --- ZeroMQ Integration End --- - -# NumPy Type Conversion Helper -def convert_numpy_to_python(obj): - if isinstance(obj, np.generic): - return obj.item() - elif isinstance(obj, list): - return [convert_numpy_to_python(item) for item in obj] - elif isinstance(obj, tuple): - return tuple(convert_numpy_to_python(item) for item in obj) - elif isinstance(obj, dict): - return {key: convert_numpy_to_python(value) for key, value in obj.items()} - else: - return obj - -def safe_literal_eval(filename, defaultValue): - try: - with open(filename, "r") as file: - return literal_eval(file.read()) - except (FileNotFoundError, SyntaxError, ValueError, Exception) as e: - logging.info(f"Error reading {filename}: {e}") - return defaultValue - -iport = safe_literal_eval("concore.iport", {}) -oport = safe_literal_eval("concore.oport", {}) +_cleanup_in_progress = False s = '' olds = '' @@ -120,59 +32,45 @@ def safe_literal_eval(filename, defaultValue): inpath = os.path.abspath("/in") outpath = os.path.abspath("/out") simtime = 0 -concore_params_file = os.path.join(inpath + "1", "concore.params") -concore_maxtime_file = os.path.join(inpath + "1", "concore.maxtime") -#9/21/22 -def parse_params(sparams): - params = {} - if not sparams: - return params +def _port_path(base, port_num): + return os.path.join(base, str(port_num)) - s = sparams.strip() +concore_params_file = os.path.join(_port_path(inpath, 1), "concore.params") +concore_maxtime_file = os.path.join(_port_path(inpath, 1), "concore.maxtime") - # full dict literal - if s.startswith("{") and s.endswith("}"): - try: - val = literal_eval(s) - if isinstance(val, dict): - return val - except (ValueError, SyntaxError): - pass +# Load input/output ports if present +iport = safe_literal_eval("concore.iport", {}) +oport = safe_literal_eval("concore.oport", {}) - # Potentially breaking backward compatibility: moving away from the comma-separated params - for item in s.split(";"): - if "=" in item: - key, value = item.split("=", 1) - key = key.strip() - value = value.strip() - #try to convert to python type (int, float, list, etc.) - # Use literal_eval to preserve backward compatibility (integers/lists) - # Fallback to string for unquoted values (paths, URLs) - try: - params[key] = literal_eval(value) - except (ValueError, SyntaxError): - params[key] = value - return params +_mod = sys.modules[__name__] -try: - with open(concore_params_file, "r") as f: - sparams = f.read().strip() +# =================================================================== +# ZeroMQ Communication Wrapper +# =================================================================== +def init_zmq_port(port_name, port_type, address, socket_type_str): + concore_base.init_zmq_port(_mod, port_name, port_type, address, socket_type_str) - if sparams and sparams[0] == '"': # windows keeps quotes - sparams = sparams[1:] - if '"' in sparams: - sparams = sparams[:sparams.find('"')] +def terminate_zmq(): + """Clean up all ZMQ sockets and contexts before exit.""" + concore_base.terminate_zmq(_mod) - if sparams: - logging.debug("parsing sparams: "+sparams) - params = parse_params(sparams) - logging.debug("parsed params: " + str(params)) - else: - params = dict() -except Exception as e: - logging.error(f"Error reading concore.params: {e}") - params = dict() +def signal_handler(sig, frame): + """Handle interrupt signals gracefully.""" + print(f"\nReceived signal {sig}, shutting down gracefully...") + try: + atexit.unregister(terminate_zmq) + except Exception: + pass + concore_base.terminate_zmq(_mod) + sys.exit(0) + +# Register cleanup handlers (docker containers receive SIGTERM from docker stop) +atexit.register(terminate_zmq) +signal.signal(signal.SIGINT, signal_handler) +signal.signal(signal.SIGTERM, signal_handler) + +params = concore_base.load_params(concore_params_file) #9/30/22 def tryparam(n, i): @@ -186,156 +84,16 @@ def default_maxtime(default): default_maxtime(100) def unchanged(): - global olds, s - if olds == s: - s = '' - return True - olds = s - return False + return concore_base.unchanged(_mod) +# =================================================================== +# I/O Handling (File + ZMQ) +# =================================================================== def read(port_identifier, name, initstr_val): - global s, simtime, retrycount - - default_return_val = initstr_val - if isinstance(initstr_val, str): - try: - default_return_val = literal_eval(initstr_val) - except (SyntaxError, ValueError): - # Failed to parse initstr_val; fall back to the original string value. - logging.debug( - "Could not parse initstr_val %r with literal_eval; using raw string as default.", - initstr_val - ) - - if isinstance(port_identifier, str) and port_identifier in zmq_ports: - zmq_p = zmq_ports[port_identifier] - try: - message = zmq_p.recv_json_with_retry() - if isinstance(message, list) and len(message) > 0: - first_element = message[0] - if isinstance(first_element, (int, float)): - simtime = max(simtime, first_element) - return message[1:] - return message - except zmq.error.ZMQError as e: - logging.error(f"ZMQ read error on port {port_identifier} (name: {name}): {e}. Returning default.") - return default_return_val - except Exception as e: - logging.error(f"Unexpected error during ZMQ read on port {port_identifier} (name: {name}): {e}. Returning default.") - return default_return_val - - try: - file_port_num = int(port_identifier) - except ValueError: - logging.error(f"Error: Invalid port identifier '{port_identifier}' for file operation. Must be integer or ZMQ name.") - return default_return_val - - time.sleep(delay) - # Construct file path consistent with other components (e.g., /in1/) - file_path = os.path.join(inpath, str(file_port_num), name) - - try: - with open(file_path, "r") as infile: - ins = infile.read() - except FileNotFoundError: - ins = str(initstr_val) - s += ins - except Exception as e: - logging.error(f"Error reading {file_path}: {e}. Using default value.") - return default_return_val - - attempts = 0 - max_retries = 5 - while len(ins) == 0 and attempts < max_retries: - time.sleep(delay) - try: - with open(file_path, "r") as infile: - ins = infile.read() - except Exception as e: - logging.warning(f"Retry {attempts + 1}: Error reading {file_path} - {e}") - attempts += 1 - retrycount += 1 - - if len(ins) == 0: - logging.error(f"Max retries reached for {file_path}, using default value.") - return default_return_val - - s += ins - try: - inval = literal_eval(ins) - if isinstance(inval, list) and len(inval) > 0: - current_simtime_from_file = inval[0] - if isinstance(current_simtime_from_file, (int, float)): - simtime = max(simtime, current_simtime_from_file) - return inval[1:] - else: - logging.warning(f"Warning: Unexpected data format in {file_path}: {ins}. Returning raw content or default.") - return inval - except Exception as e: - logging.error(f"Error parsing content from {file_path} ('{ins}'): {e}. Returning default.") - return default_return_val + return concore_base.read(_mod, port_identifier, name, initstr_val) def write(port_identifier, name, val, delta=0): - global simtime - - if isinstance(port_identifier, str) and port_identifier in zmq_ports: - zmq_p = zmq_ports[port_identifier] - try: - zmq_val = convert_numpy_to_python(val) - if isinstance(zmq_val, list): - # Prepend simtime to match file-based write behavior - payload = [simtime + delta] + zmq_val - zmq_p.send_json_with_retry(payload) - simtime += delta - else: - zmq_p.send_json_with_retry(zmq_val) - except zmq.error.ZMQError as e: - logging.error(f"ZMQ write error on port {port_identifier} (name: {name}): {e}") - except Exception as e: - logging.error(f"Unexpected error during ZMQ write on port {port_identifier} (name: {name}): {e}") - return - - try: - file_port_num = int(port_identifier) - file_path = os.path.join(outpath, str(file_port_num), name) - except ValueError: - logging.error(f"Error: Invalid port identifier '{port_identifier}' for file operation. Must be integer or ZMQ name.") - return - - if isinstance(val, str): - time.sleep(2 * delay) - elif not isinstance(val, list): - logging.error(f"File write to {file_path} must have list or str value, got {type(val)}") - return - - try: - with open(file_path, "w") as outfile: - if isinstance(val, list): - val_converted = convert_numpy_to_python(val) - data_to_write = [simtime + delta] + val_converted - outfile.write(str(data_to_write)) - simtime += delta - else: - outfile.write(val) - except Exception as e: - logging.error(f"Error writing to {file_path}: {e}") + concore_base.write(_mod, port_identifier, name, val, delta) def initval(simtime_val): - global simtime - try: - val = literal_eval(simtime_val) - if isinstance(val, list) and len(val) > 0: - first_element = val[0] - if isinstance(first_element, (int, float)): - simtime = first_element - return val[1:] - else: - logging.error(f"Error: First element in initval string '{simtime_val}' is not a number. Using data part as is or empty.") - return val[1:] if len(val) > 1 else [] - else: - logging.error(f"Error: initval string '{simtime_val}' is not a list or is empty. Returning empty list.") - return [] - except Exception as e: - logging.error(f"Error parsing simtime_val_str '{simtime_val}': {e}. Returning empty list.") - return [] - + return concore_base.initval(_mod, simtime_val) diff --git a/mkconcore.py b/mkconcore.py index fe66c7c8..2258944f 100644 --- a/mkconcore.py +++ b/mkconcore.py @@ -573,8 +573,15 @@ def cleanup_script_files(): quit() with open(outdir+"/src/concore.py","w") as fcopy: fcopy.write(fsource.read()) - fsource.close() - + fsource.close() + try: + with open(CONCOREPATH+"/concore_base.py") as fbase: + with open(outdir+"/src/concore_base.py","w") as fcopy: + fcopy.write(fbase.read()) + except (FileNotFoundError, IOError): + print(CONCOREPATH+" is not correct path to concore (missing concore_base.py)") + quit() + if 'cpp' in required_langs: try: if concoretype=="docker": @@ -767,6 +774,7 @@ def cleanup_script_files(): fbuild.write("cp ../src/"+sourcecode+" .\n") if langext == "py": #4/29/21 fbuild.write("cp ../src/concore.py .\n") + fbuild.write("cp ../src/concore_base.py .\n") elif langext == "cpp": #6/22/21 fbuild.write("cp ../src/concore.hpp .\n") elif langext == "v": #6/25/21 @@ -1005,6 +1013,7 @@ def cleanup_script_files(): fbuild.write("copy .\\src\\"+sourcecode+" .\\"+containername+"\\"+sourcecode+"\n") if langext == "py": fbuild.write("copy .\\src\\concore.py .\\" + containername + "\\concore.py\n") + fbuild.write("copy .\\src\\concore_base.py .\\" + containername + "\\concore_base.py\n") elif langext == "cpp": # 6/22/21 fbuild.write("copy .\\src\\concore.hpp .\\" + containername + "\\concore.hpp\n") @@ -1023,6 +1032,7 @@ def cleanup_script_files(): fbuild.write("cp ./src/"+sourcecode+" ./"+containername+"/"+sourcecode+"\n") if langext == "py": fbuild.write("cp ./src/concore.py ./"+containername+"/concore.py\n") + fbuild.write("cp ./src/concore_base.py ./"+containername+"/concore_base.py\n") elif langext == "cpp": fbuild.write("cp ./src/concore.hpp ./"+containername+"/concore.hpp\n") elif langext == "v": diff --git a/tests/test_concoredocker.py b/tests/test_concoredocker.py index 2b9a260d..c3743911 100644 --- a/tests/test_concoredocker.py +++ b/tests/test_concoredocker.py @@ -109,7 +109,8 @@ def test_writes_with_delta(self, temp_dir): with open(os.path.join(outdir, "testfile")) as f: content = f.read() assert content == "[12.0, 3.0]" - assert concoredocker.simtime == 12.0 + # simtime must not be mutated by write() + assert concoredocker.simtime == 10.0 concoredocker.outpath = old_outpath @@ -180,7 +181,8 @@ def send_json_with_retry(self, message): concoredocker.write("test_zmq", "data", [1.0, 2.0], delta=2) assert dummy.sent == [5.0, 1.0, 2.0] - assert concoredocker.simtime == 5.0 + # simtime must not be mutated by write() + assert concoredocker.simtime == 3.0 def test_read_strips_simtime(self): import concoredocker From 1136c2686df4819ede732b74004329043b9ed1c5 Mon Sep 17 00:00:00 2001 From: Avinash Kumar Deepak Date: Sat, 21 Feb 2026 22:44:07 +0530 Subject: [PATCH 201/275] fix: java init stuck in main and param parser broken for values with = --- concoredocker.java | 67 ++++++++++++++++++++++++---------------------- 1 file changed, 35 insertions(+), 32 deletions(-) diff --git a/concoredocker.java b/concoredocker.java index c987d278..2ac7cbd3 100644 --- a/concoredocker.java +++ b/concoredocker.java @@ -28,58 +28,61 @@ public class concoredocker { private static double simtime = 0; private static double maxtime; - public static void main(String[] args) { + // initialize on class load, same as Python module-level init + static { try { iport = parseFile("concore.iport"); } catch (IOException e) { - e.printStackTrace(); } try { oport = parseFile("concore.oport"); } catch (IOException e) { - e.printStackTrace(); } - try { String sparams = new String(Files.readAllBytes(Paths.get(inpath + "1/concore.params")), java.nio.charset.StandardCharsets.UTF_8); if (sparams.length() > 0 && sparams.charAt(0) == '"') { // windows keeps "" need to remove sparams = sparams.substring(1); sparams = sparams.substring(0, sparams.indexOf('"')); } - // Try parsing as dict literal first (matches Python parse_params logic) - sparams = sparams.trim(); - if (sparams.startsWith("{") && sparams.endsWith("}")) { - try { - Object parsed = literalEval(sparams); - if (parsed instanceof Map) { - @SuppressWarnings("unchecked") - Map parsedMap = (Map) parsed; - params = parsedMap; - } - } catch (Exception e) { - System.out.println("bad params: " + sparams); + params = parseParams(sparams); + } catch (IOException e) { + params = new HashMap<>(); + } + defaultMaxTime(100); + } + + /** + * Parses a param string into a map, matching concore_base.parse_params. + * Tries dict literal first, then falls back to semicolon-separated key=value pairs. + */ + private static Map parseParams(String sparams) { + Map result = new HashMap<>(); + if (sparams == null || sparams.isEmpty()) return result; + String trimmed = sparams.trim(); + if (trimmed.startsWith("{") && trimmed.endsWith("}")) { + try { + Object val = literalEval(trimmed); + if (val instanceof Map) { + @SuppressWarnings("unchecked") + Map map = (Map) val; + return map; } - } else if (!sparams.isEmpty()) { - // Fallback: convert key=value,key=value format to dict - System.out.println("converting sparams: " + sparams); - sparams = "{'" + sparams.replaceAll(";", ",'").replaceAll("=", "':").replaceAll(" ", "") + "}"; - System.out.println("converted sparams: " + sparams); + } catch (Exception e) { + } + } + for (String item : trimmed.split(";")) { + if (item.contains("=")) { + String[] parts = item.split("=", 2); // split on first '=' only + String key = parts[0].trim(); + String value = parts[1].trim(); try { - Object parsed = literalEval(sparams); - if (parsed instanceof Map) { - @SuppressWarnings("unchecked") - Map parsedMap = (Map) parsed; - params = parsedMap; - } + result.put(key, literalEval(value)); } catch (Exception e) { - System.out.println("bad params: " + sparams); + result.put(key, value); } } - } catch (IOException e) { - params = new HashMap<>(); } - - defaultMaxTime(100); + return result; } /** From 209297bb7a016fc93ab7d723dd1d1ae613fe1ead Mon Sep 17 00:00:00 2001 From: Avinash Kumar Deepak Date: Sun, 22 Feb 2026 22:24:12 +0530 Subject: [PATCH 202/275] concore_cli: add watch command for live simulation monitoring --- concore_cli/cli.py | 13 ++ concore_cli/commands/__init__.py | 3 +- concore_cli/commands/watch.py | 225 +++++++++++++++++++++++++++++++ 3 files changed, 240 insertions(+), 1 deletion(-) create mode 100644 concore_cli/commands/watch.py diff --git a/concore_cli/cli.py b/concore_cli/cli.py index 615cb7b9..9076e870 100644 --- a/concore_cli/cli.py +++ b/concore_cli/cli.py @@ -9,6 +9,7 @@ from .commands.status import show_status from .commands.stop import stop_all from .commands.inspect import inspect_workflow +from .commands.watch import watch_study console = Console() DEFAULT_EXEC_TYPE = 'windows' if os.name == 'nt' else 'posix' @@ -87,5 +88,17 @@ def stop(): console.print(f"[red]Error:[/red] {str(e)}") sys.exit(1) +@cli.command() +@click.argument('study_dir', type=click.Path(exists=True)) +@click.option('--interval', '-n', default=2.0, help='Refresh interval in seconds') +@click.option('--once', is_flag=True, help='Print a single snapshot and exit') +def watch(study_dir, interval, once): + """Watch a running simulation study for live monitoring""" + try: + watch_study(study_dir, interval, once, console) + except Exception as e: + console.print(f"[red]Error:[/red] {str(e)}") + sys.exit(1) + if __name__ == '__main__': cli() diff --git a/concore_cli/commands/__init__.py b/concore_cli/commands/__init__.py index dd9bae05..77820b85 100644 --- a/concore_cli/commands/__init__.py +++ b/concore_cli/commands/__init__.py @@ -3,5 +3,6 @@ from .validate import validate_workflow from .status import show_status from .stop import stop_all +from .watch import watch_study -__all__ = ['init_project', 'run_workflow', 'validate_workflow', 'show_status', 'stop_all'] +__all__ = ['init_project', 'run_workflow', 'validate_workflow', 'show_status', 'stop_all', 'watch_study'] diff --git a/concore_cli/commands/watch.py b/concore_cli/commands/watch.py new file mode 100644 index 00000000..44f39e74 --- /dev/null +++ b/concore_cli/commands/watch.py @@ -0,0 +1,225 @@ +import time +import re +from pathlib import Path +from ast import literal_eval +from datetime import datetime +from rich.table import Table +from rich.live import Live +from rich.panel import Panel +from rich.text import Text +from rich.console import Group + + +def watch_study(study_dir, interval, once, console): + study_path = Path(study_dir).resolve() + if not study_path.is_dir(): + console.print(f"[red]Error:[/red] '{study_dir}' is not a directory") + return + + nodes = _find_nodes(study_path) + edges = _find_edges(study_path, nodes) + + if not nodes and not edges: + console.print(Panel( + "[yellow]No nodes or edge directories found.[/yellow]\n" + "[dim]Make sure you point to a built study directory (run makestudy/build first).[/dim]", + title="concore watch", border_style="yellow")) + return + + if once: + output = _build_display(study_path, nodes, edges) + console.print(output) + return + + console.print(f"[cyan]Watching:[/cyan] {study_path}") + console.print(f"[dim]Refresh every {interval}s — Ctrl+C to stop[/dim]\n") + + try: + with Live(console=console, refresh_per_second=1, screen=False) as live: + while True: + nodes = _find_nodes(study_path) + edges = _find_edges(study_path, nodes) + live.update(_build_display(study_path, nodes, edges)) + time.sleep(interval) + except KeyboardInterrupt: + console.print("\n[yellow]Watch stopped.[/yellow]") + + +def _build_display(study_path, nodes, edges): + parts = [] + + header = Text() + header.append("Study ", style="bold cyan") + header.append(study_path.name, style="bold white") + header.append(f" | {len(nodes)} node(s), {len(edges)} edge(s)", style="dim") + parts.append(header) + + if edges: + parts.append(Text()) + parts.append(_edge_table(edges)) + + if nodes: + parts.append(Text()) + parts.append(_node_table(nodes)) + + if not edges and not nodes: + parts.append(Panel("[yellow]No data yet[/yellow]", + border_style="yellow")) + + return Group(*parts) + + +def _edge_table(edges): + table = Table(title="Edges (port data)", show_header=True, + title_style="bold cyan", expand=True) + table.add_column("Edge", style="green", min_width=10) + table.add_column("Port File", style="cyan") + table.add_column("Simtime", style="yellow", justify="right") + table.add_column("Value", style="white") + table.add_column("Age", style="magenta", justify="right") + + now = time.time() + for edge_name, edge_path in sorted(edges): + files = _read_edge_files(edge_path) + if not files: + table.add_row(edge_name, "[dim]—[/dim]", "", "[dim]empty[/dim]", "") + continue + first = True + for fname, simtime_val, value_str, mtime in files: + age_str = _format_age(now, mtime) + label = edge_name if first else "" + st = str(simtime_val) if simtime_val is not None else "—" + table.add_row(label, fname, st, value_str, age_str) + first = False + + return table + + +def _node_table(nodes): + table = Table(title="Nodes", show_header=True, + title_style="bold cyan", expand=True) + table.add_column("Node", style="green", min_width=10) + table.add_column("Ports (in)", style="cyan") + table.add_column("Ports (out)", style="cyan") + table.add_column("Source", style="dim") + + for node_name, node_path in sorted(nodes): + in_dirs = sorted(d.name for d in node_path.iterdir() + if d.is_dir() and re.match(r'^in\d+$', d.name)) + out_dirs = sorted(d.name for d in node_path.iterdir() + if d.is_dir() and re.match(r'^out\d+$', d.name)) + src = _detect_source(node_path) + table.add_row( + node_name, + ", ".join(in_dirs) if in_dirs else "—", + ", ".join(out_dirs) if out_dirs else "—", + src, + ) + + return table + + +def _find_nodes(study_path): + nodes = [] + port_re = re.compile(r'^(in|out)\d+$') + skip = {'src', '__pycache__', '.git'} + for entry in study_path.iterdir(): + if not entry.is_dir() or entry.name in skip or entry.name.startswith('.'): + continue + try: + has_ports = any(c.is_dir() and port_re.match(c.name) + for c in entry.iterdir()) + except PermissionError: + continue + if has_ports: + nodes.append((entry.name, entry)) + return nodes + + +def _find_edges(study_path, nodes=None): + if nodes is None: + nodes = _find_nodes(study_path) + node_names = {name for name, _ in nodes} + skip = {'src', '__pycache__', '.git'} + edges = [] + for entry in study_path.iterdir(): + if not entry.is_dir(): + continue + if entry.name in skip or entry.name in node_names or entry.name.startswith('.'): + continue + try: + has_file = any(f.is_file() for f in entry.iterdir()) + except PermissionError: + continue + if has_file: + edges.append((entry.name, entry)) + return edges + + +def _read_edge_files(edge_path): + results = [] + try: + children = sorted(edge_path.iterdir()) + except PermissionError: + return results + for f in children: + if not f.is_file(): + continue + # skip concore internal files + if f.name.startswith('concore.'): + continue + simtime_val, value_str = _parse_port_file(f) + try: + mtime = f.stat().st_mtime + except OSError: + mtime = 0 + results.append((f.name, simtime_val, value_str, mtime)) + return results + + +def _detect_source(node_path): + for ext in ('*.py', '*.m', '*.cpp', '*.v', '*.sh', '*.java'): + matches = list(node_path.glob(ext)) + for m in matches: + # skip concore library copies + if m.name.startswith('concore'): + continue + return m.name + return "—" + + +def _parse_port_file(port_file): + try: + content = port_file.read_text().strip() + if not content: + return None, "(empty)" + val = literal_eval(content) + if isinstance(val, list) and len(val) > 0: + simtime = val[0] if isinstance(val[0], (int, float)) else None + data = val[1:] if simtime is not None else val + data_str = str(data) + if len(data_str) > 50: + data_str = data_str[:47] + "..." + return simtime, data_str + raw = str(val) + return None, raw[:50] if len(raw) > 50 else raw + except Exception: + try: + raw = port_file.read_text().strip() + return None, raw[:50] if raw else "(empty)" + except Exception: + return None, "(read error)" + + +def _format_age(now, mtime): + if mtime == 0: + return "—" + age = now - mtime + if age < 3: + return "[bold green]now[/bold green]" + elif age < 60: + return f"{int(age)}s" + elif age < 3600: + return f"{int(age // 60)}m" + else: + return datetime.fromtimestamp(mtime).strftime("%H:%M:%S") From 866beed61f24ce181784781518606e1993212ead Mon Sep 17 00:00:00 2001 From: GREENRAT-K405 Date: Mon, 23 Feb 2026 02:17:04 +0530 Subject: [PATCH 203/275] add option to see realtime simulation --- tools/plotu.py | 96 +++++++++++++++++++++++++++++++--------------- tools/plotym.py | 74 +++++++++++++++++++++++++---------- tools/plotymlag.py | 67 ++++++++++++++++++++++++-------- 3 files changed, 169 insertions(+), 68 deletions(-) diff --git a/tools/plotu.py b/tools/plotu.py index 73edc6ec..25a8ab55 100644 --- a/tools/plotu.py +++ b/tools/plotu.py @@ -12,48 +12,82 @@ ut = [] ymt = [] u = concore.initval(init_simtime_u) + +# --- Real-time plotting setup --- +realtime = concore.tryparam('realtime', False) +if realtime: + plt.ion() + fig, axs = plt.subplots(3, 2, figsize=(8, 6)) + lines = [ax.plot([], [])[0] for ax in axs.flat] + ylabels = ['Pw1 (s)', 'Pf1 (Hz)', 'Pw2 (s)', 'Pf2 (Hz)', 'Pw3 (s)', 'Pf3 (Hz)'] + + for ax, ylab in zip(axs.flat, ylabels): + ax.set_ylabel(ylab) + axs[2, 0].set_xlabel('Cycles') + axs[2, 1].set_xlabel('Cycles') + plt.tight_layout() + plt.show(block=False) +# -------------------------------- + while(concore.simtime= 6: + for i in range(6): + lines[i].set_data(range(len(ut)), [x[i].item() for x in ut]) + axs.flat[i].relim() + axs.flat[i].autoscale_view() + fig.canvas.draw() + fig.canvas.flush_events() + # ----------------------------- + logging.info(f"retry={concore.retrycount}") ################# -# plot inputs and outputs -u1 = [x[0].item() for x in ut] -u2 = [x[1].item() for x in ut] -u3 = [x[2].item() for x in ut] -u4 = [x[3].item() for x in ut] -u5 = [x[4].item() for x in ut] -u6 = [x[5].item() for x in ut] - -Nsim = len(u1) -plt.figure() -plt.subplot(321) -plt.plot(range(Nsim), u1) -plt.ylabel('Pw1 (s)') -plt.subplot(322) -plt.plot(range(Nsim), u2) -plt.ylabel('Pf1 (Hz)') -plt.subplot(323) -plt.plot(range(Nsim), u3) -plt.xlabel('Cycles') -plt.ylabel('Pw2 (s)') -plt.subplot(324) -plt.plot(range(Nsim), u4) -plt.ylabel('Pf2 (Hz)') -plt.subplot(325) -plt.plot(range(Nsim), u5) -plt.ylabel('Pw3 (s)') -plt.subplot(326) -plt.plot(range(Nsim), u6) -plt.xlabel('Cycles') -plt.ylabel('Pf3 (Hz)') +# Finalize rendering +if realtime: + plt.ioff() +else: + # Original static plotting logic + u1 = [x[0].item() for x in ut] + u2 = [x[1].item() for x in ut] + u3 = [x[2].item() for x in ut] + u4 = [x[3].item() for x in ut] + u5 = [x[4].item() for x in ut] + u6 = [x[5].item() for x in ut] + + Nsim = len(u1) + plt.figure() + plt.subplot(321) + plt.plot(range(Nsim), u1) + plt.ylabel('Pw1 (s)') + plt.subplot(322) + plt.plot(range(Nsim), u2) + plt.ylabel('Pf1 (Hz)') + plt.subplot(323) + plt.plot(range(Nsim), u3) + plt.xlabel('Cycles') + plt.ylabel('Pw2 (s)') + plt.subplot(324) + plt.plot(range(Nsim), u4) + plt.ylabel('Pf2 (Hz)') + plt.subplot(325) + plt.plot(range(Nsim), u5) + plt.ylabel('Pw3 (s)') + plt.subplot(326) + plt.plot(range(Nsim), u6) + plt.xlabel('Cycles') + plt.ylabel('Pf3 (Hz)') + plt.tight_layout() + +# Save and show for both modes plt.savefig("stim.pdf") -plt.tight_layout() plt.show() diff --git a/tools/plotym.py b/tools/plotym.py index cd9bfae2..fdeac954 100644 --- a/tools/plotym.py +++ b/tools/plotym.py @@ -5,37 +5,69 @@ import time logging.info("plot ym") -concore.delay = 0.005 +concore.delay = 0.02 concore.default_maxtime(150) -init_simtime_u = "[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]" -init_simtime_ym = "[0.0, 0.0, 0.0]" -ut = [] +init_simtime_u = "[0.0, 0.0]" +init_simtime_ym = "[0.0, 0.0]" ymt = [] ym = concore.initval(init_simtime_ym) + +# 1. Fetch 'realtime' parameter passed from terminal (defaults to False) +realtime = concore.tryparam('realtime', False) + +# DEBUG: Check if the parameter was successfully caught +logging.info(f"--- Realtime mode is set to: {realtime} ---") + +# 2. Set up interactive plot before the loop if realtime is True +if realtime: + plt.ion() # Turn on interactive mode + fig, ax1 = plt.subplots(1, 1) + line1, = ax1.plot([], []) + + ax1.set_ylabel('ym') + ax1.legend(['ym'], loc=0) + ax1.set_xlabel('Cycles') + plt.show(block=False) # Ensure it does not block the script + while(concore.simtime 0 and len(ymt[-1]) >= 2: + for i in range(2): + lines[i].set_data(range(len(ymt)), [x[i].item() for x in ymt]) + axs[i].relim() + axs[i].autoscale_view() + fig.canvas.draw() + fig.canvas.flush_events() + # ----------------------------- + logging.info(f"retry={concore.retrycount}") ################# # plot inputs and outputs -ym1 = [x[0].item() for x in ymt] -ym2 = [x[1].item() for x in ymt] -Nsim = len(ym1) - -plt.figure() -plt.subplot(211) -plt.plot(range(Nsim), ym1) -plt.ylabel('MAP (mmHg)') -plt.legend(['MAP'], loc=0) -plt.subplot(212) -plt.plot(range(Nsim), ym2) -plt.xlabel('Cycles '+str(concore.params)) -plt.ylabel('HR (bpm)') -plt.legend(['HR'], loc=0) +if realtime: + plt.ioff() +else: + # Original static plotting logic + ym1 = [x[0].item() for x in ymt] + ym2 = [x[1].item() for x in ymt] + Nsim = len(ym1) + + plt.figure() + plt.subplot(211) + plt.plot(range(Nsim), ym1) + plt.ylabel('MAP (mmHg)') + plt.legend(['MAP'], loc=0) + plt.subplot(212) + plt.plot(range(Nsim), ym2) + plt.xlabel('Cycles '+str(concore.params)) + plt.ylabel('HR (bpm)') + plt.legend(['HR'], loc=0) + plt.tight_layout() + plt.savefig("hrmap.pdf") -plt.show() +plt.show() \ No newline at end of file From c28d6b0a68691e891d25c3fe9852f5bad0348068 Mon Sep 17 00:00:00 2001 From: GREENRAT-K405 Date: Mon, 23 Feb 2026 02:47:52 +0530 Subject: [PATCH 204/275] add sample for realtime simulation --- demo/controller_RT.py | 37 ++++++++ demo/plotym_RT.py | 73 +++++++++++++++ demo/pm_RT.py | 45 +++++++++ demo/sample_RT.graphml | 208 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 363 insertions(+) create mode 100644 demo/controller_RT.py create mode 100644 demo/plotym_RT.py create mode 100644 demo/pm_RT.py create mode 100644 demo/sample_RT.graphml diff --git a/demo/controller_RT.py b/demo/controller_RT.py new file mode 100644 index 00000000..5516fbcc --- /dev/null +++ b/demo/controller_RT.py @@ -0,0 +1,37 @@ +import numpy as np +from ast import literal_eval +import concore + +try: + ysp = literal_eval(open("ysp.txt").read()) +except: + ysp = 3.0 + +def controller(ym): + if ym[0] < ysp: + return 1.01 * ym + else: + return 0.9 * ym + +#//main// +concore.default_maxtime(150) ##maps to-- for i in range(0,150): +concore.delay = 0.02 + +#//initial values-- transforms to string including the simtime as the 0th entry in the list// +# u = np.array([[0.0]]) +# ym = np.array([[0.0]]) +init_simtime_u = "[0.0, 0.0]" +init_simtime_ym = "[0.0, 0.0]" + +u = np.array([concore.initval(init_simtime_u)]).T +while(concore.simtime + + + + + + + + + + + pm:pm_RT.py + + + + + + + + + + + controller:controller_RT.py + + + + + + + + + + + plotym:plotym_RT.py + + + + + + + + + + edge1 + + + + + + + + + + + + edge2 + + + + + + + + + + + + edge3 + + + + + + + + 1771794664219 + + DEL_NODE + WyI5N2M2OTRlZS0zNTAzLTRlZDctOWZhZS0xOGQ3YjgwNzA5ZmYiXQ== + + + ADD_NODE + WyJwbTpwbV9SVC5weSIseyJ3aWR0aCI6MTE4LCJoZWlnaHQiOjYwLCJzaGFwZSI6InJlY3RhbmdsZSIsIm9wYWNpdHkiOjEsImJhY2tncm91bmRDb2xvciI6IiNmZmNjMDAiLCJib3JkZXJDb2xvciI6IiMwMDAiLCJib3JkZXJXaWR0aCI6MX0sIm9yZGluIix7IngiOjExMCwieSI6MTEwfSx7fSwiOTdjNjk0ZWUtMzUwMy00ZWQ3LTlmYWUtMThkN2I4MDcwOWZmIl0= + + a95ae0297e5f0e63bfa65a29c8d14d92 + + + 1771794687823 + + DEL_NODE + WyIxYmUwMDJkYi0wZmMyLTRjM2ItOGNhMS01NGU0YmJiM2ZiY2IiXQ== + + + ADD_NODE + WyJjb250cm9sbGVyOmNvbnRyb2xsZXJfUlQucHkiLHsid2lkdGgiOjIwNiwiaGVpZ2h0Ijo2MCwic2hhcGUiOiJyZWN0YW5nbGUiLCJvcGFjaXR5IjoxLCJiYWNrZ3JvdW5kQ29sb3IiOiIjZmZjYzAwIiwiYm9yZGVyQ29sb3IiOiIjMDAwIiwiYm9yZGVyV2lkdGgiOjF9LCJvcmRpbiIseyJ4Ijo2NzAsInkiOjkwfSx7fSwiMWJlMDAyZGItMGZjMi00YzNiLThjYTEtNTRlNGJiYjNmYmNiIl0= + + 81165b0ec333e71a4ee794f887493323 + + + 1771794690698 + + SET_POS + WyIxYmUwMDJkYi0wZmMyLTRjM2ItOGNhMS01NGU0YmJiM2ZiY2IiLHsieCI6MTEwLCJ5IjoxMTB9LHsieCI6NDkwLCJ5IjoxMTB9XQ== + + + SET_POS + WyIxYmUwMDJkYi0wZmMyLTRjM2ItOGNhMS01NGU0YmJiM2ZiY2IiLHsieCI6NDkwLCJ5IjoxMTB9LHsieCI6MTEwLCJ5IjoxMTB9XQ== + + 2137de444092ba020db5d395a39a3b79 + + + 1771794720693 + + DEL_NODE + WyIyN2M1ZDUyNC1iZGM5LTQ4Y2QtOWFhNi00Y2IxODk0Y2I1ZDciXQ== + + + ADD_NODE + WyJwbG90eW06cGxvdHltX1JULnB5Iix7IndpZHRoIjoxOTYsImhlaWdodCI6NjAsInNoYXBlIjoicmVjdGFuZ2xlIiwib3BhY2l0eSI6MSwiYmFja2dyb3VuZENvbG9yIjoiI2ZmY2MwMCIsImJvcmRlckNvbG9yIjoiIzAwMCIsImJvcmRlcldpZHRoIjoxfSwib3JkaW4iLHsieCI6MzcwLCJ5Ijo0MzB9LHt9LCIyN2M1ZDUyNC1iZGM5LTQ4Y2QtOWFhNi00Y2IxODk0Y2I1ZDciXQ== + + 64c4ee3fac24cbd01e9cf06ea57dd8c2 + + + 1771794723359 + + SET_POS + WyIyN2M1ZDUyNC1iZGM5LTQ4Y2QtOWFhNi00Y2IxODk0Y2I1ZDciLHsieCI6MTEwLCJ5IjoxMTB9LHsieCI6MzcwLCJ5IjozNTB9XQ== + + + SET_POS + WyIyN2M1ZDUyNC1iZGM5LTQ4Y2QtOWFhNi00Y2IxODk0Y2I1ZDciLHsieCI6MzcwLCJ5IjozNTB9LHsieCI6MTEwLCJ5IjoxMTB9XQ== + + b92c36146ebe446c0907df865f83794d + + + 1771794740309 + + DEL_EDGE + WyJkNGRkZDA2Mi00NTUyLTRkNTktOWIzYy0wYzA0ZGU2MGQxOTciXQ== + + + ADD_EDGE + W3sic291cmNlSUQiOiIxYmUwMDJkYi0wZmMyLTRjM2ItOGNhMS01NGU0YmJiM2ZiY2IiLCJ0YXJnZXRJRCI6Ijk3YzY5NGVlLTM1MDMtNGVkNy05ZmFlLTE4ZDdiODA3MDlmZiIsImxhYmVsIjoiZWRnZTEiLCJzdHlsZSI6eyJ0aGlja25lc3MiOjIsImJhY2tncm91bmRDb2xvciI6IiM1NTUiLCJzaGFwZSI6InNvbGlkIn0sImlkIjoiZDRkZGQwNjItNDU1Mi00ZDU5LTliM2MtMGMwNGRlNjBkMTk3In1d + + 8b3c683a137808fb40dc2921313f8115 + + + 1771794751288 + + SET_POS + WyIxYmUwMDJkYi0wZmMyLTRjM2ItOGNhMS01NGU0YmJiM2ZiY2IiLHsieCI6NDkwLCJ5IjoxMTB9LHsieCI6NjMwLCJ5Ijo5MH1d + + + SET_POS + WyIxYmUwMDJkYi0wZmMyLTRjM2ItOGNhMS01NGU0YmJiM2ZiY2IiLHsieCI6NjMwLCJ5Ijo5MH0seyJ4Ijo0OTAsInkiOjExMH1d + + 0ce00744b96d9ae529645d9297b56821 + + + 1771794765726 + + DEL_EDGE + WyIwOGU0NTRhNS1mYjAwLTQ2MDEtOWMxNC00NDA4NGIyMzFiNTciXQ== + + + ADD_EDGE + W3sic291cmNlSUQiOiI5N2M2OTRlZS0zNTAzLTRlZDctOWZhZS0xOGQ3YjgwNzA5ZmYiLCJ0YXJnZXRJRCI6IjI3YzVkNTI0LWJkYzktNDhjZC05YWE2LTRjYjE4OTRjYjVkNyIsImxhYmVsIjoiZWRnZTIiLCJzdHlsZSI6eyJ0aGlja25lc3MiOjIsImJhY2tncm91bmRDb2xvciI6IiM1NTUiLCJzaGFwZSI6InNvbGlkIn0sImlkIjoiMDhlNDU0YTUtZmIwMC00NjAxLTljMTQtNDQwODRiMjMxYjU3In1d + + 901af9eaa14d4bf0fc1ba715fc5ac3d2 + + + 1771794782565 + + DEL_EDGE + WyI3ZmM3YjQxNy01NDBkLTRhMDgtYTc4Ni0wNDkwYjgyMDg4YWUiXQ== + + + ADD_EDGE + W3sic291cmNlSUQiOiIyN2M1ZDUyNC1iZGM5LTQ4Y2QtOWFhNi00Y2IxODk0Y2I1ZDciLCJ0YXJnZXRJRCI6IjFiZTAwMmRiLTBmYzItNGMzYi04Y2ExLTU0ZTRiYmIzZmJjYiIsImxhYmVsIjoiZWRnZTMiLCJzdHlsZSI6eyJ0aGlja25lc3MiOjIsImJhY2tncm91bmRDb2xvciI6IiM1NTUiLCJzaGFwZSI6InNvbGlkIn0sImlkIjoiN2ZjN2I0MTctNTQwZC00YTA4LWE3ODYtMDQ5MGI4MjA4OGFlIn1d + + e157ed01dca772c8461787c1053794ce + + + 1771794788695 + + SET_POS + WyIyN2M1ZDUyNC1iZGM5LTQ4Y2QtOWFhNi00Y2IxODk0Y2I1ZDciLHsieCI6MzcwLCJ5IjozNTB9LHsieCI6MzcwLCJ5Ijo0MzB9XQ== + + + SET_POS + WyIyN2M1ZDUyNC1iZGM5LTQ4Y2QtOWFhNi00Y2IxODk0Y2I1ZDciLHsieCI6MzcwLCJ5Ijo0MzB9LHsieCI6MzcwLCJ5IjozNTB9XQ== + + aebd758397fec30aac0b59fe643a0da2 + + + 1771794795323 + + SET_POS + WyIxYmUwMDJkYi0wZmMyLTRjM2ItOGNhMS01NGU0YmJiM2ZiY2IiLHsieCI6NjMwLCJ5Ijo5MH0seyJ4Ijo2NzAsInkiOjkwfV0= + + + SET_POS + WyIxYmUwMDJkYi0wZmMyLTRjM2ItOGNhMS01NGU0YmJiM2ZiY2IiLHsieCI6NjcwLCJ5Ijo5MH0seyJ4Ijo2MzAsInkiOjkwfV0= + + 20778f5ad2b1c7bd577f09da007bc0e0 + + + \ No newline at end of file From 73b239371768d9dfbafea88de7941628b73a23f6 Mon Sep 17 00:00:00 2001 From: GREENRAT-K405 Date: Mon, 23 Feb 2026 03:52:37 +0530 Subject: [PATCH 205/275] update tools documentation --- tools/README.md | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/tools/README.md b/tools/README.md index 5e393d6c..71ecfc2d 100644 --- a/tools/README.md +++ b/tools/README.md @@ -1,3 +1,34 @@ # _concore_ tools A collection of utility methods and toolkits for _concore_. + +## Plotting Tools + +This directory includes several standard Python scripts for visualizing your simulation data: +* **`plotym.py`**: This script visualizes the measured outputs of the physiological model over the course of the simulation. +* **`plotymlag.py`**: Plots outputs utilizing a configurable lag buffer. +* **`plotu.py`**: This script visualizes the control signals being generated by the controller node.Specifically, this script generates a 3x2 grid to track neuromodulation stimulation parameters across three different channels, displaying the Pulse Width (Pw) in seconds and Pulse Frequency (Pf) in Hz for each channel. + +### Real-Time Plotting + +By default, these scripts collect data silently during the simulation and render a single static graph when the simulation completes. + +You can enable **real-time dynamic plotting** by passing the `realtime=True` parameter to your simulation. This opens an interactive Matplotlib window that updates continuously as the simulation runs. + +#### How to enable real-time plotting + +After generating and building your workflow, navigate to your generated output directory (usually `out/`). Use the generated `params` script to set the parameter *before* starting the run script: + +**POSIX (Linux / macOS):** +```bash +cd +./params "realtime=True" +./run or ./debug +``` + +**Windows:** +```bash +cd +.\params "realtime=True" +.\run or .\debug +``` \ No newline at end of file From bc87af5dba40dc37847a9dae583baea517d65c46 Mon Sep 17 00:00:00 2001 From: Avinash Kumar Deepak Date: Mon, 23 Feb 2026 16:24:47 +0530 Subject: [PATCH 206/275] fix: packaging broken install and version duplication (#437) --- .gitignore | 5 +++++ concore_cli/__init__.py | 2 ++ concore_cli/cli.py | 3 ++- pyproject.toml | 13 ++++++++---- requirements.txt | 3 --- setup.py | 45 ++++------------------------------------- 6 files changed, 22 insertions(+), 49 deletions(-) diff --git a/.gitignore b/.gitignore index 0488ad98..81647a6f 100644 --- a/.gitignore +++ b/.gitignore @@ -19,6 +19,11 @@ ENV/ .vscode/ .idea/ +# Build/packaging +*.egg-info/ +dist/ +build/ + # Testing .pytest_cache/ htmlcov/ diff --git a/concore_cli/__init__.py b/concore_cli/__init__.py index 658e35e4..8e8a3220 100644 --- a/concore_cli/__init__.py +++ b/concore_cli/__init__.py @@ -1,3 +1,5 @@ +__version__ = "1.0.0" + from .cli import cli __all__ = ['cli'] diff --git a/concore_cli/cli.py b/concore_cli/cli.py index 9076e870..fadb8b6e 100644 --- a/concore_cli/cli.py +++ b/concore_cli/cli.py @@ -10,12 +10,13 @@ from .commands.stop import stop_all from .commands.inspect import inspect_workflow from .commands.watch import watch_study +from . import __version__ console = Console() DEFAULT_EXEC_TYPE = 'windows' if os.name == 'nt' else 'posix' @click.group() -@click.version_option(version='1.0.0', prog_name='concore') +@click.version_option(version=__version__, prog_name='concore') def cli(): pass diff --git a/pyproject.toml b/pyproject.toml index 2c12b5bc..94cb9154 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "concore" -version = "1.0.0" +dynamic = ["version"] description = "Concore workflow management CLI" readme = "README.md" requires-python = ">=3.9" @@ -17,8 +17,6 @@ dependencies = [ "psutil>=5.8.0", "numpy>=1.19.0", "pyzmq>=22.0.0", - "scipy>=1.5.0", - "matplotlib>=3.3.0", ] [project.optional-dependencies] @@ -26,10 +24,17 @@ dev = [ "pytest>=6.0.0", "pytest-cov>=2.10.0", ] +demo = [ + "scipy>=1.5.0", + "matplotlib>=3.3.0", +] [project.scripts] concore = "concore_cli.cli:cli" [tool.setuptools] packages = ["concore_cli", "concore_cli.commands"] -py-modules = ["mkconcore"] +py-modules = ["concore", "concoredocker", "concore_base", "mkconcore"] + +[tool.setuptools.dynamic] +version = {attr = "concore_cli.__version__"} diff --git a/requirements.txt b/requirements.txt index 9a3554f4..f63b0bb3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,12 +1,9 @@ beautifulsoup4 lxml -tensorflow numpy pyzmq scipy matplotlib -cvxopt -PyGithub click>=8.0.0 rich>=10.0.0 psutil>=5.8.0 \ No newline at end of file diff --git a/setup.py b/setup.py index f7bbb866..ed9acf30 100644 --- a/setup.py +++ b/setup.py @@ -1,42 +1,5 @@ -from setuptools import setup, find_packages +from setuptools import setup -with open("README.md", "r", encoding="utf-8") as fh: - long_description = fh.read() - -setup( - name="concore", - version="1.0.0", - author="ControlCore Project", - description="A command-line interface for concore neuromodulation workflows", - long_description=long_description, - long_description_content_type="text/markdown", - url="https://github.com/ControlCore-Project/concore", - packages=find_packages(), - classifiers=[ - "Development Status :: 4 - Beta", - "Intended Audience :: Science/Research", - "Topic :: Scientific/Engineering", - "License :: OSI Approved :: MIT License", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - ], - python_requires=">=3.9", - install_requires=[ - "beautifulsoup4", - "lxml", - "numpy", - "pyzmq", - "scipy", - "matplotlib", - "click>=8.0.0", - "rich>=10.0.0", - "psutil>=5.8.0", - ], - entry_points={ - "console_scripts": [ - "concore=concore_cli.cli:cli", - ], - }, -) +# All metadata and configuration is in pyproject.toml. +# This file exists only for legacy compatibility. +setup() From bafc5107467343b528233527260289a82dd03c09 Mon Sep 17 00:00:00 2001 From: Avinash Kumar Deepak Date: Mon, 23 Feb 2026 18:08:31 +0530 Subject: [PATCH 207/275] fix: dedupe C++ headers into concore_base.hpp and fix Java docker wrong paths and simtime mutation (#456, #457) --- concore.hpp | 120 ++++------------------------- concore_base.hpp | 183 +++++++++++++++++++++++++++++++++++++++++++++ concoredocker.hpp | 88 +++++----------------- concoredocker.java | 18 +++-- 4 files changed, 228 insertions(+), 181 deletions(-) create mode 100644 concore_base.hpp diff --git a/concore.hpp b/concore.hpp index e109a5d4..da2c7921 100644 --- a/concore.hpp +++ b/concore.hpp @@ -24,6 +24,8 @@ #include #include +#include "concore_base.hpp" + using namespace std; /** @@ -219,48 +221,13 @@ class Concore{ */ map mapParser(string filename){ map ans; - - ifstream portfile; - string portstr; - portfile.open(filename); - if(portfile){ - ostringstream ss; - ss << portfile.rdbuf(); - portstr = ss.str(); - portfile.close(); - } - - if (portstr.empty()) { - return ans; - } - - portstr[portstr.size()-1]=','; - portstr+='}'; - int i=0; - string portname=""; - string portnum=""; - - while(portstr[i]!='}'){ - if(portstr[i]=='\''){ - i++; - while(portstr[i]!='\''){ - portname+=portstr[i]; - i++; - } - ans.insert({portname,0}); + auto str_map = concore_base::safe_literal_eval_dict(filename, {}); + for (const auto& kv : str_map) { + try { + ans[kv.first] = std::stoi(kv.second); + } catch (...) { + ans[kv.first] = 0; } - - if(portstr[i]==':'){ - i++; - while(portstr[i]!=','){ - portnum+=portstr[i]; - i++; - } - ans[portname]=stoi(portnum); - portnum=""; - portname=""; - } - i++; } return ans; } @@ -286,25 +253,7 @@ class Concore{ * @return A vector of double values extracted from the input string. */ vector parser(string f){ - vector temp; - if(f.empty()) return temp; - string value = ""; - - //Changing last bracket to comma to use comma as a delimiter - f[f.length()-1]=','; - - for(int i=1;i= 2 && ((str.front() == '\'' && str.back() == '\'') || (str.front() == '"' && str.back() == '"'))) - return str.substr(1, str.size() - 2); - return str; + return concore_base::stripquotes(str); } /** @@ -629,21 +573,7 @@ class Concore{ * @return A map of key-value string pairs. */ map parsedict(string str){ - map result; - string trimmed = stripstr(str); - if (trimmed.size() < 2 || trimmed.front() != '{' || trimmed.back() != '}') - return result; - string inner = trimmed.substr(1, trimmed.size() - 2); - stringstream ss(inner); - string token; - while (getline(ss, token, ',')) { - size_t colon = token.find(':'); - if (colon == string::npos) continue; - string key = stripquotes(stripstr(token.substr(0, colon))); - string val = stripquotes(stripstr(token.substr(colon + 1))); - if (!key.empty()) result[key] = val; - } - return result; + return concore_base::parsedict(str); } /** @@ -651,33 +581,15 @@ class Concore{ * @param defaultValue The fallback value if the file is missing. */ void default_maxtime(int defaultValue){ - maxtime = defaultValue; - ifstream file(inpath + "/1/concore.maxtime"); - if (file) { - file >> maxtime; - } + maxtime = (int)concore_base::load_maxtime( + inpath + "/1/concore.maxtime", (double)defaultValue); } /** * @brief Loads simulation parameters from concore.params into the params map. */ void load_params(){ - ifstream file(inpath + "/1/concore.params"); - if (!file) return; - stringstream buffer; - buffer << file.rdbuf(); - string sparams = buffer.str(); - - if (!sparams.empty() && sparams[0] == '"') { - sparams = sparams.substr(1, sparams.find('"', 1) - 1); - } - - if (!sparams.empty() && sparams[0] != '{') { - sparams = "{\"" + regex_replace(regex_replace(regex_replace(sparams, regex(","), ",\""), regex("="), "\":"), regex(" "), "") + "}"; - } - try { - params = parsedict(sparams); - } catch (...) {} + params = concore_base::load_params(inpath + "/1/concore.params"); } /** @@ -687,7 +599,7 @@ class Concore{ * @return The parameter value or the default. */ string tryparam(string n, string i){ - return params.count(n) ? params[n] : i; + return concore_base::tryparam(params, n, i); } /** diff --git a/concore_base.hpp b/concore_base.hpp new file mode 100644 index 00000000..64799428 --- /dev/null +++ b/concore_base.hpp @@ -0,0 +1,183 @@ +// concore_base.hpp -- shared utilities for concore.hpp and concoredocker.hpp +// Extracted to eliminate drift between local and Docker C++ implementations. +#ifndef CONCORE_BASE_HPP +#define CONCORE_BASE_HPP + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace concore_base { + +// =================================================================== +// String Helpers +// =================================================================== +inline std::string stripstr(const std::string& str) { + size_t start = str.find_first_not_of(" \t\n\r"); + if (start == std::string::npos) return ""; + size_t end = str.find_last_not_of(" \t\n\r"); + return str.substr(start, end - start + 1); +} + +inline std::string stripquotes(const std::string& str) { + if (str.size() >= 2 && + ((str.front() == '\'' && str.back() == '\'') || + (str.front() == '"' && str.back() == '"'))) + return str.substr(1, str.size() - 2); + return str; +} + +// =================================================================== +// Parsing Utilities +// =================================================================== + +/** + * Parses a Python-style dict string into a string→string map. + * Format: {'key1': val1, 'key2': val2} + * Handles both quoted and unquoted keys/values. + */ +inline std::map parsedict(const std::string& str) { + std::map result; + std::string trimmed = stripstr(str); + if (trimmed.size() < 2 || trimmed.front() != '{' || trimmed.back() != '}') + return result; + std::string inner = trimmed.substr(1, trimmed.size() - 2); + std::stringstream ss(inner); + std::string token; + while (std::getline(ss, token, ',')) { + size_t colon = token.find(':'); + if (colon == std::string::npos) continue; + std::string key = stripquotes(stripstr(token.substr(0, colon))); + std::string val = stripquotes(stripstr(token.substr(colon + 1))); + if (!key.empty()) result[key] = val; + } + return result; +} + +/** + * Parses a Python-style list string into a vector of strings. + * Format: [val1, val2, val3] + */ +inline std::vector parselist(const std::string& str) { + std::vector result; + std::string trimmed = stripstr(str); + if (trimmed.size() < 2 || trimmed.front() != '[' || trimmed.back() != ']') + return result; + std::string inner = trimmed.substr(1, trimmed.size() - 2); + std::stringstream ss(inner); + std::string token; + while (std::getline(ss, token, ',')) { + std::string val = stripstr(token); + if (!val.empty()) result.push_back(val); + } + return result; +} + +/** + * Parses a double-valued list like "[0.0, 1.5, 2.3]" into a vector. + * Used by concore.hpp's read/write which work with numeric data. + */ +inline std::vector parselist_double(const std::string& str) { + std::vector result; + std::vector tokens = parselist(str); + for (const auto& tok : tokens) { + result.push_back(std::stod(tok)); + } + return result; +} + +/** + * Reads a file and parses its content as a dict. + * Returns defaultValue on any failure (matches Python safe_literal_eval). + */ +inline std::map safe_literal_eval_dict( + const std::string& filename, + const std::map& defaultValue) +{ + std::ifstream file(filename); + if (!file) return defaultValue; + std::stringstream buf; + buf << file.rdbuf(); + std::string content = buf.str(); + try { + return parsedict(content); + } catch (...) { + return defaultValue; + } +} + +/** + * Loads simulation parameters from a concore.params file. + * Handles Windows quote wrapping, semicolon-separated key=value, + * and dict-literal format. + */ +inline std::map load_params(const std::string& params_file) { + std::ifstream file(params_file); + if (!file) return {}; + std::stringstream buffer; + buffer << file.rdbuf(); + std::string sparams = buffer.str(); + + // Windows sometimes keeps surrounding quotes + if (!sparams.empty() && sparams[0] == '"') { + size_t closing = sparams.find('"', 1); + if (closing != std::string::npos) + sparams = sparams.substr(1, closing - 1); + } + + sparams = stripstr(sparams); + if (sparams.empty()) return {}; + + // If already a dict literal, parse directly + if (sparams.front() == '{') { + try { return parsedict(sparams); } catch (...) {} + } + + // Otherwise convert semicolon-separated key=value to dict format + // e.g. "a=1;b=2" -> {"a":"1","b":"2"} + std::string converted = "{\"" + + std::regex_replace( + std::regex_replace( + std::regex_replace(sparams, std::regex(","), ",\""), + std::regex("="), "\":"), + std::regex(" "), "") + + "}"; + try { return parsedict(converted); } catch (...) {} + + return {}; +} + +/** + * Reads maxtime from concore.maxtime file, falls back to defaultValue. + */ +inline double load_maxtime(const std::string& maxtime_file, double defaultValue) { + std::ifstream file(maxtime_file); + if (!file) return defaultValue; + double val; + if (file >> val) return val; + return defaultValue; +} + +/** + * Returns param value by name, or default if not found. + */ +inline std::string tryparam( + const std::map& params, + const std::string& name, + const std::string& defaultValue) +{ + auto it = params.find(name); + return (it != params.end()) ? it->second : defaultValue; +} + +} // namespace concore_base + +#endif // CONCORE_BASE_HPP diff --git a/concoredocker.hpp b/concoredocker.hpp index 88a7ab6e..593da876 100644 --- a/concoredocker.hpp +++ b/concoredocker.hpp @@ -1,5 +1,5 @@ -#ifndef CONCORE_HPP -#define CONCORE_HPP +#ifndef CONCOREDOCKER_HPP +#define CONCOREDOCKER_HPP #include #include @@ -14,6 +14,8 @@ #include #include +#include "concore_base.hpp" + class Concore { public: std::unordered_map iport; @@ -28,49 +30,20 @@ class Concore { std::unordered_map params; std::string stripstr(const std::string& str) { - size_t start = str.find_first_not_of(" \t\n\r"); - if (start == std::string::npos) return ""; - size_t end = str.find_last_not_of(" \t\n\r"); - return str.substr(start, end - start + 1); + return concore_base::stripstr(str); } std::string stripquotes(const std::string& str) { - if (str.size() >= 2 && ((str.front() == '\'' && str.back() == '\'') || (str.front() == '"' && str.back() == '"'))) - return str.substr(1, str.size() - 2); - return str; + return concore_base::stripquotes(str); } std::unordered_map parsedict(const std::string& str) { - std::unordered_map result; - std::string trimmed = stripstr(str); - if (trimmed.size() < 2 || trimmed.front() != '{' || trimmed.back() != '}') - return result; - std::string inner = trimmed.substr(1, trimmed.size() - 2); - std::stringstream ss(inner); - std::string token; - while (std::getline(ss, token, ',')) { - size_t colon = token.find(':'); - if (colon == std::string::npos) continue; - std::string key = stripquotes(stripstr(token.substr(0, colon))); - std::string val = stripquotes(stripstr(token.substr(colon + 1))); - if (!key.empty()) result[key] = val; - } - return result; + auto ordered = concore_base::parsedict(str); + return std::unordered_map(ordered.begin(), ordered.end()); } std::vector parselist(const std::string& str) { - std::vector result; - std::string trimmed = stripstr(str); - if (trimmed.size() < 2 || trimmed.front() != '[' || trimmed.back() != ']') - return result; - std::string inner = trimmed.substr(1, trimmed.size() - 2); - std::stringstream ss(inner); - std::string token; - while (std::getline(ss, token, ',')) { - std::string val = stripstr(token); - if (!val.empty()) result.push_back(val); - } - return result; + return concore_base::parselist(str); } Concore() { @@ -82,37 +55,17 @@ class Concore { std::unordered_map safe_literal_eval(const std::string& filename, std::unordered_map defaultValue) { std::ifstream file(filename); - if (!file) { - std::cerr << "Error reading " << filename << "\n"; - return defaultValue; - } + if (!file) return defaultValue; std::stringstream buf; buf << file.rdbuf(); - std::string content = buf.str(); - try { - return parsedict(content); - } catch (...) { - return defaultValue; - } + auto result = concore_base::parsedict(buf.str()); + if (result.empty()) return defaultValue; + return std::unordered_map(result.begin(), result.end()); } void load_params() { - std::ifstream file(inpath + "/1/concore.params"); - if (!file) return; - std::stringstream buffer; - buffer << file.rdbuf(); - std::string sparams = buffer.str(); - - if (!sparams.empty() && sparams[0] == '"') { - sparams = sparams.substr(1, sparams.find('"') - 1); - } - - if (!sparams.empty() && sparams[0] != '{') { - sparams = "{\"" + std::regex_replace(std::regex_replace(std::regex_replace(sparams, std::regex(","), ",\""), std::regex("="), "\":"), std::regex(" "), "") + "}"; - } - try { - params = parsedict(sparams); - } catch (...) {} + auto ordered = concore_base::load_params(inpath + "/1/concore.params"); + params = std::unordered_map(ordered.begin(), ordered.end()); } std::string tryparam(const std::string& n, const std::string& i) { @@ -120,11 +73,8 @@ class Concore { } void default_maxtime(double defaultValue) { - maxtime = defaultValue; - std::ifstream file(inpath + "/1/concore.maxtime"); - if (file) { - file >> maxtime; - } + maxtime = concore_base::load_maxtime( + inpath + "/1/concore.maxtime", defaultValue); } bool unchanged() { @@ -188,7 +138,7 @@ class Concore { outfile << val[i] << (i + 1 < val.size() ? ", " : ""); } outfile << "]"; - simtime += delta; + // simtime must not be mutated here (issue #385). } } @@ -204,4 +154,4 @@ class Concore { } }; -#endif +#endif // CONCOREDOCKER_HPP diff --git a/concoredocker.java b/concoredocker.java index 2ac7cbd3..58b68e78 100644 --- a/concoredocker.java +++ b/concoredocker.java @@ -39,7 +39,7 @@ public class concoredocker { } catch (IOException e) { } try { - String sparams = new String(Files.readAllBytes(Paths.get(inpath + "1/concore.params")), java.nio.charset.StandardCharsets.UTF_8); + String sparams = new String(Files.readAllBytes(Paths.get(inpath + "/1/concore.params")), java.nio.charset.StandardCharsets.UTF_8); if (sparams.length() > 0 && sparams.charAt(0) == '"') { // windows keeps "" need to remove sparams = sparams.substring(1); sparams = sparams.substring(0, sparams.indexOf('"')); @@ -114,7 +114,7 @@ private static Map parseFile(String filename) throws IOException */ private static void defaultMaxTime(double defaultValue) { try { - String content = new String(Files.readAllBytes(Paths.get(inpath + "1/concore.maxtime"))); + String content = new String(Files.readAllBytes(Paths.get(inpath + "/1/concore.maxtime"))); Object parsed = literalEval(content.trim()); if (parsed instanceof Number) { maxtime = ((Number) parsed).doubleValue(); @@ -161,7 +161,7 @@ private static List read(int port, String name, String initstr) { // initstr not parseable as list; defaultVal stays empty } - String filePath = inpath + port + "/" + name; + String filePath = inpath + "/" + port + "/" + name; try { Thread.sleep(delay); } catch (InterruptedException e) { @@ -277,7 +277,7 @@ private static String toPythonLiteral(Object obj) { */ private static void write(int port, String name, Object val, int delta) { try { - String path = outpath + port + "/" + name; + String path = outpath + "/" + port + "/" + name; StringBuilder content = new StringBuilder(); if (val instanceof String) { Thread.sleep(2 * delay); @@ -291,7 +291,8 @@ private static void write(int port, String name, Object val, int delta) { content.append(toPythonLiteral(listVal.get(i))); } content.append("]"); - simtime += delta; + // simtime must not be mutated here. + // Mutation breaks cross-language determinism (see issue #385). } else if (val instanceof Object[]) { // Legacy support for Object[] arguments Object[] arrayVal = (Object[]) val; @@ -302,7 +303,8 @@ private static void write(int port, String name, Object val, int delta) { content.append(toPythonLiteral(o)); } content.append("]"); - simtime += delta; + // simtime must not be mutated here. + // Mutation breaks cross-language determinism (see issue #385). } else { System.out.println("write must have list or str"); return; @@ -310,9 +312,9 @@ private static void write(int port, String name, Object val, int delta) { Files.write(Paths.get(path), content.toString().getBytes()); } catch (InterruptedException e) { Thread.currentThread().interrupt(); - System.out.println("skipping " + outpath + port + "/" + name); + System.out.println("skipping " + outpath + "/" + port + "/" + name); } catch (IOException e) { - System.out.println("skipping " + outpath + port + "/" + name); + System.out.println("skipping " + outpath + "/" + port + "/" + name); } } From a4162b6086586954dbc65e452518a4355c48a253 Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Tue, 24 Feb 2026 09:36:23 +0530 Subject: [PATCH 208/275] fix: clamp lag parameter to buffer size with warning (fixes #379) The circular buffer in plotymlag.py has a fixed size of 10. When a user sets lag >= size via tryparam, the modulo arithmetic silently wraps, producing incorrect data (e.g., lag=15 behaves like lag=5). Add input validation that clamps lag to size-1 and emits a logging.warning when the requested value exceeds the buffer capacity. This prevents silent data corruption while preserving backward compatibility and the existing circular buffer logic. --- tools/plotymlag.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tools/plotymlag.py b/tools/plotymlag.py index 96b5bf86..264a98d2 100644 --- a/tools/plotymlag.py +++ b/tools/plotymlag.py @@ -5,7 +5,13 @@ import time size = 10 -lag = concore.tryparam("lag", 0) +lag = concore.tryparam("lag", 0) +if lag >= size: + logging.warning( + "Requested lag (%d) exceeds buffer size (%d). Clamping to %d.", + lag, size, size - 1 + ) + lag = size - 1 logging.info(f"plot ym with lag={lag}") concore.delay = 0.005 From ab30ee73bb3235d7178c242e54527c8b54aee4db Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Tue, 24 Feb 2026 10:48:25 +0530 Subject: [PATCH 209/275] docs: add DOCKEREXE configuration section to README.md --- README.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/README.md b/README.md index ba7e8064..a44827d8 100644 --- a/README.md +++ b/README.md @@ -77,6 +77,32 @@ _concore_ supports customization through configuration files in the `CONCOREPATH Tool paths can also be set via environment variables (e.g., `CONCORE_CPPEXE=/usr/bin/g++`). Priority: config file > env var > defaults. +### Docker Executable Configuration + +The Docker executable used by generated scripts (`build`, `run`, `stop`, `maxtime`, `params`, `unlock`) is controlled by the `DOCKEREXE` variable. It defaults to `docker` and can be overridden in three ways (highest priority first): + +1. **Config file** — Write the desired command into `concore.sudo` in your `CONCOREPATH` directory: + ``` + docker + ``` + This remains the highest-priority override, preserving backward compatibility. + +2. **Environment variable** — Set `DOCKEREXE` before running `mkconcore.py`: + ```bash + # Rootless Docker / Docker Desktop (macOS, Windows) + export DOCKEREXE="docker" + + # Podman + export DOCKEREXE="podman" + + # Traditional Linux with sudo + export DOCKEREXE="sudo docker" + ``` + +3. **Default** — If neither the config file nor the environment variable is set, `docker` is used. + +> **Note:** Previous versions defaulted to `sudo docker`, which failed on Docker Desktop (macOS/Windows), rootless Docker, and Podman. The new default (`docker`) works out of the box on those platforms. If you still need `sudo`, set `DOCKEREXE="sudo docker"` via the environment variable or `concore.sudo` config file. + ### Security Configuration Set a secure secret key for the Flask server before running in production: From b6f2a0d17c5bbb1b8b0f0cb65a34187bfb0b7b65 Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Tue, 24 Feb 2026 13:14:40 +0530 Subject: [PATCH 210/275] Fix: Add explicit error signalling to read() (#390) read() now returns (data, success_flag) tuple and sets concore.last_read_status / concore_base.last_read_status to one of: SUCCESS, FILE_NOT_FOUND, TIMEOUT, PARSE_ERROR, EMPTY_DATA, RETRIES_EXCEEDED. - concore_base.read(): all return paths now yield (data, bool) - concore.py: exposes last_read_status, syncs from concore_base - concoredocker.py: same treatment - Existing tests updated for tuple returns - New test_read_status.py covers success, file-not-found, parse error, retries exceeded, ZMQ success/timeout/error, backward compatibility, and last_read_status exposure. Backward compatible: callers can use isinstance(result, tuple) or simply unpack with value, ok = concore.read(...). --- concore.py | 29 +++- concore_base.py | 74 ++++++++-- concoredocker.py | 7 +- tests/test_concore.py | 5 +- tests/test_concoredocker.py | 12 +- tests/test_read_status.py | 261 ++++++++++++++++++++++++++++++++++++ 6 files changed, 370 insertions(+), 18 deletions(-) create mode 100644 tests/test_read_status.py diff --git a/concore.py b/concore.py index 5bd112ba..2147e758 100644 --- a/concore.py +++ b/concore.py @@ -38,6 +38,8 @@ zmq_ports = {} _cleanup_in_progress = False +last_read_status = "SUCCESS" + s = '' olds = '' delay = 1 @@ -110,7 +112,32 @@ def unchanged(): # I/O Handling (File + ZMQ) # =================================================================== def read(port_identifier, name, initstr_val): - return concore_base.read(_mod, port_identifier, name, initstr_val) + """Read data from a ZMQ port or file-based port. + + Returns: + tuple: (data, success_flag) where success_flag is True if real + data was received, False if a fallback/default was used. + Also sets ``concore.last_read_status`` to one of: + SUCCESS, FILE_NOT_FOUND, TIMEOUT, PARSE_ERROR, + EMPTY_DATA, RETRIES_EXCEEDED. + + Backward compatibility: + Legacy callers that do ``value = concore.read(...)`` will + receive a tuple. They can adapt with:: + + result = concore.read(...) + if isinstance(result, tuple): + value, ok = result + else: + value, ok = result, True + + Alternatively, check ``concore.last_read_status`` after the + call. + """ + global last_read_status + result = concore_base.read(_mod, port_identifier, name, initstr_val) + last_read_status = concore_base.last_read_status + return result def write(port_identifier, name, val, delta=0): diff --git a/concore_base.py b/concore_base.py index f6b3e1b9..d64c5265 100644 --- a/concore_base.py +++ b/concore_base.py @@ -187,6 +187,11 @@ def load_params(params_file): except Exception: return dict() +# =================================================================== +# Read Status Tracking +# =================================================================== +last_read_status = "SUCCESS" + # =================================================================== # I/O Handling (File + ZMQ) # =================================================================== @@ -201,6 +206,31 @@ def unchanged(mod): def read(mod, port_identifier, name, initstr_val): + """Read data from a ZMQ port or file-based port. + + Returns: + tuple: (data, success_flag) where success_flag is True if real + data was received, False if a fallback/default was used. + Also sets ``concore.last_read_status`` (and + ``concore_base.last_read_status``) to one of: + SUCCESS, FILE_NOT_FOUND, TIMEOUT, PARSE_ERROR, + EMPTY_DATA, RETRIES_EXCEEDED. + + Backward compatibility: + Legacy callers that do ``value = concore.read(...)`` will + receive a tuple. They can adapt with:: + + result = concore.read(...) + if isinstance(result, tuple): + value, ok = result + else: + value, ok = result, True + + Alternatively, check ``concore.last_read_status`` after the + call. + """ + global last_read_status + # Default return default_return_val = initstr_val if isinstance(initstr_val, str): @@ -214,41 +244,52 @@ def read(mod, port_identifier, name, initstr_val): zmq_p = mod.zmq_ports[port_identifier] try: message = zmq_p.recv_json_with_retry() + if message is None: + last_read_status = "TIMEOUT" + return default_return_val, False # Strip simtime prefix if present (mirroring file-based read behavior) if isinstance(message, list) and len(message) > 0: first_element = message[0] if isinstance(first_element, (int, float)): mod.simtime = max(mod.simtime, first_element) - return message[1:] - return message + last_read_status = "SUCCESS" + return message[1:], True + last_read_status = "SUCCESS" + return message, True except zmq.error.ZMQError as e: logger.error(f"ZMQ read error on port {port_identifier} (name: {name}): {e}. Returning default.") - return default_return_val + last_read_status = "TIMEOUT" + return default_return_val, False except Exception as e: logger.error(f"Unexpected error during ZMQ read on port {port_identifier} (name: {name}): {e}. Returning default.") - return default_return_val + last_read_status = "PARSE_ERROR" + return default_return_val, False # Case 2: File-based port try: file_port_num = int(port_identifier) except ValueError: logger.error(f"Error: Invalid port identifier '{port_identifier}' for file operation. Must be integer or ZMQ name.") - return default_return_val + last_read_status = "PARSE_ERROR" + return default_return_val, False time.sleep(mod.delay) port_dir = mod._port_path(mod.inpath, file_port_num) file_path = os.path.join(port_dir, name) ins = "" + file_not_found = False try: with open(file_path, "r") as infile: ins = infile.read() except FileNotFoundError: + file_not_found = True ins = str(initstr_val) mod.s += ins # Update s to break unchanged() loop except Exception as e: logger.error(f"Error reading {file_path}: {e}. Using default value.") - return default_return_val + last_read_status = "FILE_NOT_FOUND" + return default_return_val, False # Retry logic if file is empty attempts = 0 @@ -265,7 +306,8 @@ def read(mod, port_identifier, name, initstr_val): if len(ins) == 0: logger.error(f"Max retries reached for {file_path}, using default value.") - return default_return_val + last_read_status = "RETRIES_EXCEEDED" + return default_return_val, False mod.s += ins @@ -276,13 +318,25 @@ def read(mod, port_identifier, name, initstr_val): current_simtime_from_file = inval[0] if isinstance(current_simtime_from_file, (int, float)): mod.simtime = max(mod.simtime, current_simtime_from_file) - return inval[1:] + if file_not_found: + last_read_status = "FILE_NOT_FOUND" + return inval[1:], False + last_read_status = "SUCCESS" + return inval[1:], True else: logger.warning(f"Warning: Unexpected data format in {file_path}: {ins}. Returning raw content or default.") - return inval + if file_not_found: + last_read_status = "FILE_NOT_FOUND" + return inval, False + last_read_status = "SUCCESS" + return inval, True except Exception as e: logger.error(f"Error parsing content from {file_path} ('{ins}'): {e}. Returning default.") - return default_return_val + if file_not_found: + last_read_status = "FILE_NOT_FOUND" + else: + last_read_status = "PARSE_ERROR" + return default_return_val, False def write(mod, port_identifier, name, val, delta=0): diff --git a/concoredocker.py b/concoredocker.py index 84d4caa5..51df0eb3 100644 --- a/concoredocker.py +++ b/concoredocker.py @@ -25,6 +25,8 @@ zmq_ports = {} _cleanup_in_progress = False +last_read_status = "SUCCESS" + s = '' olds = '' delay = 1 @@ -90,7 +92,10 @@ def unchanged(): # I/O Handling (File + ZMQ) # =================================================================== def read(port_identifier, name, initstr_val): - return concore_base.read(_mod, port_identifier, name, initstr_val) + global last_read_status + result = concore_base.read(_mod, port_identifier, name, initstr_val) + last_read_status = concore_base.last_read_status + return result def write(port_identifier, name, val, delta=0): concore_base.write(_mod, port_identifier, name, val, delta) diff --git a/tests/test_concore.py b/tests/test_concore.py index fd897291..58e99c3c 100644 --- a/tests/test_concore.py +++ b/tests/test_concore.py @@ -273,9 +273,10 @@ def recv_json_with_retry(self): original_data = [1.5, 2.5, 3.5] concore.write("roundtrip_test", "data", original_data) - # Read should return original data (simtime stripped) - result = concore.read("roundtrip_test", "data", "[]") + # Read should return original data (simtime stripped) plus success flag + result, ok = concore.read("roundtrip_test", "data", "[]") assert result == original_data + assert ok is True class TestSimtimeNotMutatedByWrite: diff --git a/tests/test_concoredocker.py b/tests/test_concoredocker.py index c3743911..91bf2fb7 100644 --- a/tests/test_concoredocker.py +++ b/tests/test_concoredocker.py @@ -130,9 +130,10 @@ def test_reads_and_parses_data(self, temp_dir): concoredocker.s = '' concoredocker.simtime = 0 - result = concoredocker.read(1, "data", "[0, 0, 0]") + result, ok = concoredocker.read(1, "data", "[0, 0, 0]") assert result == [100, 200] + assert ok is True assert concoredocker.simtime == 7.0 concoredocker.inpath = old_inpath concoredocker.delay = old_delay @@ -148,9 +149,10 @@ def test_returns_default_when_file_missing(self, temp_dir): concoredocker.s = '' concoredocker.simtime = 0 - result = concoredocker.read(1, "nofile", "[0, 5, 5]") + result, ok = concoredocker.read(1, "nofile", "[0, 5, 5]") assert result == [5, 5] + assert ok is False concoredocker.inpath = old_inpath concoredocker.delay = old_delay @@ -194,9 +196,10 @@ def recv_json_with_retry(self): concoredocker.zmq_ports["test_zmq"] = DummyPort() concoredocker.simtime = 0 - result = concoredocker.read("test_zmq", "data", "[]") + result, ok = concoredocker.read("test_zmq", "data", "[]") assert result == [4.0, 5.0] + assert ok is True assert concoredocker.simtime == 10.0 def test_read_updates_simtime_monotonically(self): @@ -232,6 +235,7 @@ def recv_json_with_retry(self): original = [1.5, 2.5, 3.5] concoredocker.write("roundtrip", "data", original) - result = concoredocker.read("roundtrip", "data", "[]") + result, ok = concoredocker.read("roundtrip", "data", "[]") assert result == original + assert ok is True diff --git a/tests/test_read_status.py b/tests/test_read_status.py new file mode 100644 index 00000000..54dc4b88 --- /dev/null +++ b/tests/test_read_status.py @@ -0,0 +1,261 @@ +"""Tests for read() error signalling (Issue #390). + +read() now returns (data, success_flag) and sets +concore.last_read_status / concore_base.last_read_status. +""" + +import os +import pytest +import numpy as np + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +class DummyZMQPort: + """Minimal stand-in for ZeroMQPort used in ZMQ read tests.""" + + def __init__(self, response=None, raise_on_recv=None): + self._response = response + self._raise_on_recv = raise_on_recv + + def send_json_with_retry(self, message): + self._response = message + + def recv_json_with_retry(self): + if self._raise_on_recv: + raise self._raise_on_recv + return self._response + + +# --------------------------------------------------------------------------- +# File-based read tests +# --------------------------------------------------------------------------- + +class TestReadFileSuccess: + """read() on a valid file returns (data, True) with SUCCESS status.""" + + @pytest.fixture(autouse=True) + def setup(self, temp_dir, monkeypatch): + import concore + self.concore = concore + monkeypatch.setattr(concore, 'delay', 0) + + # Create ./in1/ym with valid data: [simtime, value] + in_dir = os.path.join(temp_dir, "in1") + os.makedirs(in_dir, exist_ok=True) + with open(os.path.join(in_dir, "ym"), "w") as f: + f.write("[10, 3.14]") + + monkeypatch.setattr(concore, 'inpath', os.path.join(temp_dir, "in")) + + def test_returns_data_and_true(self): + data, ok = self.concore.read(1, "ym", "[0, 0.0]") + assert ok is True + assert data == [3.14] + + def test_last_read_status_is_success(self): + self.concore.read(1, "ym", "[0, 0.0]") + assert self.concore.last_read_status == "SUCCESS" + + +class TestReadFileMissing: + """read() on a missing file returns (default, False) with FILE_NOT_FOUND.""" + + @pytest.fixture(autouse=True) + def setup(self, temp_dir, monkeypatch): + import concore + self.concore = concore + monkeypatch.setattr(concore, 'delay', 0) + # Point to a directory that does NOT have the file + monkeypatch.setattr(concore, 'inpath', os.path.join(temp_dir, "in")) + + def test_returns_default_and_false(self): + data, ok = self.concore.read(1, "nonexistent", "[0, 0.0]") + assert ok is False + + def test_last_read_status_is_file_not_found(self): + self.concore.read(1, "nonexistent", "[0, 0.0]") + assert self.concore.last_read_status == "FILE_NOT_FOUND" + + +class TestReadFileParseError: + """read() returns (default, False) with PARSE_ERROR on malformed content.""" + + @pytest.fixture(autouse=True) + def setup(self, temp_dir, monkeypatch): + import concore + self.concore = concore + monkeypatch.setattr(concore, 'delay', 0) + + in_dir = os.path.join(temp_dir, "in1") + os.makedirs(in_dir, exist_ok=True) + with open(os.path.join(in_dir, "ym"), "w") as f: + f.write("NOT_VALID_PYTHON{{{") + + monkeypatch.setattr(concore, 'inpath', os.path.join(temp_dir, "in")) + + def test_returns_default_and_false(self): + data, ok = self.concore.read(1, "ym", "[0, 0.0]") + assert ok is False + + def test_last_read_status_is_parse_error(self): + self.concore.read(1, "ym", "[0, 0.0]") + assert self.concore.last_read_status == "PARSE_ERROR" + + +class TestReadFileRetriesExceeded: + """read() returns (default, False) with RETRIES_EXCEEDED when file is empty.""" + + @pytest.fixture(autouse=True) + def setup(self, temp_dir, monkeypatch): + import concore + self.concore = concore + monkeypatch.setattr(concore, 'delay', 0) + + # Create an empty file + in_dir = os.path.join(temp_dir, "in1") + os.makedirs(in_dir, exist_ok=True) + with open(os.path.join(in_dir, "ym"), "w") as f: + pass # empty + + monkeypatch.setattr(concore, 'inpath', os.path.join(temp_dir, "in")) + + def test_returns_default_and_false(self): + data, ok = self.concore.read(1, "ym", "[0, 0.0]") + assert ok is False + + def test_last_read_status_is_retries_exceeded(self): + self.concore.read(1, "ym", "[0, 0.0]") + assert self.concore.last_read_status == "RETRIES_EXCEEDED" + + +# --------------------------------------------------------------------------- +# ZMQ read tests +# --------------------------------------------------------------------------- + +class TestReadZMQSuccess: + """Successful ZMQ read returns (data, True).""" + + @pytest.fixture(autouse=True) + def setup(self, monkeypatch): + import concore + self.concore = concore + self.original_ports = concore.zmq_ports.copy() + yield + concore.zmq_ports.clear() + concore.zmq_ports.update(self.original_ports) + + def test_zmq_read_returns_data_and_true(self): + dummy = DummyZMQPort(response=[5, 1.1, 2.2]) + self.concore.zmq_ports["test_port"] = dummy + self.concore.simtime = 0 + + data, ok = self.concore.read("test_port", "ym", "[]") + assert ok is True + assert data == [1.1, 2.2] + assert self.concore.last_read_status == "SUCCESS" + + +class TestReadZMQTimeout: + """ZMQ read that returns None (timeout) yields (default, False).""" + + @pytest.fixture(autouse=True) + def setup(self, monkeypatch): + import concore + self.concore = concore + self.original_ports = concore.zmq_ports.copy() + yield + concore.zmq_ports.clear() + concore.zmq_ports.update(self.original_ports) + + def test_zmq_timeout_returns_default_and_false(self): + dummy = DummyZMQPort(response=None) # recv returns None → timeout + self.concore.zmq_ports["test_port"] = dummy + + data, ok = self.concore.read("test_port", "ym", "[]") + assert ok is False + assert self.concore.last_read_status == "TIMEOUT" + + +class TestReadZMQError: + """ZMQ read that raises ZMQError yields (default, False).""" + + @pytest.fixture(autouse=True) + def setup(self, monkeypatch): + import concore + self.concore = concore + self.original_ports = concore.zmq_ports.copy() + yield + concore.zmq_ports.clear() + concore.zmq_ports.update(self.original_ports) + + def test_zmq_error_returns_default_and_false(self): + import zmq + dummy = DummyZMQPort(raise_on_recv=zmq.error.ZMQError("test error")) + self.concore.zmq_ports["test_port"] = dummy + + data, ok = self.concore.read("test_port", "ym", "[]") + assert ok is False + assert self.concore.last_read_status == "TIMEOUT" + + +# --------------------------------------------------------------------------- +# Backward compatibility +# --------------------------------------------------------------------------- + +class TestReadBackwardCompatibility: + """Legacy callers can use isinstance check on the result.""" + + @pytest.fixture(autouse=True) + def setup(self, temp_dir, monkeypatch): + import concore + self.concore = concore + monkeypatch.setattr(concore, 'delay', 0) + + in_dir = os.path.join(temp_dir, "in1") + os.makedirs(in_dir, exist_ok=True) + with open(os.path.join(in_dir, "ym"), "w") as f: + f.write("[10, 42.0]") + + monkeypatch.setattr(concore, 'inpath', os.path.join(temp_dir, "in")) + + def test_legacy_unpack_pattern(self): + """The recommended migration pattern works correctly.""" + result = self.concore.read(1, "ym", "[0, 0.0]") + + if isinstance(result, tuple): + value, ok = result + else: + value = result + ok = True + + assert value == [42.0] + assert ok is True + + def test_tuple_unpack(self): + """New-style callers can unpack directly.""" + value, ok = self.concore.read(1, "ym", "[0, 0.0]") + assert value == [42.0] + assert ok is True + + +# --------------------------------------------------------------------------- +# last_read_status exposed on module +# --------------------------------------------------------------------------- + +class TestLastReadStatusExposed: + """concore.last_read_status is publicly accessible.""" + + def test_attribute_exists(self): + import concore + assert hasattr(concore, 'last_read_status') + + def test_initial_value_is_success(self): + import concore + # Before any read, default is SUCCESS + assert concore.last_read_status in ( + "SUCCESS", "FILE_NOT_FOUND", "TIMEOUT", + "PARSE_ERROR", "EMPTY_DATA", "RETRIES_EXCEEDED", + ) From 72f3653e99302a38af3a57cea55c9ede77710222 Mon Sep 17 00:00:00 2001 From: Avinash Kumar Deepak Date: Tue, 24 Feb 2026 13:44:50 +0530 Subject: [PATCH 211/275] fix(ci): add ruff.toml, format check, coverage, merge tests.yml, fix bare excepts (#454) --- .github/workflows/ci.yml | 74 +++--- .github/workflows/tests.yml | 33 --- .pre-commit-config.yaml | 7 + concore_cli/__init__.py | 2 +- concore_cli/cli.py | 55 +++-- concore_cli/commands/__init__.py | 9 +- concore_cli/commands/init.py | 63 ++--- concore_cli/commands/inspect.py | 270 +++++++++++----------- concore_cli/commands/run.py | 88 ++++--- concore_cli/commands/status.py | 108 +++++---- concore_cli/commands/stop.py | 104 +++++---- concore_cli/commands/validate.py | 212 +++++++++-------- concore_cli/commands/watch.py | 61 +++-- pytest.ini | 2 +- requirements-ci.txt | 1 + ruff.toml | 41 ++++ tests/conftest.py | 2 +- tests/test_cli.py | 354 +++++++++++++++++------------ tests/test_concore.py | 112 +++++---- tests/test_concoredocker.py | 32 ++- tests/test_graph.py | 267 +++++++++++----------- tests/test_openjupyter_security.py | 36 ++- tests/test_protocol_conformance.py | 12 +- tests/test_tool_config.py | 5 +- 24 files changed, 1080 insertions(+), 870 deletions(-) delete mode 100644 .github/workflows/tests.yml create mode 100644 .pre-commit-config.yaml create mode 100644 ruff.toml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3be703e1..5cfdc7e4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,67 +7,58 @@ on: branches: [main, dev] jobs: - lint-and-test: + lint: runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v4 + - uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v5 + - uses: actions/setup-python@v5 with: python-version: '3.11' cache: 'pip' + - name: Install ruff + run: pip install ruff + + - name: Check formatting + run: ruff format --check . + + - name: Lint + run: ruff check . --output-format=github + + test: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ['3.9', '3.10', '3.11', '3.12'] + + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + cache: 'pip' + - name: Install dependencies run: | python -m pip install --upgrade pip pip install -r requirements-ci.txt - # Uses minimal CI requirements (no tensorflow/heavy packages) - - name: Run linter (ruff) + - name: Run tests run: | - ruff check . --select=E9,F63,F7,F82 --output-format=github \ - --exclude="Dockerfile.*" \ - --exclude="linktest/" \ - --exclude="measurements/" \ - --exclude="0mq/" \ - --exclude="ratc/" - # E9: Runtime errors (syntax errors, etc.) - # F63: Invalid print syntax - # F7: Syntax errors in type comments - # F82: Undefined names in __all__ - # Excludes: Dockerfiles (not Python), linktest (symlinks), - # measurements/0mq/ratc (config-dependent experimental scripts) - - - name: Run tests (pytest) - run: | - set +e pytest --tb=short -q \ + --cov=concore_cli --cov=concore_base \ + --cov-report=term-missing \ --ignore=measurements/ \ --ignore=0mq/ \ --ignore=ratc/ \ --ignore=linktest/ - status=$? - set -e - # Allow success if no tests are collected (pytest exit code 5) - if [ "$status" -ne 0 ] && [ "$status" -ne 5 ]; then - exit "$status" - fi - # Fails on real test failures, passes on no tests collected docker-build: runs-on: ubuntu-latest - # Only run when Dockerfile.py or related files change - if: | - github.event_name == 'push' || - (github.event_name == 'pull_request' && - contains(github.event.pull_request.changed_files, 'Dockerfile')) - steps: - - name: Checkout repository - uses: actions/checkout@v4 + - uses: actions/checkout@v4 - name: Check if Dockerfile.py changed uses: dorny/paths-filter@v3 @@ -80,7 +71,4 @@ jobs: - name: Validate Dockerfile build if: steps.filter.outputs.dockerfile == 'true' - run: | - docker build -f Dockerfile.py -t concore-py-test . - # Validates that Dockerfile.py can be built successfully - # Does not push the image + run: docker build -f Dockerfile.py -t concore-py-test . diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml deleted file mode 100644 index 6b7cdf04..00000000 --- a/.github/workflows/tests.yml +++ /dev/null @@ -1,33 +0,0 @@ -name: Tests - -on: - push: - branches: [main, dev] - pull_request: - branches: [main, dev] - -jobs: - test: - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ['3.9', '3.10', '3.11', '3.12'] - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -r requirements.txt - pip install -r requirements-dev.txt - pip install pyzmq - - - name: Run tests - run: pytest -v \ No newline at end of file diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..461fd44e --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,7 @@ +repos: + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.11.12 + hooks: + - id: ruff + args: [--output-format=full] + - id: ruff-format diff --git a/concore_cli/__init__.py b/concore_cli/__init__.py index 8e8a3220..a5ccc5d1 100644 --- a/concore_cli/__init__.py +++ b/concore_cli/__init__.py @@ -2,4 +2,4 @@ from .cli import cli -__all__ = ['cli'] +__all__ = ["cli"] diff --git a/concore_cli/cli.py b/concore_cli/cli.py index fadb8b6e..15801838 100644 --- a/concore_cli/cli.py +++ b/concore_cli/cli.py @@ -13,16 +13,18 @@ from . import __version__ console = Console() -DEFAULT_EXEC_TYPE = 'windows' if os.name == 'nt' else 'posix' +DEFAULT_EXEC_TYPE = "windows" if os.name == "nt" else "posix" + @click.group() -@click.version_option(version=__version__, prog_name='concore') +@click.version_option(version=__version__, prog_name="concore") def cli(): pass + @cli.command() -@click.argument('name', required=True) -@click.option('--template', default='basic', help='Template type to use') +@click.argument("name", required=True) +@click.option("--template", default="basic", help="Template type to use") def init(name, template): """Create a new concore project""" try: @@ -31,12 +33,21 @@ def init(name, template): console.print(f"[red]Error:[/red] {str(e)}") sys.exit(1) + @cli.command() -@click.argument('workflow_file', type=click.Path(exists=True)) -@click.option('--source', '-s', default='src', help='Source directory') -@click.option('--output', '-o', default='out', help='Output directory') -@click.option('--type', '-t', default=DEFAULT_EXEC_TYPE, type=click.Choice(['windows', 'posix', 'docker']), help='Execution type') -@click.option('--auto-build', is_flag=True, help='Automatically run build after generation') +@click.argument("workflow_file", type=click.Path(exists=True)) +@click.option("--source", "-s", default="src", help="Source directory") +@click.option("--output", "-o", default="out", help="Output directory") +@click.option( + "--type", + "-t", + default=DEFAULT_EXEC_TYPE, + type=click.Choice(["windows", "posix", "docker"]), + help="Execution type", +) +@click.option( + "--auto-build", is_flag=True, help="Automatically run build after generation" +) def run(workflow_file, source, output, type, auto_build): """Run a concore workflow""" try: @@ -45,9 +56,10 @@ def run(workflow_file, source, output, type, auto_build): console.print(f"[red]Error:[/red] {str(e)}") sys.exit(1) + @cli.command() -@click.argument('workflow_file', type=click.Path(exists=True)) -@click.option('--source', '-s', default='src', help='Source directory') +@click.argument("workflow_file", type=click.Path(exists=True)) +@click.option("--source", "-s", default="src", help="Source directory") def validate(workflow_file, source): """Validate a workflow file""" try: @@ -58,10 +70,11 @@ def validate(workflow_file, source): console.print(f"[red]Error:[/red] {str(e)}") sys.exit(1) + @cli.command() -@click.argument('workflow_file', type=click.Path(exists=True)) -@click.option('--source', '-s', default='src', help='Source directory') -@click.option('--json', 'output_json', is_flag=True, help='Output in JSON format') +@click.argument("workflow_file", type=click.Path(exists=True)) +@click.option("--source", "-s", default="src", help="Source directory") +@click.option("--json", "output_json", is_flag=True, help="Output in JSON format") def inspect(workflow_file, source, output_json): """Inspect a workflow file and show its structure""" try: @@ -70,6 +83,7 @@ def inspect(workflow_file, source, output_json): console.print(f"[red]Error:[/red] {str(e)}") sys.exit(1) + @cli.command() def status(): """Show running concore processes""" @@ -79,8 +93,9 @@ def status(): console.print(f"[red]Error:[/red] {str(e)}") sys.exit(1) + @cli.command() -@click.confirmation_option(prompt='Stop all running concore processes?') +@click.confirmation_option(prompt="Stop all running concore processes?") def stop(): """Stop all running concore processes""" try: @@ -89,10 +104,11 @@ def stop(): console.print(f"[red]Error:[/red] {str(e)}") sys.exit(1) + @cli.command() -@click.argument('study_dir', type=click.Path(exists=True)) -@click.option('--interval', '-n', default=2.0, help='Refresh interval in seconds') -@click.option('--once', is_flag=True, help='Print a single snapshot and exit') +@click.argument("study_dir", type=click.Path(exists=True)) +@click.option("--interval", "-n", default=2.0, help="Refresh interval in seconds") +@click.option("--once", is_flag=True, help="Print a single snapshot and exit") def watch(study_dir, interval, once): """Watch a running simulation study for live monitoring""" try: @@ -101,5 +117,6 @@ def watch(study_dir, interval, once): console.print(f"[red]Error:[/red] {str(e)}") sys.exit(1) -if __name__ == '__main__': + +if __name__ == "__main__": cli() diff --git a/concore_cli/commands/__init__.py b/concore_cli/commands/__init__.py index 77820b85..e98d4cd5 100644 --- a/concore_cli/commands/__init__.py +++ b/concore_cli/commands/__init__.py @@ -5,4 +5,11 @@ from .stop import stop_all from .watch import watch_study -__all__ = ['init_project', 'run_workflow', 'validate_workflow', 'show_status', 'stop_all', 'watch_study'] +__all__ = [ + "init_project", + "run_workflow", + "validate_workflow", + "show_status", + "stop_all", + "watch_study", +] diff --git a/concore_cli/commands/init.py b/concore_cli/commands/init.py index 0b6badc3..eb73e916 100644 --- a/concore_cli/commands/init.py +++ b/concore_cli/commands/init.py @@ -1,9 +1,7 @@ -import os -import shutil from pathlib import Path from rich.panel import Panel -SAMPLE_GRAPHML = ''' +SAMPLE_GRAPHML = """ @@ -21,9 +19,9 @@ -''' +""" -SAMPLE_PYTHON = '''import concore +SAMPLE_PYTHON = """import concore concore.default_maxtime(100) concore.delay = 0.02 @@ -36,9 +34,9 @@ val = concore.read(1,"data",init_simtime_val) result = [v * 2 for v in val] concore.write(1,"result",result,delta=0) -''' +""" -README_TEMPLATE = '''# {project_name} +README_TEMPLATE = """# {project_name} A concore workflow project. @@ -63,38 +61,41 @@ - Add Python/C++/MATLAB scripts to `src/` - Use `concore validate workflow.graphml` to check your workflow - Use `concore status` to monitor running processes -''' +""" + def init_project(name, template, console): project_path = Path(name) - + if project_path.exists(): raise FileExistsError(f"Directory '{name}' already exists") - + console.print(f"[cyan]Creating project:[/cyan] {name}") - + project_path.mkdir() - (project_path / 'src').mkdir() - - workflow_file = project_path / 'workflow.graphml' - with open(workflow_file, 'w') as f: + (project_path / "src").mkdir() + + workflow_file = project_path / "workflow.graphml" + with open(workflow_file, "w") as f: f.write(SAMPLE_GRAPHML) - - sample_script = project_path / 'src' / 'script.py' - with open(sample_script, 'w') as f: + + sample_script = project_path / "src" / "script.py" + with open(sample_script, "w") as f: f.write(SAMPLE_PYTHON) - - readme_file = project_path / 'README.md' - with open(readme_file, 'w') as f: + + readme_file = project_path / "README.md" + with open(readme_file, "w") as f: f.write(README_TEMPLATE.format(project_name=name)) - + console.print() - console.print(Panel.fit( - f"[green]✓[/green] Project created successfully!\n\n" - f"Next steps:\n" - f" cd {name}\n" - f" concore validate workflow.graphml\n" - f" concore run workflow.graphml", - title="Success", - border_style="green" - )) + console.print( + Panel.fit( + f"[green]✓[/green] Project created successfully!\n\n" + f"Next steps:\n" + f" cd {name}\n" + f" concore validate workflow.graphml\n" + f" concore run workflow.graphml", + title="Success", + border_style="green", + ) + ) diff --git a/concore_cli/commands/inspect.py b/concore_cli/commands/inspect.py index c84fbed4..0dce24a8 100644 --- a/concore_cli/commands/inspect.py +++ b/concore_cli/commands/inspect.py @@ -2,262 +2,270 @@ from bs4 import BeautifulSoup from rich.table import Table from rich.tree import Tree -from rich.panel import Panel from collections import defaultdict import re + def inspect_workflow(workflow_file, source_dir, output_json, console): workflow_path = Path(workflow_file) - + if output_json: return _inspect_json(workflow_path, source_dir) - + _inspect_rich(workflow_path, source_dir, console) + def _inspect_rich(workflow_path, source_dir, console): console.print() console.print(f"[bold cyan]Workflow:[/bold cyan] {workflow_path.name}") console.print() - + try: - with open(workflow_path, 'r') as f: + with open(workflow_path, "r") as f: content = f.read() - - soup = BeautifulSoup(content, 'xml') - - if not soup.find('graphml'): + + soup = BeautifulSoup(content, "xml") + + if not soup.find("graphml"): console.print("[red]Not a valid GraphML file[/red]") return - - nodes = soup.find_all('node') - edges = soup.find_all('edge') - + + nodes = soup.find_all("node") + edges = soup.find_all("edge") + tree = Tree("📊 [bold]Workflow Overview[/bold]") - + lang_counts = defaultdict(int) node_files = [] missing_files = [] - + for node in nodes: - label_tag = node.find('y:NodeLabel') + label_tag = node.find("y:NodeLabel") if label_tag and label_tag.text: label = label_tag.text.strip() - if ':' in label: - _, filename = label.split(':', 1) + if ":" in label: + _, filename = label.split(":", 1) node_files.append(filename) - + ext = Path(filename).suffix - if ext == '.py': - lang_counts['Python'] += 1 - elif ext == '.m': - lang_counts['MATLAB'] += 1 - elif ext == '.java': - lang_counts['Java'] += 1 - elif ext == '.cpp' or ext == '.hpp': - lang_counts['C++'] += 1 - elif ext == '.v': - lang_counts['Verilog'] += 1 + if ext == ".py": + lang_counts["Python"] += 1 + elif ext == ".m": + lang_counts["MATLAB"] += 1 + elif ext == ".java": + lang_counts["Java"] += 1 + elif ext == ".cpp" or ext == ".hpp": + lang_counts["C++"] += 1 + elif ext == ".v": + lang_counts["Verilog"] += 1 else: - lang_counts['Other'] += 1 - + lang_counts["Other"] += 1 + src_dir = workflow_path.parent / source_dir if not (src_dir / filename).exists(): missing_files.append(filename) - + nodes_branch = tree.add(f"Nodes: [bold]{len(nodes)}[/bold]") if lang_counts: for lang, count in sorted(lang_counts.items(), key=lambda x: -x[1]): nodes_branch.add(f"{lang}: {count}") - + edges_branch = tree.add(f"Edges: [bold]{len(edges)}[/bold]") - + edge_label_regex = re.compile(r"0x([a-fA-F0-9]+)_(\S+)") zmq_count = 0 file_count = 0 - + for edge in edges: - label_tag = edge.find('y:EdgeLabel') + label_tag = edge.find("y:EdgeLabel") label_text = label_tag.text.strip() if label_tag and label_tag.text else "" if label_text and edge_label_regex.match(label_text): zmq_count += 1 else: file_count += 1 - + if zmq_count > 0: edges_branch.add(f"ZMQ: {zmq_count}") if file_count > 0: edges_branch.add(f"File-based: {file_count}") - - comm_type = "ZMQ (0mq)" if zmq_count > 0 else "File-based" if file_count > 0 else "None" + + comm_type = ( + "ZMQ (0mq)" if zmq_count > 0 else "File-based" if file_count > 0 else "None" + ) tree.add(f"Communication: [bold]{comm_type}[/bold]") - + if missing_files: - missing_branch = tree.add(f"[yellow]Missing files: {len(missing_files)}[/yellow]") + missing_branch = tree.add( + f"[yellow]Missing files: {len(missing_files)}[/yellow]" + ) for f in missing_files[:5]: missing_branch.add(f"[yellow]{f}[/yellow]") if len(missing_files) > 5: missing_branch.add(f"[dim]...and {len(missing_files) - 5} more[/dim]") - + console.print(tree) console.print() - + if nodes: - table = Table(title="Node Details", show_header=True, header_style="bold magenta") + table = Table( + title="Node Details", show_header=True, header_style="bold magenta" + ) table.add_column("ID", style="cyan", width=12) table.add_column("File", style="white") table.add_column("Language", style="green") table.add_column("Status", style="yellow") - + for node in nodes[:10]: - label_tag = node.find('y:NodeLabel') + label_tag = node.find("y:NodeLabel") if label_tag and label_tag.text: label = label_tag.text.strip() - if ':' in label: - node_id, filename = label.split(':', 1) - + if ":" in label: + node_id, filename = label.split(":", 1) + ext = Path(filename).suffix lang_map = { - '.py': 'Python', - '.m': 'MATLAB', - '.java': 'Java', - '.cpp': 'C++', - '.hpp': 'C++', - '.v': 'Verilog' + ".py": "Python", + ".m": "MATLAB", + ".java": "Java", + ".cpp": "C++", + ".hpp": "C++", + ".v": "Verilog", } - lang = lang_map.get(ext, 'Other') - + lang = lang_map.get(ext, "Other") + src_dir = workflow_path.parent / source_dir status = "✓" if (src_dir / filename).exists() else "✗" - + table.add_row(node_id, filename, lang, status) - + if len(nodes) > 10: table.caption = f"Showing 10 of {len(nodes)} nodes" - + console.print(table) console.print() - + if edges: - edge_table = Table(title="Edge Connections", show_header=True, header_style="bold magenta") + edge_table = Table( + title="Edge Connections", show_header=True, header_style="bold magenta" + ) edge_table.add_column("From", style="cyan", width=12) edge_table.add_column("To", style="cyan", width=12) edge_table.add_column("Type", style="green") - + for edge in edges[:10]: - source = edge.get('source', 'unknown') - target = edge.get('target', 'unknown') - - label_tag = edge.find('y:EdgeLabel') + source = edge.get("source", "unknown") + target = edge.get("target", "unknown") + + label_tag = edge.find("y:EdgeLabel") edge_type = "File" if label_tag and label_tag.text: if edge_label_regex.match(label_tag.text.strip()): edge_type = "ZMQ" - + edge_table.add_row(source, target, edge_type) - + if len(edges) > 10: edge_table.caption = f"Showing 10 of {len(edges)} edges" - + console.print(edge_table) console.print() - + except FileNotFoundError: console.print(f"[red]File not found:[/red] {workflow_path}") except Exception as e: console.print(f"[red]Inspection failed:[/red] {str(e)}") + def _inspect_json(workflow_path, source_dir): import json - + try: - with open(workflow_path, 'r') as f: + with open(workflow_path, "r") as f: content = f.read() - - soup = BeautifulSoup(content, 'xml') - - if not soup.find('graphml'): - print(json.dumps({'error': 'Not a valid GraphML file'}, indent=2)) + + soup = BeautifulSoup(content, "xml") + + if not soup.find("graphml"): + print(json.dumps({"error": "Not a valid GraphML file"}, indent=2)) return - - nodes = soup.find_all('node') - edges = soup.find_all('edge') - + + nodes = soup.find_all("node") + edges = soup.find_all("edge") + lang_counts = defaultdict(int) node_list = [] edge_list = [] missing_files = [] - + for node in nodes: - label_tag = node.find('y:NodeLabel') + label_tag = node.find("y:NodeLabel") if label_tag and label_tag.text: label = label_tag.text.strip() - if ':' in label: - node_id, filename = label.split(':', 1) - + if ":" in label: + node_id, filename = label.split(":", 1) + ext = Path(filename).suffix lang_map = { - '.py': 'python', - '.m': 'matlab', - '.java': 'java', - '.cpp': 'cpp', - '.hpp': 'cpp', - '.v': 'verilog' + ".py": "python", + ".m": "matlab", + ".java": "java", + ".cpp": "cpp", + ".hpp": "cpp", + ".v": "verilog", } - lang = lang_map.get(ext, 'other') + lang = lang_map.get(ext, "other") lang_counts[lang] += 1 - + src_dir = workflow_path.parent / source_dir exists = (src_dir / filename).exists() if not exists: missing_files.append(filename) - - node_list.append({ - 'id': node_id, - 'file': filename, - 'language': lang, - 'exists': exists - }) - + + node_list.append( + { + "id": node_id, + "file": filename, + "language": lang, + "exists": exists, + } + ) + edge_label_regex = re.compile(r"0x([a-fA-F0-9]+)_(\S+)") zmq_count = 0 file_count = 0 - + for edge in edges: - source = edge.get('source') - target = edge.get('target') - - label_tag = edge.find('y:EdgeLabel') + source = edge.get("source") + target = edge.get("target") + + label_tag = edge.find("y:EdgeLabel") label_text = label_tag.text.strip() if label_tag and label_tag.text else "" - edge_type = 'file' + edge_type = "file" if label_text and edge_label_regex.match(label_text): - edge_type = 'zmq' + edge_type = "zmq" zmq_count += 1 else: file_count += 1 - - edge_list.append({ - 'source': source, - 'target': target, - 'type': edge_type - }) - + + edge_list.append({"source": source, "target": target, "type": edge_type}) + result = { - 'workflow': str(workflow_path.name), - 'nodes': { - 'total': len(nodes), - 'by_language': dict(lang_counts), - 'list': node_list + "workflow": str(workflow_path.name), + "nodes": { + "total": len(nodes), + "by_language": dict(lang_counts), + "list": node_list, }, - 'edges': { - 'total': len(edges), - 'zmq': zmq_count, - 'file': file_count, - 'list': edge_list + "edges": { + "total": len(edges), + "zmq": zmq_count, + "file": file_count, + "list": edge_list, }, - 'missing_files': missing_files + "missing_files": missing_files, } - + print(json.dumps(result, indent=2)) - + except Exception as e: - print(json.dumps({'error': str(e)}, indent=2)) + print(json.dumps({"error": str(e)}, indent=2)) diff --git a/concore_cli/commands/run.py b/concore_cli/commands/run.py index 91a876b7..a80dbe05 100644 --- a/concore_cli/commands/run.py +++ b/concore_cli/commands/run.py @@ -1,10 +1,10 @@ -import os import sys import subprocess from pathlib import Path from rich.panel import Panel from rich.progress import Progress, SpinnerColumn, TextColumn + def _find_mkconcore_path(): for parent in Path(__file__).resolve().parents: candidate = parent / "mkconcore.py" @@ -12,73 +12,89 @@ def _find_mkconcore_path(): return candidate return None + def run_workflow(workflow_file, source, output, exec_type, auto_build, console): workflow_path = Path(workflow_file).resolve() source_path = Path(source).resolve() output_path = Path(output).resolve() - + if not source_path.exists(): raise FileNotFoundError(f"Source directory '{source}' not found") - + if output_path.exists(): - console.print(f"[yellow]Warning:[/yellow] Output directory '{output}' already exists") + console.print( + f"[yellow]Warning:[/yellow] Output directory '{output}' already exists" + ) console.print("Remove it first or choose a different output directory") return - + console.print(f"[cyan]Workflow:[/cyan] {workflow_path.name}") console.print(f"[cyan]Source:[/cyan] {source_path}") console.print(f"[cyan]Output:[/cyan] {output_path}") console.print(f"[cyan]Type:[/cyan] {exec_type}") console.print() - + mkconcore_path = _find_mkconcore_path() if mkconcore_path is None: - raise FileNotFoundError("mkconcore.py not found. Please install concore from source.") + raise FileNotFoundError( + "mkconcore.py not found. Please install concore from source." + ) with Progress( SpinnerColumn(), TextColumn("[progress.description]{task.description}"), - console=console + console=console, ) as progress: task = progress.add_task("Generating workflow...", total=None) - + try: result = subprocess.run( - [sys.executable, str(mkconcore_path), str(workflow_path), str(source_path), str(output_path), exec_type], + [ + sys.executable, + str(mkconcore_path), + str(workflow_path), + str(source_path), + str(output_path), + exec_type, + ], cwd=mkconcore_path.parent, capture_output=True, text=True, - check=True + check=True, ) - + progress.update(task, completed=True) - + if result.stdout: console.print(result.stdout) - - console.print(f"[green]✓[/green] Workflow generated in [cyan]{output_path}[/cyan]") - + + console.print( + f"[green]✓[/green] Workflow generated in [cyan]{output_path}[/cyan]" + ) + except subprocess.CalledProcessError as e: progress.stop() - console.print(f"[red]Generation failed:[/red]") + console.print("[red]Generation failed:[/red]") if e.stdout: console.print(e.stdout) if e.stderr: console.print(e.stderr) raise - + if auto_build: console.print() - build_script = output_path / ('build.bat' if exec_type == 'windows' else 'build') - + build_script = output_path / ( + "build.bat" if exec_type == "windows" else "build" + ) + if build_script.exists(): with Progress( SpinnerColumn(), TextColumn("[progress.description]{task.description}"), - console=console + console=console, ) as progress: task = progress.add_task("Building workflow...", total=None) - + try: result = subprocess.run( [str(build_script)], @@ -86,23 +102,25 @@ def run_workflow(workflow_file, source, output, exec_type, auto_build, console): capture_output=True, text=True, shell=True, - check=True + check=True, ) progress.update(task, completed=True) - console.print(f"[green]✓[/green] Build completed") + console.print("[green]✓[/green] Build completed") except subprocess.CalledProcessError as e: progress.stop() - console.print(f"[yellow]Build failed[/yellow]") + console.print("[yellow]Build failed[/yellow]") if e.stderr: console.print(e.stderr) - + console.print() - console.print(Panel.fit( - f"[green]✓[/green] Workflow ready!\n\n" - f"To run your workflow:\n" - f" cd {output_path}\n" - f" {'build.bat' if exec_type == 'windows' else './build'}\n" - f" {'run.bat' if exec_type == 'windows' else './run'}", - title="Next Steps", - border_style="green" - )) + console.print( + Panel.fit( + f"[green]✓[/green] Workflow ready!\n\n" + f"To run your workflow:\n" + f" cd {output_path}\n" + f" {'build.bat' if exec_type == 'windows' else './build'}\n" + f" {'run.bat' if exec_type == 'windows' else './run'}", + title="Next Steps", + border_style="green", + ) + ) diff --git a/concore_cli/commands/status.py b/concore_cli/commands/status.py index 6eaf6c8b..7ef1fca4 100644 --- a/concore_cli/commands/status.py +++ b/concore_cli/commands/status.py @@ -1,92 +1,106 @@ import psutil import os -from pathlib import Path from rich.table import Table from rich.panel import Panel from datetime import datetime + def show_status(console): console.print("[cyan]Scanning for concore processes...[/cyan]\n") - + concore_processes = [] - + try: current_pid = os.getpid() - - for proc in psutil.process_iter(['pid', 'name', 'cmdline', 'create_time', 'memory_info', 'cpu_percent']): + + for proc in psutil.process_iter( + ["pid", "name", "cmdline", "create_time", "memory_info", "cpu_percent"] + ): try: - cmdline = proc.info.get('cmdline') or [] - name = proc.info.get('name', '').lower() - - if proc.info['pid'] == current_pid: + cmdline = proc.info.get("cmdline") or [] + name = proc.info.get("name", "").lower() + + if proc.info["pid"] == current_pid: continue - - cmdline_str = ' '.join(cmdline) if cmdline else '' - + + cmdline_str = " ".join(cmdline) if cmdline else "" + is_concore = ( - 'concore' in cmdline_str.lower() or - 'concore.py' in cmdline_str.lower() or - any('concorekill.bat' in str(item) for item in cmdline) or - (name in ['python.exe', 'python', 'python3'] and 'concore' in cmdline_str) + "concore" in cmdline_str.lower() + or "concore.py" in cmdline_str.lower() + or any("concorekill.bat" in str(item) for item in cmdline) + or ( + name in ["python.exe", "python", "python3"] + and "concore" in cmdline_str + ) ) - + if is_concore: try: - create_time = datetime.fromtimestamp(proc.info['create_time']) + create_time = datetime.fromtimestamp(proc.info["create_time"]) uptime = datetime.now() - create_time hours, remainder = divmod(int(uptime.total_seconds()), 3600) minutes, seconds = divmod(remainder, 60) uptime_str = f"{hours}h {minutes}m {seconds}s" - except: + except (OSError, OverflowError, ValueError): # Failed to calculate uptime uptime_str = "unknown" - + try: - mem_mb = proc.info['memory_info'].rss / 1024 / 1024 + mem_mb = proc.info["memory_info"].rss / 1024 / 1024 mem_str = f"{mem_mb:.1f} MB" - except: + except (psutil.NoSuchProcess, psutil.AccessDenied, AttributeError): # Failed to get memory info mem_str = "unknown" - - command = ' '.join(cmdline[:3]) if len(cmdline) >= 3 else cmdline_str[:50] - - concore_processes.append({ - 'pid': proc.info['pid'], - 'name': proc.info.get('name', 'unknown'), - 'command': command, - 'uptime': uptime_str, - 'memory': mem_str - }) + + command = ( + " ".join(cmdline[:3]) if len(cmdline) >= 3 else cmdline_str[:50] + ) + + concore_processes.append( + { + "pid": proc.info["pid"], + "name": proc.info.get("name", "unknown"), + "command": command, + "uptime": uptime_str, + "memory": mem_str, + } + ) except (psutil.NoSuchProcess, psutil.AccessDenied): # Process may have exited or be inaccessible; safe to ignore continue - + except Exception as e: console.print(f"[red]Error scanning processes:[/red] {str(e)}") return - + if not concore_processes: - console.print(Panel.fit( - "[yellow]No concore processes currently running[/yellow]", - border_style="yellow" - )) + console.print( + Panel.fit( + "[yellow]No concore processes currently running[/yellow]", + border_style="yellow", + ) + ) else: - table = Table(title=f"Concore Processes ({len(concore_processes)} running)", show_header=True) + table = Table( + title=f"Concore Processes ({len(concore_processes)} running)", + show_header=True, + ) table.add_column("PID", style="cyan", justify="right") table.add_column("Name", style="green") table.add_column("Uptime", style="yellow") table.add_column("Memory", style="magenta") table.add_column("Command", style="white") - + for proc in concore_processes: table.add_row( - str(proc['pid']), - proc['name'], - proc['uptime'], - proc['memory'], - proc['command'] + str(proc["pid"]), + proc["name"], + proc["uptime"], + proc["memory"], + proc["command"], ) - + console.print(table) console.print() - console.print(f"[dim]Use 'concore stop' to terminate all processes[/dim]") + console.print("[dim]Use 'concore stop' to terminate all processes[/dim]") diff --git a/concore_cli/commands/stop.py b/concore_cli/commands/stop.py index 5b0a9a92..27b5796e 100644 --- a/concore_cli/commands/stop.py +++ b/concore_cli/commands/stop.py @@ -2,94 +2,106 @@ import os import subprocess import sys -from pathlib import Path from rich.panel import Panel + def stop_all(console): console.print("[cyan]Finding concore processes...[/cyan]\n") - + processes_to_kill = [] current_pid = os.getpid() - + try: - for proc in psutil.process_iter(['pid', 'name', 'cmdline']): + for proc in psutil.process_iter(["pid", "name", "cmdline"]): try: - if proc.info['pid'] == current_pid: + if proc.info["pid"] == current_pid: continue - - cmdline = proc.info.get('cmdline') or [] - name = proc.info.get('name', '').lower() - cmdline_str = ' '.join(cmdline) if cmdline else '' - + + cmdline = proc.info.get("cmdline") or [] + name = proc.info.get("name", "").lower() + cmdline_str = " ".join(cmdline) if cmdline else "" + is_concore = ( - 'concore' in cmdline_str.lower() or - 'concore.py' in cmdline_str.lower() or - any('concorekill.bat' in str(item) for item in cmdline) or - (name in ['python.exe', 'python', 'python3'] and 'concore' in cmdline_str) + "concore" in cmdline_str.lower() + or "concore.py" in cmdline_str.lower() + or any("concorekill.bat" in str(item) for item in cmdline) + or ( + name in ["python.exe", "python", "python3"] + and "concore" in cmdline_str + ) ) - + if is_concore: processes_to_kill.append(proc) except (psutil.NoSuchProcess, psutil.AccessDenied): # Process already exited or access denied; continue continue - + except Exception as e: console.print(f"[red]Error:[/red] {str(e)}") return - + if not processes_to_kill: - console.print(Panel.fit( - "[yellow]No concore processes found[/yellow]", - border_style="yellow" - )) + console.print( + Panel.fit( + "[yellow]No concore processes found[/yellow]", border_style="yellow" + ) + ) return - - console.print(f"[yellow]Stopping {len(processes_to_kill)} process(es)...[/yellow]\n") - + + console.print( + f"[yellow]Stopping {len(processes_to_kill)} process(es)...[/yellow]\n" + ) + killed_count = 0 failed_count = 0 - + for proc in processes_to_kill: try: - pid = proc.info['pid'] - name = proc.info.get('name', 'unknown') - - if sys.platform == 'win32': - result = subprocess.run(['taskkill', '/F', '/PID', str(pid)], - capture_output=True, - check=False) + pid = proc.info["pid"] + name = proc.info.get("name", "unknown") + + if sys.platform == "win32": + result = subprocess.run( + ["taskkill", "/F", "/PID", str(pid)], + capture_output=True, + check=False, + ) if result.returncode != 0: raise RuntimeError(f"taskkill failed with code {result.returncode}") else: proc.terminate() proc.wait(timeout=3) - + console.print(f" [green]✓[/green] Stopped {name} (PID: {pid})") killed_count += 1 - + except psutil.TimeoutExpired: try: proc.kill() console.print(f" [yellow]⚠[/yellow] Force killed {name} (PID: {pid})") killed_count += 1 - except: + except (psutil.NoSuchProcess, psutil.AccessDenied): console.print(f" [red]✗[/red] Failed to stop {name} (PID: {pid})") failed_count += 1 except Exception as e: console.print(f" [red]✗[/red] Failed to stop PID {pid}: {str(e)}") failed_count += 1 - + console.print() - + if failed_count == 0: - console.print(Panel.fit( - f"[green]✓[/green] Successfully stopped all {killed_count} process(es)", - border_style="green" - )) + console.print( + Panel.fit( + f"[green]✓[/green] Successfully stopped all {killed_count} process(es)", + border_style="green", + ) + ) else: - console.print(Panel.fit( - f"[yellow]Stopped {killed_count} process(es)\n" - f"Failed to stop {failed_count} process(es)[/yellow]", - border_style="yellow" - )) + console.print( + Panel.fit( + f"[yellow]Stopped {killed_count} process(es)\n" + f"Failed to stop {failed_count} process(es)[/yellow]", + border_style="yellow", + ) + ) diff --git a/concore_cli/commands/validate.py b/concore_cli/commands/validate.py index d9a39dd4..e987c8ad 100644 --- a/concore_cli/commands/validate.py +++ b/concore_cli/commands/validate.py @@ -1,114 +1,123 @@ from pathlib import Path from bs4 import BeautifulSoup from rich.panel import Panel -from rich.table import Table import re import xml.etree.ElementTree as ET + def validate_workflow(workflow_file, source_dir, console): workflow_path = Path(workflow_file) - source_root = (workflow_path.parent / source_dir) - + source_root = workflow_path.parent / source_dir + console.print(f"[cyan]Validating:[/cyan] {workflow_path.name}") console.print() - + errors = [] warnings = [] info = [] - + def finalize(): show_results(console, errors, warnings, info) return len(errors) == 0 - + try: - with open(workflow_path, 'r') as f: + with open(workflow_path, "r") as f: content = f.read() - + if not content.strip(): errors.append("File is empty") return finalize() - + # strict XML syntax check try: ET.fromstring(content) except ET.ParseError as e: errors.append(f"Invalid XML: {str(e)}") return finalize() - + try: - soup = BeautifulSoup(content, 'xml') + soup = BeautifulSoup(content, "xml") except Exception as e: errors.append(f"Invalid XML: {str(e)}") return finalize() - - root = soup.find('graphml') + + root = soup.find("graphml") if not root: errors.append("Not a valid GraphML file - missing root element") return finalize() - + # check the graph attributes - graph = soup.find('graph') + graph = soup.find("graph") if not graph: - errors.append("Missing element") + errors.append("Missing element") else: - edgedefault = graph.get('edgedefault') - if not edgedefault: - errors.append("Graph missing required 'edgedefault' attribute") - elif edgedefault not in ['directed', 'undirected']: - errors.append(f"Invalid edgedefault value '{edgedefault}' (must be 'directed' or 'undirected')") - - nodes = soup.find_all('node') - edges = soup.find_all('edge') - + edgedefault = graph.get("edgedefault") + if not edgedefault: + errors.append("Graph missing required 'edgedefault' attribute") + elif edgedefault not in ["directed", "undirected"]: + errors.append( + f"Invalid edgedefault value '{edgedefault}' (must be 'directed' or 'undirected')" + ) + + nodes = soup.find_all("node") + edges = soup.find_all("edge") + if len(nodes) == 0: warnings.append("No nodes found in workflow") else: info.append(f"Found {len(nodes)} node(s)") - + if len(edges) == 0: warnings.append("No edges found in workflow") else: info.append(f"Found {len(edges)} edge(s)") - + if not source_root.exists(): warnings.append(f"Source directory not found: {source_root}") - + node_labels = [] for node in nodes: - #check the node id - node_id = node.get('id') + # check the node id + node_id = node.get("id") if not node_id: errors.append("Node missing required 'id' attribute") - #skip further checks for this node to avoid noise + # skip further checks for this node to avoid noise continue try: - #robust find: try with namespace prefix first, then without - label_tag = node.find('y:NodeLabel') + # robust find: try with namespace prefix first, then without + label_tag = node.find("y:NodeLabel") if not label_tag: - label_tag = node.find('NodeLabel') - + label_tag = node.find("NodeLabel") + if label_tag and label_tag.text: label = label_tag.text.strip() node_labels.append(label) - + # reject shell metacharacters to prevent command injection (#251) if re.search(r'[;&|`$\'"()\\]', label): - errors.append(f"Node '{label}' contains unsafe shell characters") + errors.append( + f"Node '{label}' contains unsafe shell characters" + ) continue - - if ':' not in label: + + if ":" not in label: warnings.append(f"Node '{label}' missing format 'ID:filename'") else: - parts = label.split(':') + parts = label.split(":") if len(parts) != 2: warnings.append(f"Node '{label}' has invalid format") else: nodeId_part, filename = parts if not filename: errors.append(f"Node '{label}' has no filename") - elif not any(filename.endswith(ext) for ext in ['.py', '.cpp', '.m', '.v', '.java']): - warnings.append(f"Node '{label}' has unusual file extension") + elif not any( + filename.endswith(ext) + for ext in [".py", ".cpp", ".m", ".v", ".java"] + ): + warnings.append( + f"Node '{label}' has unusual file extension" + ) elif source_root.exists(): file_path = source_root / filename if not file_path.exists(): @@ -125,48 +134,48 @@ def finalize(): errors.append(f"Duplicate node label: '{label}'") seen.add(label) - node_ids = {node.get('id') for node in nodes if node.get('id')} + node_ids = {node.get("id") for node in nodes if node.get("id")} for edge in edges: - source = edge.get('source') - target = edge.get('target') - + source = edge.get("source") + target = edge.get("target") + if not source or not target: errors.append("Edge missing source or target") continue - + if source not in node_ids: errors.append(f"Edge references non-existent source node: {source}") if target not in node_ids: errors.append(f"Edge references non-existent target node: {target}") - + edge_label_regex = re.compile(r"0x([a-fA-F0-9]+)_(\S+)") zmq_edges = 0 file_edges = 0 - + for edge in edges: try: - label_tag = edge.find('y:EdgeLabel') + label_tag = edge.find("y:EdgeLabel") if not label_tag: - label_tag = edge.find('EdgeLabel') - + label_tag = edge.find("EdgeLabel") + if label_tag and label_tag.text: if edge_label_regex.match(label_tag.text.strip()): zmq_edges += 1 else: file_edges += 1 - except: + except Exception: pass - + if zmq_edges > 0: info.append(f"ZMQ-based edges: {zmq_edges}") if file_edges > 0: info.append(f"File-based edges: {file_edges}") - + _check_cycles(soup, errors, warnings) _check_zmq_ports(soup, errors, warnings) - + return finalize() - + except FileNotFoundError: console.print(f"[red]Error:[/red] File not found: {workflow_path}") return False @@ -174,35 +183,36 @@ def finalize(): console.print(f"[red]Validation failed:[/red] {str(e)}") return False + def _check_cycles(soup, errors, warnings): - nodes = soup.find_all('node') - edges = soup.find_all('edge') - - node_ids = [node.get('id') for node in nodes if node.get('id')] + nodes = soup.find_all("node") + edges = soup.find_all("edge") + + node_ids = [node.get("id") for node in nodes if node.get("id")] if not node_ids: return - + graph = {nid: [] for nid in node_ids} for edge in edges: - source = edge.get('source') - target = edge.get('target') + source = edge.get("source") + target = edge.get("target") if source and target and source in graph: graph[source].append(target) - + def has_cycle_from(start, visited, rec_stack): visited.add(start) rec_stack.add(start) - + for neighbor in graph.get(start, []): if neighbor not in visited: if has_cycle_from(neighbor, visited, rec_stack): return True elif neighbor in rec_stack: return True - + rec_stack.remove(start) return False - + visited = set() for node_id in node_ids: if node_id not in visited: @@ -210,41 +220,51 @@ def has_cycle_from(start, visited, rec_stack): warnings.append("Workflow contains cycles (expected for control loops)") return + def _check_zmq_ports(soup, errors, warnings): - edges = soup.find_all('edge') + edges = soup.find_all("edge") port_pattern = re.compile(r"0x([a-fA-F0-9]+)_(\S+)") - + ports_used = {} - + for edge in edges: - label_tag = edge.find('y:EdgeLabel') or edge.find('EdgeLabel') + label_tag = edge.find("y:EdgeLabel") or edge.find("EdgeLabel") if not label_tag or not label_tag.text: continue - + match = port_pattern.match(label_tag.text.strip()) if not match: continue - + port_hex = match.group(1) port_name = match.group(2) port_num = int(port_hex, 16) - + if port_num < 1: - errors.append(f"Invalid port number: {port_num} (0x{port_hex}) must be at least 1") + errors.append( + f"Invalid port number: {port_num} (0x{port_hex}) must be at least 1" + ) continue elif port_num > 65535: - errors.append(f"Invalid port number: {port_num} (0x{port_hex}) exceeds maximum (65535)") + errors.append( + f"Invalid port number: {port_num} (0x{port_hex}) exceeds maximum (65535)" + ) continue - + if port_num in ports_used: existing_name = ports_used[port_num] if existing_name != port_name: - errors.append(f"Port conflict: 0x{port_hex} used for both '{existing_name}' and '{port_name}'") + errors.append( + f"Port conflict: 0x{port_hex} used for both '{existing_name}' and '{port_name}'" + ) else: ports_used[port_num] = port_name - + if port_num < 1024: - warnings.append(f"Port {port_num} (0x{port_hex}) is in reserved range (< 1024)") + warnings.append( + f"Port {port_num} (0x{port_hex}) is in reserved range (< 1024)" + ) + def show_results(console, errors, warnings, info): if errors: @@ -253,27 +273,31 @@ def show_results(console, errors, warnings, info): console.print(f" [red]✗[/red] {error}") else: console.print("[green]✓ Validation passed[/green]\n") - + if warnings: console.print() for warning in warnings: console.print(f" [yellow]⚠[/yellow] {warning}") - + if info: console.print() for item in info: console.print(f" [blue]ℹ[/blue] {item}") - + console.print() - + if not errors: - console.print(Panel.fit( - "[green]✓[/green] Workflow is valid and ready to run", - border_style="green" - )) + console.print( + Panel.fit( + "[green]✓[/green] Workflow is valid and ready to run", + border_style="green", + ) + ) else: - console.print(Panel.fit( - f"[red]Found {len(errors)} error(s)[/red]\n" - "Fix the errors above before running the workflow", - border_style="red" - )) + console.print( + Panel.fit( + f"[red]Found {len(errors)} error(s)[/red]\n" + "Fix the errors above before running the workflow", + border_style="red", + ) + ) diff --git a/concore_cli/commands/watch.py b/concore_cli/commands/watch.py index 44f39e74..e82efbab 100644 --- a/concore_cli/commands/watch.py +++ b/concore_cli/commands/watch.py @@ -20,10 +20,14 @@ def watch_study(study_dir, interval, once, console): edges = _find_edges(study_path, nodes) if not nodes and not edges: - console.print(Panel( - "[yellow]No nodes or edge directories found.[/yellow]\n" - "[dim]Make sure you point to a built study directory (run makestudy/build first).[/dim]", - title="concore watch", border_style="yellow")) + console.print( + Panel( + "[yellow]No nodes or edge directories found.[/yellow]\n" + "[dim]Make sure you point to a built study directory (run makestudy/build first).[/dim]", + title="concore watch", + border_style="yellow", + ) + ) return if once: @@ -63,15 +67,18 @@ def _build_display(study_path, nodes, edges): parts.append(_node_table(nodes)) if not edges and not nodes: - parts.append(Panel("[yellow]No data yet[/yellow]", - border_style="yellow")) + parts.append(Panel("[yellow]No data yet[/yellow]", border_style="yellow")) return Group(*parts) def _edge_table(edges): - table = Table(title="Edges (port data)", show_header=True, - title_style="bold cyan", expand=True) + table = Table( + title="Edges (port data)", + show_header=True, + title_style="bold cyan", + expand=True, + ) table.add_column("Edge", style="green", min_width=10) table.add_column("Port File", style="cyan") table.add_column("Simtime", style="yellow", justify="right") @@ -96,18 +103,23 @@ def _edge_table(edges): def _node_table(nodes): - table = Table(title="Nodes", show_header=True, - title_style="bold cyan", expand=True) + table = Table(title="Nodes", show_header=True, title_style="bold cyan", expand=True) table.add_column("Node", style="green", min_width=10) table.add_column("Ports (in)", style="cyan") table.add_column("Ports (out)", style="cyan") table.add_column("Source", style="dim") for node_name, node_path in sorted(nodes): - in_dirs = sorted(d.name for d in node_path.iterdir() - if d.is_dir() and re.match(r'^in\d+$', d.name)) - out_dirs = sorted(d.name for d in node_path.iterdir() - if d.is_dir() and re.match(r'^out\d+$', d.name)) + in_dirs = sorted( + d.name + for d in node_path.iterdir() + if d.is_dir() and re.match(r"^in\d+$", d.name) + ) + out_dirs = sorted( + d.name + for d in node_path.iterdir() + if d.is_dir() and re.match(r"^out\d+$", d.name) + ) src = _detect_source(node_path) table.add_row( node_name, @@ -121,14 +133,15 @@ def _node_table(nodes): def _find_nodes(study_path): nodes = [] - port_re = re.compile(r'^(in|out)\d+$') - skip = {'src', '__pycache__', '.git'} + port_re = re.compile(r"^(in|out)\d+$") + skip = {"src", "__pycache__", ".git"} for entry in study_path.iterdir(): - if not entry.is_dir() or entry.name in skip or entry.name.startswith('.'): + if not entry.is_dir() or entry.name in skip or entry.name.startswith("."): continue try: - has_ports = any(c.is_dir() and port_re.match(c.name) - for c in entry.iterdir()) + has_ports = any( + c.is_dir() and port_re.match(c.name) for c in entry.iterdir() + ) except PermissionError: continue if has_ports: @@ -140,12 +153,12 @@ def _find_edges(study_path, nodes=None): if nodes is None: nodes = _find_nodes(study_path) node_names = {name for name, _ in nodes} - skip = {'src', '__pycache__', '.git'} + skip = {"src", "__pycache__", ".git"} edges = [] for entry in study_path.iterdir(): if not entry.is_dir(): continue - if entry.name in skip or entry.name in node_names or entry.name.startswith('.'): + if entry.name in skip or entry.name in node_names or entry.name.startswith("."): continue try: has_file = any(f.is_file() for f in entry.iterdir()) @@ -166,7 +179,7 @@ def _read_edge_files(edge_path): if not f.is_file(): continue # skip concore internal files - if f.name.startswith('concore.'): + if f.name.startswith("concore."): continue simtime_val, value_str = _parse_port_file(f) try: @@ -178,11 +191,11 @@ def _read_edge_files(edge_path): def _detect_source(node_path): - for ext in ('*.py', '*.m', '*.cpp', '*.v', '*.sh', '*.java'): + for ext in ("*.py", "*.m", "*.cpp", "*.v", "*.sh", "*.java"): matches = list(node_path.glob(ext)) for m in matches: # skip concore library copies - if m.name.startswith('concore'): + if m.name.startswith("concore"): continue return m.name return "—" diff --git a/pytest.ini b/pytest.ini index 13bc1da9..5881b607 100644 --- a/pytest.ini +++ b/pytest.ini @@ -3,4 +3,4 @@ testpaths = tests python_files = test_*.py python_classes = Test* python_functions = test_* -addopts = -v --tb=short \ No newline at end of file +addopts = -v --tb=short --cov=concore_cli --cov=concore_base --cov-report=term-missing \ No newline at end of file diff --git a/requirements-ci.txt b/requirements-ci.txt index 5668f6a5..1f26eb20 100644 --- a/requirements-ci.txt +++ b/requirements-ci.txt @@ -1,6 +1,7 @@ # Minimal dependencies for CI (linting and testing) # Does not include heavyweight packages like tensorflow pytest +pytest-cov ruff pyzmq numpy diff --git a/ruff.toml b/ruff.toml new file mode 100644 index 00000000..d487540d --- /dev/null +++ b/ruff.toml @@ -0,0 +1,41 @@ +exclude = [ + # legacy/experimental dirs - fix incrementally + "demo/", + "ratc/", + "ratc2/", + "example/", + "0mq/", + "measurements/", + "tools/", + "nintan/", + "testsou/", + "linktest/", + "fri/", + "gi/", + "humanc/", + # core modules - maintainer-managed, don't touch yet + "mkconcore.py", + "concore.py", + "concoredocker.py", + "concore_base.py", + "contribute.py", + "copy_with_port_portname.py", + # not valid Python (Dockerfiles) + "Dockerfile.py", + "Dockerfile.m", + "Dockerfile.sh", + "Dockerfile.v", + "Dockerfile.java", +] + +[lint] +select = ["E", "F"] +ignore = [ + "E501", # line too long - enforce incrementally + "E712", # comparison to True/False - common in tests + "E721", # type comparison - common in tests +] + +[format] +quote-style = "double" +indent-style = "space" diff --git a/tests/conftest.py b/tests/conftest.py index 10d20ac0..c303e450 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -12,4 +12,4 @@ def temp_dir(): dirpath = tempfile.mkdtemp() yield dirpath if os.path.exists(dirpath): - shutil.rmtree(dirpath) \ No newline at end of file + shutil.rmtree(dirpath) diff --git a/tests/test_cli.py b/tests/test_cli.py index 33316a71..4321e05a 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -6,190 +6,223 @@ from click.testing import CliRunner from concore_cli.cli import cli + class TestConcoreCLI(unittest.TestCase): - def setUp(self): self.runner = CliRunner() self.temp_dir = tempfile.mkdtemp() - + def tearDown(self): if Path(self.temp_dir).exists(): shutil.rmtree(self.temp_dir) - + def test_version(self): - result = self.runner.invoke(cli, ['--version']) + result = self.runner.invoke(cli, ["--version"]) self.assertEqual(result.exit_code, 0) - self.assertIn('1.0.0', result.output) - + self.assertIn("1.0.0", result.output) + def test_help(self): - result = self.runner.invoke(cli, ['--help']) + result = self.runner.invoke(cli, ["--help"]) self.assertEqual(result.exit_code, 0) - self.assertIn('Usage:', result.output) - self.assertIn('Commands:', result.output) - + self.assertIn("Usage:", result.output) + self.assertIn("Commands:", result.output) + def test_init_command(self): with self.runner.isolated_filesystem(temp_dir=self.temp_dir): - result = self.runner.invoke(cli, ['init', 'test-project']) + result = self.runner.invoke(cli, ["init", "test-project"]) self.assertEqual(result.exit_code, 0) - - project_path = Path('test-project') + + project_path = Path("test-project") self.assertTrue(project_path.exists()) - self.assertTrue((project_path / 'workflow.graphml').exists()) - self.assertTrue((project_path / 'src').exists()) - self.assertTrue((project_path / 'README.md').exists()) - self.assertTrue((project_path / 'src' / 'script.py').exists()) - + self.assertTrue((project_path / "workflow.graphml").exists()) + self.assertTrue((project_path / "src").exists()) + self.assertTrue((project_path / "README.md").exists()) + self.assertTrue((project_path / "src" / "script.py").exists()) + def test_init_existing_directory(self): with self.runner.isolated_filesystem(temp_dir=self.temp_dir): - Path('existing').mkdir() - result = self.runner.invoke(cli, ['init', 'existing']) + Path("existing").mkdir() + result = self.runner.invoke(cli, ["init", "existing"]) self.assertNotEqual(result.exit_code, 0) - self.assertIn('already exists', result.output) - + self.assertIn("already exists", result.output) + def test_validate_missing_file(self): - result = self.runner.invoke(cli, ['validate', 'nonexistent.graphml']) + result = self.runner.invoke(cli, ["validate", "nonexistent.graphml"]) self.assertNotEqual(result.exit_code, 0) - + def test_validate_valid_file(self): with self.runner.isolated_filesystem(temp_dir=self.temp_dir): - result = self.runner.invoke(cli, ['init', 'test-project']) + result = self.runner.invoke(cli, ["init", "test-project"]) self.assertEqual(result.exit_code, 0) - - result = self.runner.invoke(cli, ['validate', 'test-project/workflow.graphml']) + + result = self.runner.invoke( + cli, ["validate", "test-project/workflow.graphml"] + ) self.assertEqual(result.exit_code, 0) - self.assertIn('Validation passed', result.output) + self.assertIn("Validation passed", result.output) def test_validate_missing_node_file(self): with self.runner.isolated_filesystem(temp_dir=self.temp_dir): - result = self.runner.invoke(cli, ['init', 'test-project']) + result = self.runner.invoke(cli, ["init", "test-project"]) self.assertEqual(result.exit_code, 0) - missing_file = Path('test-project/src/script.py') + missing_file = Path("test-project/src/script.py") if missing_file.exists(): missing_file.unlink() - result = self.runner.invoke(cli, ['validate', 'test-project/workflow.graphml']) + result = self.runner.invoke( + cli, ["validate", "test-project/workflow.graphml"] + ) self.assertNotEqual(result.exit_code, 0) - self.assertIn('Missing source file', result.output) - + self.assertIn("Missing source file", result.output) + def test_status_command(self): - result = self.runner.invoke(cli, ['status']) + result = self.runner.invoke(cli, ["status"]) self.assertEqual(result.exit_code, 0) - + def test_run_command_missing_source(self): with self.runner.isolated_filesystem(temp_dir=self.temp_dir): - result = self.runner.invoke(cli, ['init', 'test-project']) - result = self.runner.invoke(cli, ['run', 'test-project/workflow.graphml', '--source', 'nonexistent']) + result = self.runner.invoke(cli, ["init", "test-project"]) + result = self.runner.invoke( + cli, ["run", "test-project/workflow.graphml", "--source", "nonexistent"] + ) self.assertNotEqual(result.exit_code, 0) def test_run_command_from_project_dir(self): with self.runner.isolated_filesystem(temp_dir=self.temp_dir): - result = self.runner.invoke(cli, ['init', 'test-project']) + result = self.runner.invoke(cli, ["init", "test-project"]) self.assertEqual(result.exit_code, 0) - result = self.runner.invoke(cli, [ - 'run', - 'test-project/workflow.graphml', - '--source', 'test-project/src', - '--output', 'out', - '--type', 'posix' - ]) + result = self.runner.invoke( + cli, + [ + "run", + "test-project/workflow.graphml", + "--source", + "test-project/src", + "--output", + "out", + "--type", + "posix", + ], + ) self.assertEqual(result.exit_code, 0) - self.assertTrue(Path('out/src/concore.py').exists()) + self.assertTrue(Path("out/src/concore.py").exists()) def test_run_command_default_type(self): with self.runner.isolated_filesystem(temp_dir=self.temp_dir): - result = self.runner.invoke(cli, ['init', 'test-project']) + result = self.runner.invoke(cli, ["init", "test-project"]) self.assertEqual(result.exit_code, 0) - result = self.runner.invoke(cli, [ - 'run', - 'test-project/workflow.graphml', - '--source', 'test-project/src', - '--output', 'out' - ]) + result = self.runner.invoke( + cli, + [ + "run", + "test-project/workflow.graphml", + "--source", + "test-project/src", + "--output", + "out", + ], + ) self.assertEqual(result.exit_code, 0) - if os.name == 'nt': - self.assertTrue(Path('out/build.bat').exists()) + if os.name == "nt": + self.assertTrue(Path("out/build.bat").exists()) else: - self.assertTrue(Path('out/build').exists()) + self.assertTrue(Path("out/build").exists()) def test_run_command_nested_output_path(self): with self.runner.isolated_filesystem(temp_dir=self.temp_dir): - result = self.runner.invoke(cli, ['init', 'test-project']) + result = self.runner.invoke(cli, ["init", "test-project"]) self.assertEqual(result.exit_code, 0) - result = self.runner.invoke(cli, [ - 'run', - 'test-project/workflow.graphml', - '--source', 'test-project/src', - '--output', 'build/out', - '--type', 'posix' - ]) + result = self.runner.invoke( + cli, + [ + "run", + "test-project/workflow.graphml", + "--source", + "test-project/src", + "--output", + "build/out", + "--type", + "posix", + ], + ) self.assertEqual(result.exit_code, 0) - self.assertTrue(Path('build/out/src/concore.py').exists()) + self.assertTrue(Path("build/out/src/concore.py").exists()) def test_run_command_subdir_source(self): with self.runner.isolated_filesystem(temp_dir=self.temp_dir): - result = self.runner.invoke(cli, ['init', 'test-project']) + result = self.runner.invoke(cli, ["init", "test-project"]) self.assertEqual(result.exit_code, 0) - subdir = Path('test-project/src/subdir') + subdir = Path("test-project/src/subdir") subdir.mkdir(parents=True, exist_ok=True) - shutil.move('test-project/src/script.py', subdir / 'script.py') + shutil.move("test-project/src/script.py", subdir / "script.py") - workflow_path = Path('test-project/workflow.graphml') + workflow_path = Path("test-project/workflow.graphml") content = workflow_path.read_text() - content = content.replace('N1:script.py', 'N1:subdir/script.py') + content = content.replace("N1:script.py", "N1:subdir/script.py") workflow_path.write_text(content) - result = self.runner.invoke(cli, [ - 'run', - 'test-project/workflow.graphml', - '--source', 'test-project/src', - '--output', 'out', - '--type', 'posix' - ]) + result = self.runner.invoke( + cli, + [ + "run", + "test-project/workflow.graphml", + "--source", + "test-project/src", + "--output", + "out", + "--type", + "posix", + ], + ) self.assertEqual(result.exit_code, 0) - self.assertTrue(Path('out/src/subdir/script.py').exists()) + self.assertTrue(Path("out/src/subdir/script.py").exists()) def test_run_command_docker_subdir_source_build_paths(self): with self.runner.isolated_filesystem(temp_dir=self.temp_dir): - result = self.runner.invoke(cli, ['init', 'test-project']) + result = self.runner.invoke(cli, ["init", "test-project"]) self.assertEqual(result.exit_code, 0) - subdir = Path('test-project/src/subdir') + subdir = Path("test-project/src/subdir") subdir.mkdir(parents=True, exist_ok=True) - shutil.move('test-project/src/script.py', subdir / 'script.py') + shutil.move("test-project/src/script.py", subdir / "script.py") - workflow_path = Path('test-project/workflow.graphml') + workflow_path = Path("test-project/workflow.graphml") content = workflow_path.read_text() - content = content.replace('N1:script.py', 'N1:subdir/script.py') + content = content.replace("N1:script.py", "N1:subdir/script.py") workflow_path.write_text(content) - result = self.runner.invoke(cli, [ - 'run', - 'test-project/workflow.graphml', - '--source', 'test-project/src', - '--output', 'out', - '--type', 'docker' - ]) + result = self.runner.invoke( + cli, + [ + "run", + "test-project/workflow.graphml", + "--source", + "test-project/src", + "--output", + "out", + "--type", + "docker", + ], + ) self.assertEqual(result.exit_code, 0) - build_script = Path('out/build').read_text() - self.assertIn('mkdir docker-subdir__script', build_script) - self.assertIn('cp ../src/Dockerfile.subdir/script Dockerfile', build_script) - self.assertIn('cp ../src/subdir/script.py .', build_script) - self.assertIn('cp ../src/subdir/script.iport concore.iport', build_script) - self.assertIn('cd ..', build_script) + build_script = Path("out/build").read_text() + self.assertIn("mkdir docker-subdir__script", build_script) + self.assertIn("cp ../src/Dockerfile.subdir/script Dockerfile", build_script) + self.assertIn("cp ../src/subdir/script.py .", build_script) + self.assertIn("cp ../src/subdir/script.iport concore.iport", build_script) + self.assertIn("cd ..", build_script) def test_run_command_shared_source_specialization_merges_edge_params(self): with self.runner.isolated_filesystem(temp_dir=self.temp_dir): - Path('src').mkdir() - Path('src/common.py').write_text( - "import concore\n\n" - "def step():\n" - " return None\n" + Path("src").mkdir() + Path("src/common.py").write_text( + "import concore\n\ndef step():\n return None\n" ) workflow = """ @@ -205,78 +238,97 @@ def test_run_command_shared_source_specialization_merges_edge_params(self): """ - Path('workflow.graphml').write_text(workflow) - - result = self.runner.invoke(cli, [ - 'run', - 'workflow.graphml', - '--source', 'src', - '--output', 'out', - '--type', 'posix' - ]) + Path("workflow.graphml").write_text(workflow) + + result = self.runner.invoke( + cli, + [ + "run", + "workflow.graphml", + "--source", + "src", + "--output", + "out", + "--type", + "posix", + ], + ) self.assertEqual(result.exit_code, 0) - specialized_script = Path('out/src/common.py') + specialized_script = Path("out/src/common.py") self.assertTrue(specialized_script.exists()) content = specialized_script.read_text() - self.assertIn('PORT_NAME_A_B', content) - self.assertIn('PORT_A_B', content) - self.assertIn('PORT_NAME_B_C', content) - self.assertIn('PORT_B_C', content) + self.assertIn("PORT_NAME_A_B", content) + self.assertIn("PORT_A_B", content) + self.assertIn("PORT_NAME_B_C", content) + self.assertIn("PORT_B_C", content) def test_run_command_existing_output(self): with self.runner.isolated_filesystem(temp_dir=self.temp_dir): - result = self.runner.invoke(cli, ['init', 'test-project']) - Path('output').mkdir() - - result = self.runner.invoke(cli, [ - 'run', - 'test-project/workflow.graphml', - '--source', 'test-project/src', - '--output', 'output' - ]) - self.assertIn('already exists', result.output.lower()) - + result = self.runner.invoke(cli, ["init", "test-project"]) + Path("output").mkdir() + + result = self.runner.invoke( + cli, + [ + "run", + "test-project/workflow.graphml", + "--source", + "test-project/src", + "--output", + "output", + ], + ) + self.assertIn("already exists", result.output.lower()) + def test_inspect_command_basic(self): with self.runner.isolated_filesystem(temp_dir=self.temp_dir): - result = self.runner.invoke(cli, ['init', 'test-project']) + result = self.runner.invoke(cli, ["init", "test-project"]) self.assertEqual(result.exit_code, 0) - - result = self.runner.invoke(cli, ['inspect', 'test-project/workflow.graphml']) + + result = self.runner.invoke( + cli, ["inspect", "test-project/workflow.graphml"] + ) self.assertEqual(result.exit_code, 0) - self.assertIn('Workflow Overview', result.output) - self.assertIn('Nodes:', result.output) - self.assertIn('Edges:', result.output) - + self.assertIn("Workflow Overview", result.output) + self.assertIn("Nodes:", result.output) + self.assertIn("Edges:", result.output) + def test_inspect_missing_file(self): - result = self.runner.invoke(cli, ['inspect', 'nonexistent.graphml']) + result = self.runner.invoke(cli, ["inspect", "nonexistent.graphml"]) self.assertNotEqual(result.exit_code, 0) - + def test_inspect_json_output(self): with self.runner.isolated_filesystem(temp_dir=self.temp_dir): - result = self.runner.invoke(cli, ['init', 'test-project']) + result = self.runner.invoke(cli, ["init", "test-project"]) self.assertEqual(result.exit_code, 0) - - result = self.runner.invoke(cli, ['inspect', 'test-project/workflow.graphml', '--json']) + + result = self.runner.invoke( + cli, ["inspect", "test-project/workflow.graphml", "--json"] + ) self.assertEqual(result.exit_code, 0) - + import json + output_data = json.loads(result.output) - self.assertIn('workflow', output_data) - self.assertIn('nodes', output_data) - self.assertIn('edges', output_data) - self.assertEqual(output_data['workflow'], 'workflow.graphml') - + self.assertIn("workflow", output_data) + self.assertIn("nodes", output_data) + self.assertIn("edges", output_data) + self.assertEqual(output_data["workflow"], "workflow.graphml") + def test_inspect_missing_source_file(self): with self.runner.isolated_filesystem(temp_dir=self.temp_dir): - result = self.runner.invoke(cli, ['init', 'test-project']) + result = self.runner.invoke(cli, ["init", "test-project"]) self.assertEqual(result.exit_code, 0) - - Path('test-project/src/script.py').unlink() - - result = self.runner.invoke(cli, ['inspect', 'test-project/workflow.graphml', '--source', 'src']) + + Path("test-project/src/script.py").unlink() + + result = self.runner.invoke( + cli, ["inspect", "test-project/workflow.graphml", "--source", "src"] + ) self.assertEqual(result.exit_code, 0) - self.assertIn('Missing files', result.output) + self.assertIn("Missing files", result.output) + -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/tests/test_concore.py b/tests/test_concore.py index fd897291..a62efe86 100644 --- a/tests/test_concore.py +++ b/tests/test_concore.py @@ -2,40 +2,43 @@ import os import numpy as np -class TestSafeLiteralEval: +class TestSafeLiteralEval: def test_reads_dictionary_from_file(self, temp_dir): test_file = os.path.join(temp_dir, "config.txt") with open(test_file, "w") as f: f.write("{'name': 'test', 'value': 123}") - + from concore import safe_literal_eval + result = safe_literal_eval(test_file, {}) - - assert result == {'name': 'test', 'value': 123} + + assert result == {"name": "test", "value": 123} def test_returns_default_when_file_missing(self): from concore import safe_literal_eval + result = safe_literal_eval("nonexistent_file.txt", "fallback") - + assert result == "fallback" def test_returns_default_for_empty_file(self, temp_dir): test_file = os.path.join(temp_dir, "empty.txt") - with open(test_file, "w") as f: + with open(test_file, "w") as _: pass - + from concore import safe_literal_eval + result = safe_literal_eval(test_file, "default") - + assert result == "default" class TestTryparam: - @pytest.fixture(autouse=True) def reset_params(self): from concore import params + original_params = params.copy() yield params.clear() @@ -43,46 +46,49 @@ def reset_params(self): def test_returns_existing_parameter(self): from concore import tryparam, params - params['my_setting'] = 'custom_value' - - result = tryparam('my_setting', 'default_value') - - assert result == 'custom_value' + + params["my_setting"] = "custom_value" + + result = tryparam("my_setting", "default_value") + + assert result == "custom_value" def test_returns_default_for_missing_parameter(self): from concore import tryparam - result = tryparam('missing_param', 'fallback') - - assert result == 'fallback' + result = tryparam("missing_param", "fallback") + + assert result == "fallback" -class TestZeroMQPort: +class TestZeroMQPort: def test_class_is_defined(self): from concore import ZeroMQPort + assert ZeroMQPort is not None class TestDefaultConfiguration: - def test_default_input_path(self): from concore import inpath + assert inpath == "./in" def test_default_output_path(self): from concore import outpath + assert outpath == "./out" class TestPublicAPI: - def test_module_imports_successfully(self): from concore import safe_literal_eval + assert safe_literal_eval is not None def test_core_functions_exist(self): from concore import safe_literal_eval, tryparam, default_maxtime - + assert callable(safe_literal_eval) assert callable(tryparam) assert callable(default_maxtime) @@ -91,6 +97,7 @@ def test_core_functions_exist(self): class TestNumpyConversion: def test_convert_scalar(self): from concore import convert_numpy_to_python + val = np.float64(3.14) res = convert_numpy_to_python(val) assert type(res) == float @@ -98,63 +105,70 @@ def test_convert_scalar(self): def test_convert_list_and_dict(self): from concore import convert_numpy_to_python - data = { - 'a': np.int32(10), - 'b': [np.float64(1.1), np.float64(2.2)] - } + + data = {"a": np.int32(10), "b": [np.float64(1.1), np.float64(2.2)]} res = convert_numpy_to_python(data) - assert type(res['a']) == int - assert type(res['b'][0]) == float - assert res['b'][1] == 2.2 + assert type(res["a"]) == int + assert type(res["b"][0]) == float + assert res["b"][1] == 2.2 + class TestInitVal: @pytest.fixture(autouse=True) def reset_simtime(self): import concore + old_simtime = concore.simtime yield concore.simtime = old_simtime def test_initval_updates_simtime(self): import concore + concore.simtime = 0 # initval takes string repr of a list [time, val1, val2...] result = concore.initval("[100, 'data']") - + assert concore.simtime == 100 - assert result == ['data'] + assert result == ["data"] def test_initval_handles_bad_input(self): import concore + concore.simtime = 0 # Input that isn't a list result = concore.initval("not_a_list") assert concore.simtime == 0 assert result == [] + class TestDefaultMaxTime: def test_uses_file_value(self, temp_dir, monkeypatch): import concore + # Mock the path to maxtime file maxtime_file = os.path.join(temp_dir, "concore.maxtime") with open(maxtime_file, "w") as f: f.write("500") - - monkeypatch.setattr(concore, 'concore_maxtime_file', maxtime_file) + + monkeypatch.setattr(concore, "concore_maxtime_file", maxtime_file) concore.default_maxtime(100) - + assert concore.maxtime == 500 def test_uses_default_when_missing(self, monkeypatch): import concore - monkeypatch.setattr(concore, 'concore_maxtime_file', "missing_file") + + monkeypatch.setattr(concore, "concore_maxtime_file", "missing_file") concore.default_maxtime(999) assert concore.maxtime == 999 + class TestUnchanged: @pytest.fixture(autouse=True) def reset_globals(self): import concore + old_s = concore.s old_olds = concore.olds yield @@ -163,52 +177,61 @@ def reset_globals(self): def test_unchanged_returns_true_if_same(self): import concore + concore.s = "same" concore.olds = "same" - + # Should return True and reset s to empty assert concore.unchanged() is True - assert concore.s == '' + assert concore.s == "" def test_unchanged_returns_false_if_diff(self): import concore + concore.s = "new" concore.olds = "old" - + assert concore.unchanged() is False assert concore.olds == "new" -class TestParseParams: + +class TestParseParams: def test_simple_key_value_pairs(self): from concore import parse_params + params = parse_params("a=1;b=2") assert params == {"a": 1, "b": 2} def test_preserves_whitespace_in_values(self): from concore import parse_params + params = parse_params("label = hello world ; x = 5") assert params["label"] == "hello world" assert params["x"] == 5 def test_embedded_equals_in_value(self): from concore import parse_params + params = parse_params("url=https://example.com?a=1&b=2") assert params["url"] == "https://example.com?a=1&b=2" def test_numeric_and_list_coercion(self): from concore import parse_params + params = parse_params("delay=5;coeffs=[1,2,3]") assert params["delay"] == 5 assert params["coeffs"] == [1, 2, 3] def test_dict_literal_backward_compatibility(self): from concore import parse_params + params = parse_params("{'a': 1, 'b': 2}") assert params == {"a": 1, "b": 2} def test_windows_quoted_input(self): from concore import parse_params - s = "\"a=1;b=2\"" + + s = '"a=1;b=2"' s = s[1:-1] # simulate quote stripping before parse_params params = parse_params(s) assert params == {"a": 1, "b": 2} @@ -218,6 +241,7 @@ class TestWriteZMQ: @pytest.fixture(autouse=True) def reset_zmq_ports(self): import concore + original_ports = concore.zmq_ports.copy() yield concore.zmq_ports.clear() @@ -289,6 +313,7 @@ class TestSimtimeNotMutatedByWrite: @pytest.fixture(autouse=True) def reset_simtime(self): import concore + old_simtime = concore.simtime yield concore.simtime = old_simtime @@ -296,6 +321,7 @@ def reset_simtime(self): @pytest.fixture(autouse=True) def reset_outpath(self): import concore + old_outpath = concore.outpath yield concore.outpath = old_outpath @@ -303,6 +329,7 @@ def reset_outpath(self): @pytest.fixture(autouse=True) def reset_zmq_ports(self): import concore + original_ports = concore.zmq_ports.copy() yield concore.zmq_ports.clear() @@ -363,10 +390,12 @@ def test_multi_port_file_writes_share_same_timestamp(self, temp_dir): # Read back the written files and compare timestamps from ast import literal_eval + payloads = [] for p in (1, 2): - with open(os.path.join(temp_dir, "out" + str(p), - ("u" if p == 1 else "v"))) as f: + with open( + os.path.join(temp_dir, "out" + str(p), ("u" if p == 1 else "v")) + ) as f: payloads.append(literal_eval(f.read())) ts1, ts2 = payloads[0][0], payloads[1][0] @@ -383,6 +412,7 @@ def test_multi_port_zmq_writes_share_same_timestamp(self): class DummyPort: def __init__(self): self.sent = None + def send_json_with_retry(self, msg): self.sent = msg diff --git a/tests/test_concoredocker.py b/tests/test_concoredocker.py index c3743911..8a2f1344 100644 --- a/tests/test_concoredocker.py +++ b/tests/test_concoredocker.py @@ -3,16 +3,16 @@ class TestSafeLiteralEval: - def test_reads_dictionary_from_file(self, temp_dir): test_file = os.path.join(temp_dir, "ports.txt") with open(test_file, "w") as f: f.write("{'a': 1, 'b': 2}") from concoredocker import safe_literal_eval + result = safe_literal_eval(test_file, {}) - assert result == {'a': 1, 'b': 2} + assert result == {"a": 1, "b": 2} def test_reads_list_from_file(self, temp_dir): test_file = os.path.join(temp_dir, "data.txt") @@ -20,15 +20,17 @@ def test_reads_list_from_file(self, temp_dir): f.write("[1, 2, 3]") from concoredocker import safe_literal_eval + result = safe_literal_eval(test_file, []) assert result == [1, 2, 3] def test_returns_default_when_file_missing(self): from concoredocker import safe_literal_eval - result = safe_literal_eval("/nonexistent.txt", {'default': True}) - assert result == {'default': True} + result = safe_literal_eval("/nonexistent.txt", {"default": True}) + + assert result == {"default": True} def test_returns_default_for_bad_syntax(self, temp_dir): test_file = os.path.join(temp_dir, "bad.txt") @@ -36,23 +38,25 @@ def test_returns_default_for_bad_syntax(self, temp_dir): f.write("not valid {{{") from concoredocker import safe_literal_eval + result = safe_literal_eval(test_file, "fallback") assert result == "fallback" class TestUnchanged: - def test_returns_true_when_unchanged(self): import concoredocker + concoredocker.s = "abc" concoredocker.olds = "abc" assert concoredocker.unchanged() == True - assert concoredocker.s == '' + assert concoredocker.s == "" def test_returns_false_when_changed(self): import concoredocker + concoredocker.s = "new" concoredocker.olds = "old" @@ -61,9 +65,9 @@ def test_returns_false_when_changed(self): class TestInitval: - def test_parses_simtime_and_values(self): import concoredocker + concoredocker.simtime = 0 result = concoredocker.initval("[5.0, 1.0, 2.0]") @@ -72,6 +76,7 @@ def test_parses_simtime_and_values(self): def test_parses_single_value(self): import concoredocker + concoredocker.simtime = 0 result = concoredocker.initval("[10.0, 99]") @@ -80,9 +85,9 @@ def test_parses_single_value(self): class TestWrite: - def test_writes_list_with_simtime(self, temp_dir): import concoredocker + old_outpath = concoredocker.outpath outdir = os.path.join(temp_dir, "1") os.makedirs(outdir) @@ -98,6 +103,7 @@ def test_writes_list_with_simtime(self, temp_dir): def test_writes_with_delta(self, temp_dir): import concoredocker + old_outpath = concoredocker.outpath outdir = os.path.join(temp_dir, "1") os.makedirs(outdir) @@ -115,9 +121,9 @@ def test_writes_with_delta(self, temp_dir): class TestRead: - def test_reads_and_parses_data(self, temp_dir): import concoredocker + old_inpath = concoredocker.inpath old_delay = concoredocker.delay indir = os.path.join(temp_dir, "1") @@ -125,10 +131,10 @@ def test_reads_and_parses_data(self, temp_dir): concoredocker.inpath = temp_dir concoredocker.delay = 0.001 - with open(os.path.join(indir, "data"), 'w') as f: + with open(os.path.join(indir, "data"), "w") as f: f.write("[7.0, 100, 200]") - concoredocker.s = '' + concoredocker.s = "" concoredocker.simtime = 0 result = concoredocker.read(1, "data", "[0, 0, 0]") @@ -139,6 +145,7 @@ def test_reads_and_parses_data(self, temp_dir): def test_returns_default_when_file_missing(self, temp_dir): import concoredocker + old_inpath = concoredocker.inpath old_delay = concoredocker.delay indir = os.path.join(temp_dir, "1") @@ -146,7 +153,7 @@ def test_returns_default_when_file_missing(self, temp_dir): concoredocker.inpath = temp_dir concoredocker.delay = 0.001 - concoredocker.s = '' + concoredocker.s = "" concoredocker.simtime = 0 result = concoredocker.read(1, "nofile", "[0, 5, 5]") @@ -159,6 +166,7 @@ class TestZMQ: @pytest.fixture(autouse=True) def reset_zmq_ports(self): import concoredocker + original_ports = concoredocker.zmq_ports.copy() yield concoredocker.zmq_ports.clear() diff --git a/tests/test_graph.py b/tests/test_graph.py index 6aef0d3c..efebe3e5 100644 --- a/tests/test_graph.py +++ b/tests/test_graph.py @@ -5,41 +5,41 @@ from click.testing import CliRunner from concore_cli.cli import cli + class TestGraphValidation(unittest.TestCase): - def setUp(self): self.runner = CliRunner() self.temp_dir = tempfile.mkdtemp() - + def tearDown(self): if Path(self.temp_dir).exists(): shutil.rmtree(self.temp_dir) - + def create_graph_file(self, filename, content): filepath = Path(self.temp_dir) / filename - with open(filepath, 'w') as f: + with open(filepath, "w") as f: f.write(content) return str(filepath) def test_validate_corrupted_xml(self): content = '' - filepath = self.create_graph_file('corrupted.graphml', content) - - result = self.runner.invoke(cli, ['validate', filepath]) - - self.assertIn('Validation failed', result.output) - self.assertIn('Invalid XML', result.output) + filepath = self.create_graph_file("corrupted.graphml", content) + + result = self.runner.invoke(cli, ["validate", filepath]) + + self.assertIn("Validation failed", result.output) + self.assertIn("Invalid XML", result.output) def test_validate_empty_file(self): - filepath = self.create_graph_file('empty.graphml', '') - - result = self.runner.invoke(cli, ['validate', filepath]) - - self.assertIn('Validation failed', result.output) - self.assertIn('File is empty', result.output) - + filepath = self.create_graph_file("empty.graphml", "") + + result = self.runner.invoke(cli, ["validate", filepath]) + + self.assertIn("Validation failed", result.output) + self.assertIn("File is empty", result.output) + def test_validate_missing_node_id(self): - content = ''' + content = """ @@ -47,14 +47,14 @@ def test_validate_missing_node_id(self): - ''' - filepath = self.create_graph_file('missing_id.graphml', content) - result = self.runner.invoke(cli, ['validate', filepath]) - self.assertIn('Validation failed', result.output) + """ + filepath = self.create_graph_file("missing_id.graphml", content) + result = self.runner.invoke(cli, ["validate", filepath]) + self.assertIn("Validation failed", result.output) self.assertIn("Node missing required 'id' attribute", result.output) def test_validate_missing_edgedefault(self): - content = ''' + content = """ @@ -62,23 +62,23 @@ def test_validate_missing_edgedefault(self): - ''' - filepath = self.create_graph_file('missing_default.graphml', content) - result = self.runner.invoke(cli, ['validate', filepath]) - self.assertIn('Validation failed', result.output) + """ + filepath = self.create_graph_file("missing_default.graphml", content) + result = self.runner.invoke(cli, ["validate", filepath]) + self.assertIn("Validation failed", result.output) self.assertIn("Graph missing required 'edgedefault'", result.output) def test_validate_missing_root_element(self): content = '' - filepath = self.create_graph_file('not_graphml.xml', content) - - result = self.runner.invoke(cli, ['validate', filepath]) - - self.assertIn('Validation failed', result.output) - self.assertIn('missing root element', result.output) + filepath = self.create_graph_file("not_graphml.xml", content) + + result = self.runner.invoke(cli, ["validate", filepath]) + + self.assertIn("Validation failed", result.output) + self.assertIn("missing root element", result.output) def test_validate_broken_edges(self): - content = ''' + content = """ @@ -87,16 +87,16 @@ def test_validate_broken_edges(self): - ''' - filepath = self.create_graph_file('bad_edge.graphml', content) - - result = self.runner.invoke(cli, ['validate', filepath]) - - self.assertIn('Validation failed', result.output) - self.assertIn('Edge references non-existent target node', result.output) + """ + filepath = self.create_graph_file("bad_edge.graphml", content) + + result = self.runner.invoke(cli, ["validate", filepath]) + + self.assertIn("Validation failed", result.output) + self.assertIn("Edge references non-existent target node", result.output) def test_validate_node_missing_filename(self): - content = ''' + content = """ @@ -104,16 +104,16 @@ def test_validate_node_missing_filename(self): - ''' - filepath = self.create_graph_file('bad_node.graphml', content) - - result = self.runner.invoke(cli, ['validate', filepath]) - - self.assertIn('Validation failed', result.output) - self.assertIn('has no filename', result.output) + """ + filepath = self.create_graph_file("bad_node.graphml", content) + + result = self.runner.invoke(cli, ["validate", filepath]) + + self.assertIn("Validation failed", result.output) + self.assertIn("has no filename", result.output) def test_validate_unsafe_node_label(self): - content = ''' + content = """ @@ -121,16 +121,16 @@ def test_validate_unsafe_node_label(self): - ''' - filepath = self.create_graph_file('injection.graphml', content) + """ + filepath = self.create_graph_file("injection.graphml", content) - result = self.runner.invoke(cli, ['validate', filepath]) + result = self.runner.invoke(cli, ["validate", filepath]) - self.assertIn('Validation failed', result.output) - self.assertIn('unsafe shell characters', result.output) + self.assertIn("Validation failed", result.output) + self.assertIn("unsafe shell characters", result.output) def test_validate_valid_graph(self): - content = ''' + content = """ @@ -138,16 +138,16 @@ def test_validate_valid_graph(self): - ''' - filepath = self.create_graph_file('valid.graphml', content) - - result = self.runner.invoke(cli, ['validate', filepath]) - - self.assertIn('Validation passed', result.output) - self.assertIn('Workflow is valid', result.output) - + """ + filepath = self.create_graph_file("valid.graphml", content) + + result = self.runner.invoke(cli, ["validate", filepath]) + + self.assertIn("Validation passed", result.output) + self.assertIn("Workflow is valid", result.output) + def test_validate_missing_source_file(self): - content = ''' + content = """ @@ -155,18 +155,20 @@ def test_validate_missing_source_file(self): - ''' - filepath = self.create_graph_file('workflow.graphml', content) - source_dir = Path(self.temp_dir) / 'src' + """ + filepath = self.create_graph_file("workflow.graphml", content) + source_dir = Path(self.temp_dir) / "src" source_dir.mkdir() - - result = self.runner.invoke(cli, ['validate', filepath, '--source', str(source_dir)]) - - self.assertIn('Validation failed', result.output) - self.assertIn('Missing source file', result.output) - + + result = self.runner.invoke( + cli, ["validate", filepath, "--source", str(source_dir)] + ) + + self.assertIn("Validation failed", result.output) + self.assertIn("Missing source file", result.output) + def test_validate_with_existing_source_file(self): - content = ''' + content = """ @@ -174,18 +176,20 @@ def test_validate_with_existing_source_file(self): - ''' - filepath = self.create_graph_file('workflow.graphml', content) - source_dir = Path(self.temp_dir) / 'src' + """ + filepath = self.create_graph_file("workflow.graphml", content) + source_dir = Path(self.temp_dir) / "src" source_dir.mkdir() - (source_dir / 'exists.py').write_text('print("hello")') - - result = self.runner.invoke(cli, ['validate', filepath, '--source', str(source_dir)]) - - self.assertIn('Validation passed', result.output) - + (source_dir / "exists.py").write_text('print("hello")') + + result = self.runner.invoke( + cli, ["validate", filepath, "--source", str(source_dir)] + ) + + self.assertIn("Validation passed", result.output) + def test_validate_zmq_port_conflict(self): - content = ''' + content = """ @@ -202,16 +206,16 @@ def test_validate_zmq_port_conflict(self): - ''' - filepath = self.create_graph_file('conflict.graphml', content) - - result = self.runner.invoke(cli, ['validate', filepath]) - - self.assertIn('Validation failed', result.output) - self.assertIn('Port conflict', result.output) - + """ + filepath = self.create_graph_file("conflict.graphml", content) + + result = self.runner.invoke(cli, ["validate", filepath]) + + self.assertIn("Validation failed", result.output) + self.assertIn("Port conflict", result.output) + def test_validate_reserved_port(self): - content = ''' + content = """ @@ -225,16 +229,16 @@ def test_validate_reserved_port(self): - ''' - filepath = self.create_graph_file('reserved.graphml', content) - - result = self.runner.invoke(cli, ['validate', filepath]) - - self.assertIn('Port 80', result.output) - self.assertIn('reserved range', result.output) - + """ + filepath = self.create_graph_file("reserved.graphml", content) + + result = self.runner.invoke(cli, ["validate", filepath]) + + self.assertIn("Port 80", result.output) + self.assertIn("reserved range", result.output) + def test_validate_cycle_detection(self): - content = ''' + content = """ @@ -251,16 +255,16 @@ def test_validate_cycle_detection(self): - ''' - filepath = self.create_graph_file('cycle.graphml', content) - - result = self.runner.invoke(cli, ['validate', filepath]) - - self.assertIn('cycles', result.output) - self.assertIn('control loops', result.output) - + """ + filepath = self.create_graph_file("cycle.graphml", content) + + result = self.runner.invoke(cli, ["validate", filepath]) + + self.assertIn("cycles", result.output) + self.assertIn("control loops", result.output) + def test_validate_port_zero(self): - content = ''' + content = """ @@ -274,16 +278,16 @@ def test_validate_port_zero(self): - ''' - filepath = self.create_graph_file('port_zero.graphml', content) - - result = self.runner.invoke(cli, ['validate', filepath]) - - self.assertIn('Validation failed', result.output) - self.assertIn('must be at least 1', result.output) - + """ + filepath = self.create_graph_file("port_zero.graphml", content) + + result = self.runner.invoke(cli, ["validate", filepath]) + + self.assertIn("Validation failed", result.output) + self.assertIn("must be at least 1", result.output) + def test_validate_port_exceeds_maximum(self): - content = ''' + content = """ @@ -297,13 +301,14 @@ def test_validate_port_exceeds_maximum(self): - ''' - filepath = self.create_graph_file('port_max.graphml', content) - - result = self.runner.invoke(cli, ['validate', filepath]) - - self.assertIn('Validation failed', result.output) - self.assertIn('exceeds maximum (65535)', result.output) - -if __name__ == '__main__': - unittest.main() \ No newline at end of file + """ + filepath = self.create_graph_file("port_max.graphml", content) + + result = self.runner.invoke(cli, ["validate", filepath]) + + self.assertIn("Validation failed", result.output) + self.assertIn("exceeds maximum (65535)", result.output) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_openjupyter_security.py b/tests/test_openjupyter_security.py index 230a4819..b046dc9b 100644 --- a/tests/test_openjupyter_security.py +++ b/tests/test_openjupyter_security.py @@ -1,4 +1,5 @@ """Tests for the secured /openJupyter/ and /stopJupyter/ endpoints.""" + import os import sys import pytest @@ -8,7 +9,9 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) # Skip entire module if flask is not installed (e.g. in CI with minimal deps) -pytest.importorskip("flask", reason="flask not installed — skipping server endpoint tests") +pytest.importorskip( + "flask", reason="flask not installed — skipping server endpoint tests" +) # Set a test API key before importing the app module TEST_API_KEY = "test-secret-key-12345" @@ -18,6 +21,7 @@ def reset_jupyter_process(): """Reset the module-level jupyter_process before each test.""" import fri.server.main as mod + mod.jupyter_process = None yield mod.jupyter_process = None @@ -29,6 +33,7 @@ def client(): with patch.dict(os.environ, {"CONCORE_API_KEY": TEST_API_KEY}): # Re-read env var after patching import fri.server.main as mod + mod.API_KEY = TEST_API_KEY mod.app.config["TESTING"] = True with mod.app.test_client() as c: @@ -39,6 +44,7 @@ def client(): def client_no_key(): """Create a Flask test client without API key configured.""" import fri.server.main as mod + mod.API_KEY = None mod.app.config["TESTING"] = True with mod.app.test_client() as c: @@ -60,9 +66,7 @@ def test_wrong_api_key_returns_403(self, client): def test_server_without_api_key_configured_returns_500(self, client_no_key): """If CONCORE_API_KEY is not set on server, return 500.""" - resp = client_no_key.post( - "/openJupyter/", headers={"X-API-KEY": "anything"} - ) + resp = client_no_key.post("/openJupyter/", headers={"X-API-KEY": "anything"}) assert resp.status_code == 500 @@ -76,9 +80,7 @@ def test_authorized_request_starts_jupyter(self, mock_popen, client): mock_proc.poll.return_value = None # process running mock_popen.return_value = mock_proc - resp = client.post( - "/openJupyter/", headers={"X-API-KEY": TEST_API_KEY} - ) + resp = client.post("/openJupyter/", headers={"X-API-KEY": TEST_API_KEY}) assert resp.status_code == 200 data = resp.get_json() assert data["message"] == "Jupyter Lab started" @@ -96,15 +98,11 @@ def test_duplicate_launch_returns_409(self, mock_popen, client): mock_popen.return_value = mock_proc # First launch - resp1 = client.post( - "/openJupyter/", headers={"X-API-KEY": TEST_API_KEY} - ) + resp1 = client.post("/openJupyter/", headers={"X-API-KEY": TEST_API_KEY}) assert resp1.status_code == 200 # Second launch should be rejected - resp2 = client.post( - "/openJupyter/", headers={"X-API-KEY": TEST_API_KEY} - ) + resp2 = client.post("/openJupyter/", headers={"X-API-KEY": TEST_API_KEY}) assert resp2.status_code == 409 data = resp2.get_json() assert data["message"] == "Jupyter already running" @@ -112,9 +110,7 @@ def test_duplicate_launch_returns_409(self, mock_popen, client): @patch("fri.server.main.subprocess.Popen", side_effect=OSError("fail")) def test_popen_failure_returns_500(self, mock_popen, client): """If Popen raises, return 500.""" - resp = client.post( - "/openJupyter/", headers={"X-API-KEY": TEST_API_KEY} - ) + resp = client.post("/openJupyter/", headers={"X-API-KEY": TEST_API_KEY}) assert resp.status_code == 500 data = resp.get_json() assert "error" in data @@ -130,9 +126,7 @@ def test_stop_without_auth_returns_403(self, client): def test_stop_when_no_process_returns_404(self, client): """Stop with no running process returns 404.""" - resp = client.post( - "/stopJupyter/", headers={"X-API-KEY": TEST_API_KEY} - ) + resp = client.post("/stopJupyter/", headers={"X-API-KEY": TEST_API_KEY}) assert resp.status_code == 404 @patch("fri.server.main.subprocess.Popen") @@ -146,9 +140,7 @@ def test_stop_running_process_returns_200(self, mock_popen, client): client.post("/openJupyter/", headers={"X-API-KEY": TEST_API_KEY}) # Stop - resp = client.post( - "/stopJupyter/", headers={"X-API-KEY": TEST_API_KEY} - ) + resp = client.post("/stopJupyter/", headers={"X-API-KEY": TEST_API_KEY}) assert resp.status_code == 200 data = resp.get_json() assert data["message"] == "Jupyter stopped" diff --git a/tests/test_protocol_conformance.py b/tests/test_protocol_conformance.py index e831165a..9bb5ab3c 100644 --- a/tests/test_protocol_conformance.py +++ b/tests/test_protocol_conformance.py @@ -21,11 +21,17 @@ def _validate_fixture_document_shape(doc): required_top = {"schema_version", "runtime", "mode", "cases"} missing = required_top - set(doc.keys()) if missing: - raise AssertionError(f"Fixture document missing required top-level keys: {sorted(missing)}") + raise AssertionError( + f"Fixture document missing required top-level keys: {sorted(missing)}" + ) if doc["runtime"] != "python": - raise AssertionError(f"Phase-1 fixture runtime must be 'python', found: {doc['runtime']}") + raise AssertionError( + f"Phase-1 fixture runtime must be 'python', found: {doc['runtime']}" + ) if doc["mode"] != "report_only": - raise AssertionError(f"Phase-1 fixture mode must be 'report_only', found: {doc['mode']}") + raise AssertionError( + f"Phase-1 fixture mode must be 'report_only', found: {doc['mode']}" + ) if not isinstance(doc["cases"], list) or not doc["cases"]: raise AssertionError("Fixture document must contain a non-empty 'cases' list") diff --git a/tests/test_tool_config.py b/tests/test_tool_config.py index 58adc903..7baa25e3 100644 --- a/tests/test_tool_config.py +++ b/tests/test_tool_config.py @@ -1,6 +1,6 @@ -import pytest import os + # can't import mkconcore directly (sys.argv at module level), so we duplicate the parser def _load_tool_config(filepath): tools = {} @@ -17,7 +17,6 @@ def _load_tool_config(filepath): class TestLoadToolConfig: - def test_basic_overrides(self, temp_dir): cfg = os.path.join(temp_dir, "concore.tools") with open(cfg, "w") as f: @@ -69,7 +68,7 @@ def test_whitespace_around_key_value(self, temp_dir): def test_empty_file(self, temp_dir): cfg = os.path.join(temp_dir, "concore.tools") - with open(cfg, "w") as f: + with open(cfg, "w") as _: pass tools = _load_tool_config(cfg) From 6ecbbd854ea69fcf3de35237ab430515393c78b1 Mon Sep 17 00:00:00 2001 From: Titas-Ghosh Date: Wed, 25 Feb 2026 01:44:39 +0530 Subject: [PATCH 212/275] test: add phase-2 cross-runtime conformance mapping baseline --- tests/protocol_fixtures/PROTOCOL_FIXTURES.md | 7 + .../cross_runtime_matrix.phase2.json | 211 ++++++++++++++++++ tests/test_protocol_conformance_phase2.py | 57 +++++ 3 files changed, 275 insertions(+) create mode 100644 tests/protocol_fixtures/cross_runtime_matrix.phase2.json create mode 100644 tests/test_protocol_conformance_phase2.py diff --git a/tests/protocol_fixtures/PROTOCOL_FIXTURES.md b/tests/protocol_fixtures/PROTOCOL_FIXTURES.md index 0b85c8bc..1bf953e7 100644 --- a/tests/protocol_fixtures/PROTOCOL_FIXTURES.md +++ b/tests/protocol_fixtures/PROTOCOL_FIXTURES.md @@ -4,9 +4,16 @@ This directory contains the phase-1 protocol conformance baseline for Python. - `schema.phase1.json`: fixture document shape and supported case targets. - `python_phase1_cases.json`: initial baseline cases (report-only mode metadata). +- `cross_runtime_matrix.phase2.json`: phase-2 cross-runtime mapping matrix in report-only mode. Phase-1 scope: - No runtime behavior changes. - Python-only execution through `tests/test_protocol_conformance.py`. - Fixture format is language-neutral to enable future cross-binding runners. + +Phase-2 scope (mapping only): + +- No runtime behavior changes. +- Adds a cross-runtime matrix to track per-case audit status and classification. +- Keeps CI non-blocking for non-Python runtimes by marking them as `not_audited` until adapters are added. diff --git a/tests/protocol_fixtures/cross_runtime_matrix.phase2.json b/tests/protocol_fixtures/cross_runtime_matrix.phase2.json new file mode 100644 index 00000000..746f2f1a --- /dev/null +++ b/tests/protocol_fixtures/cross_runtime_matrix.phase2.json @@ -0,0 +1,211 @@ +{ + "schema_version": "1.0", + "phase": "2", + "mode": "report_only", + "source_fixture": "python_phase1_cases.json", + "runtimes": [ + "python", + "cpp", + "matlab", + "octave", + "verilog" + ], + "classifications": [ + "required", + "implementation_defined", + "known_deviation" + ], + "statuses": [ + "observed_pass", + "observed_fail", + "not_audited" + ], + "cases": [ + { + "id": "parse_params/simple_types_and_whitespace", + "target": "parse_params", + "runtime_results": { + "python": { + "status": "observed_pass", + "classification": "required", + "note": "Phase-1 baseline execution." + }, + "cpp": { + "status": "not_audited", + "classification": "required", + "note": "Audit planned in phase 2." + }, + "matlab": { + "status": "not_audited", + "classification": "required", + "note": "Audit planned in phase 2." + }, + "octave": { + "status": "not_audited", + "classification": "required", + "note": "Audit planned in phase 2." + }, + "verilog": { + "status": "not_audited", + "classification": "implementation_defined", + "note": "May require binding-specific interpretation." + } + } + }, + { + "id": "parse_params/embedded_equals_not_split", + "target": "parse_params", + "runtime_results": { + "python": { + "status": "observed_pass", + "classification": "required", + "note": "Phase-1 baseline execution." + }, + "cpp": { + "status": "not_audited", + "classification": "required", + "note": "Audit planned in phase 2." + }, + "matlab": { + "status": "not_audited", + "classification": "required", + "note": "Audit planned in phase 2." + }, + "octave": { + "status": "not_audited", + "classification": "required", + "note": "Audit planned in phase 2." + }, + "verilog": { + "status": "not_audited", + "classification": "implementation_defined", + "note": "May require binding-specific interpretation." + } + } + }, + { + "id": "initval/valid_list_sets_simtime", + "target": "initval", + "runtime_results": { + "python": { + "status": "observed_pass", + "classification": "required", + "note": "Phase-1 baseline execution." + }, + "cpp": { + "status": "not_audited", + "classification": "required", + "note": "Audit planned in phase 2." + }, + "matlab": { + "status": "not_audited", + "classification": "required", + "note": "Audit planned in phase 2." + }, + "octave": { + "status": "not_audited", + "classification": "required", + "note": "Audit planned in phase 2." + }, + "verilog": { + "status": "not_audited", + "classification": "implementation_defined", + "note": "May require binding-specific interpretation." + } + } + }, + { + "id": "initval/invalid_input_returns_empty_and_preserves_simtime", + "target": "initval", + "runtime_results": { + "python": { + "status": "observed_pass", + "classification": "required", + "note": "Phase-1 baseline execution." + }, + "cpp": { + "status": "not_audited", + "classification": "required", + "note": "Audit planned in phase 2." + }, + "matlab": { + "status": "not_audited", + "classification": "required", + "note": "Audit planned in phase 2." + }, + "octave": { + "status": "not_audited", + "classification": "required", + "note": "Audit planned in phase 2." + }, + "verilog": { + "status": "not_audited", + "classification": "implementation_defined", + "note": "May require binding-specific interpretation." + } + } + }, + { + "id": "write_zmq/list_payload_prepends_timestamp_without_mutation", + "target": "write_zmq", + "runtime_results": { + "python": { + "status": "observed_pass", + "classification": "required", + "note": "Phase-1 baseline execution." + }, + "cpp": { + "status": "not_audited", + "classification": "required", + "note": "Audit planned in phase 2." + }, + "matlab": { + "status": "not_audited", + "classification": "required", + "note": "Audit planned in phase 2." + }, + "octave": { + "status": "not_audited", + "classification": "required", + "note": "Audit planned in phase 2." + }, + "verilog": { + "status": "not_audited", + "classification": "implementation_defined", + "note": "May require binding-specific interpretation." + } + } + }, + { + "id": "write_zmq/non_list_payload_forwarded_as_is", + "target": "write_zmq", + "runtime_results": { + "python": { + "status": "observed_pass", + "classification": "required", + "note": "Phase-1 baseline execution." + }, + "cpp": { + "status": "not_audited", + "classification": "required", + "note": "Audit planned in phase 2." + }, + "matlab": { + "status": "not_audited", + "classification": "required", + "note": "Audit planned in phase 2." + }, + "octave": { + "status": "not_audited", + "classification": "required", + "note": "Audit planned in phase 2." + }, + "verilog": { + "status": "not_audited", + "classification": "implementation_defined", + "note": "May require binding-specific interpretation." + } + } + } + ] +} diff --git a/tests/test_protocol_conformance_phase2.py b/tests/test_protocol_conformance_phase2.py new file mode 100644 index 00000000..01d8c3f8 --- /dev/null +++ b/tests/test_protocol_conformance_phase2.py @@ -0,0 +1,57 @@ +import json +from pathlib import Path + + +FIXTURE_DIR = Path(__file__).parent / "protocol_fixtures" +PHASE1_CASES_PATH = FIXTURE_DIR / "python_phase1_cases.json" +PHASE2_MATRIX_PATH = FIXTURE_DIR / "cross_runtime_matrix.phase2.json" + +EXPECTED_RUNTIMES = {"python", "cpp", "matlab", "octave", "verilog"} +EXPECTED_CLASSIFICATIONS = {"required", "implementation_defined", "known_deviation"} +EXPECTED_STATUSES = {"observed_pass", "observed_fail", "not_audited"} + + +def _load_json(path): + with path.open("r", encoding="utf-8") as f: + return json.load(f) + + +def _phase1_cases(): + doc = _load_json(PHASE1_CASES_PATH) + return {case["id"]: case for case in doc["cases"]} + + +def _phase2_matrix(): + return _load_json(PHASE2_MATRIX_PATH) + + +def test_phase2_matrix_metadata_and_enums(): + doc = _phase2_matrix() + assert doc["phase"] == "2" + assert doc["mode"] == "report_only" + assert doc["source_fixture"] == "python_phase1_cases.json" + assert set(doc["runtimes"]) == EXPECTED_RUNTIMES + assert set(doc["classifications"]) == EXPECTED_CLASSIFICATIONS + assert set(doc["statuses"]) == EXPECTED_STATUSES + + +def test_phase2_matrix_covers_all_phase1_cases(): + phase1 = _phase1_cases() + matrix_cases = _phase2_matrix()["cases"] + matrix_ids = {case["id"] for case in matrix_cases} + assert matrix_ids == set(phase1.keys()) + + +def test_phase2_matrix_rows_have_consistent_shape(): + phase1 = _phase1_cases() + for row in _phase2_matrix()["cases"]: + assert row["id"] in phase1 + assert row["target"] == phase1[row["id"]]["target"] + assert set(row["runtime_results"].keys()) == EXPECTED_RUNTIMES + + for runtime, result in row["runtime_results"].items(): + assert result["status"] in EXPECTED_STATUSES + assert result["classification"] in EXPECTED_CLASSIFICATIONS + assert isinstance(result["note"], str) and result["note"].strip() + if runtime == "python": + assert result["status"] == "observed_pass" From c6cfa942172eab05a85cf9d00e0ac9f8d6acc1b7 Mon Sep 17 00:00:00 2001 From: Avinash Kumar Deepak Date: Wed, 25 Feb 2026 14:33:56 +0530 Subject: [PATCH 213/275] fix: make concoredocker API methods public static (closes #463) --- concoredocker.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/concoredocker.java b/concoredocker.java index 58b68e78..96cf6eeb 100644 --- a/concoredocker.java +++ b/concoredocker.java @@ -112,7 +112,7 @@ private static Map parseFile(String filename) throws IOException * Sets maxtime from concore.maxtime file, or uses defaultValue if file not found. * Catches both IOException and RuntimeException to match Python safe_literal_eval. */ - private static void defaultMaxTime(double defaultValue) { + public static void defaultMaxTime(double defaultValue) { try { String content = new String(Files.readAllBytes(Paths.get(inpath + "/1/concore.maxtime"))); Object parsed = literalEval(content.trim()); @@ -126,7 +126,7 @@ private static void defaultMaxTime(double defaultValue) { } } - private static boolean unchanged() { + public static boolean unchanged() { if (olds.equals(s)) { s = ""; return true; @@ -135,7 +135,7 @@ private static boolean unchanged() { return false; } - private static Object tryParam(String n, Object i) { + public static Object tryParam(String n, Object i) { if (params.containsKey(n)) { return params.get(n); } else { @@ -149,7 +149,7 @@ private static Object tryParam(String n, Object i) { * Returns: list of values after simtime * Includes max retry limit to avoid infinite blocking (matches Python behavior). */ - private static List read(int port, String name, String initstr) { + public static List read(int port, String name, String initstr) { // Parse default value upfront for consistent return type List defaultVal = new ArrayList<>(); try { @@ -275,7 +275,7 @@ private static String toPythonLiteral(Object obj) { * Prepends simtime+delta to the value list, then serializes to Python-literal format. * Accepts List or String values (matching Python implementation). */ - private static void write(int port, String name, Object val, int delta) { + public static void write(int port, String name, Object val, int delta) { try { String path = outpath + "/" + port + "/" + name; StringBuilder content = new StringBuilder(); @@ -322,7 +322,7 @@ private static void write(int port, String name, Object val, int delta) { * Parses an initial value string like "[0.0, 1.0, 2.0]". * Extracts simtime from position 0 and returns the remaining values as a List. */ - private static List initVal(String simtimeVal) { + public static List initVal(String simtimeVal) { List val = new ArrayList<>(); try { List inval = (List) literalEval(simtimeVal); From ade6ab3d69f5571fc09ae3072958cec04765d87b Mon Sep 17 00:00:00 2001 From: Avinash Kumar Deepak Date: Sat, 28 Feb 2026 21:09:32 +0530 Subject: [PATCH 214/275] fix: delete copy ops and add move semantics to Concore (#469) --- concore.hpp | 72 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/concore.hpp b/concore.hpp index da2c7921..1f6c835b 100644 --- a/concore.hpp +++ b/concore.hpp @@ -123,6 +123,78 @@ class Concore{ #endif } + /** + * @brief Concore is not copyable as it owns shared memory handles. + */ + Concore(const Concore&) = delete; + Concore& operator=(const Concore&) = delete; + + /** + * @brief Move constructor. Transfers SHM handle ownership to the new instance. + */ + Concore(Concore&& other) noexcept + : s(std::move(other.s)), olds(std::move(other.olds)), + inpath(std::move(other.inpath)), outpath(std::move(other.outpath)), + shmId_create(other.shmId_create), shmId_get(other.shmId_get), + sharedData_create(other.sharedData_create), sharedData_get(other.sharedData_get), + communication_iport(other.communication_iport), communication_oport(other.communication_oport), + delay(other.delay), retrycount(other.retrycount), simtime(other.simtime), + maxtime(other.maxtime), iport(std::move(other.iport)), oport(std::move(other.oport)), + params(std::move(other.params)) + { + other.shmId_create = -1; + other.shmId_get = -1; + other.sharedData_create = nullptr; + other.sharedData_get = nullptr; + other.communication_iport = 0; + other.communication_oport = 0; + } + + /** + * @brief Move assignment. Cleans up current SHM resources, then takes ownership from other. + */ + Concore& operator=(Concore&& other) noexcept + { + if (this == &other) + return *this; + +#ifdef __linux__ + if (communication_oport == 1 && sharedData_create != nullptr) + shmdt(sharedData_create); + if (communication_iport == 1 && sharedData_get != nullptr) + shmdt(sharedData_get); + if (shmId_create != -1) + shmctl(shmId_create, IPC_RMID, nullptr); +#endif + + s = std::move(other.s); + olds = std::move(other.olds); + inpath = std::move(other.inpath); + outpath = std::move(other.outpath); + shmId_create = other.shmId_create; + shmId_get = other.shmId_get; + sharedData_create = other.sharedData_create; + sharedData_get = other.sharedData_get; + communication_iport = other.communication_iport; + communication_oport = other.communication_oport; + delay = other.delay; + retrycount = other.retrycount; + simtime = other.simtime; + maxtime = other.maxtime; + iport = std::move(other.iport); + oport = std::move(other.oport); + params = std::move(other.params); + + other.shmId_create = -1; + other.shmId_get = -1; + other.sharedData_create = nullptr; + other.sharedData_get = nullptr; + other.communication_iport = 0; + other.communication_oport = 0; + + return *this; + } + /** * @brief Extracts the numeric part from a string. * @param str The input string. From 90f9cad1c2022961307be6e3e5f5ccd119e6e785 Mon Sep 17 00:00:00 2001 From: GREENRAT-K405 Date: Sun, 1 Mar 2026 22:51:39 +0530 Subject: [PATCH 215/275] fix redundant zmq.Context() Instances --- concore_base.py | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/concore_base.py b/concore_base.py index f6b3e1b9..728d8074 100644 --- a/concore_base.py +++ b/concore_base.py @@ -11,15 +11,18 @@ # =================================================================== # ZeroMQ Communication Wrapper # =================================================================== +# single shared ZMQ context for the entire process. +_zmq_context = zmq.Context() + class ZeroMQPort: - def __init__(self, port_type, address, zmq_socket_type): + def __init__(self, port_type, address, zmq_socket_type, context): """ port_type: "bind" or "connect" address: ZeroMQ address (e.g., "tcp://*:5555") zmq_socket_type: zmq.REQ, zmq.REP, zmq.PUB, zmq.SUB etc. + context: shared zmq.Context() for the process (do not create one per port) """ - self.context = zmq.Context() - self.socket = self.context.socket(zmq_socket_type) + self.socket = context.socket(zmq_socket_type) self.port_type = port_type # "bind" or "connect" self.address = address @@ -76,7 +79,7 @@ def init_zmq_port(mod, port_name, port_type, address, socket_type_str): try: # Map socket type string to actual ZMQ constant (e.g., zmq.REQ, zmq.REP) zmq_socket_type = getattr(zmq, socket_type_str.upper()) - mod.zmq_ports[port_name] = ZeroMQPort(port_type, address, zmq_socket_type) + mod.zmq_ports[port_name] = ZeroMQPort(port_type, address, zmq_socket_type, _zmq_context) logger.info(f"Initialized ZMQ port: {port_name} ({socket_type_str}) on {address}") except AttributeError: logger.error(f"Error: Invalid ZMQ socket type string '{socket_type_str}'.") @@ -86,23 +89,31 @@ def init_zmq_port(mod, port_name, port_type, address, socket_type_str): logger.error(f"An unexpected error occurred during ZMQ port initialization for {port_name}: {e}") def terminate_zmq(mod): - """Clean up all ZMQ sockets and contexts before exit.""" + """Clean up all ZMQ sockets, then terminate the shared context once.""" if mod._cleanup_in_progress: return # Already cleaning up, prevent reentrant calls - + if not mod.zmq_ports: return # No ports to clean up - + mod._cleanup_in_progress = True print("\nCleaning up ZMQ resources...") + + # all sockets must be closed before context.term() is called. for port_name, port in mod.zmq_ports.items(): try: port.socket.close() - port.context.term() print(f"Closed ZMQ port: {port_name}") except Exception as e: logger.error(f"Error while terminating ZMQ port {port.address}: {e}") mod.zmq_ports.clear() + + # terminate the single shared context exactly once. + try: + _zmq_context.term() + except Exception as e: + logger.error(f"Error while terminating shared ZMQ context: {e}") + mod._cleanup_in_progress = False # --- ZeroMQ Integration End --- From 7736a97d5f4041c37eae0b8c80ca53fecc7dee68 Mon Sep 17 00:00:00 2001 From: GREENRAT-K405 Date: Sun, 1 Mar 2026 23:07:06 +0530 Subject: [PATCH 216/275] fix redundant zmq.Context() Instances --- concore_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/concore_base.py b/concore_base.py index 728d8074..fddb1f69 100644 --- a/concore_base.py +++ b/concore_base.py @@ -20,7 +20,7 @@ def __init__(self, port_type, address, zmq_socket_type, context): port_type: "bind" or "connect" address: ZeroMQ address (e.g., "tcp://*:5555") zmq_socket_type: zmq.REQ, zmq.REP, zmq.PUB, zmq.SUB etc. - context: shared zmq.Context() for the process (do not create one per port) + context: shared zmq.Context() for the process """ self.socket = context.socket(zmq_socket_type) self.port_type = port_type # "bind" or "connect" From 0c4ec88003243759c1c85eeb84e36e4bb09652d4 Mon Sep 17 00:00:00 2001 From: GREENRAT-K405 Date: Sun, 1 Mar 2026 23:22:05 +0530 Subject: [PATCH 217/275] azy-init shared context instead of one per port --- concore_base.py | 35 +++++++++++++++++++++++++---------- 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/concore_base.py b/concore_base.py index fddb1f69..b7feefda 100644 --- a/concore_base.py +++ b/concore_base.py @@ -11,17 +11,28 @@ # =================================================================== # ZeroMQ Communication Wrapper # =================================================================== -# single shared ZMQ context for the entire process. -_zmq_context = zmq.Context() +# lazy-initialized shared ZMQ context for the entire process. +# using None until first ZMQ port is created, so file-only workflows +# never spawn ZMQ I/O threads at import time. +_zmq_context = None + +def _get_zmq_context(): + """Return the process-level shared ZMQ context, creating it on first call.""" + global _zmq_context + if _zmq_context is None or _zmq_context.closed: + _zmq_context = zmq.Context() + return _zmq_context class ZeroMQPort: - def __init__(self, port_type, address, zmq_socket_type, context): + def __init__(self, port_type, address, zmq_socket_type, context=None): """ port_type: "bind" or "connect" address: ZeroMQ address (e.g., "tcp://*:5555") zmq_socket_type: zmq.REQ, zmq.REP, zmq.PUB, zmq.SUB etc. - context: shared zmq.Context() for the process + context: optional zmq.Context() for the process; defaults to the shared _zmq_context. """ + if context is None: + context = _get_zmq_context() self.socket = context.socket(zmq_socket_type) self.port_type = port_type # "bind" or "connect" self.address = address @@ -79,7 +90,7 @@ def init_zmq_port(mod, port_name, port_type, address, socket_type_str): try: # Map socket type string to actual ZMQ constant (e.g., zmq.REQ, zmq.REP) zmq_socket_type = getattr(zmq, socket_type_str.upper()) - mod.zmq_ports[port_name] = ZeroMQPort(port_type, address, zmq_socket_type, _zmq_context) + mod.zmq_ports[port_name] = ZeroMQPort(port_type, address, zmq_socket_type, _get_zmq_context()) logger.info(f"Initialized ZMQ port: {port_name} ({socket_type_str}) on {address}") except AttributeError: logger.error(f"Error: Invalid ZMQ socket type string '{socket_type_str}'.") @@ -108,11 +119,15 @@ def terminate_zmq(mod): logger.error(f"Error while terminating ZMQ port {port.address}: {e}") mod.zmq_ports.clear() - # terminate the single shared context exactly once. - try: - _zmq_context.term() - except Exception as e: - logger.error(f"Error while terminating shared ZMQ context: {e}") + # terminate the single shared context exactly once, then reset so it + # can be safely recreated if init_zmq_port is called again later. + global _zmq_context + if _zmq_context is not None and not _zmq_context.closed: + try: + _zmq_context.term() + except Exception as e: + logger.error(f"Error while terminating shared ZMQ context: {e}") + _zmq_context = None mod._cleanup_in_progress = False From fb27fa9c80a6a4223ff9a0e56e417009f4690d75 Mon Sep 17 00:00:00 2001 From: PARAM KANADA Date: Sun, 1 Mar 2026 23:30:50 +0530 Subject: [PATCH 218/275] Update concore_base.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- concore_base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/concore_base.py b/concore_base.py index b7feefda..e8f61974 100644 --- a/concore_base.py +++ b/concore_base.py @@ -104,8 +104,8 @@ def terminate_zmq(mod): if mod._cleanup_in_progress: return # Already cleaning up, prevent reentrant calls - if not mod.zmq_ports: - return # No ports to clean up + if not mod.zmq_ports and (_zmq_context is None or _zmq_context.closed): + return # Nothing to clean up: no ports and no active context mod._cleanup_in_progress = True print("\nCleaning up ZMQ resources...") From 0081e834abadfc6378f8b2ab198f98a4344d32e2 Mon Sep 17 00:00:00 2001 From: GREENRAT-K405 Date: Sun, 1 Mar 2026 23:37:54 +0530 Subject: [PATCH 219/275] fix: move global _zmq_context declaration before first use in terminate_zmq --- concore_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/concore_base.py b/concore_base.py index e8f61974..24547b01 100644 --- a/concore_base.py +++ b/concore_base.py @@ -101,6 +101,7 @@ def init_zmq_port(mod, port_name, port_type, address, socket_type_str): def terminate_zmq(mod): """Clean up all ZMQ sockets, then terminate the shared context once.""" + global _zmq_context # declared first — used both in the early-return guard and reset below if mod._cleanup_in_progress: return # Already cleaning up, prevent reentrant calls @@ -121,7 +122,6 @@ def terminate_zmq(mod): # terminate the single shared context exactly once, then reset so it # can be safely recreated if init_zmq_port is called again later. - global _zmq_context if _zmq_context is not None and not _zmq_context.closed: try: _zmq_context.term() From d8f3e6cf52572e0c8fbd9ee5c1f0f95cb00fb057 Mon Sep 17 00:00:00 2001 From: Avinash Kumar Deepak Date: Mon, 2 Mar 2026 01:01:47 +0530 Subject: [PATCH 220/275] feat: add ZMQ transport to concore.hpp/concore_base.hpp (fixes #474) --- concore.hpp | 120 +++++++++++++++++++++++++++++++++++++++++++++++ concore_base.hpp | 111 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 231 insertions(+) diff --git a/concore.hpp b/concore.hpp index da2c7921..134bf229 100644 --- a/concore.hpp +++ b/concore.hpp @@ -49,6 +49,10 @@ class Concore{ int communication_iport = 0; // iport refers to input port int communication_oport = 0; // oport refers to input port +#ifdef CONCORE_USE_ZMQ + map zmq_ports; +#endif + public: double delay = 1; int retrycount = 0; @@ -107,6 +111,11 @@ class Concore{ */ ~Concore() { +#ifdef CONCORE_USE_ZMQ + for (auto& kv : zmq_ports) + delete kv.second; + zmq_ports.clear(); +#endif #ifdef __linux__ // Detach the shared memory segment from the process if (communication_oport == 1 && sharedData_create != nullptr) { @@ -549,6 +558,117 @@ class Concore{ } } +#ifdef CONCORE_USE_ZMQ + /** + * @brief Registers a ZMQ port for use with read()/write(). + * @param port_name The ZMQ port name. + * @param port_type "bind" or "connect". + * @param address The ZMQ address. + * @param socket_type_str The socket type string. + */ + void init_zmq_port(string port_name, string port_type, string address, string socket_type_str) { + if (zmq_ports.count(port_name)) return; + int sock_type = concore_base::zmq_socket_type_from_string(socket_type_str); + if (sock_type == -1) { + cerr << "init_zmq_port: unknown socket type '" << socket_type_str << "'" << endl; + return; + } + zmq_ports[port_name] = new concore_base::ZeroMQPort(port_type, address, sock_type); + } + + /** + * @brief Reads data from a ZMQ port. Strips simtime prefix, updates simtime. + * @param port_name The ZMQ port name. + * @param name The name of the file. + * @param initstr The initial string. + * @return a vector of double values + */ + vector read_ZMQ(string port_name, string name, string initstr) { + auto it = zmq_ports.find(port_name); + if (it == zmq_ports.end()) { + cerr << "read_ZMQ: port '" << port_name << "' not initialized" << endl; + return parser(initstr); + } + vector inval = it->second->recv_with_retry(); + if (inval.empty()) + inval = parser(initstr); + if (inval.empty()) return inval; + simtime = simtime > inval[0] ? simtime : inval[0]; + s += port_name; + inval.erase(inval.begin()); + return inval; + } + + /** + * @brief Writes a vector of double values to a ZMQ port. Prepends simtime+delta. + * @param port_name The ZMQ port name. + * @param name The name of the file. + * @param val The vector of double values to write. + * @param delta The delta value (default: 0). + */ + void write_ZMQ(string port_name, string name, vector val, int delta=0) { + auto it = zmq_ports.find(port_name); + if (it == zmq_ports.end()) { + cerr << "write_ZMQ: port '" << port_name << "' not initialized" << endl; + return; + } + val.insert(val.begin(), simtime + delta); + it->second->send_with_retry(val); + // simtime must not be mutated here (issue #385). + } + + /** + * @brief Writes a string to a ZMQ port. + * @param port_name The ZMQ port name. + * @param name The name of the file. + * @param val The string to write. + * @param delta The delta value (default: 0). + */ + void write_ZMQ(string port_name, string name, string val, int delta=0) { + auto it = zmq_ports.find(port_name); + if (it == zmq_ports.end()) { + cerr << "write_ZMQ: port '" << port_name << "' not initialized" << endl; + return; + } + chrono::milliseconds timespan((int)(2000*delay)); + this_thread::sleep_for(timespan); + it->second->send_string_with_retry(val); + } + + /** + * @brief deviate the read to ZMQ communication protocol when port identifier is a string key. + * @param port_name The ZMQ port name. + * @param name The name of the file. + * @param initstr The initial string. + * @return + */ + vector read(string port_name, string name, string initstr) { + return read_ZMQ(port_name, name, initstr); + } + + /** + * @brief deviate the write to ZMQ communication protocol when port identifier is a string key. + * @param port_name The ZMQ port name. + * @param name The name of the file. + * @param val The vector of double values to write. + * @param delta The delta value (default: 0). + */ + void write(string port_name, string name, vector val, int delta=0) { + return write_ZMQ(port_name, name, val, delta); + } + + /** + * @brief deviate the write to ZMQ communication protocol when port identifier is a string key. + * @param port_name The ZMQ port name. + * @param name The name of the file. + * @param val The string to write. + * @param delta The delta value (default: 0). + */ + void write(string port_name, string name, string val, int delta=0) { + return write_ZMQ(port_name, name, val, delta); + } +#endif // CONCORE_USE_ZMQ + /** * @brief Strips leading and trailing whitespace from a string. * @param str The input string. diff --git a/concore_base.hpp b/concore_base.hpp index 64799428..9018f61e 100644 --- a/concore_base.hpp +++ b/concore_base.hpp @@ -178,6 +178,117 @@ inline std::string tryparam( return (it != params.end()) ? it->second : defaultValue; } + +// =================================================================== +// ZeroMQ Transport (opt-in: compile with -DCONCORE_USE_ZMQ) +// =================================================================== +#ifdef CONCORE_USE_ZMQ +#include + +/** + * ZMQ socket wrapper with bind/connect, timeouts, and retry. + */ +class ZeroMQPort { +public: + zmq::context_t context; + zmq::socket_t socket; + std::string port_type; + std::string address; + + ZeroMQPort(const std::string& port_type_, const std::string& address_, int socket_type) + : context(1), socket(context, socket_type), + port_type(port_type_), address(address_) + { + socket.setsockopt(ZMQ_RCVTIMEO, 2000); + socket.setsockopt(ZMQ_SNDTIMEO, 2000); + socket.setsockopt(ZMQ_LINGER, 0); + + if (port_type == "bind") + socket.bind(address); + else + socket.connect(address); + } + + ZeroMQPort(const ZeroMQPort&) = delete; + ZeroMQPort& operator=(const ZeroMQPort&) = delete; + + /** + * Sends a vector as "[v0, v1, ...]" with retry on timeout. + */ + void send_with_retry(const std::vector& payload) { + std::ostringstream ss; + ss << "["; + for (size_t i = 0; i < payload.size(); ++i) { + if (i) ss << ", "; + ss << payload[i]; + } + ss << "]"; + std::string msg = ss.str(); + for (int attempt = 0; attempt < 5; ++attempt) { + try { + zmq::message_t zmsg(msg.begin(), msg.end()); + socket.send(zmsg, zmq::send_flags::none); + return; + } catch (const zmq::error_t&) { + std::this_thread::sleep_for(std::chrono::milliseconds(500)); + } + } + std::cerr << "ZMQ send failed after retries." << std::endl; + } + + /** + * Sends a raw string with retry on timeout. + */ + void send_string_with_retry(const std::string& msg) { + for (int attempt = 0; attempt < 5; ++attempt) { + try { + zmq::message_t zmsg(msg.begin(), msg.end()); + socket.send(zmsg, zmq::send_flags::none); + return; + } catch (const zmq::error_t&) { + std::this_thread::sleep_for(std::chrono::milliseconds(500)); + } + } + std::cerr << "ZMQ send failed after retries." << std::endl; + } + + /** + * Receives and parses "[v0, v1, ...]" back to vector. + */ + std::vector recv_with_retry() { + for (int attempt = 0; attempt < 5; ++attempt) { + try { + zmq::message_t zmsg; + auto res = socket.recv(zmsg, zmq::recv_flags::none); + if (res) { + std::string data(static_cast(zmsg.data()), zmsg.size()); + return parselist_double(data); + } + } catch (const zmq::error_t&) { + std::this_thread::sleep_for(std::chrono::milliseconds(500)); + } + } + std::cerr << "ZMQ recv failed after retries." << std::endl; + return {}; + } +}; + +/** + * Maps socket type string ("REQ", "REP", etc.) to ZMQ constant. + * Returns -1 on unknown type. + */ +inline int zmq_socket_type_from_string(const std::string& s) { + if (s == "REQ") return ZMQ_REQ; + if (s == "REP") return ZMQ_REP; + if (s == "PUB") return ZMQ_PUB; + if (s == "SUB") return ZMQ_SUB; + if (s == "PUSH") return ZMQ_PUSH; + if (s == "PULL") return ZMQ_PULL; + if (s == "PAIR") return ZMQ_PAIR; + return -1; +} +#endif // CONCORE_USE_ZMQ + } // namespace concore_base #endif // CONCORE_BASE_HPP From cb9e43cb6297748b1a5992b0e9f2d9fec81acad9 Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Tue, 24 Feb 2026 10:34:45 +0530 Subject: [PATCH 221/275] fix(parser): add Python-compatible literal parser for C++ concore nodes - Replace stod-only parser with recursive descent parser in concore_base.hpp - Introduce ConcoreValue variant type supporting numbers, booleans, strings, nested arrays, and tuples (matching Python ast.literal_eval output) - Add parse_literal() and flatten_numeric() APIs to concore.hpp - Maintain full backward compatibility for flat numeric payloads - Add TestLiteralEvalCpp.cpp with 79 tests covering all payload types, error cases, and cross-language round-trip scenarios - Document wire format in README.md - Prevents silent cross-language data loss Fixes #389 --- README.md | 10 ++ TestLiteralEvalCpp.cpp | 307 +++++++++++++++++++++++++++++++++++++++++ concore.hpp | 21 +++ concore_base.hpp | 247 ++++++++++++++++++++++++++++++++- 4 files changed, 580 insertions(+), 5 deletions(-) create mode 100644 TestLiteralEvalCpp.cpp diff --git a/README.md b/README.md index a44827d8..60c94e71 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,16 @@ The CONTROL-CORE framework consists of the below projects. _concore_ enables composing studies from programs developed in different languages. Currently supported languages are, Python, Matlab/Octave, Verilog, and C++. The studies are designed through the visual _concore_ Editor (DHGWorkflow) and interpreted into _concore_ through its parser. Neural control systems consist of loops (dicycles). Therefore, they cannot be represented by classic workflow standards (such as CWL or WDL). Therefore, _concore_ addresses a significant research gap to model closed-loop neuromodulation control systems. The _concore_ protocol shares data between the programs through file sharing, with no centralized entity (a broker or an orchestrator) to arbitrate communications between the programs. (In the distributed executions, the CONTROL-CORE Mediator enables connecting the disjoint pieces of the study through REST APIs). +## Wire Format + +Concore payloads follow Python literal syntax compatible with `ast.literal_eval()`. All language implementations (Python, C++, Java, MATLAB) parse this shared format. Supported value types include: + +* **Numbers** — integers and floats, including scientific notation (e.g., `1e3`, `-2.5`) +* **Booleans** — `True` / `False` (converted to `1.0` / `0.0` in numeric contexts) +* **Strings** — single- or double-quoted (e.g., `"start"`, `'label'`) +* **Nested arrays** — `[1, [2, 3]]` +* **Tuples** — `(1.0, 2.0)` (treated identically to arrays) + # Installation and Getting Started Guide diff --git a/TestLiteralEvalCpp.cpp b/TestLiteralEvalCpp.cpp new file mode 100644 index 00000000..5746837a --- /dev/null +++ b/TestLiteralEvalCpp.cpp @@ -0,0 +1,307 @@ +/** + * TestLiteralEvalCpp.cpp + * + * Test suite for the C++ Python-literal-compatible parser in concore_base.hpp. + * Validates Issue #389 fix: C++ parser must accept all valid concore payloads + * that Python's ast.literal_eval() accepts. + * + * Compile: g++ -std=c++11 -o TestLiteralEvalCpp TestLiteralEvalCpp.cpp + * Run: ./TestLiteralEvalCpp (Linux/macOS) + * TestLiteralEvalCpp.exe (Windows) + */ + +#include +#include +#include +#include +#include +#include + +#include "concore_base.hpp" + +using namespace concore_base; + +static int passed = 0; +static int failed = 0; + +// ------------- helpers ------------------------------------------------- + +static void check(const std::string& testName, bool condition) { + if (condition) { + std::cout << "PASS: " << testName << std::endl; + ++passed; + } else { + std::cout << "FAIL: " << testName << std::endl; + ++failed; + } +} + +static bool approx(double a, double b, double eps = 1e-9) { + return std::fabs(a - b) < eps; +} + +// ------------- backward-compatibility tests ---------------------------- + +static void test_flat_numeric_list() { + std::vector v = parselist_double("[10.0, 0.5, 2.3]"); + check("flat_numeric size==3", v.size() == 3); + check("flat_numeric[0]==10.0", approx(v[0], 10.0)); + check("flat_numeric[1]==0.5", approx(v[1], 0.5)); + check("flat_numeric[2]==2.3", approx(v[2], 2.3)); +} + +static void test_empty_list() { + std::vector v = parselist_double("[]"); + check("empty_list size==0", v.size() == 0); +} + +static void test_single_element() { + std::vector v = parselist_double("[42.0]"); + check("single_element size==1", v.size() == 1); + check("single_element[0]==42", approx(v[0], 42.0)); +} + +static void test_negative_numbers() { + std::vector v = parselist_double("[-1.5, -3.0, 2.0]"); + check("negative size==3", v.size() == 3); + check("negative[0]==-1.5", approx(v[0], -1.5)); + check("negative[1]==-3.0", approx(v[1], -3.0)); +} + +static void test_scientific_notation() { + std::vector v = parselist_double("[1e3, 2.5E-2, -1.0e+1]"); + check("sci size==3", v.size() == 3); + check("sci[0]==1000", approx(v[0], 1000.0)); + check("sci[1]==0.025", approx(v[1], 0.025)); + check("sci[2]==-10", approx(v[2], -10.0)); +} + +static void test_integer_values() { + std::vector v = parselist_double("[1, 2, 3]"); + check("int size==3", v.size() == 3); + check("int[0]==1", approx(v[0], 1.0)); + check("int[2]==3", approx(v[2], 3.0)); +} + +// ------------- mixed-type payload tests (Issue #389 core) -------------- + +static void test_string_element() { + // [10.0, "start", 0.5] – string should be skipped in numeric flatten + std::vector v = parselist_double("[10.0, \"start\", 0.5]"); + check("string_elem size==2", v.size() == 2); + check("string_elem[0]==10.0", approx(v[0], 10.0)); + check("string_elem[1]==0.5", approx(v[1], 0.5)); +} + +static void test_boolean_element() { + // [10.0, True, 0.5] + std::vector v = parselist_double("[10.0, True, 0.5]"); + check("bool_elem size==3", v.size() == 3); + check("bool_elem[0]==10.0", approx(v[0], 10.0)); + check("bool_elem[1]==1.0 (True)", approx(v[1], 1.0)); + check("bool_elem[2]==0.5", approx(v[2], 0.5)); +} + +static void test_bool_false() { + std::vector v = parselist_double("[False, 5.0]"); + check("bool_false size==2", v.size() == 2); + check("bool_false[0]==0.0", approx(v[0], 0.0)); +} + +static void test_nested_list() { + // [10.0, [0.5, 0.3], 0.1] – nested list flattened to [10.0, 0.5, 0.3, 0.1] + std::vector v = parselist_double("[10.0, [0.5, 0.3], 0.1]"); + check("nested size==4", v.size() == 4); + check("nested[0]==10.0", approx(v[0], 10.0)); + check("nested[1]==0.5", approx(v[1], 0.5)); + check("nested[2]==0.3", approx(v[2], 0.3)); + check("nested[3]==0.1", approx(v[3], 0.1)); +} + +static void test_tuple_payload() { + // (10.0, 0.3) – tuple treated as array + std::vector v = parselist_double("(10.0, 0.3)"); + check("tuple size==2", v.size() == 2); + check("tuple[0]==10.0", approx(v[0], 10.0)); + check("tuple[1]==0.3", approx(v[1], 0.3)); +} + +static void test_nested_tuple() { + // [10.0, (0.5, 0.3)] + std::vector v = parselist_double("[10.0, (0.5, 0.3)]"); + check("nested_tuple size==3", v.size() == 3); + check("nested_tuple[0]==10.0", approx(v[0], 10.0)); + check("nested_tuple[1]==0.5", approx(v[1], 0.5)); + check("nested_tuple[2]==0.3", approx(v[2], 0.3)); +} + +static void test_mixed_types() { + // [10.0, "label", True, [1, 2], (3,), False, "end"] + std::vector v = parselist_double("[10.0, \"label\", True, [1, 2], (3,), False, \"end\"]"); + // numeric values: 10.0, 1.0(True), 1, 2, 3, 0.0(False) = 6 values + check("mixed size==6", v.size() == 6); + check("mixed[0]==10.0", approx(v[0], 10.0)); + check("mixed[1]==1.0", approx(v[1], 1.0)); // True + check("mixed[2]==1.0", approx(v[2], 1.0)); // nested [1,...] + check("mixed[3]==2.0", approx(v[3], 2.0)); // nested [...,2] + check("mixed[4]==3.0", approx(v[4], 3.0)); // tuple (3,) + check("mixed[5]==0.0", approx(v[5], 0.0)); // False +} + +// ------------- full ConcoreValue parse tests --------------------------- + +static void test_parse_literal_string() { + ConcoreValue v = parse_literal("[10.0, \"start\", 0.5]"); + check("literal_string is ARRAY", v.type == ConcoreValueType::ARRAY); + check("literal_string len==3", v.array.size() == 3); + check("literal_string[0] NUMBER", v.array[0].type == ConcoreValueType::NUMBER); + check("literal_string[1] STRING", v.array[1].type == ConcoreValueType::STRING); + check("literal_string[1]==\"start\"", v.array[1].str == "start"); + check("literal_string[2] NUMBER", v.array[2].type == ConcoreValueType::NUMBER); +} + +static void test_parse_literal_bool() { + ConcoreValue v = parse_literal("[True, False]"); + check("literal_bool is ARRAY", v.type == ConcoreValueType::ARRAY); + check("literal_bool[0] BOOL", v.array[0].type == ConcoreValueType::BOOL); + check("literal_bool[0]==true", v.array[0].boolean == true); + check("literal_bool[1]==false", v.array[1].boolean == false); +} + +static void test_parse_literal_nested() { + ConcoreValue v = parse_literal("[1, [2, [3]]]"); + check("literal_nested outer ARRAY", v.type == ConcoreValueType::ARRAY); + check("literal_nested[1] ARRAY", v.array[1].type == ConcoreValueType::ARRAY); + check("literal_nested[1][1] ARRAY", v.array[1].array[1].type == ConcoreValueType::ARRAY); + check("literal_nested[1][1][0]==3", approx(v.array[1].array[1].array[0].number, 3.0)); +} + +static void test_parse_single_quoted_string() { + ConcoreValue v = parse_literal("['hello']"); + check("single_quote ARRAY", v.type == ConcoreValueType::ARRAY); + check("single_quote[0] STRING", v.array[0].type == ConcoreValueType::STRING); + check("single_quote[0]=='hello'", v.array[0].str == "hello"); +} + +static void test_parse_escape_sequences() { + ConcoreValue v = parse_literal("[\"line\\none\"]"); + check("escape STRING", v.array[0].type == ConcoreValueType::STRING); + check("escape has newline", v.array[0].str == "line\none"); +} + +static void test_parse_none() { + ConcoreValue v = parse_literal("[None, 1]"); + check("none[0] STRING", v.array[0].type == ConcoreValueType::STRING); + check("none[0]==\"None\"", v.array[0].str == "None"); +} + +static void test_trailing_comma() { + // Python allows trailing comma: [1, 2,] + std::vector v = parselist_double("[1, 2,]"); + check("trailing_comma size==2", v.size() == 2); + check("trailing_comma[1]==2", approx(v[1], 2.0)); +} + +// ------------- error / failure case tests ------------------------------ + +static void test_malformed_bracket() { + bool caught = false; + try { + parse_literal("[1, 2"); + } catch (const std::runtime_error&) { + caught = true; + } + check("malformed_bracket throws", caught); +} + +static void test_malformed_string() { + bool caught = false; + try { + parse_literal("[\"unterminated]"); + } catch (const std::runtime_error&) { + caught = true; + } + check("malformed_string throws", caught); +} + +static void test_unsupported_object() { + bool caught = false; + try { + parse_literal("{1: 2}"); + } catch (const std::runtime_error&) { + caught = true; + } + check("unsupported_object throws", caught); +} + +static void test_empty_string_input() { + std::vector v = parselist_double(""); + check("empty_input size==0", v.size() == 0); +} + +// ------------- cross-language round-trip tests ------------------------- + +static void test_python_write_cpp_read_flat() { + // Simulate Python write: "[5.0, 1.0, 2.0]" + std::vector v = parselist_double("[5.0, 1.0, 2.0]"); + check("py2cpp_flat size==3", v.size() == 3); + check("py2cpp_flat[0]==5.0", approx(v[0], 5.0)); +} + +static void test_python_write_cpp_read_mixed() { + // Simulate Python write: "[5.0, 'sensor_a', True, [0.1, 0.2]]" + std::vector v = parselist_double("[5.0, 'sensor_a', True, [0.1, 0.2]]"); + // numeric: 5.0, 1.0(True), 0.1, 0.2 = 4 + check("py2cpp_mixed size==4", v.size() == 4); + check("py2cpp_mixed[0]==5.0", approx(v[0], 5.0)); + check("py2cpp_mixed[1]==1.0", approx(v[1], 1.0)); + check("py2cpp_mixed[2]==0.1", approx(v[2], 0.1)); + check("py2cpp_mixed[3]==0.2", approx(v[3], 0.2)); +} + +// ------------- main ---------------------------------------------------- + +int main() { + std::cout << "===== C++ Literal Parser Tests (Issue #389) =====\n\n"; + + // Backward compatibility + test_flat_numeric_list(); + test_empty_list(); + test_single_element(); + test_negative_numbers(); + test_scientific_notation(); + test_integer_values(); + + // Mixed-type payloads (core of Issue #389) + test_string_element(); + test_boolean_element(); + test_bool_false(); + test_nested_list(); + test_tuple_payload(); + test_nested_tuple(); + test_mixed_types(); + + // Full ConcoreValue structure tests + test_parse_literal_string(); + test_parse_literal_bool(); + test_parse_literal_nested(); + test_parse_single_quoted_string(); + test_parse_escape_sequences(); + test_parse_none(); + test_trailing_comma(); + + // Error / failure cases + test_malformed_bracket(); + test_malformed_string(); + test_unsupported_object(); + test_empty_string_input(); + + // Cross-language round-trip + test_python_write_cpp_read_flat(); + test_python_write_cpp_read_mixed(); + + std::cout << "\n=== Results: " << passed << " passed, " << failed + << " failed out of " << (passed + failed) << " tests ===\n"; + + return (failed > 0) ? 1 : 0; +} diff --git a/concore.hpp b/concore.hpp index 3e666b38..5016a18a 100644 --- a/concore.hpp +++ b/concore.hpp @@ -337,6 +337,27 @@ class Concore{ return concore_base::parselist_double(f); } + /** + * @brief Parses a Python-literal payload into a structured ConcoreValue. + * Supports numbers, booleans, strings, nested arrays, and tuples. + * Use this when you need the full parsed structure, not just doubles. + * @param f The input string to parse. + * @return A ConcoreValue representing the parsed literal. + * @throws std::runtime_error on malformed input. + */ + concore_base::ConcoreValue parse_literal(string f){ + return concore_base::parse_literal(f); + } + + /** + * @brief Recursively extracts all numeric values from a ConcoreValue. + * @param v The ConcoreValue to flatten. + * @return A flat vector of doubles. + */ + vector flatten_numeric(const concore_base::ConcoreValue& v){ + return concore_base::flatten_numeric(v); + } + /** * @brief deviate the read to either the SM (Shared Memory) or FM (File Method) communication protocol based on iport and oport. * @param port The port number. diff --git a/concore_base.hpp b/concore_base.hpp index 9018f61e..d03a0e03 100644 --- a/concore_base.hpp +++ b/concore_base.hpp @@ -84,14 +84,251 @@ inline std::vector parselist(const std::string& str) { /** * Parses a double-valued list like "[0.0, 1.5, 2.3]" into a vector. * Used by concore.hpp's read/write which work with numeric data. + * Now delegates to the full literal parser to handle mixed-type payloads + * (strings, booleans, nested lists, tuples) without crashing. + * See Issue #389. */ +inline std::vector parselist_double(const std::string& str); // forward decl; defined after ConcoreValue + +// =================================================================== +// Python-Literal-Compatible Value Type and Parser (Issue #389) +// =================================================================== + +/** + * Tag for ConcoreValue discriminated union. + */ +enum class ConcoreValueType { NUMBER, BOOL, STRING, ARRAY }; + +/** + * A recursive value type that mirrors Python's ast.literal_eval output. + * Supported: numbers, booleans, strings, and nested arrays / tuples. + */ +struct ConcoreValue { + ConcoreValueType type; + double number; + bool boolean; + std::string str; + std::vector array; + + ConcoreValue() : type(ConcoreValueType::NUMBER), number(0.0), boolean(false) {} + + static ConcoreValue make_number(double v) { + ConcoreValue cv; + cv.type = ConcoreValueType::NUMBER; + cv.number = v; + return cv; + } + static ConcoreValue make_bool(bool v) { + ConcoreValue cv; + cv.type = ConcoreValueType::BOOL; + cv.boolean = v; + cv.number = v ? 1.0 : 0.0; // Python: True == 1, False == 0 + return cv; + } + static ConcoreValue make_string(const std::string& v) { + ConcoreValue cv; + cv.type = ConcoreValueType::STRING; + cv.str = v; + return cv; + } + static ConcoreValue make_array(const std::vector& v) { + ConcoreValue cv; + cv.type = ConcoreValueType::ARRAY; + cv.array = v; + return cv; + } +}; + +// --------------- internal helpers (anonymous-namespace-like) -------- + +inline void skip_ws(const std::string& s, size_t& pos) { + while (pos < s.size() && std::isspace(static_cast(s[pos]))) + ++pos; +} + +inline ConcoreValue parse_literal_value(const std::string& s, size_t& pos); + +inline ConcoreValue parse_literal_string(const std::string& s, size_t& pos) { + char quote = s[pos]; // ' or " + ++pos; + std::string result; + while (pos < s.size() && s[pos] != quote) { + if (s[pos] == '\\' && pos + 1 < s.size()) { + ++pos; + switch (s[pos]) { + case 'n': result += '\n'; break; + case 't': result += '\t'; break; + case '\\': result += '\\'; break; + case '\'': result += '\''; break; + case '"': result += '"'; break; + default: result += '\\'; result += s[pos]; break; + } + } else { + result += s[pos]; + } + ++pos; + } + if (pos >= s.size()) + throw std::runtime_error("Invalid concore payload: unterminated string"); + ++pos; // skip closing quote + return ConcoreValue::make_string(result); +} + +inline ConcoreValue parse_literal_array(const std::string& s, size_t& pos) { + char open = s[pos]; + char close = (open == '[') ? ']' : ')'; + ++pos; + std::vector elements; + skip_ws(s, pos); + if (pos < s.size() && s[pos] == close) { ++pos; return ConcoreValue::make_array(elements); } + while (pos < s.size()) { + elements.push_back(parse_literal_value(s, pos)); + skip_ws(s, pos); + if (pos < s.size() && s[pos] == ',') { ++pos; skip_ws(s, pos); } + if (pos < s.size() && s[pos] == close) { ++pos; return ConcoreValue::make_array(elements); } + } + throw std::runtime_error("Invalid concore payload: unterminated array/tuple"); +} + +/** + * Recursive descent parser entry for a single Python literal value. + * Advances `pos` past the consumed token. + */ +inline ConcoreValue parse_literal_value(const std::string& s, size_t& pos) { + skip_ws(s, pos); + if (pos >= s.size()) + throw std::runtime_error("Invalid concore payload: unexpected end of input"); + + char c = s[pos]; + + // Array / Tuple + if (c == '[' || c == '(') + return parse_literal_array(s, pos); + + // String + if (c == '\'' || c == '"') + return parse_literal_string(s, pos); + + // Boolean True + if (s.compare(pos, 4, "True") == 0 && + (pos + 4 >= s.size() || !std::isalnum(static_cast(s[pos + 4])))) { + pos += 4; + return ConcoreValue::make_bool(true); + } + // Boolean False + if (s.compare(pos, 5, "False") == 0 && + (pos + 5 >= s.size() || !std::isalnum(static_cast(s[pos + 5])))) { + pos += 5; + return ConcoreValue::make_bool(false); + } + // None → treat as string "None" (no numeric equivalent) + if (s.compare(pos, 4, "None") == 0 && + (pos + 4 >= s.size() || !std::isalnum(static_cast(s[pos + 4])))) { + pos += 4; + return ConcoreValue::make_string("None"); + } + + // Number (int, float, negative, scientific notation) + { + size_t start = pos; + if (pos < s.size() && (s[pos] == '+' || s[pos] == '-')) ++pos; + bool has_digits = false; + while (pos < s.size() && std::isdigit(static_cast(s[pos]))) { + ++pos; has_digits = true; + } + if (pos < s.size() && s[pos] == '.') { + ++pos; + while (pos < s.size() && std::isdigit(static_cast(s[pos]))) { + ++pos; has_digits = true; + } + } + if (has_digits && pos < s.size() && (s[pos] == 'e' || s[pos] == 'E')) { + ++pos; + if (pos < s.size() && (s[pos] == '+' || s[pos] == '-')) ++pos; + while (pos < s.size() && std::isdigit(static_cast(s[pos]))) ++pos; + } + if (has_digits && pos > start) { + std::string numstr = s.substr(start, pos - start); + try { + double val = std::stod(numstr); + return ConcoreValue::make_number(val); + } catch (...) { + throw std::runtime_error( + "Invalid concore payload: bad number '" + numstr + "'"); + } + } + pos = start; // backtrack + } + + throw std::runtime_error( + std::string("Invalid concore payload: unsupported literal at position ") + + std::to_string(pos)); +} + +/** + * Parses a complete Python literal string and returns a ConcoreValue. + * Trailing content after the value (other than whitespace) is an error. + */ +inline ConcoreValue parse_literal(const std::string& s) { + size_t pos = 0; + ConcoreValue v = parse_literal_value(s, pos); + skip_ws(s, pos); + if (pos != s.size()) + throw std::runtime_error( + "Invalid concore payload: unexpected trailing content"); + return v; +} + +/** + * Recursively extracts all numeric values from a ConcoreValue. + * Booleans convert to 1.0 / 0.0 (matching Python's int(True) / int(False)). + * Strings are skipped. + * Nested arrays are flattened. + */ +inline void flatten_numeric_impl(const ConcoreValue& v, std::vector& out) { + switch (v.type) { + case ConcoreValueType::NUMBER: + out.push_back(v.number); + break; + case ConcoreValueType::BOOL: + out.push_back(v.boolean ? 1.0 : 0.0); + break; + case ConcoreValueType::STRING: + // Skip non-numeric tokens + break; + case ConcoreValueType::ARRAY: + for (const auto& elem : v.array) + flatten_numeric_impl(elem, out); + break; + } +} + +inline std::vector flatten_numeric(const ConcoreValue& v) { + std::vector out; + flatten_numeric_impl(v, out); + return out; +} + +// --------------- parselist_double (full definition) ----------------- + inline std::vector parselist_double(const std::string& str) { - std::vector result; - std::vector tokens = parselist(str); - for (const auto& tok : tokens) { - result.push_back(std::stod(tok)); + std::string trimmed = stripstr(str); + if (trimmed.empty()) return {}; + try { + ConcoreValue v = parse_literal(trimmed); + return flatten_numeric(v); + } catch (...) { + // Fall back to the simple comma-split parser for edge cases + std::vector result; + if (trimmed.size() < 2) return result; + if (trimmed.front() == '[' || trimmed.front() == '(') { + std::vector tokens = parselist(trimmed); + for (const auto& tok : tokens) { + try { result.push_back(std::stod(tok)); } catch (...) {} + } + } + return result; } - return result; } /** From 3d35efd2d6f09f85af35a7d0fc99c9897395a1a8 Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Tue, 24 Feb 2026 10:39:35 +0530 Subject: [PATCH 222/275] style: remove comments from new parser code --- concore.hpp | 13 ------------ concore_base.hpp | 54 +++++------------------------------------------- 2 files changed, 5 insertions(+), 62 deletions(-) diff --git a/concore.hpp b/concore.hpp index 5016a18a..b8653cbf 100644 --- a/concore.hpp +++ b/concore.hpp @@ -337,23 +337,10 @@ class Concore{ return concore_base::parselist_double(f); } - /** - * @brief Parses a Python-literal payload into a structured ConcoreValue. - * Supports numbers, booleans, strings, nested arrays, and tuples. - * Use this when you need the full parsed structure, not just doubles. - * @param f The input string to parse. - * @return A ConcoreValue representing the parsed literal. - * @throws std::runtime_error on malformed input. - */ concore_base::ConcoreValue parse_literal(string f){ return concore_base::parse_literal(f); } - /** - * @brief Recursively extracts all numeric values from a ConcoreValue. - * @param v The ConcoreValue to flatten. - * @return A flat vector of doubles. - */ vector flatten_numeric(const concore_base::ConcoreValue& v){ return concore_base::flatten_numeric(v); } diff --git a/concore_base.hpp b/concore_base.hpp index d03a0e03..3cb93e27 100644 --- a/concore_base.hpp +++ b/concore_base.hpp @@ -81,28 +81,10 @@ inline std::vector parselist(const std::string& str) { return result; } -/** - * Parses a double-valued list like "[0.0, 1.5, 2.3]" into a vector. - * Used by concore.hpp's read/write which work with numeric data. - * Now delegates to the full literal parser to handle mixed-type payloads - * (strings, booleans, nested lists, tuples) without crashing. - * See Issue #389. - */ -inline std::vector parselist_double(const std::string& str); // forward decl; defined after ConcoreValue - -// =================================================================== -// Python-Literal-Compatible Value Type and Parser (Issue #389) -// =================================================================== +inline std::vector parselist_double(const std::string& str); -/** - * Tag for ConcoreValue discriminated union. - */ enum class ConcoreValueType { NUMBER, BOOL, STRING, ARRAY }; -/** - * A recursive value type that mirrors Python's ast.literal_eval output. - * Supported: numbers, booleans, strings, and nested arrays / tuples. - */ struct ConcoreValue { ConcoreValueType type; double number; @@ -122,7 +104,7 @@ struct ConcoreValue { ConcoreValue cv; cv.type = ConcoreValueType::BOOL; cv.boolean = v; - cv.number = v ? 1.0 : 0.0; // Python: True == 1, False == 0 + cv.number = v ? 1.0 : 0.0; return cv; } static ConcoreValue make_string(const std::string& v) { @@ -139,8 +121,6 @@ struct ConcoreValue { } }; -// --------------- internal helpers (anonymous-namespace-like) -------- - inline void skip_ws(const std::string& s, size_t& pos) { while (pos < s.size() && std::isspace(static_cast(s[pos]))) ++pos; @@ -149,7 +129,7 @@ inline void skip_ws(const std::string& s, size_t& pos) { inline ConcoreValue parse_literal_value(const std::string& s, size_t& pos); inline ConcoreValue parse_literal_string(const std::string& s, size_t& pos) { - char quote = s[pos]; // ' or " + char quote = s[pos]; ++pos; std::string result; while (pos < s.size() && s[pos] != quote) { @@ -170,7 +150,7 @@ inline ConcoreValue parse_literal_string(const std::string& s, size_t& pos) { } if (pos >= s.size()) throw std::runtime_error("Invalid concore payload: unterminated string"); - ++pos; // skip closing quote + ++pos; return ConcoreValue::make_string(result); } @@ -190,10 +170,6 @@ inline ConcoreValue parse_literal_array(const std::string& s, size_t& pos) { throw std::runtime_error("Invalid concore payload: unterminated array/tuple"); } -/** - * Recursive descent parser entry for a single Python literal value. - * Advances `pos` past the consumed token. - */ inline ConcoreValue parse_literal_value(const std::string& s, size_t& pos) { skip_ws(s, pos); if (pos >= s.size()) @@ -201,34 +177,28 @@ inline ConcoreValue parse_literal_value(const std::string& s, size_t& pos) { char c = s[pos]; - // Array / Tuple if (c == '[' || c == '(') return parse_literal_array(s, pos); - // String if (c == '\'' || c == '"') return parse_literal_string(s, pos); - // Boolean True if (s.compare(pos, 4, "True") == 0 && (pos + 4 >= s.size() || !std::isalnum(static_cast(s[pos + 4])))) { pos += 4; return ConcoreValue::make_bool(true); } - // Boolean False if (s.compare(pos, 5, "False") == 0 && (pos + 5 >= s.size() || !std::isalnum(static_cast(s[pos + 5])))) { pos += 5; return ConcoreValue::make_bool(false); } - // None → treat as string "None" (no numeric equivalent) if (s.compare(pos, 4, "None") == 0 && (pos + 4 >= s.size() || !std::isalnum(static_cast(s[pos + 4])))) { pos += 4; return ConcoreValue::make_string("None"); } - // Number (int, float, negative, scientific notation) { size_t start = pos; if (pos < s.size() && (s[pos] == '+' || s[pos] == '-')) ++pos; @@ -257,7 +227,7 @@ inline ConcoreValue parse_literal_value(const std::string& s, size_t& pos) { "Invalid concore payload: bad number '" + numstr + "'"); } } - pos = start; // backtrack + pos = start; } throw std::runtime_error( @@ -265,10 +235,6 @@ inline ConcoreValue parse_literal_value(const std::string& s, size_t& pos) { std::to_string(pos)); } -/** - * Parses a complete Python literal string and returns a ConcoreValue. - * Trailing content after the value (other than whitespace) is an error. - */ inline ConcoreValue parse_literal(const std::string& s) { size_t pos = 0; ConcoreValue v = parse_literal_value(s, pos); @@ -279,12 +245,6 @@ inline ConcoreValue parse_literal(const std::string& s) { return v; } -/** - * Recursively extracts all numeric values from a ConcoreValue. - * Booleans convert to 1.0 / 0.0 (matching Python's int(True) / int(False)). - * Strings are skipped. - * Nested arrays are flattened. - */ inline void flatten_numeric_impl(const ConcoreValue& v, std::vector& out) { switch (v.type) { case ConcoreValueType::NUMBER: @@ -294,7 +254,6 @@ inline void flatten_numeric_impl(const ConcoreValue& v, std::vector& out out.push_back(v.boolean ? 1.0 : 0.0); break; case ConcoreValueType::STRING: - // Skip non-numeric tokens break; case ConcoreValueType::ARRAY: for (const auto& elem : v.array) @@ -309,8 +268,6 @@ inline std::vector flatten_numeric(const ConcoreValue& v) { return out; } -// --------------- parselist_double (full definition) ----------------- - inline std::vector parselist_double(const std::string& str) { std::string trimmed = stripstr(str); if (trimmed.empty()) return {}; @@ -318,7 +275,6 @@ inline std::vector parselist_double(const std::string& str) { ConcoreValue v = parse_literal(trimmed); return flatten_numeric(v); } catch (...) { - // Fall back to the simple comma-split parser for edge cases std::vector result; if (trimmed.size() < 2) return result; if (trimmed.front() == '[' || trimmed.front() == '(') { From 7849594d5c293783d05f768b345d30a8b2456070 Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Mon, 2 Mar 2026 10:59:07 +0530 Subject: [PATCH 223/275] fix: restore original parselist_double doc comment --- concore_base.hpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/concore_base.hpp b/concore_base.hpp index 3cb93e27..f034f0e3 100644 --- a/concore_base.hpp +++ b/concore_base.hpp @@ -81,6 +81,10 @@ inline std::vector parselist(const std::string& str) { return result; } +/** + * Parses a double-valued list like "[0.0, 1.5, 2.3]" into a vector. + * Used by concore.hpp's read/write which work with numeric data. + */ inline std::vector parselist_double(const std::string& str); enum class ConcoreValueType { NUMBER, BOOL, STRING, ARRAY }; From f9e17b90f7bf5cb2731e4d5a6aa7ff7b7256ca1a Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Mon, 2 Mar 2026 11:13:51 +0530 Subject: [PATCH 224/275] fix: apply Copilot review suggestions - README.md: clarify MATLAB/Verilog only support flat numeric arrays - concore_base.hpp: add #include and for portability - concore_base.hpp: fix keyword boundary checks to also exclude underscore - concore.hpp: add Doxygen docs for parse_literal and flatten_numeric --- README.md | 2 +- concore.hpp | 10 ++++++++++ concore_base.hpp | 11 ++++++++--- 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 60c94e71..6c5c195c 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ _concore_ enables composing studies from programs developed in different languag ## Wire Format -Concore payloads follow Python literal syntax compatible with `ast.literal_eval()`. All language implementations (Python, C++, Java, MATLAB) parse this shared format. Supported value types include: +Concore payloads follow Python literal syntax compatible with `ast.literal_eval()`. The Python, C++, and Java implementations parse this shared format; the MATLAB and Verilog implementations currently support only flat numeric arrays derived from it. Supported value types include: * **Numbers** — integers and floats, including scientific notation (e.g., `1e3`, `-2.5`) * **Booleans** — `True` / `False` (converted to `1.0` / `0.0` in numeric contexts) diff --git a/concore.hpp b/concore.hpp index b8653cbf..5eb2b7ed 100644 --- a/concore.hpp +++ b/concore.hpp @@ -337,10 +337,20 @@ class Concore{ return concore_base::parselist_double(f); } + /** + * @brief Parses a literal string into a ConcoreValue representation. + * @param f The input string to parse. + * @return A ConcoreValue obtained by parsing the input string. + */ concore_base::ConcoreValue parse_literal(string f){ return concore_base::parse_literal(f); } + /** + * @brief Flattens a ConcoreValue into a vector of numeric (double) values. + * @param v The ConcoreValue to flatten. + * @return A vector of double values obtained by flattening the input. + */ vector flatten_numeric(const concore_base::ConcoreValue& v){ return concore_base::flatten_numeric(v); } diff --git a/concore_base.hpp b/concore_base.hpp index f034f0e3..b4d96892 100644 --- a/concore_base.hpp +++ b/concore_base.hpp @@ -14,6 +14,8 @@ #include #include #include +#include +#include namespace concore_base { @@ -188,17 +190,20 @@ inline ConcoreValue parse_literal_value(const std::string& s, size_t& pos) { return parse_literal_string(s, pos); if (s.compare(pos, 4, "True") == 0 && - (pos + 4 >= s.size() || !std::isalnum(static_cast(s[pos + 4])))) { + (pos + 4 >= s.size() || + (!std::isalnum(static_cast(s[pos + 4])) && s[pos + 4] != '_'))) { pos += 4; return ConcoreValue::make_bool(true); } if (s.compare(pos, 5, "False") == 0 && - (pos + 5 >= s.size() || !std::isalnum(static_cast(s[pos + 5])))) { + (pos + 5 >= s.size() || + (!std::isalnum(static_cast(s[pos + 5])) && s[pos + 5] != '_'))) { pos += 5; return ConcoreValue::make_bool(false); } if (s.compare(pos, 4, "None") == 0 && - (pos + 4 >= s.size() || !std::isalnum(static_cast(s[pos + 4])))) { + (pos + 4 >= s.size() || + (!std::isalnum(static_cast(s[pos + 4])) && s[pos + 4] != '_'))) { pos += 4; return ConcoreValue::make_string("None"); } From 2625e85afc2aeec36b7487201a3c75989df9e8a0 Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Mon, 2 Mar 2026 11:18:32 +0530 Subject: [PATCH 225/275] style: format test_read_status.py with ruff --- tests/test_read_status.py | 46 ++++++++++++++++++++++++++++----------- 1 file changed, 33 insertions(+), 13 deletions(-) diff --git a/tests/test_read_status.py b/tests/test_read_status.py index 54dc4b88..df3b7efa 100644 --- a/tests/test_read_status.py +++ b/tests/test_read_status.py @@ -13,6 +13,7 @@ # Helpers # --------------------------------------------------------------------------- + class DummyZMQPort: """Minimal stand-in for ZeroMQPort used in ZMQ read tests.""" @@ -33,14 +34,16 @@ def recv_json_with_retry(self): # File-based read tests # --------------------------------------------------------------------------- + class TestReadFileSuccess: """read() on a valid file returns (data, True) with SUCCESS status.""" @pytest.fixture(autouse=True) def setup(self, temp_dir, monkeypatch): import concore + self.concore = concore - monkeypatch.setattr(concore, 'delay', 0) + monkeypatch.setattr(concore, "delay", 0) # Create ./in1/ym with valid data: [simtime, value] in_dir = os.path.join(temp_dir, "in1") @@ -48,7 +51,7 @@ def setup(self, temp_dir, monkeypatch): with open(os.path.join(in_dir, "ym"), "w") as f: f.write("[10, 3.14]") - monkeypatch.setattr(concore, 'inpath', os.path.join(temp_dir, "in")) + monkeypatch.setattr(concore, "inpath", os.path.join(temp_dir, "in")) def test_returns_data_and_true(self): data, ok = self.concore.read(1, "ym", "[0, 0.0]") @@ -66,10 +69,11 @@ class TestReadFileMissing: @pytest.fixture(autouse=True) def setup(self, temp_dir, monkeypatch): import concore + self.concore = concore - monkeypatch.setattr(concore, 'delay', 0) + monkeypatch.setattr(concore, "delay", 0) # Point to a directory that does NOT have the file - monkeypatch.setattr(concore, 'inpath', os.path.join(temp_dir, "in")) + monkeypatch.setattr(concore, "inpath", os.path.join(temp_dir, "in")) def test_returns_default_and_false(self): data, ok = self.concore.read(1, "nonexistent", "[0, 0.0]") @@ -86,15 +90,16 @@ class TestReadFileParseError: @pytest.fixture(autouse=True) def setup(self, temp_dir, monkeypatch): import concore + self.concore = concore - monkeypatch.setattr(concore, 'delay', 0) + monkeypatch.setattr(concore, "delay", 0) in_dir = os.path.join(temp_dir, "in1") os.makedirs(in_dir, exist_ok=True) with open(os.path.join(in_dir, "ym"), "w") as f: f.write("NOT_VALID_PYTHON{{{") - monkeypatch.setattr(concore, 'inpath', os.path.join(temp_dir, "in")) + monkeypatch.setattr(concore, "inpath", os.path.join(temp_dir, "in")) def test_returns_default_and_false(self): data, ok = self.concore.read(1, "ym", "[0, 0.0]") @@ -111,8 +116,9 @@ class TestReadFileRetriesExceeded: @pytest.fixture(autouse=True) def setup(self, temp_dir, monkeypatch): import concore + self.concore = concore - monkeypatch.setattr(concore, 'delay', 0) + monkeypatch.setattr(concore, "delay", 0) # Create an empty file in_dir = os.path.join(temp_dir, "in1") @@ -120,7 +126,7 @@ def setup(self, temp_dir, monkeypatch): with open(os.path.join(in_dir, "ym"), "w") as f: pass # empty - monkeypatch.setattr(concore, 'inpath', os.path.join(temp_dir, "in")) + monkeypatch.setattr(concore, "inpath", os.path.join(temp_dir, "in")) def test_returns_default_and_false(self): data, ok = self.concore.read(1, "ym", "[0, 0.0]") @@ -135,12 +141,14 @@ def test_last_read_status_is_retries_exceeded(self): # ZMQ read tests # --------------------------------------------------------------------------- + class TestReadZMQSuccess: """Successful ZMQ read returns (data, True).""" @pytest.fixture(autouse=True) def setup(self, monkeypatch): import concore + self.concore = concore self.original_ports = concore.zmq_ports.copy() yield @@ -164,6 +172,7 @@ class TestReadZMQTimeout: @pytest.fixture(autouse=True) def setup(self, monkeypatch): import concore + self.concore = concore self.original_ports = concore.zmq_ports.copy() yield @@ -185,6 +194,7 @@ class TestReadZMQError: @pytest.fixture(autouse=True) def setup(self, monkeypatch): import concore + self.concore = concore self.original_ports = concore.zmq_ports.copy() yield @@ -193,6 +203,7 @@ def setup(self, monkeypatch): def test_zmq_error_returns_default_and_false(self): import zmq + dummy = DummyZMQPort(raise_on_recv=zmq.error.ZMQError("test error")) self.concore.zmq_ports["test_port"] = dummy @@ -205,21 +216,23 @@ def test_zmq_error_returns_default_and_false(self): # Backward compatibility # --------------------------------------------------------------------------- + class TestReadBackwardCompatibility: """Legacy callers can use isinstance check on the result.""" @pytest.fixture(autouse=True) def setup(self, temp_dir, monkeypatch): import concore + self.concore = concore - monkeypatch.setattr(concore, 'delay', 0) + monkeypatch.setattr(concore, "delay", 0) in_dir = os.path.join(temp_dir, "in1") os.makedirs(in_dir, exist_ok=True) with open(os.path.join(in_dir, "ym"), "w") as f: f.write("[10, 42.0]") - monkeypatch.setattr(concore, 'inpath', os.path.join(temp_dir, "in")) + monkeypatch.setattr(concore, "inpath", os.path.join(temp_dir, "in")) def test_legacy_unpack_pattern(self): """The recommended migration pattern works correctly.""" @@ -245,17 +258,24 @@ def test_tuple_unpack(self): # last_read_status exposed on module # --------------------------------------------------------------------------- + class TestLastReadStatusExposed: """concore.last_read_status is publicly accessible.""" def test_attribute_exists(self): import concore - assert hasattr(concore, 'last_read_status') + + assert hasattr(concore, "last_read_status") def test_initial_value_is_success(self): import concore + # Before any read, default is SUCCESS assert concore.last_read_status in ( - "SUCCESS", "FILE_NOT_FOUND", "TIMEOUT", - "PARSE_ERROR", "EMPTY_DATA", "RETRIES_EXCEEDED", + "SUCCESS", + "FILE_NOT_FOUND", + "TIMEOUT", + "PARSE_ERROR", + "EMPTY_DATA", + "RETRIES_EXCEEDED", ) From 37082417a133d688bd34a7287f8059f49ca2ec43 Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Mon, 2 Mar 2026 11:21:40 +0530 Subject: [PATCH 226/275] style: fix ruff lint errors in test_read_status.py --- tests/test_read_status.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_read_status.py b/tests/test_read_status.py index df3b7efa..29e64340 100644 --- a/tests/test_read_status.py +++ b/tests/test_read_status.py @@ -6,7 +6,6 @@ import os import pytest -import numpy as np # --------------------------------------------------------------------------- @@ -123,7 +122,7 @@ def setup(self, temp_dir, monkeypatch): # Create an empty file in_dir = os.path.join(temp_dir, "in1") os.makedirs(in_dir, exist_ok=True) - with open(os.path.join(in_dir, "ym"), "w") as f: + with open(os.path.join(in_dir, "ym"), "w") as _f: pass # empty monkeypatch.setattr(concore, "inpath", os.path.join(temp_dir, "in")) From a84ae7bbf3fbaa3e77f84900fa45c18db546b68e Mon Sep 17 00:00:00 2001 From: Avinash Kumar Deepak Date: Mon, 2 Mar 2026 12:15:15 +0530 Subject: [PATCH 227/275] docs: add ZMQ transport section to README (closes #476) --- README.md | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/README.md b/README.md index a44827d8..5458c435 100644 --- a/README.md +++ b/README.md @@ -115,6 +115,31 @@ Do **NOT** commit your secret key to version control. If `FLASK_SECRET_KEY` is n For a detailed and more scientific documentation, please read our extensive [open-access research paper on CONTROL-CORE](https://doi.org/10.1109/ACCESS.2022.3161471). This paper has a complete discussion on the CONTROL-CORE architecture and deployment, together with the commands to execute the studies in different programming languages and programming environments (Ubuntu, Windows, MacOS, Docker, and distributed execution). +## C++ ZMQ Transport + +`concore.hpp` supports ZMQ-based communication as an opt-in transport alongside the default file-based I/O. + +To enable it, compile with `-DCONCORE_USE_ZMQ` and link against cppzmq: + +```bash +g++ -DCONCORE_USE_ZMQ my_node.cpp -lzmq -o my_node +``` + +In your C++ node, register a ZMQ port before reading or writing: + +```cpp +#include "concore.hpp" + +Concore c; +c.init_zmq_port("in1", "bind", "tcp://*:5555", "REP"); +c.init_zmq_port("out1", "connect", "tcp://localhost:5556", "REQ"); + +vector val = c.read("in1", "", "0.0"); +c.write("out1", "", val, 0); +``` + +Builds without `-DCONCORE_USE_ZMQ` are unaffected. + # The _concore_ Repository From e61fea007add703a6c6dc96ef2a03e64b83315d9 Mon Sep 17 00:00:00 2001 From: Avinash Kumar Deepak Date: Mon, 2 Mar 2026 12:39:16 +0530 Subject: [PATCH 228/275] fix: rule of five for concoredocker.hpp --- concoredocker.hpp | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/concoredocker.hpp b/concoredocker.hpp index 593da876..bcd2bcb4 100644 --- a/concoredocker.hpp +++ b/concoredocker.hpp @@ -53,6 +53,38 @@ class Concore { load_params(); } + Concore(const Concore&) = delete; + Concore& operator=(const Concore&) = delete; + + Concore(Concore&& other) noexcept + : iport(std::move(other.iport)), oport(std::move(other.oport)), + s(std::move(other.s)), olds(std::move(other.olds)), + delay(other.delay), retrycount(other.retrycount), + inpath(std::move(other.inpath)), outpath(std::move(other.outpath)), + simtime(other.simtime), maxtime(other.maxtime), + params(std::move(other.params)) + {} + + Concore& operator=(Concore&& other) noexcept + { + if (this == &other) + return *this; + + iport = std::move(other.iport); + oport = std::move(other.oport); + s = std::move(other.s); + olds = std::move(other.olds); + delay = other.delay; + retrycount = other.retrycount; + inpath = std::move(other.inpath); + outpath = std::move(other.outpath); + simtime = other.simtime; + maxtime = other.maxtime; + params = std::move(other.params); + + return *this; + } + std::unordered_map safe_literal_eval(const std::string& filename, std::unordered_map defaultValue) { std::ifstream file(filename); if (!file) return defaultValue; From c0ce0ded2a05c62510d8fa7deee9f764a01e5de4 Mon Sep 17 00:00:00 2001 From: Avinash Kumar Deepak Date: Mon, 2 Mar 2026 14:02:17 +0530 Subject: [PATCH 229/275] feat: add ZMQ support to concoredocker.hpp, align read/write/initval to vector --- concoredocker.hpp | 116 ++++++++++++++++++++++++++++++++++++---------- 1 file changed, 91 insertions(+), 25 deletions(-) diff --git a/concoredocker.hpp b/concoredocker.hpp index bcd2bcb4..1e23aff1 100644 --- a/concoredocker.hpp +++ b/concoredocker.hpp @@ -13,6 +13,7 @@ #include #include #include +#include #include "concore_base.hpp" @@ -28,6 +29,9 @@ class Concore { double simtime = 0; double maxtime = 100; std::unordered_map params; +#ifdef CONCORE_USE_ZMQ + std::map zmq_ports; +#endif std::string stripstr(const std::string& str) { return concore_base::stripstr(str); @@ -53,6 +57,14 @@ class Concore { load_params(); } + ~Concore() { +#ifdef CONCORE_USE_ZMQ + for (auto& kv : zmq_ports) + delete kv.second; + zmq_ports.clear(); +#endif + } + Concore(const Concore&) = delete; Concore& operator=(const Concore&) = delete; @@ -63,13 +75,23 @@ class Concore { inpath(std::move(other.inpath)), outpath(std::move(other.outpath)), simtime(other.simtime), maxtime(other.maxtime), params(std::move(other.params)) - {} + { +#ifdef CONCORE_USE_ZMQ + zmq_ports = std::move(other.zmq_ports); +#endif + } Concore& operator=(Concore&& other) noexcept { if (this == &other) return *this; +#ifdef CONCORE_USE_ZMQ + for (auto& kv : zmq_ports) + delete kv.second; + zmq_ports = std::move(other.zmq_ports); +#endif + iport = std::move(other.iport); oport = std::move(other.oport); s = std::move(other.s); @@ -118,7 +140,7 @@ class Concore { return false; } - std::vector read(int port, const std::string& name, const std::string& initstr) { + std::vector read(int port, const std::string& name, const std::string& initstr) { std::this_thread::sleep_for(std::chrono::seconds(delay)); std::string file_path = inpath + "/" + std::to_string(port) + "/" + name; std::ifstream infile(file_path); @@ -126,10 +148,10 @@ class Concore { if (!infile) { std::cerr << "File " << file_path << " not found, using default value.\n"; - return {initstr}; + return concore_base::parselist_double(initstr); } std::getline(infile, ins); - + int attempts = 0, max_retries = 5; while (ins.empty() && attempts < max_retries) { std::this_thread::sleep_for(std::chrono::seconds(delay)); @@ -142,22 +164,21 @@ class Concore { if (ins.empty()) { std::cerr << "Max retries reached for " << file_path << ", using default value.\n"; - return {initstr}; + return concore_base::parselist_double(initstr); } - + s += ins; - try { - std::vector inval = parselist(ins); - if (!inval.empty()) { - double file_simtime = std::stod(inval[0]); - simtime = std::max(simtime, file_simtime); - return std::vector(inval.begin() + 1, inval.end()); - } - } catch (...) {} - return {ins}; + std::vector inval = concore_base::parselist_double(ins); + if (inval.empty()) + inval = concore_base::parselist_double(initstr); + if (inval.empty()) + return inval; + simtime = simtime > inval[0] ? simtime : inval[0]; + inval.erase(inval.begin()); + return inval; } - void write(int port, const std::string& name, const std::vector& val, int delta = 0) { + void write(int port, const std::string& name, const std::vector& val, int delta = 0) { std::string file_path = outpath + "/" + std::to_string(port) + "/" + name; std::ofstream outfile(file_path); if (!outfile) { @@ -174,15 +195,60 @@ class Concore { } } - std::vector initval(const std::string& simtime_val) { - try { - std::vector val = parselist(simtime_val); - if (!val.empty()) { - simtime = std::stod(val[0]); - return std::vector(val.begin() + 1, val.end()); - } - } catch (...) {} - return {}; +#ifdef CONCORE_USE_ZMQ + void init_zmq_port(const std::string& port_name, const std::string& port_type, + const std::string& address, const std::string& socket_type_str) { + if (zmq_ports.count(port_name)) return; + int sock_type = concore_base::zmq_socket_type_from_string(socket_type_str); + if (sock_type == -1) { + std::cerr << "init_zmq_port: unknown socket type '" << socket_type_str << "'\n"; + return; + } + zmq_ports[port_name] = new concore_base::ZeroMQPort(port_type, address, sock_type); + } + + std::vector read_ZMQ(const std::string& port_name, const std::string& name, const std::string& initstr) { + auto it = zmq_ports.find(port_name); + if (it == zmq_ports.end()) { + std::cerr << "read_ZMQ: port '" << port_name << "' not initialized\n"; + return concore_base::parselist_double(initstr); + } + std::vector inval = it->second->recv_with_retry(); + if (inval.empty()) + inval = concore_base::parselist_double(initstr); + if (inval.empty()) return inval; + simtime = simtime > inval[0] ? simtime : inval[0]; + s += port_name; + inval.erase(inval.begin()); + return inval; + } + + void write_ZMQ(const std::string& port_name, const std::string& name, std::vector val, int delta = 0) { + auto it = zmq_ports.find(port_name); + if (it == zmq_ports.end()) { + std::cerr << "write_ZMQ: port '" << port_name << "' not initialized\n"; + return; + } + val.insert(val.begin(), simtime + delta); + it->second->send_with_retry(val); + // simtime must not be mutated here (issue #385). + } + + std::vector read(const std::string& port_name, const std::string& name, const std::string& initstr) { + return read_ZMQ(port_name, name, initstr); + } + + void write(const std::string& port_name, const std::string& name, std::vector val, int delta = 0) { + return write_ZMQ(port_name, name, val, delta); + } +#endif // CONCORE_USE_ZMQ + + std::vector initval(const std::string& simtime_val) { + std::vector val = concore_base::parselist_double(simtime_val); + if (val.empty()) return val; + simtime = val[0]; + val.erase(val.begin()); + return val; } }; From 34b5687ed3fd9d843a787d5e48b81e799514eb1d Mon Sep 17 00:00:00 2001 From: Avinash Kumar Deepak Date: Mon, 2 Mar 2026 14:17:42 +0530 Subject: [PATCH 230/275] remove issue numbers from inline comments --- concoredocker.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/concoredocker.hpp b/concoredocker.hpp index 1e23aff1..b4fa69b9 100644 --- a/concoredocker.hpp +++ b/concoredocker.hpp @@ -231,7 +231,7 @@ class Concore { } val.insert(val.begin(), simtime + delta); it->second->send_with_retry(val); - // simtime must not be mutated here (issue #385). + // simtime must not be mutated here. } std::vector read(const std::string& port_name, const std::string& name, const std::string& initstr) { From de6ca5d74ddc4a78a2663ae2d18db16295022a90 Mon Sep 17 00:00:00 2001 From: Avinash Kumar Deepak Date: Tue, 3 Mar 2026 11:27:00 +0530 Subject: [PATCH 231/275] feat: add ZMQ transport to concoredocker.java via JeroMQ --- Dockerfile.java | 19 +++--- concoredocker.java | 153 ++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 159 insertions(+), 13 deletions(-) diff --git a/Dockerfile.java b/Dockerfile.java index a3eb3ad0..b78f1e6b 100644 --- a/Dockerfile.java +++ b/Dockerfile.java @@ -1,17 +1,14 @@ #build stage -FROM maven:3.9-eclipse-temurin-17-alpine AS builder +FROM eclipse-temurin:17-jdk-alpine AS builder WORKDIR /build -COPY pom.xml . -RUN mvn dependency:go-offline -B -COPY src ./src -RUN mvn clean package -DskipTests +RUN apk add --no-cache wget +RUN wget -q "https://search.maven.org/remotecontent?filepath=org/zeromq/jeromq/0.6.0/jeromq-0.6.0.jar" -O /opt/jeromq.jar +COPY *.java . +RUN javac -cp /opt/jeromq.jar *.java #runtime stage -FROM eclipse-temurin:17-jdk-alpine +FROM eclipse-temurin:17-jre-alpine WORKDIR /app - -# Copy the JAR from the build stage -COPY --from=builder /build/target/concore-*.jar /app/concore.jar -EXPOSE 3000 -CMD ["java", "-jar", "/app/concore.jar"] \ No newline at end of file +COPY --from=builder /build/*.class /app/ +COPY --from=builder /opt/jeromq.jar /app/jeromq.jar \ No newline at end of file diff --git a/concoredocker.java b/concoredocker.java index 96cf6eeb..226865f4 100644 --- a/concoredocker.java +++ b/concoredocker.java @@ -5,6 +5,7 @@ import java.util.Map; import java.util.ArrayList; import java.util.List; +import org.zeromq.ZMQ; /** * Java implementation of concore Docker communication. @@ -24,6 +25,8 @@ public class concoredocker { private static String inpath = "/in"; private static String outpath = "/out"; private static Map params = new HashMap<>(); + private static Map zmqPorts = new HashMap<>(); + private static ZMQ.Context zmqContext = null; // simtime as double to preserve fractional values (e.g. "[0.0, ...]") private static double simtime = 0; private static double maxtime; @@ -49,6 +52,7 @@ public class concoredocker { params = new HashMap<>(); } defaultMaxTime(100); + Runtime.getRuntime().addShutdownHook(new Thread(concoredocker::terminateZmq)); } /** @@ -292,7 +296,7 @@ public static void write(int port, String name, Object val, int delta) { } content.append("]"); // simtime must not be mutated here. - // Mutation breaks cross-language determinism (see issue #385). + // Mutation breaks cross-language determinism. } else if (val instanceof Object[]) { // Legacy support for Object[] arguments Object[] arrayVal = (Object[]) val; @@ -304,7 +308,7 @@ public static void write(int port, String name, Object val, int delta) { } content.append("]"); // simtime must not be mutated here. - // Mutation breaks cross-language determinism (see issue #385). + // Mutation breaks cross-language determinism. } else { System.out.println("write must have list or str"); return; @@ -336,6 +340,113 @@ public static List initVal(String simtimeVal) { return val; } + private static ZMQ.Context getZmqContext() { + if (zmqContext == null) { + zmqContext = ZMQ.context(1); + } + return zmqContext; + } + + public static void initZmqPort(String portName, String portType, String address, String socketTypeStr) { + if (zmqPorts.containsKey(portName)) return; + int sockType = zmqSocketTypeFromString(socketTypeStr); + if (sockType == -1) { + System.err.println("initZmqPort: unknown socket type '" + socketTypeStr + "'"); + return; + } + zmqPorts.put(portName, new ZeroMQPort(portType, address, sockType)); + } + + public static void terminateZmq() { + for (Map.Entry entry : zmqPorts.entrySet()) { + entry.getValue().socket.close(); + } + zmqPorts.clear(); + if (zmqContext != null) { + zmqContext.term(); + zmqContext = null; + } + } + + private static int zmqSocketTypeFromString(String s) { + switch (s.toUpperCase()) { + case "REQ": return ZMQ.REQ; + case "REP": return ZMQ.REP; + case "PUB": return ZMQ.PUB; + case "SUB": return ZMQ.SUB; + case "PUSH": return ZMQ.PUSH; + case "PULL": return ZMQ.PULL; + case "PAIR": return ZMQ.PAIR; + default: return -1; + } + } + + /** + * Reads data from a ZMQ port. Same wire format as file-based read: + * expects [simtime, val1, val2, ...], strips simtime, returns the rest. + */ + public static List read(String portName, String name, String initstr) { + List defaultVal = new ArrayList<>(); + try { + List parsed = (List) literalEval(initstr); + if (parsed.size() > 1) { + defaultVal = new ArrayList<>(parsed.subList(1, parsed.size())); + } + } catch (Exception e) { + } + ZeroMQPort port = zmqPorts.get(portName); + if (port == null) { + System.err.println("read: ZMQ port '" + portName + "' not initialized"); + return defaultVal; + } + String msg = port.recvWithRetry(); + if (msg == null) { + System.err.println("read: ZMQ recv timeout on port '" + portName + "'"); + return defaultVal; + } + s += msg; + try { + List inval = (List) literalEval(msg); + if (!inval.isEmpty()) { + simtime = Math.max(simtime, ((Number) inval.get(0)).doubleValue()); + return new ArrayList<>(inval.subList(1, inval.size())); + } + } catch (Exception e) { + System.out.println("Error parsing ZMQ message '" + msg + "': " + e.getMessage()); + } + return defaultVal; + } + + /** + * Writes data to a ZMQ port. Prepends [simtime+delta] to match file-based write behavior. + */ + public static void write(String portName, String name, Object val, int delta) { + ZeroMQPort port = zmqPorts.get(portName); + if (port == null) { + System.err.println("write: ZMQ port '" + portName + "' not initialized"); + return; + } + String payload; + if (val instanceof List) { + List listVal = (List) val; + StringBuilder sb = new StringBuilder("["); + sb.append(toPythonLiteral(simtime + delta)); + for (Object o : listVal) { + sb.append(", "); + sb.append(toPythonLiteral(o)); + } + sb.append("]"); + payload = sb.toString(); + // simtime must not be mutated here + } else if (val instanceof String) { + payload = (String) val; + } else { + System.out.println("write must have list or str"); + return; + } + port.sendWithRetry(payload); + } + /** * Parses a Python-literal string into Java objects using a recursive descent parser. * Supports: dict, list, int, float, string (single/double quoted), bool, None, nested structures. @@ -354,6 +465,44 @@ static Object literalEval(String s) { return result; } + /** + * ZMQ socket wrapper with bind/connect, timeouts, and retry. + */ + private static class ZeroMQPort { + final ZMQ.Socket socket; + final String address; + + ZeroMQPort(String portType, String address, int socketType) { + this.address = address; + ZMQ.Context ctx = getZmqContext(); + this.socket = ctx.socket(socketType); + this.socket.setReceiveTimeOut(2000); + this.socket.setSendTimeOut(2000); + this.socket.setLinger(0); + if (portType.equals("bind")) { + this.socket.bind(address); + } else { + this.socket.connect(address); + } + } + + String recvWithRetry() { + for (int attempt = 0; attempt < 5; attempt++) { + String msg = socket.recvStr(); + if (msg != null) return msg; + try { Thread.sleep(500); } catch (InterruptedException e) { Thread.currentThread().interrupt(); break; } + } + return null; + } + + void sendWithRetry(String message) { + for (int attempt = 0; attempt < 5; attempt++) { + if (socket.send(message)) return; + try { Thread.sleep(500); } catch (InterruptedException e) { Thread.currentThread().interrupt(); break; } + } + } + } + /** * Recursive descent parser for Python literal expressions. * Handles: dicts, lists, tuples, strings, numbers, booleans, None. From 41489331cad83e1de08ef7f4bf9474f9cb14f1ba Mon Sep 17 00:00:00 2001 From: Pradeeban Kathiravelu Date: Mon, 2 Mar 2026 20:59:02 -0900 Subject: [PATCH 232/275] Update citation details in README.md Add url. --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index be2042b8..0f0ba7a5 100644 --- a/README.md +++ b/README.md @@ -170,5 +170,6 @@ Please make sure to send your _concore_ pull requests to the [dev branch](https: If you use _concore_ in your research, please cite the below papers: -* Kathiravelu, P., Arnold, M., Vijay, S., Jagwani, R., Goyal, P., Goel, A.K., Li, N., Horn, C., Pan, T., Kothare, M. V., and Mahmoudi, B. **Distributed Executions with CONTROL-CORE Integrated Development Environment (IDE) for Closed-loop Neuromodulation Control Systems.** In Cluster Computing – The Journal of Networks Software Tools and Applications (CLUSTER). May 2025. Accepted. Springer. +* Kathiravelu, P., Arnold, M., Vijay, S., Jagwani, R., Goyal, P., Goel, A.K., Li, N., Horn, C., Pan, T., Kothare, M. V., and Mahmoudi, B. **Distributed Executions with CONTROL-CORE Integrated Development Environment (IDE) for Closed-loop Neuromodulation Control Systems.** In Cluster Computing – The Journal of Networks Software Tools and Applications (CLUSTER). 28, 697. September 2025. Springer. https://doi.org/10.1007/s10586-025-05476-w + * Kathiravelu, P., Arnold, M., Fleischer, J., Yao, Y., Awasthi, S., Goel, A. K., Branen, A., Sarikhani, P., Kumar, G., Kothare, M. V., and Mahmoudi, B. **CONTROL-CORE: A Framework for Simulation and Design of Closed-Loop Peripheral Neuromodulation Control Systems**. In IEEE Access. March 2022. https://doi.org/10.1109/ACCESS.2022.3161471 From 4bc59b8e22f76e28e072b8a00b3cd836f50d57e1 Mon Sep 17 00:00:00 2001 From: Avinash Kumar Deepak Date: Tue, 3 Mar 2026 13:03:05 +0530 Subject: [PATCH 233/275] test: add Concore class API tests for concore.hpp (#484) --- TestConcoreHpp.cpp | 315 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 315 insertions(+) create mode 100644 TestConcoreHpp.cpp diff --git a/TestConcoreHpp.cpp b/TestConcoreHpp.cpp new file mode 100644 index 00000000..54df52b2 --- /dev/null +++ b/TestConcoreHpp.cpp @@ -0,0 +1,315 @@ +/** + * TestConcoreHpp.cpp + * + * Test suite for the Concore class API in concore.hpp. + * Covers: read_FM/write_FM round-trip, unchanged(), initval(), + * mapParser(), default_maxtime(), and tryparam(). + * + * Addresses Issue #484: adds coverage for the Concore class API. + * + * Compile: g++ -std=c++11 -o TestConcoreHpp TestConcoreHpp.cpp + * Run: ./TestConcoreHpp (Linux/macOS) + * TestConcoreHpp.exe (Windows) + */ + +#include +#include +#include +#include +#include +#include +#include + +#ifdef _WIN32 + #include + #define MAKE_DIR(p) _mkdir(p) +#else + #include + #define MAKE_DIR(p) mkdir(p, 0755) +#endif + +#include "concore.hpp" + +static int passed = 0; +static int failed = 0; + +// ------------- helpers --------------------------------------------------- + +static void check(const std::string& testName, bool condition) { + if (condition) { + std::cout << "PASS: " << testName << std::endl; + ++passed; + } else { + std::cout << "FAIL: " << testName << std::endl; + ++failed; + } +} + +static bool approx(double a, double b, double eps = 1e-9) { + return std::fabs(a - b) < eps; +} + +static void write_file(const std::string& path, const std::string& data) { + std::ofstream f(path); + f << data; +} + +static void rm(const std::string& path) { + std::remove(path.c_str()); +} + +static void setup_dirs() { + MAKE_DIR("in"); + MAKE_DIR("in/1"); + MAKE_DIR("in1"); + MAKE_DIR("out1"); +} + +// ------------- read_FM --------------------------------------------------- + +static void test_read_FM_file() { + setup_dirs(); + write_file("in1/v", "[3.0,0.5,1.5]"); + + Concore c; + c.delay = 0; + c.simtime = 0.0; + + std::vector v = c.read_FM(1, "v", "[0.0]"); + check("read_FM size==2", v.size() == 2); + check("read_FM[0]==0.5", approx(v[0], 0.5)); + check("read_FM[1]==1.5", approx(v[1], 1.5)); + check("read_FM simtime updated", approx(c.simtime, 3.0)); + + rm("in1/v"); +} + +static void test_read_FM_missing_file_uses_initstr() { + setup_dirs(); + rm("in1/no_port"); + + Concore c; + c.delay = 0; + c.simtime = 0.0; + + std::vector v = c.read_FM(1, "no_port", "[9.0,3.0]"); + check("missing_file fallback size==1", v.size() == 1); + check("missing_file fallback val==3.0", approx(v[0], 3.0)); +} + +// ------------- write_FM -------------------------------------------------- + +static void test_write_FM_creates_file() { + setup_dirs(); + + Concore c; + c.delay = 0; + c.simtime = 2.0; + + c.write_FM(1, "w_out", {10.0, 20.0}); + + std::ifstream f("out1/w_out"); + std::ostringstream ss; + ss << f.rdbuf(); + std::string content = ss.str(); + + check("write_FM file not empty", !content.empty()); + check("write_FM contains value 10", content.find("10") != std::string::npos); + check("write_FM contains simtime 2", content.find("2") != std::string::npos); + + rm("out1/w_out"); +} + +// ------------- unchanged() ----------------------------------------------- + +static void test_unchanged_after_read_is_false() { + setup_dirs(); + write_file("in1/uc", "[1.0,0.1]"); + + Concore c; + c.delay = 0; + c.simtime = 0.0; + + c.read_FM(1, "uc", "[0.0]"); + check("unchanged after read is false", !c.unchanged()); + + rm("in1/uc"); +} + +static void test_unchanged_fresh_object_is_true() { + Concore c; + c.delay = 0; + check("unchanged fresh object is true", c.unchanged()); +} + +static void test_unchanged_second_call_after_false_is_true() { + setup_dirs(); + write_file("in1/uc2", "[5.0,7.0]"); + + Concore c; + c.delay = 0; + c.simtime = 0.0; + + c.read_FM(1, "uc2", "[0.0]"); + c.unchanged(); + bool u = c.unchanged(); + check("unchanged second call is true", u); + + rm("in1/uc2"); +} + +// ------------- retry exhaustion on empty file ---------------------------- + +static void test_retry_empty_file_falls_back_to_initstr() { + setup_dirs(); + write_file("in1/empty_port", ""); + + Concore c; + c.delay = 0; + c.simtime = 0.0; + + std::vector v = c.read_FM(1, "empty_port", "[7.0,5.0]"); + check("retry exhaustion: initstr used", v.size() == 1 && approx(v[0], 5.0)); + check("retry exhaustion: retrycount>0", c.retrycount > 0); + + rm("in1/empty_port"); +} + +// ------------- initval() ------------------------------------------------- + +static void test_initval_parses_simtime_and_data() { + Concore c; + c.delay = 0; + + std::vector v = c.initval("[4.0,1.0,2.0]"); + check("initval size==2", v.size() == 2); + check("initval[0]==1.0", approx(v[0], 1.0)); + check("initval[1]==2.0", approx(v[1], 2.0)); + check("initval simtime==4.0", approx(c.simtime, 4.0)); +} + +static void test_initval_empty_input_returns_empty() { + Concore c; + c.delay = 0; + + std::vector v = c.initval("[]"); + check("initval empty returns empty", v.empty()); +} + +// ------------- mapParser() ----------------------------------------------- + +static void test_mapParser_reads_file() { + setup_dirs(); + write_file("tmp_iport_test.txt", "{'u': 1}"); + + Concore c; + c.delay = 0; + + auto m = c.mapParser("tmp_iport_test.txt"); + check("mapParser has key u", m.count("u") == 1); + check("mapParser value==1", m["u"] == 1); + + rm("tmp_iport_test.txt"); +} + +static void test_mapParser_missing_file_is_empty() { + rm("concore_noport_tmp.txt"); + + Concore c; + c.delay = 0; + + auto m = c.mapParser("concore_noport_tmp.txt"); + check("mapParser missing file is empty", m.empty()); +} + +// ------------- default_maxtime() ----------------------------------------- + +static void test_default_maxtime_reads_file() { + setup_dirs(); + write_file("in/1/concore.maxtime", "200"); + + Concore c; + c.delay = 0; + c.default_maxtime(100); + + check("default_maxtime from file==200", c.maxtime == 200); + + rm("in/1/concore.maxtime"); +} + +static void test_default_maxtime_fallback() { + rm("in/1/concore.maxtime"); + + Concore c; + c.delay = 0; + c.default_maxtime(42); + + check("default_maxtime fallback==42", c.maxtime == 42); +} + +// ------------- tryparam() ------------------------------------------------ + +static void test_tryparam_found() { + setup_dirs(); + write_file("in/1/concore.params", "{'alpha': '0.5', 'beta': '1.0'}"); + + Concore c; + c.delay = 0; + c.load_params(); + + check("tryparam found alpha", c.tryparam("alpha", "0.0") == "0.5"); + check("tryparam found beta", c.tryparam("beta", "0.0") == "1.0"); + + rm("in/1/concore.params"); +} + +static void test_tryparam_missing_key_uses_default() { + rm("in/1/concore.params"); + + Concore c; + c.delay = 0; + c.load_params(); + + check("tryparam missing key uses default", + c.tryparam("no_key", "def_val") == "def_val"); +} + +// ------------- main ------------------------------------------------------- + +int main() { + std::cout << "===== Concore API Tests (Issue #484) =====\n\n"; + + // read_FM / write_FM + test_read_FM_file(); + test_read_FM_missing_file_uses_initstr(); + test_write_FM_creates_file(); + + // unchanged() + test_unchanged_after_read_is_false(); + test_unchanged_fresh_object_is_true(); + test_unchanged_second_call_after_false_is_true(); + + // retry exhaustion on empty file + test_retry_empty_file_falls_back_to_initstr(); + + // initval() + test_initval_parses_simtime_and_data(); + test_initval_empty_input_returns_empty(); + + // mapParser() + test_mapParser_reads_file(); + test_mapParser_missing_file_is_empty(); + + // default_maxtime() + test_default_maxtime_reads_file(); + test_default_maxtime_fallback(); + + // tryparam() + test_tryparam_found(); + test_tryparam_missing_key_uses_default(); + + std::cout << "\n=== Results: " << passed << " passed, " << failed + << " failed out of " << (passed + failed) << " tests ===\n"; + + return (failed > 0) ? 1 : 0; +} From 7e8090eb33705f7f010359dc5eaf41339bca197a Mon Sep 17 00:00:00 2001 From: Avinash Kumar Deepak Date: Tue, 3 Mar 2026 23:08:43 +0530 Subject: [PATCH 234/275] feat: add shared memory transport to concoredocker.hpp (#486) --- concoredocker.hpp | 197 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 196 insertions(+), 1 deletion(-) diff --git a/concoredocker.hpp b/concoredocker.hpp index b4fa69b9..651e5b33 100644 --- a/concoredocker.hpp +++ b/concoredocker.hpp @@ -14,10 +14,25 @@ #include #include #include +#include + +#ifdef __linux__ +#include +#include +#include +#endif #include "concore_base.hpp" class Concore { +private: + int shmId_create = -1; + int shmId_get = -1; + char* sharedData_create = nullptr; + char* sharedData_get = nullptr; + int communication_iport = 0; // iport refers to input port + int communication_oport = 0; // oport refers to output port + public: std::unordered_map iport; std::unordered_map oport; @@ -55,6 +70,25 @@ class Concore { oport = safe_literal_eval("concore.oport", {}); default_maxtime(100); load_params(); + +#ifdef __linux__ + int iport_number = -1; + int oport_number = -1; + + if (!iport.empty()) + iport_number = ExtractNumeric(iport.begin()->first); + if (!oport.empty()) + oport_number = ExtractNumeric(oport.begin()->first); + + if (oport_number != -1) { + communication_oport = 1; + createSharedMemory(oport_number); + } + if (iport_number != -1) { + communication_iport = 1; + getSharedMemory(iport_number); + } +#endif } ~Concore() { @@ -62,6 +96,14 @@ class Concore { for (auto& kv : zmq_ports) delete kv.second; zmq_ports.clear(); +#endif +#ifdef __linux__ + if (communication_oport == 1 && sharedData_create != nullptr) + shmdt(sharedData_create); + if (communication_iport == 1 && sharedData_get != nullptr) + shmdt(sharedData_get); + if (shmId_create != -1) + shmctl(shmId_create, IPC_RMID, nullptr); #endif } @@ -74,11 +116,20 @@ class Concore { delay(other.delay), retrycount(other.retrycount), inpath(std::move(other.inpath)), outpath(std::move(other.outpath)), simtime(other.simtime), maxtime(other.maxtime), - params(std::move(other.params)) + params(std::move(other.params)), + shmId_create(other.shmId_create), shmId_get(other.shmId_get), + sharedData_create(other.sharedData_create), sharedData_get(other.sharedData_get), + communication_iport(other.communication_iport), communication_oport(other.communication_oport) { #ifdef CONCORE_USE_ZMQ zmq_ports = std::move(other.zmq_ports); #endif + other.shmId_create = -1; + other.shmId_get = -1; + other.sharedData_create = nullptr; + other.sharedData_get = nullptr; + other.communication_iport = 0; + other.communication_oport = 0; } Concore& operator=(Concore&& other) noexcept @@ -91,6 +142,14 @@ class Concore { delete kv.second; zmq_ports = std::move(other.zmq_ports); #endif +#ifdef __linux__ + if (communication_oport == 1 && sharedData_create != nullptr) + shmdt(sharedData_create); + if (communication_iport == 1 && sharedData_get != nullptr) + shmdt(sharedData_get); + if (shmId_create != -1) + shmctl(shmId_create, IPC_RMID, nullptr); +#endif iport = std::move(other.iport); oport = std::move(other.oport); @@ -103,6 +162,19 @@ class Concore { simtime = other.simtime; maxtime = other.maxtime; params = std::move(other.params); + shmId_create = other.shmId_create; + shmId_get = other.shmId_get; + sharedData_create = other.sharedData_create; + sharedData_get = other.sharedData_get; + communication_iport = other.communication_iport; + communication_oport = other.communication_oport; + + other.shmId_create = -1; + other.shmId_get = -1; + other.sharedData_create = nullptr; + other.sharedData_get = nullptr; + other.communication_iport = 0; + other.communication_oport = 0; return *this; } @@ -131,6 +203,57 @@ class Concore { inpath + "/1/concore.maxtime", defaultValue); } + key_t ExtractNumeric(const std::string& str) { + std::string numberString; + size_t numDigits = 0; + while (numDigits < str.length() && std::isdigit(str[numDigits])) { + numberString += str[numDigits]; + ++numDigits; + } + if (numDigits == 0) + return -1; + if (numDigits == 1 && std::stoi(numberString) <= 0) + return -1; + return std::stoi(numberString); + } + +#ifdef __linux__ + void createSharedMemory(key_t key) { + shmId_create = shmget(key, 256, IPC_CREAT | 0666); + if (shmId_create == -1) { + std::cerr << "Failed to create shared memory segment.\n"; + return; + } + sharedData_create = static_cast(shmat(shmId_create, NULL, 0)); + if (sharedData_create == reinterpret_cast(-1)) { + std::cerr << "Failed to attach shared memory segment.\n"; + sharedData_create = nullptr; + } + } + + void getSharedMemory(key_t key) { + int retry = 0; + const int MAX_RETRY = 100; + while (retry < MAX_RETRY) { + shmId_get = shmget(key, 256, 0666); + if (shmId_get != -1) + break; + std::cout << "Shared memory does not exist. Make sure the writer process is running.\n"; + sleep(1); + retry++; + } + if (shmId_get == -1) { + std::cerr << "Failed to get shared memory segment after max retries.\n"; + return; + } + sharedData_get = static_cast(shmat(shmId_get, NULL, 0)); + if (sharedData_get == reinterpret_cast(-1)) { + std::cerr << "Failed to attach shared memory segment.\n"; + sharedData_get = nullptr; + } + } +#endif + bool unchanged() { if (olds == s) { s.clear(); @@ -141,6 +264,10 @@ class Concore { } std::vector read(int port, const std::string& name, const std::string& initstr) { +#ifdef __linux__ + if (communication_iport == 1) + return read_SM(port, name, initstr); +#endif std::this_thread::sleep_for(std::chrono::seconds(delay)); std::string file_path = inpath + "/" + std::to_string(port) + "/" + name; std::ifstream infile(file_path); @@ -178,7 +305,56 @@ class Concore { return inval; } +#ifdef __linux__ + std::vector read_SM(int port, const std::string& name, const std::string& initstr) { + std::this_thread::sleep_for(std::chrono::seconds(delay)); + std::string ins; + try { + if (shmId_get != -1 && sharedData_get && sharedData_get[0] != '\0') + ins = std::string(sharedData_get, strnlen(sharedData_get, 256)); + else + throw 505; + } catch (...) { + ins = initstr; + } + + int retry = 0; + const int MAX_RETRY = 100; + while ((int)ins.length() == 0 && retry < MAX_RETRY) { + std::this_thread::sleep_for(std::chrono::seconds(delay)); + try { + if (shmId_get != -1 && sharedData_get) { + ins = std::string(sharedData_get, strnlen(sharedData_get, 256)); + retrycount++; + } else { + retrycount++; + throw 505; + } + } catch (...) { + std::cerr << "Read error\n"; + } + retry++; + } + + s += ins; + std::vector inval = concore_base::parselist_double(ins); + if (inval.empty()) + inval = concore_base::parselist_double(initstr); + if (inval.empty()) + return inval; + simtime = simtime > inval[0] ? simtime : inval[0]; + inval.erase(inval.begin()); + return inval; + } +#endif + void write(int port, const std::string& name, const std::vector& val, int delta = 0) { +#ifdef __linux__ + if (communication_oport == 1) { + write_SM(port, name, val, delta); + return; + } +#endif std::string file_path = outpath + "/" + std::to_string(port) + "/" + name; std::ofstream outfile(file_path); if (!outfile) { @@ -195,6 +371,25 @@ class Concore { } } +#ifdef __linux__ + void write_SM(int port, const std::string& name, std::vector val, int delta = 0) { + try { + if (shmId_create == -1) + throw 505; + val.insert(val.begin(), simtime + delta); + std::ostringstream outfile; + outfile << '['; + for (size_t i = 0; i < val.size() - 1; i++) + outfile << val[i] << ','; + outfile << val[val.size() - 1] << ']'; + std::string result = outfile.str(); + std::strncpy(sharedData_create, result.c_str(), 256 - 1); + } catch (...) { + std::cerr << "skipping +" << outpath << port << "/" << name << "\n"; + } + } +#endif + #ifdef CONCORE_USE_ZMQ void init_zmq_port(const std::string& port_name, const std::string& port_type, const std::string& address, const std::string& socket_type_str) { From 30ecc5e40a441063896f98bddba203f0109cd0b8 Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Wed, 4 Mar 2026 17:44:20 +0530 Subject: [PATCH 235/275] fix: raise TimeoutError on ZMQ retry exhaustion (#393) send_json_with_retry() and recv_json_with_retry() now raise TimeoutError instead of silently returning None when all 5 retries are exhausted. read() and write() catch the new exception so callers never see an unhandled crash. - send_json_with_retry: raise TimeoutError (was: return) - recv_json_with_retry: raise TimeoutError (was: return None) - read(): except TimeoutError -> (default, False), status TIMEOUT - read(): remove dead 'message is None' guard (now unreachable) - write(): except TimeoutError -> log and continue - tests: save/restore global state in try/finally to prevent leakage - test_read_status: use TimeoutError instead of response=None Tests added in test_concore.py and test_concoredocker.py. All 75 tests pass. Closes #393 --- concore_base.py | 15 +++-- tests/test_concore.py | 129 ++++++++++++++++++++++++++++++++++++ tests/test_concoredocker.py | 91 +++++++++++++++++++++++++ tests/test_read_status.py | 4 +- 4 files changed, 231 insertions(+), 8 deletions(-) diff --git a/concore_base.py b/concore_base.py index 35f2c349..9173289b 100644 --- a/concore_base.py +++ b/concore_base.py @@ -59,8 +59,7 @@ def send_json_with_retry(self, message): except zmq.Again: logger.warning(f"Send timeout (attempt {attempt + 1}/5)") time.sleep(0.5) - logger.error("Failed to send after retries.") - return + raise TimeoutError(f"ZMQ send failed after 5 retries on {self.address}") def recv_json_with_retry(self): """Receive JSON message with retries if timeout occurs.""" @@ -70,8 +69,7 @@ def recv_json_with_retry(self): except zmq.Again: logger.warning(f"Receive timeout (attempt {attempt + 1}/5)") time.sleep(0.5) - logger.error("Failed to receive after retries.") - return None + raise TimeoutError(f"ZMQ recv failed after 5 retries on {self.address}") def init_zmq_port(mod, port_name, port_type, address, socket_type_str): @@ -270,9 +268,6 @@ def read(mod, port_identifier, name, initstr_val): zmq_p = mod.zmq_ports[port_identifier] try: message = zmq_p.recv_json_with_retry() - if message is None: - last_read_status = "TIMEOUT" - return default_return_val, False # Strip simtime prefix if present (mirroring file-based read behavior) if isinstance(message, list) and len(message) > 0: first_element = message[0] @@ -282,6 +277,10 @@ def read(mod, port_identifier, name, initstr_val): return message[1:], True last_read_status = "SUCCESS" return message, True + except TimeoutError as e: + logger.error(f"ZMQ read timeout on port {port_identifier} (name: {name}): {e}. Returning default.") + last_read_status = "TIMEOUT" + return default_return_val, False except zmq.error.ZMQError as e: logger.error(f"ZMQ read error on port {port_identifier} (name: {name}): {e}. Returning default.") last_read_status = "TIMEOUT" @@ -384,6 +383,8 @@ def write(mod, port_identifier, name, val, delta=0): # Mutation breaks cross-language determinism (see issue #385). else: zmq_p.send_json_with_retry(zmq_val) + except TimeoutError as e: + logger.error(f"ZMQ write timeout on port {port_identifier} (name: {name}): {e}") except zmq.error.ZMQError as e: logger.error(f"ZMQ write error on port {port_identifier} (name: {name}): {e}") except Exception as e: diff --git a/tests/test_concore.py b/tests/test_concore.py index b1a980e6..fb67054f 100644 --- a/tests/test_concore.py +++ b/tests/test_concore.py @@ -1,6 +1,7 @@ import pytest import os import numpy as np +from unittest.mock import patch class TestSafeLiteralEval: @@ -450,3 +451,131 @@ def test_write_timestamp_matches_cpp_semantics(self, temp_dir): "After 3 writes with delta=1 simtime must remain 0 " "(matching C++/MATLAB/Verilog); got %s" % concore.simtime ) + + +class TestZMQRetryExhaustion: + """Issue #393 – recv_json_with_retry / send_json_with_retry must raise + TimeoutError instead of silently returning None when retries are + exhausted, and callers (read / write) must handle it gracefully.""" + + @patch("concore_base.time.sleep") + def test_recv_raises_timeout_error(self, _mock_sleep): + """recv_json_with_retry must raise TimeoutError after 5 failed attempts.""" + import concore_base + + port = concore_base.ZeroMQPort.__new__(concore_base.ZeroMQPort) + port.address = "tcp://127.0.0.1:9999" + + class FakeSocket: + def recv_json(self, flags=0): + import zmq + + raise zmq.Again("Resource temporarily unavailable") + + port.socket = FakeSocket() + with pytest.raises(TimeoutError): + port.recv_json_with_retry() + + @patch("concore_base.time.sleep") + def test_send_raises_timeout_error(self, _mock_sleep): + """send_json_with_retry must raise TimeoutError after 5 failed attempts.""" + import concore_base + + port = concore_base.ZeroMQPort.__new__(concore_base.ZeroMQPort) + port.address = "tcp://127.0.0.1:9999" + + class FakeSocket: + def send_json(self, data, flags=0): + import zmq + + raise zmq.Again("Resource temporarily unavailable") + + port.socket = FakeSocket() + with pytest.raises(TimeoutError): + port.send_json_with_retry([42]) + + def test_read_returns_default_on_timeout(self, temp_dir): + """read() must return (default, False) when ZMQ recv times out.""" + import concore + import concore_base + + # Save original global state to avoid leaking into other tests. + had_inpath = hasattr(concore, "inpath") + orig_inpath = concore.inpath if had_inpath else None + orig_zmq_ports = dict(concore.zmq_ports) + had_simtime = hasattr(concore, "simtime") + orig_simtime = concore.simtime if had_simtime else None + + try: + concore.inpath = os.path.join(temp_dir, "in") + + class TimeoutPort: + address = "tcp://127.0.0.1:0" + socket = None + + def recv_json_with_retry(self): + raise TimeoutError("ZMQ recv failed after 5 retries") + + concore.zmq_ports["t_in"] = TimeoutPort() + concore.simtime = 0 + + result, ok = concore.read("t_in", "x", "[0.0]") + + assert result == [0.0] + assert ok is False + assert concore_base.last_read_status == "TIMEOUT" + finally: + # Restore zmq_ports and other globals to their original state. + concore.zmq_ports.clear() + concore.zmq_ports.update(orig_zmq_ports) + + if had_inpath: + concore.inpath = orig_inpath + elif hasattr(concore, "inpath"): + delattr(concore, "inpath") + + if had_simtime: + concore.simtime = orig_simtime + elif hasattr(concore, "simtime"): + delattr(concore, "simtime") + + def test_write_does_not_crash_on_timeout(self, temp_dir): + """write() must not propagate TimeoutError to the caller.""" + import concore + + # Save original global state to avoid leaking into other tests. + had_outpath = hasattr(concore, "outpath") + orig_outpath = concore.outpath if had_outpath else None + orig_zmq_ports = dict(concore.zmq_ports) + had_simtime = hasattr(concore, "simtime") + orig_simtime = concore.simtime if had_simtime else None + + try: + concore.outpath = os.path.join(temp_dir, "out") + os.makedirs(os.path.join(temp_dir, "out_t_out"), exist_ok=True) + + class TimeoutPort: + address = "tcp://127.0.0.1:0" + socket = None + + def send_json_with_retry(self, message): + raise TimeoutError("ZMQ send failed after 5 retries") + + concore.zmq_ports["t_out"] = TimeoutPort() + concore.simtime = 0 + + concore.write("t_out", "y", [1.0], delta=1) + finally: + # Restore zmq_ports and other globals to their original state. + concore.zmq_ports.clear() + concore.zmq_ports.update(orig_zmq_ports) + + if had_outpath: + concore.outpath = orig_outpath + elif hasattr(concore, "outpath"): + delattr(concore, "outpath") + + if had_simtime: + concore.simtime = orig_simtime + elif hasattr(concore, "simtime"): + delattr(concore, "simtime") diff --git a/tests/test_concoredocker.py b/tests/test_concoredocker.py index 40d68082..fe19dd1b 100644 --- a/tests/test_concoredocker.py +++ b/tests/test_concoredocker.py @@ -247,3 +247,94 @@ def recv_json_with_retry(self): assert result == original assert ok is True + + +class TestZMQRetryExhaustion: + """Issue #393 – concoredocker read/write must handle TimeoutError + raised by ZMQ retry exhaustion without crashing.""" + + def test_read_returns_default_on_timeout(self, temp_dir): + """read() must return (default, False) when ZMQ recv times out.""" + import concoredocker + import concore_base + + # Save original global state to avoid leaking into other tests. + had_inpath = hasattr(concoredocker, "inpath") + orig_inpath = concoredocker.inpath if had_inpath else None + orig_zmq_ports = dict(concoredocker.zmq_ports) + had_simtime = hasattr(concoredocker, "simtime") + orig_simtime = concoredocker.simtime if had_simtime else None + + try: + concoredocker.inpath = os.path.join(temp_dir, "in") + + class TimeoutPort: + address = "tcp://127.0.0.1:0" + socket = None + + def recv_json_with_retry(self): + raise TimeoutError("ZMQ recv failed after 5 retries") + + concoredocker.zmq_ports["t_in"] = TimeoutPort() + concoredocker.simtime = 0 + + result, ok = concoredocker.read("t_in", "x", "[0.0]") + + assert result == [0.0] + assert ok is False + assert concore_base.last_read_status == "TIMEOUT" + finally: + # Restore zmq_ports and other globals to their original state. + concoredocker.zmq_ports.clear() + concoredocker.zmq_ports.update(orig_zmq_ports) + + if had_inpath: + concoredocker.inpath = orig_inpath + elif hasattr(concoredocker, "inpath"): + delattr(concoredocker, "inpath") + + if had_simtime: + concoredocker.simtime = orig_simtime + elif hasattr(concoredocker, "simtime"): + delattr(concoredocker, "simtime") + + def test_write_does_not_crash_on_timeout(self, temp_dir): + """write() must not propagate TimeoutError to the caller.""" + import concoredocker + + # Save original global state to avoid leaking into other tests. + had_outpath = hasattr(concoredocker, "outpath") + orig_outpath = concoredocker.outpath if had_outpath else None + orig_zmq_ports = dict(concoredocker.zmq_ports) + had_simtime = hasattr(concoredocker, "simtime") + orig_simtime = concoredocker.simtime if had_simtime else None + + try: + concoredocker.outpath = os.path.join(temp_dir, "out") + os.makedirs(os.path.join(temp_dir, "out_t_out"), exist_ok=True) + + class TimeoutPort: + address = "tcp://127.0.0.1:0" + socket = None + + def send_json_with_retry(self, message): + raise TimeoutError("ZMQ send failed after 5 retries") + + concoredocker.zmq_ports["t_out"] = TimeoutPort() + concoredocker.simtime = 0 + + concoredocker.write("t_out", "y", [1.0], delta=1) + finally: + # Restore zmq_ports and other globals to their original state. + concoredocker.zmq_ports.clear() + concoredocker.zmq_ports.update(orig_zmq_ports) + + if had_outpath: + concoredocker.outpath = orig_outpath + elif hasattr(concoredocker, "outpath"): + delattr(concoredocker, "outpath") + + if had_simtime: + concoredocker.simtime = orig_simtime + elif hasattr(concoredocker, "simtime"): + delattr(concoredocker, "simtime") diff --git a/tests/test_read_status.py b/tests/test_read_status.py index 29e64340..b9fe33d9 100644 --- a/tests/test_read_status.py +++ b/tests/test_read_status.py @@ -179,7 +179,9 @@ def setup(self, monkeypatch): concore.zmq_ports.update(self.original_ports) def test_zmq_timeout_returns_default_and_false(self): - dummy = DummyZMQPort(response=None) # recv returns None → timeout + dummy = DummyZMQPort( + raise_on_recv=TimeoutError("ZMQ recv failed after 5 retries") + ) self.concore.zmq_ports["test_port"] = dummy data, ok = self.concore.read("test_port", "ym", "[]") From 9c900b00b02b8eb9865ece8769243a33cefaa8dd Mon Sep 17 00:00:00 2001 From: Avinash Kumar Deepak Date: Thu, 5 Mar 2026 14:08:52 +0530 Subject: [PATCH 236/275] test: add TestConcoredockerApi covering read/write/unchanged/initVal closes #464 --- TestConcoredockerApi.java | 198 ++++++++++++++++++++++++++++++++++++++ concoredocker.java | 7 ++ 2 files changed, 205 insertions(+) create mode 100644 TestConcoredockerApi.java diff --git a/TestConcoredockerApi.java b/TestConcoredockerApi.java new file mode 100644 index 00000000..21934441 --- /dev/null +++ b/TestConcoredockerApi.java @@ -0,0 +1,198 @@ +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.*; + +/** + * Tests for concoredocker read(), write(), unchanged(), initVal() + * using temp directories for file-based IPC. + */ +public class TestConcoredockerApi { + static int passed = 0; + static int failed = 0; + + public static void main(String[] args) { + // zero delay so tests don't sleep for 1s per read() + concoredocker.setDelay(0); + + testWriteProducesCorrectFormat(); + testReadParsesFileAndStripsSimtime(); + testReadWriteRoundtrip(); + testSimtimeAdvancesWithDelta(); + testUnchangedReturnsFalseAfterRead(); + testUnchangedReturnsTrueOnSameData(); + testInitValExtractsSimtime(); + testInitValReturnsRemainingValues(); + testOutputFileMatchesPythonWireFormat(); + + System.out.println("\n=== Results: " + passed + " passed, " + failed + " failed out of " + (passed + failed) + " tests ==="); + if (failed > 0) { + System.exit(1); + } + } + + static void check(String testName, Object expected, Object actual) { + if (Objects.equals(expected, actual)) { + System.out.println("PASS: " + testName); + passed++; + } else { + System.out.println("FAIL: " + testName + " | expected: " + expected + " | actual: " + actual); + failed++; + } + } + + static Path makeTempDir() { + try { + return Files.createTempDirectory("concore_test_"); + } catch (IOException e) { + throw new RuntimeException("Failed to create temp dir", e); + } + } + + /** Creates temp dir with port subdirectory ready for write(). */ + static Path makeTempDir(int port) { + Path tmp = makeTempDir(); + try { + Files.createDirectories(tmp.resolve(String.valueOf(port))); + } catch (IOException e) { + throw new RuntimeException(e); + } + return tmp; + } + + static void writeFile(Path base, int port, String name, String content) { + try { + Path dir = base.resolve(String.valueOf(port)); + Files.createDirectories(dir); + Files.write(dir.resolve(name), content.getBytes()); + } catch (IOException e) { + throw new RuntimeException("Failed to write test file", e); + } + } + + static String readFile(Path base, int port, String name) { + try { + return new String(Files.readAllBytes(base.resolve(String.valueOf(port)).resolve(name))); + } catch (IOException e) { + throw new RuntimeException("Failed to read test file", e); + } + } + + static void testWriteProducesCorrectFormat() { + Path tmp = makeTempDir(1); + concoredocker.resetState(); + concoredocker.setOutPath(tmp.toString()); + + List vals = new ArrayList<>(); + vals.add(10.0); + vals.add(20.0); + concoredocker.write(1, "signal", vals, 1); + + String content = readFile(tmp, 1, "signal"); + @SuppressWarnings("unchecked") + List parsed = (List) concoredocker.literalEval(content); + check("write: simtime+delta as first element", 1.0, parsed.get(0)); + check("write: val1 correct", 10.0, parsed.get(1)); + check("write: val2 correct", 20.0, parsed.get(2)); + } + + static void testReadParsesFileAndStripsSimtime() { + Path tmp = makeTempDir(); + concoredocker.resetState(); + concoredocker.setInPath(tmp.toString()); + writeFile(tmp, 1, "sensor", "[0.0, 42.0, 99.0]"); + + List result = concoredocker.read(1, "sensor", "[0.0, 0.0, 0.0]"); + check("read: strips simtime, size=2", 2, result.size()); + check("read: val1 correct", 42.0, result.get(0)); + check("read: val2 correct", 99.0, result.get(1)); + } + + static void testReadWriteRoundtrip() { + Path tmp = makeTempDir(1); + concoredocker.resetState(); + concoredocker.setInPath(tmp.toString()); + concoredocker.setOutPath(tmp.toString()); + + List outVals = new ArrayList<>(); + outVals.add(7.0); + outVals.add(8.0); + concoredocker.write(1, "data", outVals, 1); + + List inVals = concoredocker.read(1, "data", "[0.0, 0.0, 0.0]"); + check("roundtrip: size", 2, inVals.size()); + check("roundtrip: val1", 7.0, inVals.get(0)); + check("roundtrip: val2", 8.0, inVals.get(1)); + } + + static void testSimtimeAdvancesWithDelta() { + Path tmp = makeTempDir(1); + concoredocker.resetState(); + concoredocker.setInPath(tmp.toString()); + concoredocker.setOutPath(tmp.toString()); + + List v = Collections.singletonList((Object) 1.0); + + // iteration 1: simtime=0, delta=1 -> file has [1.0, 1.0], read -> simtime becomes 1.0 + concoredocker.write(1, "tick", v, 1); + concoredocker.read(1, "tick", "[0.0, 0.0]"); + check("simtime after iter 1", 1.0, concoredocker.getSimtime()); + + // iteration 2: write again with delta=1 -> file has [2.0, 1.0], read -> simtime becomes 2.0 + concoredocker.write(1, "tick", v, 1); + concoredocker.read(1, "tick", "[0.0, 0.0]"); + check("simtime after iter 2", 2.0, concoredocker.getSimtime()); + } + + static void testUnchangedReturnsFalseAfterRead() { + Path tmp = makeTempDir(); + concoredocker.resetState(); + concoredocker.setInPath(tmp.toString()); + writeFile(tmp, 1, "sig", "[0.0, 5.0]"); + + concoredocker.read(1, "sig", "[0.0, 0.0]"); + check("unchanged: false right after read", false, concoredocker.unchanged()); + } + + static void testUnchangedReturnsTrueOnSameData() { + Path tmp = makeTempDir(); + concoredocker.resetState(); + concoredocker.setInPath(tmp.toString()); + writeFile(tmp, 1, "sig", "[0.0, 5.0]"); + + concoredocker.read(1, "sig", "[0.0, 0.0]"); + concoredocker.unchanged(); // first call: false, locks olds = s + check("unchanged: true on second call with same data", true, concoredocker.unchanged()); + } + + static void testInitValExtractsSimtime() { + concoredocker.resetState(); + concoredocker.initVal("[2.0, 10, 20]"); + check("initVal: simtime extracted", 2.0, concoredocker.getSimtime()); + } + + static void testInitValReturnsRemainingValues() { + concoredocker.resetState(); + List result = concoredocker.initVal("[3.5, 100, 200]"); + check("initVal: size of returned list", 2, result.size()); + check("initVal: first remaining val", 100, result.get(0)); + check("initVal: second remaining val", 200, result.get(1)); + } + + static void testOutputFileMatchesPythonWireFormat() { + Path tmp = makeTempDir(1); + concoredocker.resetState(); + concoredocker.setOutPath(tmp.toString()); + + List vals = new ArrayList<>(); + vals.add(1.0); + vals.add(2.0); + concoredocker.write(1, "out", vals, 0); + + String raw = readFile(tmp, 1, "out"); + check("wire format: starts with '['", true, raw.startsWith("[")); + check("wire format: ends with ']'", true, raw.endsWith("]")); + Object reparsed = concoredocker.literalEval(raw); + check("wire format: re-parseable as list", true, reparsed instanceof List); + } +} diff --git a/concoredocker.java b/concoredocker.java index 226865f4..1e66bcad 100644 --- a/concoredocker.java +++ b/concoredocker.java @@ -130,6 +130,13 @@ public static void defaultMaxTime(double defaultValue) { } } + // package-level helpers for testing with temp directories + static void setInPath(String path) { inpath = path; } + static void setOutPath(String path) { outpath = path; } + static void setDelay(int ms) { delay = ms; } + static double getSimtime() { return simtime; } + static void resetState() { s = ""; olds = ""; simtime = 0; } + public static boolean unchanged() { if (olds.equals(s)) { s = ""; From 4534bf2bc576a5747d41cd317a4d6bb7fe049e42 Mon Sep 17 00:00:00 2001 From: Avinash Kumar Deepak Date: Fri, 6 Mar 2026 23:00:30 +0530 Subject: [PATCH 237/275] fix(zmq): use JSON on wire, accept JSON keywords in parser (#492) --- TestLiteralEval.java | 43 +++++++++++++++++++++++++++++ concoredocker.java | 64 ++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 102 insertions(+), 5 deletions(-) diff --git a/TestLiteralEval.java b/TestLiteralEval.java index 97d80f3c..5e36d325 100644 --- a/TestLiteralEval.java +++ b/TestLiteralEval.java @@ -37,6 +37,11 @@ public static void main(String[] args) { testUnterminatedDict(); testUnterminatedTuple(); testNonStringDictKey(); + testJsonKeywordTrue(); + testJsonKeywordFalse(); + testJsonKeywordNull(); + testJsonMixedList(); + testJsonRoundTrip(); System.out.println("\n=== Results: " + passed + " passed, " + failed + " failed out of " + (passed + failed) + " tests ==="); if (failed > 0) { @@ -267,4 +272,42 @@ static void testNonStringDictKey() { check("numeric dict key throws", true, e.getMessage().contains("Dict keys must be non-null strings")); } } + + // --- JSON keyword interop tests --- + + static void testJsonKeywordTrue() { + Object result = concoredocker.literalEval("true"); + check("json true -> Boolean.TRUE", Boolean.TRUE, result); + } + + static void testJsonKeywordFalse() { + Object result = concoredocker.literalEval("false"); + check("json false -> Boolean.FALSE", Boolean.FALSE, result); + } + + static void testJsonKeywordNull() { + Object result = concoredocker.literalEval("null"); + check("json null -> null", null, result); + } + + static void testJsonMixedList() { + // Python sends [0.0, true, null] via json.dumps — Java must parse it + @SuppressWarnings("unchecked") + List result = (List) concoredocker.literalEval("[0.0, true, null]"); + check("json list[0] simtime", 0.0, result.get(0)); + check("json list[1] true", Boolean.TRUE, result.get(1)); + check("json list[2] null", null, result.get(2)); + } + + static void testJsonRoundTrip() { + // JSON-style payload: true/false/null and double-quoted strings + String jsonPayload = "[0.0, 1.5, true, false, null, \"hello\"]"; + @SuppressWarnings("unchecked") + List result = (List) concoredocker.literalEval(jsonPayload); + check("json round-trip double", 1.5, result.get(1)); + check("json round-trip true", Boolean.TRUE, result.get(2)); + check("json round-trip false", Boolean.FALSE, result.get(3)); + check("json round-trip null", null, result.get(4)); + check("json round-trip string", "hello", result.get(5)); + } } diff --git a/concoredocker.java b/concoredocker.java index 1e66bcad..80baf5b8 100644 --- a/concoredocker.java +++ b/concoredocker.java @@ -281,6 +281,60 @@ private static String toPythonLiteral(Object obj) { return obj.toString(); } + /** + * Escapes a Java string so it can be safely embedded in a JSON double-quoted string. + * Escapes backslash, double quote, newline, carriage return, and tab. + */ + private static String escapeJsonString(String s) { + StringBuilder sb = new StringBuilder(s.length()); + for (int i = 0; i < s.length(); i++) { + char c = s.charAt(i); + switch (c) { + case '\\': sb.append("\\\\"); break; + case '"': sb.append("\\\""); break; + case '\n': sb.append("\\n"); break; + case '\r': sb.append("\\r"); break; + case '\t': sb.append("\\t"); break; + default: sb.append(c); break; + } + } + return sb.toString(); + } + + /** + * Converts a Java object to its JSON string representation. + * true/false/null instead of True/False/None; strings double-quoted. + */ + private static String toJsonLiteral(Object obj) { + if (obj == null) return "null"; + if (obj instanceof Boolean) return ((Boolean) obj) ? "true" : "false"; + if (obj instanceof String) return "\"" + escapeJsonString((String) obj) + "\""; + if (obj instanceof Number) return obj.toString(); + if (obj instanceof List) { + List list = (List) obj; + StringBuilder sb = new StringBuilder("["); + for (int i = 0; i < list.size(); i++) { + if (i > 0) sb.append(", "); + sb.append(toJsonLiteral(list.get(i))); + } + sb.append("]"); + return sb.toString(); + } + if (obj instanceof Map) { + Map map = (Map) obj; + StringBuilder sb = new StringBuilder("{"); + boolean first = true; + for (Map.Entry entry : map.entrySet()) { + if (!first) sb.append(", "); + sb.append(toJsonLiteral(entry.getKey())).append(": ").append(toJsonLiteral(entry.getValue())); + first = false; + } + sb.append("}"); + return sb.toString(); + } + return obj.toString(); + } + /** * Writes data to a port file. * Prepends simtime+delta to the value list, then serializes to Python-literal format. @@ -437,10 +491,10 @@ public static void write(String portName, String name, Object val, int delta) { if (val instanceof List) { List listVal = (List) val; StringBuilder sb = new StringBuilder("["); - sb.append(toPythonLiteral(simtime + delta)); + sb.append(toJsonLiteral(simtime + delta)); for (Object o : listVal) { sb.append(", "); - sb.append(toPythonLiteral(o)); + sb.append(toJsonLiteral(o)); } sb.append("]"); payload = sb.toString(); @@ -750,9 +804,9 @@ Object parseKeyword() { } String word = input.substring(start, pos); switch (word) { - case "True": return Boolean.TRUE; - case "False": return Boolean.FALSE; - case "None": return null; + case "True": case "true": return Boolean.TRUE; + case "False": case "false": return Boolean.FALSE; + case "None": case "null": return null; default: throw new IllegalArgumentException("Unknown keyword: '" + word + "' at position " + start); } } From 484593e476423b1174cd2e3001740d1ce7cdb420 Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Wed, 4 Mar 2026 18:07:40 +0530 Subject: [PATCH 238/275] fix: replace concorekill.bat single-PID overwrite with safe PID registry (#391) On Windows, concore.py overwrote concorekill.bat at import time with a single PID, creating race conditions when multiple nodes launched simultaneously and leaving stale PIDs after crashes. Replace with append-based PID registry (concorekill_pids.txt): - _register_pid(): appends current PID (append mode, no overwrite) - _cleanup_pid(): removes current PID on exit with file locking - _write_kill_script(): generates concorekill.bat that validates each PID via tasklist before issuing taskkill Users still run concorekill.bat as before (backward compatible). --- concore.py | 79 ++++++++++++++++++++++-- tests/test_concore.py | 140 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 213 insertions(+), 6 deletions(-) diff --git a/concore.py b/concore.py index 2147e758..865c42b5 100644 --- a/concore.py +++ b/concore.py @@ -21,13 +21,80 @@ logging.getLogger('requests').setLevel(logging.WARNING) -# if windows, create script to kill this process -# because batch files don't provide easy way to know pid of last command -# ignored for posix != windows, because "concorepid" is handled by script -# ignored for docker (linux != windows), because handled by docker stop +# if windows, register this process PID for safe termination +# Previous approach: single "concorekill.bat" overwritten by each node (race condition). +# New approach: append PID to shared registry; generate validated kill script. +# See: https://github.com/ControlCore-Project/concore/issues/391 + +_PID_REGISTRY_FILE = "concorekill_pids.txt" +_KILL_SCRIPT_FILE = "concorekill.bat" + +def _register_pid(): + """Append current PID to the shared registry file.""" + try: + with open(_PID_REGISTRY_FILE, "a") as f: + f.write(str(os.getpid()) + "\n") + except OSError: + pass + +def _cleanup_pid(): + """Remove current PID from registry on exit. Uses file locking on Windows.""" + pid = str(os.getpid()) + try: + if not os.path.exists(_PID_REGISTRY_FILE): + return + with open(_PID_REGISTRY_FILE, "r+") as f: + if hasattr(sys, 'getwindowsversion'): + import msvcrt + msvcrt.locking(f.fileno(), msvcrt.LK_LOCK, 1) + pids = [line.strip() for line in f if line.strip()] + remaining = [p for p in pids if p != pid] + if remaining: + f.seek(0) + f.truncate() + for p in remaining: + f.write(p + "\n") + else: + f.close() + try: + os.remove(_PID_REGISTRY_FILE) + except OSError: + pass + try: + os.remove(_KILL_SCRIPT_FILE) + except OSError: + pass + except OSError: + pass + +def _write_kill_script(): + """Generate concorekill.bat that validates each PID before killing.""" + try: + script = "@echo off\r\n" + script += 'if not exist "%~dp0' + _PID_REGISTRY_FILE + '" (\r\n' + script += " echo No PID registry found. Nothing to kill.\r\n" + script += " exit /b 0\r\n" + script += ")\r\n" + script += 'for /f "usebackq tokens=*" %%p in ("%~dp0' + _PID_REGISTRY_FILE + '") do (\r\n' + script += ' tasklist /FI "PID eq %%p" 2>nul | find /i "python" >nul\r\n' + script += " if not errorlevel 1 (\r\n" + script += " echo Killing Python process %%p\r\n" + script += " taskkill /F /PID %%p >nul 2>&1\r\n" + script += " ) else (\r\n" + script += " echo Skipping PID %%p - not a Python process or not running\r\n" + script += " )\r\n" + script += ")\r\n" + script += 'del /q "%~dp0' + _PID_REGISTRY_FILE + '" 2>nul\r\n' + script += 'del /q "%~dp0' + _KILL_SCRIPT_FILE + '" 2>nul\r\n' + with open(_KILL_SCRIPT_FILE, "w", newline="") as f: + f.write(script) + except OSError: + pass + if hasattr(sys, 'getwindowsversion'): - with open("concorekill.bat","w") as fpid: - fpid.write("taskkill /F /PID "+str(os.getpid())+"\n") + _register_pid() + _write_kill_script() + atexit.register(_cleanup_pid) ZeroMQPort = concore_base.ZeroMQPort convert_numpy_to_python = concore_base.convert_numpy_to_python diff --git a/tests/test_concore.py b/tests/test_concore.py index fb67054f..dc98ced5 100644 --- a/tests/test_concore.py +++ b/tests/test_concore.py @@ -1,5 +1,6 @@ import pytest import os +import sys import numpy as np from unittest.mock import patch @@ -579,3 +580,142 @@ def send_json_with_retry(self, message): concore.simtime = orig_simtime elif hasattr(concore, "simtime"): delattr(concore, "simtime") + + +class TestPidRegistry: + """Tests for the Windows PID registry mechanism (Issue #391).""" + + @pytest.fixture(autouse=True) + def use_temp_dir(self, temp_dir, monkeypatch): + self.temp_dir = temp_dir + monkeypatch.chdir(temp_dir) + import concore + + monkeypatch.setattr(concore, "_BASE_DIR", temp_dir) + monkeypatch.setattr( + concore, + "_PID_REGISTRY_FILE", + os.path.join(temp_dir, "concorekill_pids.txt"), + ) + monkeypatch.setattr( + concore, + "_KILL_SCRIPT_FILE", + os.path.join(temp_dir, "concorekill.bat"), + ) + + def test_register_pid_creates_registry_file(self): + from concore import _register_pid, _PID_REGISTRY_FILE + + _register_pid() + assert os.path.exists(_PID_REGISTRY_FILE) + with open(_PID_REGISTRY_FILE) as f: + pids = [line.strip() for line in f if line.strip()] + assert str(os.getpid()) in pids + + def test_register_pid_appends_not_overwrites(self): + from concore import _register_pid, _PID_REGISTRY_FILE + + with open(_PID_REGISTRY_FILE, "w") as f: + f.write("11111\n") + f.write("22222\n") + _register_pid() + with open(_PID_REGISTRY_FILE) as f: + pids = [line.strip() for line in f if line.strip()] + assert "11111" in pids + assert "22222" in pids + assert str(os.getpid()) in pids + assert len(pids) == 3 + + def test_cleanup_pid_removes_current_pid(self): + from concore import _cleanup_pid, _PID_REGISTRY_FILE + + current_pid = str(os.getpid()) + with open(_PID_REGISTRY_FILE, "w") as f: + f.write("99999\n") + f.write(current_pid + "\n") + f.write("88888\n") + _cleanup_pid() + with open(_PID_REGISTRY_FILE) as f: + pids = [line.strip() for line in f if line.strip()] + assert current_pid not in pids + assert "99999" in pids + assert "88888" in pids + + def test_cleanup_pid_deletes_files_when_last_pid(self): + from concore import _cleanup_pid, _PID_REGISTRY_FILE, _KILL_SCRIPT_FILE + + current_pid = str(os.getpid()) + with open(_PID_REGISTRY_FILE, "w") as f: + f.write(current_pid + "\n") + with open(_KILL_SCRIPT_FILE, "w") as f: + f.write("@echo off\n") + _cleanup_pid() + assert not os.path.exists(_PID_REGISTRY_FILE) + assert not os.path.exists(_KILL_SCRIPT_FILE) + + def test_cleanup_pid_handles_missing_registry(self): + from concore import _cleanup_pid, _PID_REGISTRY_FILE + + assert not os.path.exists(_PID_REGISTRY_FILE) + _cleanup_pid() # Should not raise + + def test_write_kill_script_generates_bat_file(self): + from concore import _write_kill_script, _KILL_SCRIPT_FILE, _PID_REGISTRY_FILE + + _write_kill_script() + assert os.path.exists(_KILL_SCRIPT_FILE) + with open(_KILL_SCRIPT_FILE) as f: + content = f.read() + assert os.path.basename(_PID_REGISTRY_FILE) in content + assert "wmic" in content + assert "taskkill" in content + assert "concore" in content.lower() + + def test_multi_node_registration(self): + from concore import _register_pid, _PID_REGISTRY_FILE + + fake_pids = ["1204", "1932", "8120"] + with open(_PID_REGISTRY_FILE, "w") as f: + for pid in fake_pids: + f.write(pid + "\n") + _register_pid() + with open(_PID_REGISTRY_FILE) as f: + pids = [line.strip() for line in f if line.strip()] + for pid in fake_pids: + assert pid in pids + assert str(os.getpid()) in pids + assert len(pids) == 4 + + def test_cleanup_preserves_other_pids(self): + from concore import _cleanup_pid, _PID_REGISTRY_FILE + + current_pid = str(os.getpid()) + other_pids = ["1111", "2222", "3333"] + with open(_PID_REGISTRY_FILE, "w") as f: + for pid in other_pids: + f.write(pid + "\n") + f.write(current_pid + "\n") + _cleanup_pid() + with open(_PID_REGISTRY_FILE) as f: + pids = [line.strip() for line in f if line.strip()] + assert len(pids) == 3 + assert current_pid not in pids + for pid in other_pids: + assert pid in pids + + @pytest.mark.skipif( + not hasattr(sys, "getwindowsversion"), reason="Windows-only test" + ) + def test_import_registers_pid_on_windows(self): + """Verify module-level PID registration on Windows.""" + import importlib + + for mod_name in ("concore", "concore_base"): + sys.modules.pop(mod_name, None) + import concore + + assert os.path.exists(concore._PID_REGISTRY_FILE) + with open(concore._PID_REGISTRY_FILE) as f: + pids = [line.strip() for line in f if line.strip()] + assert str(os.getpid()) in pids + importlib.reload(concore) From 26aa8f869a07dde57befc973968d33e4848607f5 Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Wed, 4 Mar 2026 18:22:08 +0530 Subject: [PATCH 239/275] Address Copilot review: wmic validation, locking in register, larger lock range with finally unlock, absolute paths - Kill script uses wmic+concore instead of tasklist+python for precise PID validation - _register_pid() now acquires file lock (same as _cleanup_pid) - Lock range changed from 1 byte to 0x7FFFFFFF with LK_UNLCK in finally block - _PID_REGISTRY_FILE and _KILL_SCRIPT_FILE use os.path.abspath at module init - Tests updated: monkeypatch module-level paths to temp_dir, assert wmic/concore --- concore.py | 98 ++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 70 insertions(+), 28 deletions(-) diff --git a/concore.py b/concore.py index 865c42b5..d2f5b6ff 100644 --- a/concore.py +++ b/concore.py @@ -26,14 +26,28 @@ # New approach: append PID to shared registry; generate validated kill script. # See: https://github.com/ControlCore-Project/concore/issues/391 -_PID_REGISTRY_FILE = "concorekill_pids.txt" -_KILL_SCRIPT_FILE = "concorekill.bat" +_LOCK_LEN = 0x7FFFFFFF # lock range large enough to cover entire file +_BASE_DIR = os.path.abspath(".") # capture CWD before atexit can shift it +_PID_REGISTRY_FILE = os.path.join(_BASE_DIR, "concorekill_pids.txt") +_KILL_SCRIPT_FILE = os.path.join(_BASE_DIR, "concorekill.bat") def _register_pid(): - """Append current PID to the shared registry file.""" + """Append current PID to the shared registry file. Uses file locking on Windows.""" try: with open(_PID_REGISTRY_FILE, "a") as f: - f.write(str(os.getpid()) + "\n") + if hasattr(sys, 'getwindowsversion'): + import msvcrt + try: + msvcrt.locking(f.fileno(), msvcrt.LK_LOCK, _LOCK_LEN) + f.write(str(os.getpid()) + "\n") + finally: + try: + f.seek(0) + msvcrt.locking(f.fileno(), msvcrt.LK_UNLCK, _LOCK_LEN) + except OSError: + pass + else: + f.write(str(os.getpid()) + "\n") except OSError: pass @@ -46,46 +60,74 @@ def _cleanup_pid(): with open(_PID_REGISTRY_FILE, "r+") as f: if hasattr(sys, 'getwindowsversion'): import msvcrt - msvcrt.locking(f.fileno(), msvcrt.LK_LOCK, 1) - pids = [line.strip() for line in f if line.strip()] - remaining = [p for p in pids if p != pid] - if remaining: - f.seek(0) - f.truncate() - for p in remaining: - f.write(p + "\n") - else: - f.close() - try: - os.remove(_PID_REGISTRY_FILE) - except OSError: - pass try: - os.remove(_KILL_SCRIPT_FILE) - except OSError: - pass + msvcrt.locking(f.fileno(), msvcrt.LK_LOCK, _LOCK_LEN) + pids = [line.strip() for line in f if line.strip()] + remaining = [p for p in pids if p != pid] + if remaining: + f.seek(0) + f.truncate() + for p in remaining: + f.write(p + "\n") + else: + f.close() + try: + os.remove(_PID_REGISTRY_FILE) + except OSError: + pass + try: + os.remove(_KILL_SCRIPT_FILE) + except OSError: + pass + return + finally: + try: + f.seek(0) + msvcrt.locking(f.fileno(), msvcrt.LK_UNLCK, _LOCK_LEN) + except (OSError, ValueError): + pass + else: + pids = [line.strip() for line in f if line.strip()] + remaining = [p for p in pids if p != pid] + if remaining: + f.seek(0) + f.truncate() + for p in remaining: + f.write(p + "\n") + else: + f.close() + try: + os.remove(_PID_REGISTRY_FILE) + except OSError: + pass + try: + os.remove(_KILL_SCRIPT_FILE) + except OSError: + pass except OSError: pass def _write_kill_script(): """Generate concorekill.bat that validates each PID before killing.""" try: + reg_name = os.path.basename(_PID_REGISTRY_FILE) + bat_name = os.path.basename(_KILL_SCRIPT_FILE) script = "@echo off\r\n" - script += 'if not exist "%~dp0' + _PID_REGISTRY_FILE + '" (\r\n' + script += 'if not exist "%~dp0' + reg_name + '" (\r\n' script += " echo No PID registry found. Nothing to kill.\r\n" script += " exit /b 0\r\n" script += ")\r\n" - script += 'for /f "usebackq tokens=*" %%p in ("%~dp0' + _PID_REGISTRY_FILE + '") do (\r\n' - script += ' tasklist /FI "PID eq %%p" 2>nul | find /i "python" >nul\r\n' + script += 'for /f "usebackq tokens=*" %%p in ("%~dp0' + reg_name + '") do (\r\n' + script += ' wmic process where "ProcessId=%%p" get CommandLine /value 2>nul | find /i "concore" >nul\r\n' script += " if not errorlevel 1 (\r\n" - script += " echo Killing Python process %%p\r\n" + script += " echo Killing concore process %%p\r\n" script += " taskkill /F /PID %%p >nul 2>&1\r\n" script += " ) else (\r\n" - script += " echo Skipping PID %%p - not a Python process or not running\r\n" + script += " echo Skipping PID %%p - not a concore process or not running\r\n" script += " )\r\n" script += ")\r\n" - script += 'del /q "%~dp0' + _PID_REGISTRY_FILE + '" 2>nul\r\n' - script += 'del /q "%~dp0' + _KILL_SCRIPT_FILE + '" 2>nul\r\n' + script += 'del /q "%~dp0' + reg_name + '" 2>nul\r\n' + script += 'del /q "%~dp0' + bat_name + '" 2>nul\r\n' with open(_KILL_SCRIPT_FILE, "w", newline="") as f: f.write(script) except OSError: From f3299c011bb60b84007421a0104cfb3f10f4ef5b Mon Sep 17 00:00:00 2001 From: Avinash Kumar Deepak Date: Sun, 8 Mar 2026 23:27:50 +0530 Subject: [PATCH 240/275] fix #491: read() returns ReadResult with ReadStatus instead of silent default --- TestConcoredockerApi.java | 51 +++++++++++++++++++++++++++++++++------ concoredocker.java | 37 +++++++++++++++++++--------- 2 files changed, 68 insertions(+), 20 deletions(-) diff --git a/TestConcoredockerApi.java b/TestConcoredockerApi.java index 21934441..90472430 100644 --- a/TestConcoredockerApi.java +++ b/TestConcoredockerApi.java @@ -24,6 +24,9 @@ public static void main(String[] args) { testInitValExtractsSimtime(); testInitValReturnsRemainingValues(); testOutputFileMatchesPythonWireFormat(); + testReadFileNotFound(); + testReadRetriesExceeded(); + testReadParseError(); System.out.println("\n=== Results: " + passed + " passed, " + failed + " failed out of " + (passed + failed) + " tests ==="); if (failed > 0) { @@ -102,10 +105,11 @@ static void testReadParsesFileAndStripsSimtime() { concoredocker.setInPath(tmp.toString()); writeFile(tmp, 1, "sensor", "[0.0, 42.0, 99.0]"); - List result = concoredocker.read(1, "sensor", "[0.0, 0.0, 0.0]"); - check("read: strips simtime, size=2", 2, result.size()); - check("read: val1 correct", 42.0, result.get(0)); - check("read: val2 correct", 99.0, result.get(1)); + concoredocker.ReadResult result = concoredocker.read(1, "sensor", "[0.0, 0.0, 0.0]"); + check("read: status SUCCESS", concoredocker.ReadStatus.SUCCESS, result.status); + check("read: strips simtime, size=2", 2, result.data.size()); + check("read: val1 correct", 42.0, result.data.get(0)); + check("read: val2 correct", 99.0, result.data.get(1)); } static void testReadWriteRoundtrip() { @@ -119,10 +123,11 @@ static void testReadWriteRoundtrip() { outVals.add(8.0); concoredocker.write(1, "data", outVals, 1); - List inVals = concoredocker.read(1, "data", "[0.0, 0.0, 0.0]"); - check("roundtrip: size", 2, inVals.size()); - check("roundtrip: val1", 7.0, inVals.get(0)); - check("roundtrip: val2", 8.0, inVals.get(1)); + concoredocker.ReadResult inVals = concoredocker.read(1, "data", "[0.0, 0.0, 0.0]"); + check("roundtrip: status", concoredocker.ReadStatus.SUCCESS, inVals.status); + check("roundtrip: size", 2, inVals.data.size()); + check("roundtrip: val1", 7.0, inVals.data.get(0)); + check("roundtrip: val2", 8.0, inVals.data.get(1)); } static void testSimtimeAdvancesWithDelta() { @@ -195,4 +200,34 @@ static void testOutputFileMatchesPythonWireFormat() { Object reparsed = concoredocker.literalEval(raw); check("wire format: re-parseable as list", true, reparsed instanceof List); } + + static void testReadFileNotFound() { + Path tmp = makeTempDir(); + concoredocker.resetState(); + concoredocker.setInPath(tmp.toString()); + // no file written, port 1/missing does not exist + concoredocker.ReadResult result = concoredocker.read(1, "missing", "[0.0, 0.0]"); + check("read file not found: status", concoredocker.ReadStatus.FILE_NOT_FOUND, result.status); + check("read file not found: data is default", 1, result.data.size()); + } + + static void testReadRetriesExceeded() { + Path tmp = makeTempDir(); + concoredocker.resetState(); + concoredocker.setInPath(tmp.toString()); + writeFile(tmp, 1, "empty", ""); // always empty, exhausts retries + concoredocker.ReadResult result = concoredocker.read(1, "empty", "[0.0, 0.0]"); + check("read retries exceeded: status", concoredocker.ReadStatus.RETRIES_EXCEEDED, result.status); + check("read retries exceeded: data is default", 1, result.data.size()); + } + + static void testReadParseError() { + Path tmp = makeTempDir(); + concoredocker.resetState(); + concoredocker.setInPath(tmp.toString()); + writeFile(tmp, 1, "bad", "not_a_valid_list"); + concoredocker.ReadResult result = concoredocker.read(1, "bad", "[0.0, 0.0]"); + check("read parse error: status", concoredocker.ReadStatus.PARSE_ERROR, result.status); + check("read parse error: data is default", 1, result.data.size()); + } } diff --git a/concoredocker.java b/concoredocker.java index 1e66bcad..8dbbaedb 100644 --- a/concoredocker.java +++ b/concoredocker.java @@ -160,7 +160,7 @@ public static Object tryParam(String n, Object i) { * Returns: list of values after simtime * Includes max retry limit to avoid infinite blocking (matches Python behavior). */ - public static List read(int port, String name, String initstr) { + public static ReadResult read(int port, String name, String initstr) { // Parse default value upfront for consistent return type List defaultVal = new ArrayList<>(); try { @@ -178,7 +178,7 @@ public static List read(int port, String name, String initstr) { } catch (InterruptedException e) { Thread.currentThread().interrupt(); s += initstr; - return defaultVal; + return new ReadResult(ReadStatus.TIMEOUT, defaultVal); } String ins; @@ -187,7 +187,7 @@ public static List read(int port, String name, String initstr) { } catch (IOException e) { System.out.println("File " + filePath + " not found, using default value."); s += initstr; - return defaultVal; + return new ReadResult(ReadStatus.FILE_NOT_FOUND, defaultVal); } int attempts = 0; @@ -197,7 +197,7 @@ public static List read(int port, String name, String initstr) { } catch (InterruptedException e) { Thread.currentThread().interrupt(); s += initstr; - return defaultVal; + return new ReadResult(ReadStatus.TIMEOUT, defaultVal); } try { ins = new String(Files.readAllBytes(Paths.get(filePath))); @@ -210,7 +210,7 @@ public static List read(int port, String name, String initstr) { if (ins.length() == 0) { System.out.println("Max retries reached for " + filePath + ", using default value."); - return defaultVal; + return new ReadResult(ReadStatus.RETRIES_EXCEEDED, defaultVal); } s += ins; @@ -219,12 +219,12 @@ public static List read(int port, String name, String initstr) { if (!inval.isEmpty()) { double firstSimtime = ((Number) inval.get(0)).doubleValue(); simtime = Math.max(simtime, firstSimtime); - return new ArrayList<>(inval.subList(1, inval.size())); + return new ReadResult(ReadStatus.SUCCESS, new ArrayList<>(inval.subList(1, inval.size()))); } } catch (Exception e) { System.out.println("Error parsing " + ins + ": " + e.getMessage()); } - return defaultVal; + return new ReadResult(ReadStatus.PARSE_ERROR, defaultVal); } /** @@ -392,7 +392,7 @@ private static int zmqSocketTypeFromString(String s) { * Reads data from a ZMQ port. Same wire format as file-based read: * expects [simtime, val1, val2, ...], strips simtime, returns the rest. */ - public static List read(String portName, String name, String initstr) { + public static ReadResult read(String portName, String name, String initstr) { List defaultVal = new ArrayList<>(); try { List parsed = (List) literalEval(initstr); @@ -404,24 +404,24 @@ public static List read(String portName, String name, String initstr) { ZeroMQPort port = zmqPorts.get(portName); if (port == null) { System.err.println("read: ZMQ port '" + portName + "' not initialized"); - return defaultVal; + return new ReadResult(ReadStatus.FILE_NOT_FOUND, defaultVal); } String msg = port.recvWithRetry(); if (msg == null) { System.err.println("read: ZMQ recv timeout on port '" + portName + "'"); - return defaultVal; + return new ReadResult(ReadStatus.TIMEOUT, defaultVal); } s += msg; try { List inval = (List) literalEval(msg); if (!inval.isEmpty()) { simtime = Math.max(simtime, ((Number) inval.get(0)).doubleValue()); - return new ArrayList<>(inval.subList(1, inval.size())); + return new ReadResult(ReadStatus.SUCCESS, new ArrayList<>(inval.subList(1, inval.size()))); } } catch (Exception e) { System.out.println("Error parsing ZMQ message '" + msg + "': " + e.getMessage()); } - return defaultVal; + return new ReadResult(ReadStatus.PARSE_ERROR, defaultVal); } /** @@ -472,6 +472,19 @@ static Object literalEval(String s) { return result; } + public enum ReadStatus { + SUCCESS, FILE_NOT_FOUND, TIMEOUT, PARSE_ERROR, RETRIES_EXCEEDED + } + + public static class ReadResult { + public final ReadStatus status; + public final List data; + ReadResult(ReadStatus status, List data) { + this.status = status; + this.data = data; + } + } + /** * ZMQ socket wrapper with bind/connect, timeouts, and retry. */ From ae39be00c6f6000638c0996dabf80e8b8221e3c9 Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Mon, 9 Mar 2026 08:49:25 +0530 Subject: [PATCH 241/275] feat: add concore doctor command for system readiness checks Implements a new 'concore doctor' CLI command that performs comprehensive system readiness diagnostics including: - Core checks: Python version, concore installation, CONCOREPATH - Tool detection: C++, Python, Verilog, Octave, MATLAB, Docker - Configuration file validation (concore.tools, .octave, .mcr, .sudo) - Environment variable detection for tool overrides - Dependency checks for required and optional packages Includes 23 unit tests covering all diagnostic functions. Closes #495 --- concore_cli/cli.py | 13 + concore_cli/commands/__init__.py | 2 + concore_cli/commands/doctor.py | 428 +++++++++++++++++++++++++++++++ tests/test_doctor.py | 222 ++++++++++++++++ 4 files changed, 665 insertions(+) create mode 100644 concore_cli/commands/doctor.py create mode 100644 tests/test_doctor.py diff --git a/concore_cli/cli.py b/concore_cli/cli.py index 15801838..92ca6e50 100644 --- a/concore_cli/cli.py +++ b/concore_cli/cli.py @@ -10,6 +10,7 @@ from .commands.stop import stop_all from .commands.inspect import inspect_workflow from .commands.watch import watch_study +from .commands.doctor import doctor_check from . import __version__ console = Console() @@ -118,5 +119,17 @@ def watch(study_dir, interval, once): sys.exit(1) +@cli.command() +def doctor(): + """Check system readiness for running concore studies""" + try: + ok = doctor_check(console) + if not ok: + sys.exit(1) + except Exception as e: + console.print(f"[red]Error:[/red] {str(e)}") + sys.exit(1) + + if __name__ == "__main__": cli() diff --git a/concore_cli/commands/__init__.py b/concore_cli/commands/__init__.py index e98d4cd5..b9771e4c 100644 --- a/concore_cli/commands/__init__.py +++ b/concore_cli/commands/__init__.py @@ -4,6 +4,7 @@ from .status import show_status from .stop import stop_all from .watch import watch_study +from .doctor import doctor_check __all__ = [ "init_project", @@ -12,4 +13,5 @@ "show_status", "stop_all", "watch_study", + "doctor_check", ] diff --git a/concore_cli/commands/doctor.py b/concore_cli/commands/doctor.py new file mode 100644 index 00000000..98ea356f --- /dev/null +++ b/concore_cli/commands/doctor.py @@ -0,0 +1,428 @@ +import shutil +import subprocess +import sys +import os +import platform +from pathlib import Path +from rich.panel import Panel + +# Map of tool keys to their lookup names per platform +TOOL_DEFINITIONS = { + "C++ compiler": { + "names": { + "posix": ["g++", "clang++"], + "windows": ["g++", "cl"], + }, + "version_flag": "--version", + "config_keys": ["CPPEXE", "CPPWIN"], + "install_hints": { + "Linux": "sudo apt install g++", + "Darwin": "brew install gcc", + "Windows": "winget install -e --id GnuWin32.Gcc", + }, + }, + "Python": { + "names": { + "posix": ["python3", "python"], + "windows": ["python", "python3"], + }, + "version_flag": "--version", + "config_keys": ["PYTHONEXE", "PYTHONWIN"], + "install_hints": { + "Linux": "sudo apt install python3", + "Darwin": "brew install python3", + "Windows": "winget install -e --id Python.Python.3.11", + }, + }, + "Verilog (iverilog)": { + "names": { + "posix": ["iverilog"], + "windows": ["iverilog"], + }, + "version_flag": "-V", + "config_keys": ["VEXE", "VWIN"], + "install_hints": { + "Linux": "sudo apt install iverilog", + "Darwin": "brew install icarus-verilog", + "Windows": "Download from http://bleyer.org/icarus/", + }, + }, + "Octave": { + "names": { + "posix": ["octave", "octave-cli"], + "windows": ["octave", "octave-cli"], + }, + "version_flag": "--version", + "config_keys": ["OCTAVEEXE", "OCTAVEWIN"], + "install_hints": { + "Linux": "sudo apt install octave", + "Darwin": "brew install octave", + "Windows": "winget install -e --id JohnWHiggins.Octave", + }, + }, + "MATLAB": { + "names": { + "posix": ["matlab"], + "windows": ["matlab"], + }, + "version_flag": "-batch \"disp('ok')\"", + "config_keys": ["MATLABEXE", "MATLABWIN"], + "install_hints": { + "Linux": "Install from https://mathworks.com/downloads/", + "Darwin": "Install from https://mathworks.com/downloads/", + "Windows": "Install from https://mathworks.com/downloads/", + }, + }, + "Docker": { + "names": { + "posix": ["docker", "podman"], + "windows": ["docker", "podman"], + }, + "version_flag": "--version", + "config_keys": [], + "install_hints": { + "Linux": "sudo apt install docker.io", + "Darwin": "brew install --cask docker", + "Windows": "winget install -e --id Docker.DockerDesktop", + }, + }, +} + +REQUIRED_PACKAGES = [ + "click", + "rich", + "beautifulsoup4", + "lxml", + "psutil", + "numpy", + "pyzmq", +] + +OPTIONAL_PACKAGES = { + "scipy": "pip install concore[demo]", + "matplotlib": "pip install concore[demo]", +} + +# Map import names that differ from package names +IMPORT_NAME_MAP = { + "beautifulsoup4": "bs4", + "pyzmq": "zmq", +} + + +def _get_platform_key(): + """Return 'posix' or 'windows' based on OS.""" + return "windows" if os.name == "nt" else "posix" + + +def _get_platform_name(): + """Return platform name for install hint lookup.""" + return platform.system() + + +def _resolve_concore_path(): + """Resolve CONCOREPATH the same way mkconcore.py does.""" + script_dir = Path(__file__).resolve().parent.parent.parent + if (script_dir / "concore.py").exists(): + return script_dir + cwd = Path.cwd() + if (cwd / "concore.py").exists(): + return cwd + return script_dir + + +def _detect_tool(names): + """Try to find a tool by checking a list of candidate names. + + Returns (path, name) of the first match, or (None, None). + """ + for name in names: + path = shutil.which(name) + if path: + return path, name + return None, None + + +def _get_version(path, version_flag): + """Run tool with version flag and return first line of output.""" + try: + result = subprocess.run( + [path, version_flag], + capture_output=True, + text=True, + timeout=10, + ) + output = result.stdout.strip() or result.stderr.strip() + if output: + return output.splitlines()[0] + except Exception: + pass + return None + + +def _check_docker_daemon(docker_path): + """Check if Docker daemon is running.""" + try: + result = subprocess.run( + [docker_path, "info"], + capture_output=True, + text=True, + timeout=15, + ) + return result.returncode == 0 + except Exception: + return False + + +def _check_package(package_name): + """Check if a Python package is importable and get its version.""" + import_name = IMPORT_NAME_MAP.get(package_name, package_name) + try: + mod = __import__(import_name) + version = getattr(mod, "__version__", None) + if version is None: + version = getattr(mod, "VERSION", "installed") + return True, str(version) + except ImportError: + return False, None + + +def doctor_check(console): + """Run system readiness checks and display results.""" + passed = 0 + warnings = 0 + errors = 0 + + console.print() + console.print( + Panel.fit( + "[bold]concore Doctor — System Readiness Report[/bold]", + border_style="cyan", + ) + ) + console.print() + + # === Core Checks === + console.print("[bold cyan]Core Checks[/bold cyan]") + + # Python version + py_version = platform.python_version() + py_major, py_minor = sys.version_info.major, sys.version_info.minor + if py_major >= 3 and py_minor >= 9: + console.print(f" [green]✓[/green] Python {py_version} (>= 3.9 required)") + passed += 1 + else: + console.print( + f" [red]✗[/red] Python {py_version} — " + f"concore requires Python >= 3.9" + ) + errors += 1 + + # concore installation + try: + from concore_cli import __version__ + console.print(f" [green]✓[/green] concore {__version__} installed") + passed += 1 + except ImportError: + console.print(" [red]✗[/red] concore package not found") + errors += 1 + + # CONCOREPATH + concore_path = _resolve_concore_path() + if concore_path.exists(): + writable = os.access(str(concore_path), os.W_OK) + status = "writable" if writable else "read-only" + if writable: + console.print( + f" [green]✓[/green] CONCOREPATH: {concore_path} ({status})" + ) + passed += 1 + else: + console.print( + f" [yellow]![/yellow] CONCOREPATH: {concore_path} ({status})" + ) + warnings += 1 + else: + console.print(f" [red]✗[/red] CONCOREPATH: {concore_path} (not found)") + errors += 1 + + console.print() + + # === Tool Detection === + console.print("[bold cyan]Tools[/bold cyan]") + + plat_key = _get_platform_key() + plat_name = _get_platform_name() + + for tool_label, tool_def in TOOL_DEFINITIONS.items(): + candidates = tool_def["names"].get(plat_key, []) + path, found_name = _detect_tool(candidates) + + if path: + version = _get_version(path, tool_def["version_flag"]) + version_str = f" ({version})" if version else "" + extra = "" + if tool_label == "Docker": + daemon_ok = _check_docker_daemon(path) + extra = ( + " [green](daemon running)[/green]" + if daemon_ok + else " [yellow](daemon not running)[/yellow]" + ) + if not daemon_ok: + warnings += 1 + console.print( + f" [yellow]![/yellow] {tool_label}{version_str} " + f"→ {path}{extra}" + ) + continue + console.print( + f" [green]✓[/green] {tool_label}{version_str} → {path}{extra}" + ) + passed += 1 + else: + hint = tool_def["install_hints"].get(plat_name, "") + hint_str = f" (install: {hint})" if hint else "" + # MATLAB is optional if Octave is available, show as warning + if tool_label == "MATLAB": + console.print( + f" [yellow]![/yellow] {tool_label} → Not found{hint_str}" + ) + warnings += 1 + elif tool_label == "Verilog (iverilog)": + console.print( + f" [yellow]![/yellow] {tool_label} → Not found{hint_str}" + ) + warnings += 1 + else: + console.print( + f" [red]✗[/red] {tool_label} → Not found{hint_str}" + ) + errors += 1 + + console.print() + + # === Configuration Checks === + console.print("[bold cyan]Configuration[/bold cyan]") + + config_files = { + "concore.tools": "Tool path overrides", + "concore.octave": "Treat .m files as Octave", + "concore.mcr": "MATLAB Compiler Runtime path", + "concore.sudo": "Docker executable override", + } + + for filename, description in config_files.items(): + filepath = concore_path / filename + if filepath.exists(): + try: + content = filepath.read_text().strip() + if filename == "concore.tools": + line_count = len( + [ln for ln in content.splitlines() + if ln.strip() and not ln.strip().startswith("#")] + ) + console.print( + f" [green]✓[/green] {filename} → " + f"{line_count} tool path(s) configured" + ) + elif filename == "concore.mcr": + if os.path.exists(os.path.expanduser(content)): + console.print( + f" [green]✓[/green] {filename} → {content}" + ) + else: + console.print( + f" [yellow]![/yellow] {filename} → " + f"path does not exist: {content}" + ) + warnings += 1 + continue + elif filename == "concore.sudo": + console.print( + f" [green]✓[/green] {filename} → {content}" + ) + else: + console.print( + f" [green]✓[/green] {filename} → Enabled" + ) + passed += 1 + except Exception: + console.print( + f" [yellow]![/yellow] {filename} → Could not read" + ) + warnings += 1 + else: + console.print( + f" [dim]—[/dim] {filename} → Not set ({description})" + ) + + # Check environment variables + env_vars = [ + "CONCORE_CPPEXE", "CONCORE_PYTHONEXE", "CONCORE_VEXE", + "CONCORE_OCTAVEEXE", "CONCORE_MATLABEXE", "DOCKEREXE", + ] + env_set = [v for v in env_vars if os.environ.get(v)] + if env_set: + console.print( + f" [green]✓[/green] Environment variables: " + f"{', '.join(env_set)}" + ) + passed += 1 + else: + console.print(" [dim]—[/dim] No concore environment variables set") + + console.print() + + # === Dependency Checks === + console.print("[bold cyan]Dependencies[/bold cyan]") + + for pkg in REQUIRED_PACKAGES: + found, version = _check_package(pkg) + if found: + console.print(f" [green]✓[/green] {pkg} {version}") + passed += 1 + else: + console.print( + f" [red]✗[/red] {pkg} → Not installed " + f"(pip install {pkg})" + ) + errors += 1 + + for pkg, install_hint in OPTIONAL_PACKAGES.items(): + found, version = _check_package(pkg) + if found: + console.print(f" [green]✓[/green] {pkg} {version}") + passed += 1 + else: + console.print( + f" [yellow]![/yellow] {pkg} → Not installed " + f"({install_hint})" + ) + warnings += 1 + + console.print() + + # === Summary === + summary_parts = [] + if passed: + summary_parts.append(f"[green]{passed} passed[/green]") + if warnings: + summary_parts.append(f"[yellow]{warnings} warning(s)[/yellow]") + if errors: + summary_parts.append(f"[red]{errors} error(s)[/red]") + + console.print(f"[bold]Summary:[/bold] {', '.join(summary_parts)}") + + if errors == 0: + console.print() + console.print( + Panel.fit( + "[green]System is ready to run concore studies![/green]", + border_style="green", + ) + ) + + console.print() + + return errors == 0 diff --git a/tests/test_doctor.py b/tests/test_doctor.py new file mode 100644 index 00000000..b1b66a53 --- /dev/null +++ b/tests/test_doctor.py @@ -0,0 +1,222 @@ +import unittest +import tempfile +import shutil +from pathlib import Path +from unittest.mock import patch +from click.testing import CliRunner +from concore_cli.cli import cli +from concore_cli.commands.doctor import ( + _detect_tool, + _get_platform_key, + _check_package, + _resolve_concore_path, + doctor_check, +) + + +class TestDoctorCommand(unittest.TestCase): + """Tests for the concore doctor CLI command.""" + + def setUp(self): + self.runner = CliRunner() + + def test_doctor_command_runs(self): + """Doctor command should run and produce output.""" + result = self.runner.invoke(cli, ["doctor"]) + self.assertIn("concore Doctor", result.output) + self.assertIn("Core Checks", result.output) + self.assertIn("Tools", result.output) + self.assertIn("Configuration", result.output) + self.assertIn("Dependencies", result.output) + self.assertIn("Summary", result.output) + + def test_doctor_help(self): + """Doctor command should have help text.""" + result = self.runner.invoke(cli, ["doctor", "--help"]) + self.assertEqual(result.exit_code, 0) + self.assertIn("Check system readiness", result.output) + + def test_doctor_shows_python_version(self): + """Doctor should show the current Python version.""" + result = self.runner.invoke(cli, ["doctor"]) + import platform + py_version = platform.python_version() + self.assertIn(py_version, result.output) + + def test_doctor_shows_concore_version(self): + """Doctor should detect and show concore version.""" + result = self.runner.invoke(cli, ["doctor"]) + self.assertIn("concore", result.output) + self.assertIn("1.0.0", result.output) + + def test_doctor_shows_concorepath(self): + """Doctor should show the CONCOREPATH.""" + result = self.runner.invoke(cli, ["doctor"]) + self.assertIn("CONCOREPATH", result.output) + + def test_doctor_checks_dependencies(self): + """Doctor should check required Python packages.""" + result = self.runner.invoke(cli, ["doctor"]) + # These should be installed since we're running tests + self.assertIn("click", result.output) + self.assertIn("rich", result.output) + + def test_doctor_shows_summary(self): + """Doctor should show a summary with pass/warn/error counts.""" + result = self.runner.invoke(cli, ["doctor"]) + self.assertIn("Summary", result.output) + self.assertIn("passed", result.output) + + +class TestDetectTool(unittest.TestCase): + """Tests for tool detection helpers.""" + + def test_detect_python(self): + """Should detect the currently running Python.""" + # python or python3 should be findable + path, name = _detect_tool(["python3", "python"]) + self.assertIsNotNone(path) + self.assertIn(name, ["python3", "python"]) + + def test_detect_nonexistent_tool(self): + """Should return None for a tool that doesn't exist.""" + path, name = _detect_tool(["nonexistent_tool_abc123"]) + self.assertIsNone(path) + self.assertIsNone(name) + + def test_detect_tool_tries_multiple_names(self): + """Should try all candidate names and return the first match.""" + path, name = _detect_tool( + ["nonexistent_tool_abc123", "python3", "python"] + ) + self.assertIsNotNone(path) + + def test_detect_tool_empty_list(self): + """Should handle an empty candidate list gracefully.""" + path, name = _detect_tool([]) + self.assertIsNone(path) + self.assertIsNone(name) + + +class TestGetPlatformKey(unittest.TestCase): + """Tests for platform detection.""" + + def test_returns_valid_key(self): + """Should return 'posix' or 'windows'.""" + key = _get_platform_key() + self.assertIn(key, ["posix", "windows"]) + + @patch("os.name", "nt") + def test_windows_detection(self): + """Should return 'windows' when os.name is 'nt'.""" + key = _get_platform_key() + self.assertEqual(key, "windows") + + @patch("os.name", "posix") + def test_posix_detection(self): + """Should return 'posix' when os.name is 'posix'.""" + key = _get_platform_key() + self.assertEqual(key, "posix") + + +class TestCheckPackage(unittest.TestCase): + """Tests for package checking.""" + + def test_check_installed_package(self): + """Should detect an installed package.""" + found, version = _check_package("click") + self.assertTrue(found) + self.assertIsNotNone(version) + + def test_check_missing_package(self): + """Should return False for a package that isn't installed.""" + found, version = _check_package("nonexistent_package_abc123") + self.assertFalse(found) + self.assertIsNone(version) + + def test_check_package_with_import_name_map(self): + """Should use the correct import name for beautifulsoup4 (bs4).""" + found, version = _check_package("beautifulsoup4") + self.assertTrue(found) + + def test_check_pyzmq_import_name(self): + """Should use 'zmq' as import name for pyzmq.""" + found, version = _check_package("pyzmq") + self.assertTrue(found) + + +class TestResolveConCorePath(unittest.TestCase): + """Tests for CONCOREPATH resolution.""" + + def test_resolves_to_existing_path(self): + """Should return a Path object.""" + result = _resolve_concore_path() + self.assertIsInstance(result, Path) + + def test_resolved_path_contains_concore(self): + """Resolved path should contain concore.py if run from repo.""" + result = _resolve_concore_path() + # In the test environment (run from repo), concore.py should exist + # This may not hold in all CI environments, so we just check it's a Path + self.assertIsInstance(result, Path) + + +class TestDoctorWithConfig(unittest.TestCase): + """Tests for doctor command with config files present.""" + + def setUp(self): + self.runner = CliRunner() + self.temp_dir = tempfile.mkdtemp() + + def tearDown(self): + shutil.rmtree(self.temp_dir) + + @patch( + "concore_cli.commands.doctor._resolve_concore_path" + ) + def test_doctor_with_concore_tools(self, mock_path): + """Doctor should detect and report concore.tools.""" + mock_path.return_value = Path(self.temp_dir) + tools_file = Path(self.temp_dir) / "concore.tools" + tools_file.write_text("CPPEXE=/usr/bin/g++\nPYTHONEXE=/usr/bin/python3\n") + + from rich.console import Console + import io + console = Console(file=io.StringIO(), force_terminal=True) + result = doctor_check(console) + # Just verify it doesn't crash + self.assertIsInstance(result, bool) + + @patch( + "concore_cli.commands.doctor._resolve_concore_path" + ) + def test_doctor_with_concore_octave(self, mock_path): + """Doctor should detect concore.octave flag.""" + mock_path.return_value = Path(self.temp_dir) + octave_file = Path(self.temp_dir) / "concore.octave" + octave_file.write_text("") + + from rich.console import Console + import io + console = Console(file=io.StringIO(), force_terminal=True) + result = doctor_check(console) + self.assertIsInstance(result, bool) + + @patch( + "concore_cli.commands.doctor._resolve_concore_path" + ) + def test_doctor_with_concore_sudo(self, mock_path): + """Doctor should detect concore.sudo config.""" + mock_path.return_value = Path(self.temp_dir) + sudo_file = Path(self.temp_dir) / "concore.sudo" + sudo_file.write_text("docker") + + from rich.console import Console + import io + console = Console(file=io.StringIO(), force_terminal=True) + result = doctor_check(console) + self.assertIsInstance(result, bool) + + +if __name__ == "__main__": + unittest.main() From a87437f3b9ca3dc9fb9e84b6ab0d614bc50eb3ce Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Mon, 9 Mar 2026 12:28:46 +0530 Subject: [PATCH 242/275] refactor: apply review suggestions to concore doctor - Use importlib.metadata instead of __import__ for package version checks - Fix Python version check to use tuple comparison (sys.version_info >= (3, 9)) - Make MATLAB version_flag a list to handle multi-arg flags correctly - Support list-type version_flag in _get_version() - Show found executable name (e.g., [g++]) in tool output - Treat Docker-not-found as warning instead of error (optional tool) - Add concore.repo to config file checks - Fix concore.mcr passed counter not incrementing on valid path - Build env var list dynamically from TOOL_DEFINITIONS config_keys - Handle empty summary_parts gracefully - Remove unused IMPORT_NAME_MAP (replaced by importlib.metadata) - Remove duplicate test (test_resolved_path_contains_concore) - Patch os.name via correct module path in platform key tests - Use __version__ import instead of hardcoded version in test --- concore_cli/commands/doctor.py | 72 +++++++++++++++++----------------- tests/test_doctor.py | 14 ++----- 2 files changed, 41 insertions(+), 45 deletions(-) diff --git a/concore_cli/commands/doctor.py b/concore_cli/commands/doctor.py index 98ea356f..67b07920 100644 --- a/concore_cli/commands/doctor.py +++ b/concore_cli/commands/doctor.py @@ -1,3 +1,4 @@ +import importlib.metadata import shutil import subprocess import sys @@ -65,7 +66,7 @@ "posix": ["matlab"], "windows": ["matlab"], }, - "version_flag": "-batch \"disp('ok')\"", + "version_flag": ["-batch", "disp('ok')"], "config_keys": ["MATLABEXE", "MATLABWIN"], "install_hints": { "Linux": "Install from https://mathworks.com/downloads/", @@ -103,13 +104,6 @@ "matplotlib": "pip install concore[demo]", } -# Map import names that differ from package names -IMPORT_NAME_MAP = { - "beautifulsoup4": "bs4", - "pyzmq": "zmq", -} - - def _get_platform_key(): """Return 'posix' or 'windows' based on OS.""" return "windows" if os.name == "nt" else "posix" @@ -146,8 +140,12 @@ def _detect_tool(names): def _get_version(path, version_flag): """Run tool with version flag and return first line of output.""" try: + if isinstance(version_flag, list): + cmd = [path] + version_flag + else: + cmd = [path, version_flag] result = subprocess.run( - [path, version_flag], + cmd, capture_output=True, text=True, timeout=10, @@ -176,14 +174,10 @@ def _check_docker_daemon(docker_path): def _check_package(package_name): """Check if a Python package is importable and get its version.""" - import_name = IMPORT_NAME_MAP.get(package_name, package_name) try: - mod = __import__(import_name) - version = getattr(mod, "__version__", None) - if version is None: - version = getattr(mod, "VERSION", "installed") - return True, str(version) - except ImportError: + version = importlib.metadata.version(package_name) + return True, version + except importlib.metadata.PackageNotFoundError: return False, None @@ -207,8 +201,7 @@ def doctor_check(console): # Python version py_version = platform.python_version() - py_major, py_minor = sys.version_info.major, sys.version_info.minor - if py_major >= 3 and py_minor >= 9: + if sys.version_info >= (3, 9): console.print(f" [green]✓[/green] Python {py_version} (>= 3.9 required)") passed += 1 else: @@ -261,6 +254,7 @@ def doctor_check(console): if path: version = _get_version(path, tool_def["version_flag"]) version_str = f" ({version})" if version else "" + exe_info = f" [{found_name}]" if found_name else "" extra = "" if tool_label == "Docker": daemon_ok = _check_docker_daemon(path) @@ -272,24 +266,20 @@ def doctor_check(console): if not daemon_ok: warnings += 1 console.print( - f" [yellow]![/yellow] {tool_label}{version_str} " - f"→ {path}{extra}" + f" [yellow]![/yellow] {tool_label}{exe_info}" + f"{version_str} → {path}{extra}" ) continue console.print( - f" [green]✓[/green] {tool_label}{version_str} → {path}{extra}" + f" [green]✓[/green] {tool_label}{exe_info}" + f"{version_str} → {path}{extra}" ) passed += 1 else: hint = tool_def["install_hints"].get(plat_name, "") hint_str = f" (install: {hint})" if hint else "" - # MATLAB is optional if Octave is available, show as warning - if tool_label == "MATLAB": - console.print( - f" [yellow]![/yellow] {tool_label} → Not found{hint_str}" - ) - warnings += 1 - elif tool_label == "Verilog (iverilog)": + # Docker, MATLAB, Verilog are optional — show as warning + if tool_label in ("MATLAB", "Verilog (iverilog)", "Docker"): console.print( f" [yellow]![/yellow] {tool_label} → Not found{hint_str}" ) @@ -309,6 +299,7 @@ def doctor_check(console): "concore.tools": "Tool path overrides", "concore.octave": "Treat .m files as Octave", "concore.mcr": "MATLAB Compiler Runtime path", + "concore.repo": "Docker repository path", "concore.sudo": "Docker executable override", } @@ -331,17 +322,22 @@ def doctor_check(console): console.print( f" [green]✓[/green] {filename} → {content}" ) + passed += 1 else: console.print( f" [yellow]![/yellow] {filename} → " f"path does not exist: {content}" ) warnings += 1 - continue + continue elif filename == "concore.sudo": console.print( f" [green]✓[/green] {filename} → {content}" ) + elif filename == "concore.repo": + console.print( + f" [green]✓[/green] {filename} → {content}" + ) else: console.print( f" [green]✓[/green] {filename} → Enabled" @@ -357,11 +353,12 @@ def doctor_check(console): f" [dim]—[/dim] {filename} → Not set ({description})" ) - # Check environment variables - env_vars = [ - "CONCORE_CPPEXE", "CONCORE_PYTHONEXE", "CONCORE_VEXE", - "CONCORE_OCTAVEEXE", "CONCORE_MATLABEXE", "DOCKEREXE", - ] + # Build environment variable list from TOOL_DEFINITIONS config_keys + env_vars = [] + for tool_def in TOOL_DEFINITIONS.values(): + for key in tool_def.get("config_keys", []): + env_vars.append(f"CONCORE_{key}") + env_vars.append("DOCKEREXE") env_set = [v for v in env_vars if os.environ.get(v)] if env_set: console.print( @@ -412,7 +409,12 @@ def doctor_check(console): if errors: summary_parts.append(f"[red]{errors} error(s)[/red]") - console.print(f"[bold]Summary:[/bold] {', '.join(summary_parts)}") + if summary_parts: + summary_text = ", ".join(summary_parts) + else: + summary_text = "[yellow]No checks were run.[/yellow]" + + console.print(f"[bold]Summary:[/bold] {summary_text}") if errors == 0: console.print() diff --git a/tests/test_doctor.py b/tests/test_doctor.py index b1b66a53..6722ae53 100644 --- a/tests/test_doctor.py +++ b/tests/test_doctor.py @@ -45,9 +45,10 @@ def test_doctor_shows_python_version(self): def test_doctor_shows_concore_version(self): """Doctor should detect and show concore version.""" + from concore_cli import __version__ result = self.runner.invoke(cli, ["doctor"]) self.assertIn("concore", result.output) - self.assertIn("1.0.0", result.output) + self.assertIn(__version__, result.output) def test_doctor_shows_concorepath(self): """Doctor should show the CONCOREPATH.""" @@ -106,13 +107,13 @@ def test_returns_valid_key(self): key = _get_platform_key() self.assertIn(key, ["posix", "windows"]) - @patch("os.name", "nt") + @patch("concore_cli.commands.doctor.os.name", "nt") def test_windows_detection(self): """Should return 'windows' when os.name is 'nt'.""" key = _get_platform_key() self.assertEqual(key, "windows") - @patch("os.name", "posix") + @patch("concore_cli.commands.doctor.os.name", "posix") def test_posix_detection(self): """Should return 'posix' when os.name is 'posix'.""" key = _get_platform_key() @@ -153,13 +154,6 @@ def test_resolves_to_existing_path(self): result = _resolve_concore_path() self.assertIsInstance(result, Path) - def test_resolved_path_contains_concore(self): - """Resolved path should contain concore.py if run from repo.""" - result = _resolve_concore_path() - # In the test environment (run from repo), concore.py should exist - # This may not hold in all CI environments, so we just check it's a Path - self.assertIsInstance(result, Path) - class TestDoctorWithConfig(unittest.TestCase): """Tests for doctor command with config files present.""" From 243ab44805d9277823a0208337f67b5de893af4c Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Mon, 9 Mar 2026 12:32:04 +0530 Subject: [PATCH 243/275] style: apply ruff format to doctor.py and test_doctor.py --- concore_cli/commands/doctor.py | 57 ++++++++++++---------------------- tests/test_doctor.py | 21 ++++++------- 2 files changed, 28 insertions(+), 50 deletions(-) diff --git a/concore_cli/commands/doctor.py b/concore_cli/commands/doctor.py index 67b07920..0d5183d4 100644 --- a/concore_cli/commands/doctor.py +++ b/concore_cli/commands/doctor.py @@ -104,6 +104,7 @@ "matplotlib": "pip install concore[demo]", } + def _get_platform_key(): """Return 'posix' or 'windows' based on OS.""" return "windows" if os.name == "nt" else "posix" @@ -206,14 +207,14 @@ def doctor_check(console): passed += 1 else: console.print( - f" [red]✗[/red] Python {py_version} — " - f"concore requires Python >= 3.9" + f" [red]✗[/red] Python {py_version} — concore requires Python >= 3.9" ) errors += 1 # concore installation try: from concore_cli import __version__ + console.print(f" [green]✓[/green] concore {__version__} installed") passed += 1 except ImportError: @@ -226,9 +227,7 @@ def doctor_check(console): writable = os.access(str(concore_path), os.W_OK) status = "writable" if writable else "read-only" if writable: - console.print( - f" [green]✓[/green] CONCOREPATH: {concore_path} ({status})" - ) + console.print(f" [green]✓[/green] CONCOREPATH: {concore_path} ({status})") passed += 1 else: console.print( @@ -285,9 +284,7 @@ def doctor_check(console): ) warnings += 1 else: - console.print( - f" [red]✗[/red] {tool_label} → Not found{hint_str}" - ) + console.print(f" [red]✗[/red] {tool_label} → Not found{hint_str}") errors += 1 console.print() @@ -310,8 +307,11 @@ def doctor_check(console): content = filepath.read_text().strip() if filename == "concore.tools": line_count = len( - [ln for ln in content.splitlines() - if ln.strip() and not ln.strip().startswith("#")] + [ + ln + for ln in content.splitlines() + if ln.strip() and not ln.strip().startswith("#") + ] ) console.print( f" [green]✓[/green] {filename} → " @@ -319,9 +319,7 @@ def doctor_check(console): ) elif filename == "concore.mcr": if os.path.exists(os.path.expanduser(content)): - console.print( - f" [green]✓[/green] {filename} → {content}" - ) + console.print(f" [green]✓[/green] {filename} → {content}") passed += 1 else: console.print( @@ -331,27 +329,17 @@ def doctor_check(console): warnings += 1 continue elif filename == "concore.sudo": - console.print( - f" [green]✓[/green] {filename} → {content}" - ) + console.print(f" [green]✓[/green] {filename} → {content}") elif filename == "concore.repo": - console.print( - f" [green]✓[/green] {filename} → {content}" - ) + console.print(f" [green]✓[/green] {filename} → {content}") else: - console.print( - f" [green]✓[/green] {filename} → Enabled" - ) + console.print(f" [green]✓[/green] {filename} → Enabled") passed += 1 except Exception: - console.print( - f" [yellow]![/yellow] {filename} → Could not read" - ) + console.print(f" [yellow]![/yellow] {filename} → Could not read") warnings += 1 else: - console.print( - f" [dim]—[/dim] {filename} → Not set ({description})" - ) + console.print(f" [dim]—[/dim] {filename} → Not set ({description})") # Build environment variable list from TOOL_DEFINITIONS config_keys env_vars = [] @@ -361,10 +349,7 @@ def doctor_check(console): env_vars.append("DOCKEREXE") env_set = [v for v in env_vars if os.environ.get(v)] if env_set: - console.print( - f" [green]✓[/green] Environment variables: " - f"{', '.join(env_set)}" - ) + console.print(f" [green]✓[/green] Environment variables: {', '.join(env_set)}") passed += 1 else: console.print(" [dim]—[/dim] No concore environment variables set") @@ -380,10 +365,7 @@ def doctor_check(console): console.print(f" [green]✓[/green] {pkg} {version}") passed += 1 else: - console.print( - f" [red]✗[/red] {pkg} → Not installed " - f"(pip install {pkg})" - ) + console.print(f" [red]✗[/red] {pkg} → Not installed (pip install {pkg})") errors += 1 for pkg, install_hint in OPTIONAL_PACKAGES.items(): @@ -393,8 +375,7 @@ def doctor_check(console): passed += 1 else: console.print( - f" [yellow]![/yellow] {pkg} → Not installed " - f"({install_hint})" + f" [yellow]![/yellow] {pkg} → Not installed ({install_hint})" ) warnings += 1 diff --git a/tests/test_doctor.py b/tests/test_doctor.py index 6722ae53..e9ec5846 100644 --- a/tests/test_doctor.py +++ b/tests/test_doctor.py @@ -40,12 +40,14 @@ def test_doctor_shows_python_version(self): """Doctor should show the current Python version.""" result = self.runner.invoke(cli, ["doctor"]) import platform + py_version = platform.python_version() self.assertIn(py_version, result.output) def test_doctor_shows_concore_version(self): """Doctor should detect and show concore version.""" from concore_cli import __version__ + result = self.runner.invoke(cli, ["doctor"]) self.assertIn("concore", result.output) self.assertIn(__version__, result.output) @@ -87,9 +89,7 @@ def test_detect_nonexistent_tool(self): def test_detect_tool_tries_multiple_names(self): """Should try all candidate names and return the first match.""" - path, name = _detect_tool( - ["nonexistent_tool_abc123", "python3", "python"] - ) + path, name = _detect_tool(["nonexistent_tool_abc123", "python3", "python"]) self.assertIsNotNone(path) def test_detect_tool_empty_list(self): @@ -165,9 +165,7 @@ def setUp(self): def tearDown(self): shutil.rmtree(self.temp_dir) - @patch( - "concore_cli.commands.doctor._resolve_concore_path" - ) + @patch("concore_cli.commands.doctor._resolve_concore_path") def test_doctor_with_concore_tools(self, mock_path): """Doctor should detect and report concore.tools.""" mock_path.return_value = Path(self.temp_dir) @@ -176,14 +174,13 @@ def test_doctor_with_concore_tools(self, mock_path): from rich.console import Console import io + console = Console(file=io.StringIO(), force_terminal=True) result = doctor_check(console) # Just verify it doesn't crash self.assertIsInstance(result, bool) - @patch( - "concore_cli.commands.doctor._resolve_concore_path" - ) + @patch("concore_cli.commands.doctor._resolve_concore_path") def test_doctor_with_concore_octave(self, mock_path): """Doctor should detect concore.octave flag.""" mock_path.return_value = Path(self.temp_dir) @@ -192,13 +189,12 @@ def test_doctor_with_concore_octave(self, mock_path): from rich.console import Console import io + console = Console(file=io.StringIO(), force_terminal=True) result = doctor_check(console) self.assertIsInstance(result, bool) - @patch( - "concore_cli.commands.doctor._resolve_concore_path" - ) + @patch("concore_cli.commands.doctor._resolve_concore_path") def test_doctor_with_concore_sudo(self, mock_path): """Doctor should detect concore.sudo config.""" mock_path.return_value = Path(self.temp_dir) @@ -207,6 +203,7 @@ def test_doctor_with_concore_sudo(self, mock_path): from rich.console import Console import io + console = Console(file=io.StringIO(), force_terminal=True) result = doctor_check(console) self.assertIsInstance(result, bool) From 888f143cadbcdbe2e6d88942225c2f1dffdda37d Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Mon, 9 Mar 2026 12:57:09 +0530 Subject: [PATCH 244/275] fix: replace Unicode symbols with ASCII for Windows cp1252 compatibility Replace non-ASCII characters (U+2713, U+2717, U+2192, U+2014) with ASCII equivalents (+, x, ->, -) to prevent UnicodeEncodeError on Windows terminals using legacy cp1252 encoding. Rich color markup still conveys pass/fail/warn semantics via green/red/yellow styling. --- concore_cli/commands/doctor.py | 54 +++++++++++++++++----------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/concore_cli/commands/doctor.py b/concore_cli/commands/doctor.py index 0d5183d4..254edfd0 100644 --- a/concore_cli/commands/doctor.py +++ b/concore_cli/commands/doctor.py @@ -191,7 +191,7 @@ def doctor_check(console): console.print() console.print( Panel.fit( - "[bold]concore Doctor — System Readiness Report[/bold]", + "[bold]concore Doctor - System Readiness Report[/bold]", border_style="cyan", ) ) @@ -203,11 +203,11 @@ def doctor_check(console): # Python version py_version = platform.python_version() if sys.version_info >= (3, 9): - console.print(f" [green]✓[/green] Python {py_version} (>= 3.9 required)") + console.print(f" [green]+[/green] Python {py_version} (>= 3.9 required)") passed += 1 else: console.print( - f" [red]✗[/red] Python {py_version} — concore requires Python >= 3.9" + f" [red]x[/red] Python {py_version} - concore requires Python >= 3.9" ) errors += 1 @@ -215,10 +215,10 @@ def doctor_check(console): try: from concore_cli import __version__ - console.print(f" [green]✓[/green] concore {__version__} installed") + console.print(f" [green]+[/green] concore {__version__} installed") passed += 1 except ImportError: - console.print(" [red]✗[/red] concore package not found") + console.print(" [red]x[/red] concore package not found") errors += 1 # CONCOREPATH @@ -227,7 +227,7 @@ def doctor_check(console): writable = os.access(str(concore_path), os.W_OK) status = "writable" if writable else "read-only" if writable: - console.print(f" [green]✓[/green] CONCOREPATH: {concore_path} ({status})") + console.print(f" [green]+[/green] CONCOREPATH: {concore_path} ({status})") passed += 1 else: console.print( @@ -235,7 +235,7 @@ def doctor_check(console): ) warnings += 1 else: - console.print(f" [red]✗[/red] CONCOREPATH: {concore_path} (not found)") + console.print(f" [red]x[/red] CONCOREPATH: {concore_path} (not found)") errors += 1 console.print() @@ -266,25 +266,25 @@ def doctor_check(console): warnings += 1 console.print( f" [yellow]![/yellow] {tool_label}{exe_info}" - f"{version_str} → {path}{extra}" + f"{version_str} -> {path}{extra}" ) continue console.print( - f" [green]✓[/green] {tool_label}{exe_info}" - f"{version_str} → {path}{extra}" + f" [green]+[/green] {tool_label}{exe_info}" + f"{version_str} -> {path}{extra}" ) passed += 1 else: hint = tool_def["install_hints"].get(plat_name, "") hint_str = f" (install: {hint})" if hint else "" - # Docker, MATLAB, Verilog are optional — show as warning + # Docker, MATLAB, Verilog are optional - show as warning if tool_label in ("MATLAB", "Verilog (iverilog)", "Docker"): console.print( - f" [yellow]![/yellow] {tool_label} → Not found{hint_str}" + f" [yellow]![/yellow] {tool_label} -> Not found{hint_str}" ) warnings += 1 else: - console.print(f" [red]✗[/red] {tool_label} → Not found{hint_str}") + console.print(f" [red]x[/red] {tool_label} -> Not found{hint_str}") errors += 1 console.print() @@ -314,32 +314,32 @@ def doctor_check(console): ] ) console.print( - f" [green]✓[/green] {filename} → " + f" [green]+[/green] {filename} -> " f"{line_count} tool path(s) configured" ) elif filename == "concore.mcr": if os.path.exists(os.path.expanduser(content)): - console.print(f" [green]✓[/green] {filename} → {content}") + console.print(f" [green]+[/green] {filename} -> {content}") passed += 1 else: console.print( - f" [yellow]![/yellow] {filename} → " + f" [yellow]![/yellow] {filename} -> " f"path does not exist: {content}" ) warnings += 1 continue elif filename == "concore.sudo": - console.print(f" [green]✓[/green] {filename} → {content}") + console.print(f" [green]+[/green] {filename} -> {content}") elif filename == "concore.repo": - console.print(f" [green]✓[/green] {filename} → {content}") + console.print(f" [green]+[/green] {filename} -> {content}") else: - console.print(f" [green]✓[/green] {filename} → Enabled") + console.print(f" [green]+[/green] {filename} -> Enabled") passed += 1 except Exception: - console.print(f" [yellow]![/yellow] {filename} → Could not read") + console.print(f" [yellow]![/yellow] {filename} -> Could not read") warnings += 1 else: - console.print(f" [dim]—[/dim] {filename} → Not set ({description})") + console.print(f" [dim]-[/dim] {filename} -> Not set ({description})") # Build environment variable list from TOOL_DEFINITIONS config_keys env_vars = [] @@ -349,10 +349,10 @@ def doctor_check(console): env_vars.append("DOCKEREXE") env_set = [v for v in env_vars if os.environ.get(v)] if env_set: - console.print(f" [green]✓[/green] Environment variables: {', '.join(env_set)}") + console.print(f" [green]+[/green] Environment variables: {', '.join(env_set)}") passed += 1 else: - console.print(" [dim]—[/dim] No concore environment variables set") + console.print(" [dim]-[/dim] No concore environment variables set") console.print() @@ -362,20 +362,20 @@ def doctor_check(console): for pkg in REQUIRED_PACKAGES: found, version = _check_package(pkg) if found: - console.print(f" [green]✓[/green] {pkg} {version}") + console.print(f" [green]+[/green] {pkg} {version}") passed += 1 else: - console.print(f" [red]✗[/red] {pkg} → Not installed (pip install {pkg})") + console.print(f" [red]x[/red] {pkg} -> Not installed (pip install {pkg})") errors += 1 for pkg, install_hint in OPTIONAL_PACKAGES.items(): found, version = _check_package(pkg) if found: - console.print(f" [green]✓[/green] {pkg} {version}") + console.print(f" [green]+[/green] {pkg} {version}") passed += 1 else: console.print( - f" [yellow]![/yellow] {pkg} → Not installed ({install_hint})" + f" [yellow]![/yellow] {pkg} -> Not installed ({install_hint})" ) warnings += 1 From 686ff9aa7358c49c5dad74940d46e88e4119365f Mon Sep 17 00:00:00 2001 From: Avinash Kumar Deepak Date: Mon, 9 Mar 2026 18:37:59 +0530 Subject: [PATCH 245/275] fix #496: add concore.java for local Java nodes --- concore.java | 923 +++++++++++++++++++++++++++++++++++++++++++++++++++ mkconcore.py | 2 +- 2 files changed, 924 insertions(+), 1 deletion(-) create mode 100644 concore.java diff --git a/concore.java b/concore.java new file mode 100644 index 00000000..3cb7d021 --- /dev/null +++ b/concore.java @@ -0,0 +1,923 @@ +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.channels.FileLock; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; +import java.util.HashMap; +import java.util.Map; +import java.util.ArrayList; +import java.util.List; +import org.zeromq.ZMQ; + +/** + * Java implementation of concore local communication. + * + * This class provides file-based inter-process communication for control systems, + * mirroring the functionality of concore.py. + */ +public class concore { + private static Map iport = new HashMap<>(); + private static Map oport = new HashMap<>(); + private static String s = ""; + private static String olds = ""; + // delay in milliseconds (Python uses time.sleep(1) = 1 second) + private static int delay = 1000; + private static int retrycount = 0; + private static int maxRetries = 5; + private static String inpath = "./in"; + private static String outpath = "./out"; + private static Map params = new HashMap<>(); + private static Map zmqPorts = new HashMap<>(); + private static ZMQ.Context zmqContext = null; + // simtime as double to preserve fractional values (e.g. "[0.0, ...]") + private static double simtime = 0; + private static double maxtime; + + private static final Path BASE_DIR = Paths.get("").toAbsolutePath().normalize(); + private static final Path PID_REGISTRY_FILE = BASE_DIR.resolve("concorekill_pids.txt"); + private static final Path KILL_SCRIPT_FILE = BASE_DIR.resolve("concorekill.bat"); + + // initialize on class load, same as Python module-level init + static { + if (isWindows()) { + registerPid(); + writeKillScript(); + Runtime.getRuntime().addShutdownHook(new Thread(concore::cleanupPid)); + } + try { + iport = parseFile("concore.iport"); + } catch (IOException e) { + } + try { + oport = parseFile("concore.oport"); + } catch (IOException e) { + } + try { + String paramsFile = Paths.get(portPath(inpath, 1), "concore.params").toString(); + String sparams = new String(Files.readAllBytes(Paths.get(paramsFile)), java.nio.charset.StandardCharsets.UTF_8); + if (sparams.length() > 0 && sparams.charAt(0) == '"') { // windows keeps "" need to remove + sparams = sparams.substring(1); + sparams = sparams.substring(0, sparams.indexOf('"')); + } + params = parseParams(sparams); + } catch (IOException e) { + params = new HashMap<>(); + } + defaultMaxTime(100); + Runtime.getRuntime().addShutdownHook(new Thread(concore::terminateZmq)); + } + + private static boolean isWindows() { + String os = System.getProperty("os.name"); + return os != null && os.toLowerCase().contains("win"); + } + + private static void registerPid() { + try { + String pid = String.valueOf(ProcessHandle.current().pid()); + try (FileChannel channel = FileChannel.open(PID_REGISTRY_FILE, + StandardOpenOption.CREATE, StandardOpenOption.WRITE, StandardOpenOption.APPEND)) { + try (FileLock lock = channel.lock()) { + channel.write(ByteBuffer.wrap((pid + System.lineSeparator()).getBytes(java.nio.charset.StandardCharsets.UTF_8))); + } + } + } catch (IOException e) { + } + } + + private static void cleanupPid() { + String pid = String.valueOf(ProcessHandle.current().pid()); + if (!Files.exists(PID_REGISTRY_FILE)) return; + List remaining = new ArrayList<>(); + try (FileChannel channel = FileChannel.open(PID_REGISTRY_FILE, + StandardOpenOption.READ, StandardOpenOption.WRITE)) { + try (FileLock lock = channel.lock()) { + ByteBuffer buf = ByteBuffer.allocate((int) channel.size()); + channel.read(buf); + buf.flip(); + String content = java.nio.charset.StandardCharsets.UTF_8.decode(buf).toString(); + for (String line : content.split("\\R")) { + String trimmed = line.trim(); + if (!trimmed.isEmpty() && !trimmed.equals(pid)) { + remaining.add(trimmed); + } + } + channel.truncate(0); + channel.position(0); + if (!remaining.isEmpty()) { + StringBuilder sb = new StringBuilder(); + for (String r : remaining) sb.append(r).append(System.lineSeparator()); + channel.write(ByteBuffer.wrap(sb.toString().getBytes(java.nio.charset.StandardCharsets.UTF_8))); + } + } + } catch (IOException e) { + } + if (remaining.isEmpty()) { + try { Files.deleteIfExists(PID_REGISTRY_FILE); } catch (IOException e) {} + try { Files.deleteIfExists(KILL_SCRIPT_FILE); } catch (IOException e) {} + } + } + + private static void writeKillScript() { + try { + String regName = PID_REGISTRY_FILE.getFileName().toString(); + String batName = KILL_SCRIPT_FILE.getFileName().toString(); + String script = "@echo off\r\n"; + script += "if not exist \"%~dp0" + regName + "\" (\r\n"; + script += " echo No PID registry found. Nothing to kill.\r\n"; + script += " exit /b 0\r\n"; + script += ")\r\n"; + script += "for /f \"usebackq tokens=*\" %%p in (\"%~dp0" + regName + "\") do (\r\n"; + script += " wmic process where \"ProcessId=%%p\" get CommandLine /value 2>nul | find /i \"concore\" >nul\r\n"; + script += " if not errorlevel 1 (\r\n"; + script += " echo Killing concore process %%p\r\n"; + script += " taskkill /F /PID %%p >nul 2>&1\r\n"; + script += " ) else (\r\n"; + script += " echo Skipping PID %%p - not a concore process or not running\r\n"; + script += " )\r\n"; + script += ")\r\n"; + script += "del /q \"%~dp0" + regName + "\" 2>nul\r\n"; + script += "del /q \"%~dp0" + batName + "\" 2>nul\r\n"; + Files.write(KILL_SCRIPT_FILE, script.getBytes(java.nio.charset.StandardCharsets.UTF_8)); + } catch (IOException e) { + } + } + + /** + * Parses a param string into a map, matching concore_base.parse_params. + * Tries dict literal first, then falls back to semicolon-separated key=value pairs. + */ + private static Map parseParams(String sparams) { + Map result = new HashMap<>(); + if (sparams == null || sparams.isEmpty()) return result; + String trimmed = sparams.trim(); + if (trimmed.startsWith("{") && trimmed.endsWith("}")) { + try { + Object val = literalEval(trimmed); + if (val instanceof Map) { + @SuppressWarnings("unchecked") + Map map = (Map) val; + return map; + } + } catch (Exception e) { + } + } + for (String item : trimmed.split(";")) { + if (item.contains("=")) { + String[] parts = item.split("=", 2); // split on first '=' only + String key = parts[0].trim(); + String value = parts[1].trim(); + try { + result.put(key, literalEval(value)); + } catch (Exception e) { + result.put(key, value); + } + } + } + return result; + } + + /** + * Parses a file containing a Python-style dictionary literal. + * Returns empty map if file is empty or malformed (matches Python safe_literal_eval). + */ + private static Map parseFile(String filename) throws IOException { + String content = new String(Files.readAllBytes(Paths.get(filename)), java.nio.charset.StandardCharsets.UTF_8); + content = content.trim(); + if (content.isEmpty()) { + return new HashMap<>(); + } + try { + Object result = literalEval(content); + if (result instanceof Map) { + @SuppressWarnings("unchecked") + Map map = (Map) result; + return map; + } + } catch (IllegalArgumentException e) { + System.err.println("Failed to parse file as map: " + filename + " (" + e.getMessage() + ")"); + } + return new HashMap<>(); + } + + /** + * Sets maxtime from concore.maxtime file, or uses defaultValue if file not found. + * Catches both IOException and RuntimeException to match Python safe_literal_eval. + */ + public static void defaultMaxTime(double defaultValue) { + try { + String maxtimeFile = Paths.get(portPath(inpath, 1), "concore.maxtime").toString(); + String content = new String(Files.readAllBytes(Paths.get(maxtimeFile))); + Object parsed = literalEval(content.trim()); + if (parsed instanceof Number) { + maxtime = ((Number) parsed).doubleValue(); + } else { + maxtime = defaultValue; + } + } catch (IOException | RuntimeException e) { + maxtime = defaultValue; + } + } + + private static String portPath(String base, int portNum) { + return base + portNum; + } + + // package-level helpers for testing with temp directories + static void setInPath(String path) { inpath = path; } + static void setOutPath(String path) { outpath = path; } + static void setDelay(int ms) { delay = ms; } + static double getSimtime() { return simtime; } + static void resetState() { s = ""; olds = ""; simtime = 0; } + + public static boolean unchanged() { + if (olds.equals(s)) { + s = ""; + return true; + } + olds = s; + return false; + } + + public static Object tryParam(String n, Object i) { + if (params.containsKey(n)) { + return params.get(n); + } else { + return i; + } + } + + /** + * Reads data from a port file. Returns the values after extracting simtime. + * Input format: [simtime, val1, val2, ...] + * Returns: list of values after simtime + * Includes max retry limit to avoid infinite blocking (matches Python behavior). + */ + public static ReadResult read(int port, String name, String initstr) { + // Parse default value upfront for consistent return type + List defaultVal = new ArrayList<>(); + try { + List parsed = (List) literalEval(initstr); + if (parsed.size() > 1) { + defaultVal = new ArrayList<>(parsed.subList(1, parsed.size())); + } + } catch (Exception e) { + // initstr not parseable as list; defaultVal stays empty + } + + String filePath = Paths.get(portPath(inpath, port), name).toString(); + try { + Thread.sleep(delay); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + s += initstr; + return new ReadResult(ReadStatus.TIMEOUT, defaultVal); + } + + String ins; + try { + ins = new String(Files.readAllBytes(Paths.get(filePath))); + } catch (IOException e) { + System.out.println("File " + filePath + " not found, using default value."); + s += initstr; + return new ReadResult(ReadStatus.FILE_NOT_FOUND, defaultVal); + } + + int attempts = 0; + while (ins.length() == 0 && attempts < maxRetries) { + try { + Thread.sleep(delay); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + s += initstr; + return new ReadResult(ReadStatus.TIMEOUT, defaultVal); + } + try { + ins = new String(Files.readAllBytes(Paths.get(filePath))); + } catch (IOException e) { + System.out.println("Retry " + (attempts + 1) + ": Error reading " + filePath); + } + attempts++; + retrycount++; + } + + if (ins.length() == 0) { + System.out.println("Max retries reached for " + filePath + ", using default value."); + return new ReadResult(ReadStatus.RETRIES_EXCEEDED, defaultVal); + } + + s += ins; + try { + List inval = (List) literalEval(ins); + if (!inval.isEmpty()) { + double firstSimtime = ((Number) inval.get(0)).doubleValue(); + simtime = Math.max(simtime, firstSimtime); + return new ReadResult(ReadStatus.SUCCESS, new ArrayList<>(inval.subList(1, inval.size()))); + } + } catch (Exception e) { + System.out.println("Error parsing " + ins + ": " + e.getMessage()); + } + return new ReadResult(ReadStatus.PARSE_ERROR, defaultVal); + } + + /** + * Escapes a Java string so it can be safely used as a single-quoted Python string literal. + * At minimum, escapes backslash, single quote, newline, carriage return, and tab. + */ + private static String escapePythonString(String s) { + StringBuilder sb = new StringBuilder(s.length()); + for (int i = 0; i < s.length(); i++) { + char c = s.charAt(i); + switch (c) { + case '\\': sb.append("\\\\"); break; + case '\'': sb.append("\\'"); break; + case '\n': sb.append("\\n"); break; + case '\r': sb.append("\\r"); break; + case '\t': sb.append("\\t"); break; + default: sb.append(c); break; + } + } + return sb.toString(); + } + + /** + * Converts a Java object to its Python-literal string representation. + * True/False/None instead of true/false/null; strings single-quoted. + */ + private static String toPythonLiteral(Object obj) { + if (obj == null) return "None"; + if (obj instanceof Boolean) return ((Boolean) obj) ? "True" : "False"; + if (obj instanceof String) return "'" + escapePythonString((String) obj) + "'"; + if (obj instanceof Number) return obj.toString(); + if (obj instanceof List) { + List list = (List) obj; + StringBuilder sb = new StringBuilder("["); + for (int i = 0; i < list.size(); i++) { + if (i > 0) sb.append(", "); + sb.append(toPythonLiteral(list.get(i))); + } + sb.append("]"); + return sb.toString(); + } + if (obj instanceof Map) { + Map map = (Map) obj; + StringBuilder sb = new StringBuilder("{"); + boolean first = true; + for (Map.Entry entry : map.entrySet()) { + if (!first) sb.append(", "); + sb.append(toPythonLiteral(entry.getKey())).append(": ").append(toPythonLiteral(entry.getValue())); + first = false; + } + sb.append("}"); + return sb.toString(); + } + return obj.toString(); + } + + /** + * Escapes a Java string so it can be safely embedded in a JSON double-quoted string. + * Escapes backslash, double quote, newline, carriage return, and tab. + */ + private static String escapeJsonString(String s) { + StringBuilder sb = new StringBuilder(s.length()); + for (int i = 0; i < s.length(); i++) { + char c = s.charAt(i); + switch (c) { + case '\\': sb.append("\\\\"); break; + case '"': sb.append("\\\""); break; + case '\n': sb.append("\\n"); break; + case '\r': sb.append("\\r"); break; + case '\t': sb.append("\\t"); break; + default: sb.append(c); break; + } + } + return sb.toString(); + } + + /** + * Converts a Java object to its JSON string representation. + * true/false/null instead of True/False/None; strings double-quoted. + */ + private static String toJsonLiteral(Object obj) { + if (obj == null) return "null"; + if (obj instanceof Boolean) return ((Boolean) obj) ? "true" : "false"; + if (obj instanceof String) return "\"" + escapeJsonString((String) obj) + "\""; + if (obj instanceof Number) return obj.toString(); + if (obj instanceof List) { + List list = (List) obj; + StringBuilder sb = new StringBuilder("["); + for (int i = 0; i < list.size(); i++) { + if (i > 0) sb.append(", "); + sb.append(toJsonLiteral(list.get(i))); + } + sb.append("]"); + return sb.toString(); + } + if (obj instanceof Map) { + Map map = (Map) obj; + StringBuilder sb = new StringBuilder("{"); + boolean first = true; + for (Map.Entry entry : map.entrySet()) { + if (!first) sb.append(", "); + sb.append(toJsonLiteral(entry.getKey())).append(": ").append(toJsonLiteral(entry.getValue())); + first = false; + } + sb.append("}"); + return sb.toString(); + } + return obj.toString(); + } + + /** + * Writes data to a port file. + * Prepends simtime+delta to the value list, then serializes to Python-literal format. + * Accepts List or String values (matching Python implementation). + */ + public static void write(int port, String name, Object val, int delta) { + try { + String path = Paths.get(portPath(outpath, port), name).toString(); + StringBuilder content = new StringBuilder(); + if (val instanceof String) { + Thread.sleep(2 * delay); + content.append(val); + } else if (val instanceof List) { + List listVal = (List) val; + content.append("["); + content.append(toPythonLiteral(simtime + delta)); + for (int i = 0; i < listVal.size(); i++) { + content.append(", "); + content.append(toPythonLiteral(listVal.get(i))); + } + content.append("]"); + // simtime must not be mutated here. + // Mutation breaks cross-language determinism. + } else if (val instanceof Object[]) { + // Legacy support for Object[] arguments + Object[] arrayVal = (Object[]) val; + content.append("["); + content.append(toPythonLiteral(simtime + delta)); + for (Object o : arrayVal) { + content.append(", "); + content.append(toPythonLiteral(o)); + } + content.append("]"); + // simtime must not be mutated here. + // Mutation breaks cross-language determinism. + } else { + System.out.println("write must have list or str"); + return; + } + Files.write(Paths.get(path), content.toString().getBytes()); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + System.out.println("skipping " + outpath + "/" + port + "/" + name); + } catch (IOException e) { + System.out.println("skipping " + outpath + "/" + port + "/" + name); + } + } + + /** + * Parses an initial value string like "[0.0, 1.0, 2.0]". + * Extracts simtime from position 0 and returns the remaining values as a List. + */ + public static List initVal(String simtimeVal) { + List val = new ArrayList<>(); + try { + List inval = (List) literalEval(simtimeVal); + if (!inval.isEmpty()) { + simtime = ((Number) inval.get(0)).doubleValue(); + val = new ArrayList<>(inval.subList(1, inval.size())); + } + } catch (Exception e) { + System.out.println("Error parsing initVal: " + e.getMessage()); + } + return val; + } + + private static ZMQ.Context getZmqContext() { + if (zmqContext == null) { + zmqContext = ZMQ.context(1); + } + return zmqContext; + } + + public static void initZmqPort(String portName, String portType, String address, String socketTypeStr) { + if (zmqPorts.containsKey(portName)) return; + int sockType = zmqSocketTypeFromString(socketTypeStr); + if (sockType == -1) { + System.err.println("initZmqPort: unknown socket type '" + socketTypeStr + "'"); + return; + } + zmqPorts.put(portName, new ZeroMQPort(portType, address, sockType)); + } + + public static void terminateZmq() { + for (Map.Entry entry : zmqPorts.entrySet()) { + entry.getValue().socket.close(); + } + zmqPorts.clear(); + if (zmqContext != null) { + zmqContext.term(); + zmqContext = null; + } + } + + private static int zmqSocketTypeFromString(String s) { + switch (s.toUpperCase()) { + case "REQ": return ZMQ.REQ; + case "REP": return ZMQ.REP; + case "PUB": return ZMQ.PUB; + case "SUB": return ZMQ.SUB; + case "PUSH": return ZMQ.PUSH; + case "PULL": return ZMQ.PULL; + case "PAIR": return ZMQ.PAIR; + default: return -1; + } + } + + /** + * Reads data from a ZMQ port. Same wire format as file-based read: + * expects [simtime, val1, val2, ...], strips simtime, returns the rest. + */ + public static ReadResult read(String portName, String name, String initstr) { + List defaultVal = new ArrayList<>(); + try { + List parsed = (List) literalEval(initstr); + if (parsed.size() > 1) { + defaultVal = new ArrayList<>(parsed.subList(1, parsed.size())); + } + } catch (Exception e) { + } + ZeroMQPort port = zmqPorts.get(portName); + if (port == null) { + System.err.println("read: ZMQ port '" + portName + "' not initialized"); + return new ReadResult(ReadStatus.FILE_NOT_FOUND, defaultVal); + } + String msg = port.recvWithRetry(); + if (msg == null) { + System.err.println("read: ZMQ recv timeout on port '" + portName + "'"); + return new ReadResult(ReadStatus.TIMEOUT, defaultVal); + } + s += msg; + try { + List inval = (List) literalEval(msg); + if (!inval.isEmpty()) { + simtime = Math.max(simtime, ((Number) inval.get(0)).doubleValue()); + return new ReadResult(ReadStatus.SUCCESS, new ArrayList<>(inval.subList(1, inval.size()))); + } + } catch (Exception e) { + System.out.println("Error parsing ZMQ message '" + msg + "': " + e.getMessage()); + } + return new ReadResult(ReadStatus.PARSE_ERROR, defaultVal); + } + + /** + * Writes data to a ZMQ port. Prepends [simtime+delta] to match file-based write behavior. + */ + public static void write(String portName, String name, Object val, int delta) { + ZeroMQPort port = zmqPorts.get(portName); + if (port == null) { + System.err.println("write: ZMQ port '" + portName + "' not initialized"); + return; + } + String payload; + if (val instanceof List) { + List listVal = (List) val; + StringBuilder sb = new StringBuilder("["); + sb.append(toJsonLiteral(simtime + delta)); + for (Object o : listVal) { + sb.append(", "); + sb.append(toJsonLiteral(o)); + } + sb.append("]"); + payload = sb.toString(); + // simtime must not be mutated here + } else if (val instanceof String) { + payload = (String) val; + } else { + System.out.println("write must have list or str"); + return; + } + port.sendWithRetry(payload); + } + + /** + * Parses a Python-literal string into Java objects using a recursive descent parser. + * Supports: dict, list, int, float, string (single/double quoted), bool, None, nested structures. + * This replaces the broken split-based parser that could not handle quoted commas or nesting. + */ + static Object literalEval(String s) { + if (s == null) throw new IllegalArgumentException("Input cannot be null"); + s = s.trim(); + if (s.isEmpty()) throw new IllegalArgumentException("Input cannot be empty"); + Parser parser = new Parser(s); + Object result = parser.parseExpression(); + parser.skipWhitespace(); + if (parser.pos < parser.input.length()) { + throw new IllegalArgumentException("Unexpected trailing content at position " + parser.pos); + } + return result; + } + + public enum ReadStatus { + SUCCESS, FILE_NOT_FOUND, TIMEOUT, PARSE_ERROR, RETRIES_EXCEEDED + } + + public static class ReadResult { + public final ReadStatus status; + public final List data; + ReadResult(ReadStatus status, List data) { + this.status = status; + this.data = data; + } + } + + /** + * ZMQ socket wrapper with bind/connect, timeouts, and retry. + */ + private static class ZeroMQPort { + final ZMQ.Socket socket; + final String address; + + ZeroMQPort(String portType, String address, int socketType) { + this.address = address; + ZMQ.Context ctx = getZmqContext(); + this.socket = ctx.socket(socketType); + this.socket.setReceiveTimeOut(2000); + this.socket.setSendTimeOut(2000); + this.socket.setLinger(0); + if (portType.equals("bind")) { + this.socket.bind(address); + } else { + this.socket.connect(address); + } + } + + String recvWithRetry() { + for (int attempt = 0; attempt < 5; attempt++) { + String msg = socket.recvStr(); + if (msg != null) return msg; + try { Thread.sleep(500); } catch (InterruptedException e) { Thread.currentThread().interrupt(); break; } + } + return null; + } + + void sendWithRetry(String message) { + for (int attempt = 0; attempt < 5; attempt++) { + if (socket.send(message)) return; + try { Thread.sleep(500); } catch (InterruptedException e) { Thread.currentThread().interrupt(); break; } + } + } + } + + /** + * Recursive descent parser for Python literal expressions. + * Handles: dicts, lists, tuples, strings, numbers, booleans, None. + */ + private static class Parser { + final String input; + int pos; + + Parser(String input) { + this.input = input; + this.pos = 0; + } + + void skipWhitespace() { + while (pos < input.length() && Character.isWhitespace(input.charAt(pos))) { + pos++; + } + } + + char peek() { + skipWhitespace(); + if (pos >= input.length()) throw new IllegalArgumentException("Unexpected end of input"); + return input.charAt(pos); + } + + char advance() { + char c = input.charAt(pos); + pos++; + return c; + } + + boolean hasMore() { + skipWhitespace(); + return pos < input.length(); + } + + Object parseExpression() { + skipWhitespace(); + if (pos >= input.length()) throw new IllegalArgumentException("Unexpected end of input"); + char c = input.charAt(pos); + + if (c == '{') return parseDict(); + if (c == '[') return parseList(); + if (c == '(') return parseTuple(); + if (c == '\'' || c == '"') return parseString(); + if (c == '-' || c == '+' || Character.isDigit(c)) return parseNumber(); + return parseKeyword(); + } + + Map parseDict() { + Map map = new HashMap<>(); + pos++; // skip '{' + skipWhitespace(); + if (hasMore() && input.charAt(pos) == '}') { + pos++; + return map; + } + while (true) { + skipWhitespace(); + Object key = parseExpression(); + skipWhitespace(); + if (pos >= input.length() || input.charAt(pos) != ':') { + throw new IllegalArgumentException("Expected ':' in dict at position " + pos); + } + pos++; // skip ':' + skipWhitespace(); + Object value = parseExpression(); + if (!(key instanceof String)) { + throw new IllegalArgumentException( + "Dict keys must be non-null strings, but got: " + + (key == null ? "null" : key.getClass().getSimpleName())); + } + map.put((String) key, value); + skipWhitespace(); + if (pos >= input.length()) { + throw new IllegalArgumentException("Unterminated dict: missing '}'"); + } + if (input.charAt(pos) == '}') { + pos++; + break; + } + if (input.charAt(pos) == ',') { + pos++; + skipWhitespace(); + // trailing comma before close + if (hasMore() && input.charAt(pos) == '}') { + pos++; + break; + } + } else { + throw new IllegalArgumentException("Expected ',' or '}' in dict at position " + pos); + } + } + return map; + } + + List parseList() { + List list = new ArrayList<>(); + pos++; // skip '[' + skipWhitespace(); + if (hasMore() && input.charAt(pos) == ']') { + pos++; + return list; + } + while (true) { + skipWhitespace(); + list.add(parseExpression()); + skipWhitespace(); + if (pos >= input.length()) { + throw new IllegalArgumentException("Unterminated list: missing ']'"); + } + if (input.charAt(pos) == ']') { + pos++; + break; + } + if (input.charAt(pos) == ',') { + pos++; + skipWhitespace(); + // trailing comma before close + if (hasMore() && input.charAt(pos) == ']') { + pos++; + break; + } + } else { + throw new IllegalArgumentException("Expected ',' or ']' in list at position " + pos); + } + } + return list; + } + + List parseTuple() { + List list = new ArrayList<>(); + pos++; // skip '(' + skipWhitespace(); + if (hasMore() && input.charAt(pos) == ')') { + pos++; + return list; + } + while (true) { + skipWhitespace(); + list.add(parseExpression()); + skipWhitespace(); + if (pos >= input.length()) { + throw new IllegalArgumentException("Unterminated tuple: missing ')'"); + } + if (input.charAt(pos) == ')') { + pos++; + break; + } + if (input.charAt(pos) == ',') { + pos++; + skipWhitespace(); + // trailing comma before close + if (hasMore() && input.charAt(pos) == ')') { + pos++; + break; + } + } else { + throw new IllegalArgumentException("Expected ',' or ')' in tuple at position " + pos); + } + } + return list; + } + + String parseString() { + char quote = advance(); // opening quote + StringBuilder sb = new StringBuilder(); + while (pos < input.length()) { + char c = input.charAt(pos); + if (c == '\\' && pos + 1 < input.length()) { + pos++; + char escaped = input.charAt(pos); + switch (escaped) { + case 'n': sb.append('\n'); break; + case 't': sb.append('\t'); break; + case 'r': sb.append('\r'); break; + case '\\': sb.append('\\'); break; + case '\'': sb.append('\''); break; + case '"': sb.append('"'); break; + default: sb.append('\\').append(escaped); break; + } + pos++; + } else if (c == quote) { + pos++; + return sb.toString(); + } else { + sb.append(c); + pos++; + } + } + throw new IllegalArgumentException("Unterminated string starting at position " + (pos - sb.length() - 1)); + } + + Number parseNumber() { + int start = pos; + if (pos < input.length() && (input.charAt(pos) == '-' || input.charAt(pos) == '+')) { + pos++; + } + boolean hasDecimal = false; + boolean hasExponent = false; + while (pos < input.length()) { + char c = input.charAt(pos); + if (Character.isDigit(c)) { + pos++; + } else if (c == '.' && !hasDecimal && !hasExponent) { + hasDecimal = true; + pos++; + } else if ((c == 'e' || c == 'E') && !hasExponent) { + hasExponent = true; + pos++; + if (pos < input.length() && (input.charAt(pos) == '+' || input.charAt(pos) == '-')) { + pos++; + } + } else { + break; + } + } + String numStr = input.substring(start, pos); + try { + if (hasDecimal || hasExponent) { + return Double.parseDouble(numStr); + } else { + try { + return Integer.parseInt(numStr); + } catch (NumberFormatException e) { + return Long.parseLong(numStr); + } + } + } catch (NumberFormatException e) { + throw new IllegalArgumentException("Invalid number: '" + numStr + "' at position " + start); + } + } + + Object parseKeyword() { + int start = pos; + while (pos < input.length() && Character.isLetterOrDigit(input.charAt(pos)) || (pos < input.length() && input.charAt(pos) == '_')) { + pos++; + } + String word = input.substring(start, pos); + switch (word) { + case "True": case "true": return Boolean.TRUE; + case "False": case "false": return Boolean.FALSE; + case "None": case "null": return null; + default: throw new IllegalArgumentException("Unknown keyword: '" + word + "' at position " + start); + } + } + } +} diff --git a/mkconcore.py b/mkconcore.py index 8f40ccf7..28fab9df 100644 --- a/mkconcore.py +++ b/mkconcore.py @@ -1095,7 +1095,7 @@ def cleanup_script_files(): containername,sourcecode = nodes_dict[node].split(':') if len(sourcecode)!=0: dockername,langext = sourcecode.rsplit(".", 1) - if not (langext in ["py","m","sh","cpp","v"]): # 6/22/21 + if not (langext in ["py","m","sh","cpp","v","java"]): # 6/22/21 logging.error(f"Extension .{langext} is unsupported") quit() if concoretype=="windows": From 8b28ad56f73d1fdb745fb4601fd53f46b278c138 Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Sun, 15 Mar 2026 19:41:38 +0530 Subject: [PATCH 246/275] feat(cli): generate STUDY.json for init and run outputs --- concore_cli/commands/init.py | 8 ++++ concore_cli/commands/metadata.py | 75 ++++++++++++++++++++++++++++++ concore_cli/commands/run.py | 9 ++++ tests/test_cli.py | 15 ++++++ tests/test_openjupyter_security.py | 26 +++++++++-- 5 files changed, 128 insertions(+), 5 deletions(-) create mode 100644 concore_cli/commands/metadata.py diff --git a/concore_cli/commands/init.py b/concore_cli/commands/init.py index eb73e916..c8ce87cd 100644 --- a/concore_cli/commands/init.py +++ b/concore_cli/commands/init.py @@ -1,6 +1,8 @@ from pathlib import Path from rich.panel import Panel +from .metadata import write_study_metadata + SAMPLE_GRAPHML = """ @@ -87,10 +89,16 @@ def init_project(name, template, console): with open(readme_file, "w") as f: f.write(README_TEMPLATE.format(project_name=name)) + metadata_path = write_study_metadata( + project_path, generated_by="concore init", workflow_file=workflow_file + ) + console.print() console.print( Panel.fit( f"[green]✓[/green] Project created successfully!\n\n" + f"Metadata:\n" + f" {metadata_path.name}\n\n" f"Next steps:\n" f" cd {name}\n" f" concore validate workflow.graphml\n" diff --git a/concore_cli/commands/metadata.py b/concore_cli/commands/metadata.py new file mode 100644 index 00000000..2c44adbf --- /dev/null +++ b/concore_cli/commands/metadata.py @@ -0,0 +1,75 @@ +import hashlib +import json +import platform +import shutil +from datetime import datetime, timezone +from pathlib import Path + +from concore_cli import __version__ + + +def _checksum_file(path: Path) -> str: + hasher = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(8192), b""): + hasher.update(chunk) + return f"sha256:{hasher.hexdigest()}" + + +def _detect_tools() -> dict: + tool_candidates = { + "python": ["python", "python3"], + "g++": ["g++"], + "docker": ["docker"], + "octave": ["octave"], + "iverilog": ["iverilog"], + } + detected = {} + for tool_name, candidates in tool_candidates.items(): + detected_path = None + for candidate in candidates: + detected_path = shutil.which(candidate) + if detected_path: + break + detected[tool_name] = detected_path or "not found" + return detected + + +def write_study_metadata(study_path: Path, generated_by: str, workflow_file: Path = None): + checksums = {} + checksum_candidates = [ + "workflow.graphml", + "docker-compose.yml", + "concore.toml", + "runner.py", + "README.md", + "build", + "run", + "build.bat", + "run.bat", + ] + + if workflow_file is not None and workflow_file.exists(): + checksums[workflow_file.name] = _checksum_file(workflow_file) + + for relative_name in checksum_candidates: + file_path = study_path / relative_name + if file_path.exists() and file_path.is_file(): + checksums[relative_name] = _checksum_file(file_path) + + metadata = { + "generated_by": generated_by, + "concore_version": __version__, + "timestamp": datetime.now(timezone.utc).replace(microsecond=0).isoformat(), + "python_version": platform.python_version(), + "platform": platform.platform(), + "study_name": study_path.name, + "working_directory": str(study_path.resolve()), + "tools_detected": _detect_tools(), + "checksums": checksums, + "schema_version": 1, + } + + metadata_path = study_path / "STUDY.json" + metadata_path.write_text(json.dumps(metadata, indent=2) + "\n", encoding="utf-8") + return metadata_path \ No newline at end of file diff --git a/concore_cli/commands/run.py b/concore_cli/commands/run.py index a80dbe05..2653809c 100644 --- a/concore_cli/commands/run.py +++ b/concore_cli/commands/run.py @@ -4,6 +4,8 @@ from rich.panel import Panel from rich.progress import Progress, SpinnerColumn, TextColumn +from .metadata import write_study_metadata + def _find_mkconcore_path(): for parent in Path(__file__).resolve().parents: @@ -68,9 +70,16 @@ def run_workflow(workflow_file, source, output, exec_type, auto_build, console): if result.stdout: console.print(result.stdout) + metadata_path = write_study_metadata( + output_path, + generated_by="concore run", + workflow_file=workflow_path, + ) + console.print( f"[green]✓[/green] Workflow generated in [cyan]{output_path}[/cyan]" ) + console.print(f"[green]✓[/green] Metadata written to [cyan]{metadata_path}[/cyan]") except subprocess.CalledProcessError as e: progress.stop() diff --git a/tests/test_cli.py b/tests/test_cli.py index 4321e05a..63cd0f23 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -2,6 +2,7 @@ import tempfile import shutil import os +import json from pathlib import Path from click.testing import CliRunner from concore_cli.cli import cli @@ -38,6 +39,13 @@ def test_init_command(self): self.assertTrue((project_path / "src").exists()) self.assertTrue((project_path / "README.md").exists()) self.assertTrue((project_path / "src" / "script.py").exists()) + self.assertTrue((project_path / "STUDY.json").exists()) + + metadata = json.loads((project_path / "STUDY.json").read_text()) + self.assertEqual(metadata["generated_by"], "concore init") + self.assertEqual(metadata["study_name"], "test-project") + self.assertEqual(metadata["schema_version"], 1) + self.assertIn("workflow.graphml", metadata["checksums"]) def test_init_existing_directory(self): with self.runner.isolated_filesystem(temp_dir=self.temp_dir): @@ -108,6 +116,13 @@ def test_run_command_from_project_dir(self): ) self.assertEqual(result.exit_code, 0) self.assertTrue(Path("out/src/concore.py").exists()) + self.assertTrue(Path("out/STUDY.json").exists()) + + metadata = json.loads(Path("out/STUDY.json").read_text()) + self.assertEqual(metadata["generated_by"], "concore run") + self.assertEqual(metadata["study_name"], "out") + self.assertEqual(metadata["schema_version"], 1) + self.assertIn("workflow.graphml", metadata["checksums"]) def test_run_command_default_type(self): with self.runner.isolated_filesystem(temp_dir=self.temp_dir): diff --git a/tests/test_openjupyter_security.py b/tests/test_openjupyter_security.py index b046dc9b..06b29065 100644 --- a/tests/test_openjupyter_security.py +++ b/tests/test_openjupyter_security.py @@ -20,17 +20,33 @@ @pytest.fixture(autouse=True) def reset_jupyter_process(): """Reset the module-level jupyter_process before each test.""" - import fri.server.main as mod + with patch.dict( + os.environ, + { + "CONCORE_API_KEY": TEST_API_KEY, + "FLASK_SECRET_KEY": "test-flask-secret-key", + }, + clear=False, + ): + import fri.server.main as mod - mod.jupyter_process = None - yield - mod.jupyter_process = None + mod.API_KEY = TEST_API_KEY + mod.jupyter_process = None + yield + mod.jupyter_process = None @pytest.fixture def client(): """Create a Flask test client with the API key configured.""" - with patch.dict(os.environ, {"CONCORE_API_KEY": TEST_API_KEY}): + with patch.dict( + os.environ, + { + "CONCORE_API_KEY": TEST_API_KEY, + "FLASK_SECRET_KEY": "test-flask-secret-key", + }, + clear=False, + ): # Re-read env var after patching import fri.server.main as mod From b593e8eca670091d1a3e383bf1ba7272a6a64298 Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Sun, 15 Mar 2026 19:50:31 +0530 Subject: [PATCH 247/275] style: apply ruff formatting for CLI metadata changes --- concore_cli/commands/init.py | 6 +++--- concore_cli/commands/metadata.py | 6 ++++-- concore_cli/commands/run.py | 4 +++- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/concore_cli/commands/init.py b/concore_cli/commands/init.py index c8ce87cd..30ca9782 100644 --- a/concore_cli/commands/init.py +++ b/concore_cli/commands/init.py @@ -90,15 +90,15 @@ def init_project(name, template, console): f.write(README_TEMPLATE.format(project_name=name)) metadata_path = write_study_metadata( - project_path, generated_by="concore init", workflow_file=workflow_file + project_path, generated_by="concore init", workflow_file=workflow_file ) console.print() console.print( Panel.fit( f"[green]✓[/green] Project created successfully!\n\n" - f"Metadata:\n" - f" {metadata_path.name}\n\n" + f"Metadata:\n" + f" {metadata_path.name}\n\n" f"Next steps:\n" f" cd {name}\n" f" concore validate workflow.graphml\n" diff --git a/concore_cli/commands/metadata.py b/concore_cli/commands/metadata.py index 2c44adbf..4ccd1082 100644 --- a/concore_cli/commands/metadata.py +++ b/concore_cli/commands/metadata.py @@ -35,7 +35,9 @@ def _detect_tools() -> dict: return detected -def write_study_metadata(study_path: Path, generated_by: str, workflow_file: Path = None): +def write_study_metadata( + study_path: Path, generated_by: str, workflow_file: Path = None +): checksums = {} checksum_candidates = [ "workflow.graphml", @@ -72,4 +74,4 @@ def write_study_metadata(study_path: Path, generated_by: str, workflow_file: Pat metadata_path = study_path / "STUDY.json" metadata_path.write_text(json.dumps(metadata, indent=2) + "\n", encoding="utf-8") - return metadata_path \ No newline at end of file + return metadata_path diff --git a/concore_cli/commands/run.py b/concore_cli/commands/run.py index 2653809c..d4cd7d85 100644 --- a/concore_cli/commands/run.py +++ b/concore_cli/commands/run.py @@ -79,7 +79,9 @@ def run_workflow(workflow_file, source, output, exec_type, auto_build, console): console.print( f"[green]✓[/green] Workflow generated in [cyan]{output_path}[/cyan]" ) - console.print(f"[green]✓[/green] Metadata written to [cyan]{metadata_path}[/cyan]") + console.print( + f"[green]✓[/green] Metadata written to [cyan]{metadata_path}[/cyan]" + ) except subprocess.CalledProcessError as e: progress.stop() From 937648c130f45433230d3672e9d71e356f5a1bea Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Sun, 15 Mar 2026 19:55:48 +0530 Subject: [PATCH 248/275] fix(cli): keep metadata generation non-fatal in init/run --- concore_cli/commands/init.py | 19 ++++++++++++++----- concore_cli/commands/run.py | 23 ++++++++++++++--------- 2 files changed, 28 insertions(+), 14 deletions(-) diff --git a/concore_cli/commands/init.py b/concore_cli/commands/init.py index 30ca9782..85519723 100644 --- a/concore_cli/commands/init.py +++ b/concore_cli/commands/init.py @@ -89,16 +89,25 @@ def init_project(name, template, console): with open(readme_file, "w") as f: f.write(README_TEMPLATE.format(project_name=name)) - metadata_path = write_study_metadata( - project_path, generated_by="concore init", workflow_file=workflow_file - ) + metadata_info = "" + try: + metadata_path = write_study_metadata( + project_path, + generated_by="concore init", + workflow_file=workflow_file, + ) + metadata_info = f"Metadata:\n {metadata_path.name}\n\n" + except Exception as exc: + # Metadata is additive, so project creation should still succeed on failure. + console.print( + f"[yellow]Warning:[/yellow] Failed to write study metadata: {exc}" + ) console.print() console.print( Panel.fit( f"[green]✓[/green] Project created successfully!\n\n" - f"Metadata:\n" - f" {metadata_path.name}\n\n" + f"{metadata_info}" f"Next steps:\n" f" cd {name}\n" f" concore validate workflow.graphml\n" diff --git a/concore_cli/commands/run.py b/concore_cli/commands/run.py index d4cd7d85..ad1c23c6 100644 --- a/concore_cli/commands/run.py +++ b/concore_cli/commands/run.py @@ -70,18 +70,23 @@ def run_workflow(workflow_file, source, output, exec_type, auto_build, console): if result.stdout: console.print(result.stdout) - metadata_path = write_study_metadata( - output_path, - generated_by="concore run", - workflow_file=workflow_path, - ) - console.print( f"[green]✓[/green] Workflow generated in [cyan]{output_path}[/cyan]" ) - console.print( - f"[green]✓[/green] Metadata written to [cyan]{metadata_path}[/cyan]" - ) + try: + metadata_path = write_study_metadata( + output_path, + generated_by="concore run", + workflow_file=workflow_path, + ) + console.print( + f"[green]✓[/green] Metadata written to [cyan]{metadata_path}[/cyan]" + ) + except Exception as exc: + # Metadata is additive, so workflow generation should still succeed on failure. + console.print( + f"[yellow]Warning:[/yellow] Failed to write study metadata for [cyan]{output_path}[/cyan]: {exc}" + ) except subprocess.CalledProcessError as e: progress.stop() From 04f86fa464a7c7c5917b1faf225dc3aad18f2287 Mon Sep 17 00:00:00 2001 From: Avinash Kumar Deepak Date: Mon, 16 Mar 2026 14:04:07 +0530 Subject: [PATCH 249/275] cpp: add read status result and params parse fix --- TestConcoreHpp.cpp | 30 +++++++++++++++++ concore.hpp | 80 ++++++++++++++++++++++++++++++++++++++++++---- concore_base.hpp | 3 +- concoredocker.hpp | 77 +++++++++++++++++++++++++++++++++++++++----- 4 files changed, 175 insertions(+), 15 deletions(-) diff --git a/TestConcoreHpp.cpp b/TestConcoreHpp.cpp index 54df52b2..64c77224 100644 --- a/TestConcoreHpp.cpp +++ b/TestConcoreHpp.cpp @@ -97,6 +97,20 @@ static void test_read_FM_missing_file_uses_initstr() { check("missing_file fallback val==3.0", approx(v[0], 3.0)); } +static void test_read_result_missing_file_status() { + setup_dirs(); + rm("in1/no_port_status"); + + Concore c; + c.delay = 0; + c.simtime = 0.0; + + Concore::ReadResult r = c.read_result(1, "no_port_status", "[9.0,3.0]"); + check("read_result missing status FILE_NOT_FOUND", + r.status == Concore::ReadStatus::FILE_NOT_FOUND); + check("read_result missing uses initstr", r.data.size() == 1 && approx(r.data[0], 3.0)); +} + // ------------- write_FM -------------------------------------------------- static void test_write_FM_creates_file() { @@ -274,6 +288,20 @@ static void test_tryparam_missing_key_uses_default() { c.tryparam("no_key", "def_val") == "def_val"); } +static void test_load_params_semicolon_format() { + setup_dirs(); + write_file("in/1/concore.params", "a=1;b=2"); + + Concore c; + c.delay = 0; + c.load_params(); + + check("load_params semicolon parses a", c.tryparam("a", "") == "1"); + check("load_params semicolon parses b", c.tryparam("b", "") == "2"); + + rm("in/1/concore.params"); +} + // ------------- main ------------------------------------------------------- int main() { @@ -282,6 +310,7 @@ int main() { // read_FM / write_FM test_read_FM_file(); test_read_FM_missing_file_uses_initstr(); + test_read_result_missing_file_status(); test_write_FM_creates_file(); // unchanged() @@ -307,6 +336,7 @@ int main() { // tryparam() test_tryparam_found(); test_tryparam_missing_key_uses_default(); + test_load_params_semicolon_format(); std::cout << "\n=== Results: " << passed << " passed, " << failed << " failed out of " << (passed + failed) << " tests ===\n"; diff --git a/concore.hpp b/concore.hpp index 5eb2b7ed..eb88108e 100644 --- a/concore.hpp +++ b/concore.hpp @@ -54,6 +54,19 @@ class Concore{ #endif public: + enum class ReadStatus { + SUCCESS, + TIMEOUT, + PARSE_ERROR, + FILE_NOT_FOUND, + RETRIES_EXCEEDED + }; + + struct ReadResult { + ReadStatus status; + vector data; + }; + double delay = 1; int retrycount = 0; double simtime; @@ -61,6 +74,7 @@ class Concore{ map iport; map oport; map params; + ReadStatus last_read_status = ReadStatus::SUCCESS; /** * @brief Constructor for Concore class. @@ -372,6 +386,14 @@ class Concore{ return read_FM(port, name, initstr); } + ReadResult read_result(int port, string name, string initstr) + { + ReadResult result; + result.data = read(port, name, initstr); + result.status = last_read_status; + return result; + } + /** * @brief Reads data from a specified port and name using the FM (File Method) communication protocol. * @param port The port number. @@ -383,6 +405,7 @@ class Concore{ chrono::milliseconds timespan((int)(1000*delay)); this_thread::sleep_for(timespan); string ins; + ReadStatus status = ReadStatus::SUCCESS; try { ifstream infile; infile.open(inpath+to_string(port)+"/"+name, ios::in); @@ -393,10 +416,13 @@ class Concore{ infile.close(); } else { + status = ReadStatus::FILE_NOT_FOUND; throw 505;} } catch (...) { ins = initstr; + if (status == ReadStatus::SUCCESS) + status = ReadStatus::FILE_NOT_FOUND; } int retry = 0; @@ -424,14 +450,24 @@ class Concore{ } retry++; } + if ((int)ins.length()==0) + status = ReadStatus::RETRIES_EXCEEDED; s += ins; vector inval = parser(ins); - if(inval.empty()) + if(inval.empty()) { + if (status == ReadStatus::SUCCESS) + status = ReadStatus::PARSE_ERROR; inval = parser(initstr); - if(inval.empty()) + } + if(inval.empty()) { + if (status == ReadStatus::SUCCESS) + status = ReadStatus::PARSE_ERROR; + last_read_status = status; return inval; + } simtime = simtime > inval[0] ? simtime : inval[0]; + last_read_status = status; //returning a string with data excluding simtime inval.erase(inval.begin()); @@ -450,6 +486,7 @@ class Concore{ chrono::milliseconds timespan((int)(1000*delay)); this_thread::sleep_for(timespan); string ins = ""; + ReadStatus status = ReadStatus::SUCCESS; try { if (shmId_get != -1) { if (sharedData_get && sharedData_get[0] != '\0') { @@ -463,10 +500,13 @@ class Concore{ } else { + status = ReadStatus::FILE_NOT_FOUND; throw 505; } } catch (...) { ins = initstr; + if (status == ReadStatus::SUCCESS) + status = ReadStatus::FILE_NOT_FOUND; } int retry = 0; @@ -490,14 +530,24 @@ class Concore{ } retry++; } + if ((int)ins.length()==0) + status = ReadStatus::RETRIES_EXCEEDED; s += ins; vector inval = parser(ins); - if(inval.empty()) + if(inval.empty()) { + if (status == ReadStatus::SUCCESS) + status = ReadStatus::PARSE_ERROR; inval = parser(initstr); - if(inval.empty()) + } + if(inval.empty()) { + if (status == ReadStatus::SUCCESS) + status = ReadStatus::PARSE_ERROR; + last_read_status = status; return inval; + } simtime = simtime > inval[0] ? simtime : inval[0]; + last_read_status = status; //returning a string with data excluding simtime inval.erase(inval.begin()); @@ -674,15 +724,26 @@ class Concore{ * @return a vector of double values */ vector read_ZMQ(string port_name, string name, string initstr) { + ReadStatus status = ReadStatus::SUCCESS; auto it = zmq_ports.find(port_name); if (it == zmq_ports.end()) { cerr << "read_ZMQ: port '" << port_name << "' not initialized" << endl; + status = ReadStatus::FILE_NOT_FOUND; + last_read_status = status; return parser(initstr); } vector inval = it->second->recv_with_retry(); - if (inval.empty()) + if (inval.empty()) { + status = ReadStatus::TIMEOUT; inval = parser(initstr); - if (inval.empty()) return inval; + } + if (inval.empty()) { + if (status == ReadStatus::SUCCESS) + status = ReadStatus::PARSE_ERROR; + last_read_status = status; + return inval; + } + last_read_status = status; simtime = simtime > inval[0] ? simtime : inval[0]; s += port_name; inval.erase(inval.begin()); @@ -736,6 +797,13 @@ class Concore{ return read_ZMQ(port_name, name, initstr); } + ReadResult read_result(string port_name, string name, string initstr) { + ReadResult result; + result.data = read(port_name, name, initstr); + result.status = last_read_status; + return result; + } + /** * @brief deviate the write to ZMQ communication protocol when port identifier is a string key. * @param port_name The ZMQ port name. diff --git a/concore_base.hpp b/concore_base.hpp index b4d96892..f38b6edb 100644 --- a/concore_base.hpp +++ b/concore_base.hpp @@ -345,10 +345,11 @@ inline std::map load_params(const std::string& params_ // Otherwise convert semicolon-separated key=value to dict format // e.g. "a=1;b=2" -> {"a":"1","b":"2"} + std::string normalized = std::regex_replace(sparams, std::regex(";"), ","); std::string converted = "{\"" + std::regex_replace( std::regex_replace( - std::regex_replace(sparams, std::regex(","), ",\""), + std::regex_replace(normalized, std::regex(","), ",\""), std::regex("="), "\":"), std::regex(" "), "") + "}"; diff --git a/concoredocker.hpp b/concoredocker.hpp index 651e5b33..51d6ca5f 100644 --- a/concoredocker.hpp +++ b/concoredocker.hpp @@ -34,6 +34,19 @@ class Concore { int communication_oport = 0; // oport refers to output port public: + enum class ReadStatus { + SUCCESS, + TIMEOUT, + PARSE_ERROR, + FILE_NOT_FOUND, + RETRIES_EXCEEDED + }; + + struct ReadResult { + ReadStatus status; + std::vector data; + }; + std::unordered_map iport; std::unordered_map oport; std::string s, olds; @@ -44,6 +57,7 @@ class Concore { double simtime = 0; double maxtime = 100; std::unordered_map params; + ReadStatus last_read_status = ReadStatus::SUCCESS; #ifdef CONCORE_USE_ZMQ std::map zmq_ports; #endif @@ -268,6 +282,7 @@ class Concore { if (communication_iport == 1) return read_SM(port, name, initstr); #endif + ReadStatus status = ReadStatus::SUCCESS; std::this_thread::sleep_for(std::chrono::seconds(delay)); std::string file_path = inpath + "/" + std::to_string(port) + "/" + name; std::ifstream infile(file_path); @@ -275,7 +290,10 @@ class Concore { if (!infile) { std::cerr << "File " << file_path << " not found, using default value.\n"; - return concore_base::parselist_double(initstr); + status = ReadStatus::FILE_NOT_FOUND; + std::vector fallback = concore_base::parselist_double(initstr); + last_read_status = status; + return fallback; } std::getline(infile, ins); @@ -291,22 +309,38 @@ class Concore { if (ins.empty()) { std::cerr << "Max retries reached for " << file_path << ", using default value.\n"; - return concore_base::parselist_double(initstr); + status = ReadStatus::RETRIES_EXCEEDED; + std::vector fallback = concore_base::parselist_double(initstr); + last_read_status = status; + return fallback; } s += ins; std::vector inval = concore_base::parselist_double(ins); - if (inval.empty()) + if (inval.empty()) { + status = ReadStatus::PARSE_ERROR; inval = concore_base::parselist_double(initstr); - if (inval.empty()) + } + if (inval.empty()) { + last_read_status = status; return inval; + } + last_read_status = status; simtime = simtime > inval[0] ? simtime : inval[0]; inval.erase(inval.begin()); return inval; } + ReadResult read_result(int port, const std::string& name, const std::string& initstr) { + ReadResult result; + result.data = read(port, name, initstr); + result.status = last_read_status; + return result; + } + #ifdef __linux__ std::vector read_SM(int port, const std::string& name, const std::string& initstr) { + ReadStatus status = ReadStatus::SUCCESS; std::this_thread::sleep_for(std::chrono::seconds(delay)); std::string ins; try { @@ -315,6 +349,7 @@ class Concore { else throw 505; } catch (...) { + status = ReadStatus::FILE_NOT_FOUND; ins = initstr; } @@ -335,13 +370,21 @@ class Concore { } retry++; } + if ((int)ins.length() == 0) + status = ReadStatus::RETRIES_EXCEEDED; s += ins; std::vector inval = concore_base::parselist_double(ins); - if (inval.empty()) + if (inval.empty()) { + if (status == ReadStatus::SUCCESS) + status = ReadStatus::PARSE_ERROR; inval = concore_base::parselist_double(initstr); - if (inval.empty()) + } + if (inval.empty()) { + last_read_status = status; return inval; + } + last_read_status = status; simtime = simtime > inval[0] ? simtime : inval[0]; inval.erase(inval.begin()); return inval; @@ -403,15 +446,26 @@ class Concore { } std::vector read_ZMQ(const std::string& port_name, const std::string& name, const std::string& initstr) { + ReadStatus status = ReadStatus::SUCCESS; auto it = zmq_ports.find(port_name); if (it == zmq_ports.end()) { std::cerr << "read_ZMQ: port '" << port_name << "' not initialized\n"; + status = ReadStatus::FILE_NOT_FOUND; + last_read_status = status; return concore_base::parselist_double(initstr); } std::vector inval = it->second->recv_with_retry(); - if (inval.empty()) + if (inval.empty()) { + status = ReadStatus::TIMEOUT; inval = concore_base::parselist_double(initstr); - if (inval.empty()) return inval; + } + if (inval.empty()) { + if (status == ReadStatus::SUCCESS) + status = ReadStatus::PARSE_ERROR; + last_read_status = status; + return inval; + } + last_read_status = status; simtime = simtime > inval[0] ? simtime : inval[0]; s += port_name; inval.erase(inval.begin()); @@ -433,6 +487,13 @@ class Concore { return read_ZMQ(port_name, name, initstr); } + ReadResult read_result(const std::string& port_name, const std::string& name, const std::string& initstr) { + ReadResult result; + result.data = read(port_name, name, initstr); + result.status = last_read_status; + return result; + } + void write(const std::string& port_name, const std::string& name, std::vector val, int delta = 0) { return write_ZMQ(port_name, name, val, delta); } From 0d1d68d69eed21b01c1be75d18c3b4226f89b897 Mon Sep 17 00:00:00 2001 From: Avinash Kumar Deepak Date: Wed, 18 Mar 2026 21:56:00 +0530 Subject: [PATCH 250/275] mkconcore: add local Java build/run/debug (#507) --- mkconcore.py | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/mkconcore.py b/mkconcore.py index 28fab9df..514e384d 100644 --- a/mkconcore.py +++ b/mkconcore.py @@ -156,6 +156,10 @@ def _resolve_concore_path(): MATLABWIN = os.environ.get("CONCORE_MATLABWIN", "matlab") #Windows matlab OCTAVEEXE = os.environ.get("CONCORE_OCTAVEEXE", "octave") #Ubuntu/macOS octave OCTAVEWIN = os.environ.get("CONCORE_OCTAVEWIN", "octave") #Windows octave +JAVACEXE = os.environ.get("CONCORE_JAVACEXE", "javac") #Ubuntu/macOS javac +JAVACWIN = os.environ.get("CONCORE_JAVACWIN", "javac") #Windows javac +JAVAEXE = os.environ.get("CONCORE_JAVAEXE", "java") #Ubuntu/macOS java +JAVAWIN = os.environ.get("CONCORE_JAVAWIN", "java") #Windows java M_IS_OCTAVE = False #treat .m as octave MCRPATH = "~/MATLAB/R2021a" #path to local Ubunta Matlab Compiler Runtime DOCKEREXE = os.environ.get("DOCKEREXE", "docker")#default to docker, allow env override @@ -196,6 +200,10 @@ def _resolve_concore_path(): MATLABWIN = _tools.get("MATLABWIN", MATLABWIN) OCTAVEEXE = _tools.get("OCTAVEEXE", OCTAVEEXE) OCTAVEWIN = _tools.get("OCTAVEWIN", OCTAVEWIN) + JAVACEXE = _tools.get("JAVACEXE", JAVACEXE) + JAVACWIN = _tools.get("JAVACWIN", JAVACWIN) + JAVAEXE = _tools.get("JAVAEXE", JAVAEXE) + JAVAWIN = _tools.get("JAVAWIN", JAVAWIN) prefixedgenode = "" sourcedir = os.path.abspath(sys.argv[2]) @@ -608,6 +616,16 @@ def cleanup_script_files(): fcopy.write(fsource.read()) fsource.close() +if 'java' in required_langs and concoretype != "docker": + try: + fsource = open(CONCOREPATH+"/concore.java") + except (FileNotFoundError, IOError): + print(CONCOREPATH+" is not correct path to concore (missing Java files)") + quit() + with open(outdir+"/src/concore.java","w") as fcopy: + fcopy.write(fsource.read()) + fsource.close() + if 'm' in required_langs: try: fsource = open(CONCOREPATH+"/concore_default_maxtime.m") @@ -1020,6 +1038,8 @@ def cleanup_script_files(): elif langext == "v": # 6/25/21 fbuild.write("copy .\\src\\concore.v .\\" + containername + "\\concore.v\n") + elif langext == "java": + fbuild.write("copy .\\src\\concore.java .\\" + containername + "\\concore.java\n") elif langext == "m": # 4/2/21 fbuild.write("copy .\\src\\concore_*.m .\\" + containername + "\\\n") fbuild.write("copy .\\src\\import_concore.m .\\" + containername + "\\\n") @@ -1037,6 +1057,8 @@ def cleanup_script_files(): fbuild.write("cp ./src/concore.hpp ./"+containername+"/concore.hpp\n") elif langext == "v": fbuild.write("cp ./src/concore.v ./"+containername+"/concore.v\n") + elif langext == "java": + fbuild.write("cp ./src/concore.java ./"+containername+"/concore.java\n") elif langext == "m": # 4/2/21 fbuild.write("cp ./src/concore_*.m ./"+containername+"/\n") fbuild.write("cp ./src/import_concore.m ./"+containername+"/\n") @@ -1127,6 +1149,16 @@ def cleanup_script_files(): fdebug.write('cd ..\n') fdebug.write('start /D '+q_container+' cmd /K vvp a.out\n') #fdebug.write('start /D '+containername+' cmd /K "'+CPPWIN+' '+sourcecode+'|a"\n') + elif langext=="java": + javaclass = os.path.splitext(os.path.basename(sourcecode))[0] + frun.write('cd '+q_container+'\n') + frun.write(JAVACWIN+' '+q_source+'\n') + frun.write('cd ..\n') + frun.write('start /B /D '+q_container+' cmd /c '+JAVAWIN+' -cp .;..\\src\\jeromq.jar '+javaclass+' >'+q_container+'\\concoreout.txt\n') + fdebug.write('cd '+q_container+'\n') + fdebug.write(JAVACWIN+' '+q_source+'\n') + fdebug.write('cd ..\n') + fdebug.write('start /D '+q_container+' cmd /K '+JAVAWIN+' -cp .;..\\src\\jeromq.jar '+javaclass+'\n') elif langext=="m": #3/23/21 # Use q_source in Windows commands to ensure quoting consistency if M_IS_OCTAVE: @@ -1168,6 +1200,17 @@ def cleanup_script_files(): fdebug.write('concorewd="$(pwd)"\n') fdebug.write('osascript -e "tell application \\"Terminal\\" to do script \\"cd \\\\\\"$concorewd/' + safe_container + '\\\\\\"; ' + VEXE + ' ' + safe_source + '; vvp a.out\\"" \n') + elif langext == "java": + javaclass = os.path.splitext(os.path.basename(sourcecode))[0] + safe_javaclass = shlex.quote(javaclass) + frun.write('(cd ' + safe_container + '; ' + JAVACEXE + ' ' + safe_source + '; ' + JAVAEXE + ' -cp .:../src/jeromq.jar ' + safe_javaclass + ' >concoreout.txt & echo $! >concorepid) &\n') + if ubuntu: + fdebug.write('concorewd="$(pwd)"\n') + fdebug.write('xterm -e bash -c "cd \\"$concorewd/' + safe_container + '\\"; ' + JAVACEXE + ' ' + safe_source + '; ' + JAVAEXE + ' -cp .:../src/jeromq.jar ' + safe_javaclass + '; bash" &\n') + else: + fdebug.write('concorewd="$(pwd)"\n') + fdebug.write('osascript -e "tell application \\"Terminal\\" to do script \\"cd \\\\\\"$concorewd/' + safe_container + '\\\\\\\"; ' + JAVACEXE + ' ' + safe_source + '; ' + JAVAEXE + ' -cp .:../src/jeromq.jar ' + safe_javaclass + '\\"" \n') + elif langext == "sh": # 5/19/21 # FIX: Escape MCRPATH to prevent shell injection safe_mcr = shlex.quote(MCRPATH) From b44e9a138ac798812566759bab60fcde214e0db4 Mon Sep 17 00:00:00 2001 From: GREENRAT-K405 Date: Wed, 25 Mar 2026 23:13:13 +0530 Subject: [PATCH 251/275] Add interactive mode to concore init CLI --- concore_cli/cli.py | 25 ++- concore_cli/commands/init.py | 294 ++++++++++++++++++++++++++++++++--- 2 files changed, 294 insertions(+), 25 deletions(-) diff --git a/concore_cli/cli.py b/concore_cli/cli.py index 92ca6e50..3f4f9634 100644 --- a/concore_cli/cli.py +++ b/concore_cli/cli.py @@ -3,7 +3,7 @@ import os import sys -from .commands.init import init_project +from .commands.init import init_project, init_project_interactive, run_wizard from .commands.run import run_workflow from .commands.validate import validate_workflow from .commands.status import show_status @@ -24,12 +24,29 @@ def cli(): @cli.command() -@click.argument("name", required=True) +@click.argument("name", required=False, default=None) @click.option("--template", default="basic", help="Template type to use") -def init(name, template): +@click.option( + "--interactive", "-i", + is_flag=True, + help="Launch guided wizard to select node types", +) +def init(name, template, interactive): """Create a new concore project""" try: - init_project(name, template, console) + if interactive: + if not name: + name = console.input("[cyan]Project name:[/cyan] ").strip() + if not name: + console.print("[red]Error:[/red] Project name is required.") + sys.exit(1) + selected = run_wizard(console) + init_project_interactive(name, selected, console) + else: + if not name: + console.print("[red]Error:[/red] Provide a project name or use --interactive.") + sys.exit(1) + init_project(name, template, console) except Exception as e: console.print(f"[red]Error:[/red] {str(e)}") sys.exit(1) diff --git a/concore_cli/commands/init.py b/concore_cli/commands/init.py index 85519723..388871d3 100644 --- a/concore_cli/commands/init.py +++ b/concore_cli/commands/init.py @@ -3,6 +3,36 @@ from .metadata import write_study_metadata +# --------------------------------------------------------------------------- +# GraphML templates +# --------------------------------------------------------------------------- + +GRAPHML_HEADER = """ + + + + +{nodes} + + +""" + +GRAPHML_NODE = """ + + + + + + N{idx}:{filename} + + + + """ + +# Single-node fallback used by non-interactive init SAMPLE_GRAPHML = """ @@ -23,20 +53,124 @@ """ -SAMPLE_PYTHON = """import concore - -concore.default_maxtime(100) -concore.delay = 0.02 - -init_simtime_val = "[0.0, 0.0]" -val = concore.initval(init_simtime_val) +# --------------------------------------------------------------------------- +# Per-language metadata: label, filename, node colour, source stub +# --------------------------------------------------------------------------- -while(concore.simtime\n\n" + "int main() {\n" + " Concore concore;\n" + " concore.default_maxtime(100);\n" + " concore.delay = 0.02;\n\n" + ' std::string init_val = "[0.0, 0.0]";\n' + " std::vector val = concore.initval(init_val);\n\n" + " while (concore.simtime < concore.maxtime) {\n" + " while (concore.unchanged()) {\n" + ' val = concore.read(1, "data", init_val);\n' + " }\n" + ' concore.write(1, "result", val, 0);\n' + " }\n" + " return 0;\n" + "}\n" + ), + }, + "octave": { + "label": "Octave/MATLAB", + "filename": "script.m", + "color": "#6db3f2", + "stub": ( + "global concore;\n" + "import_concore;\n\n" + "concore.delay = 0.02;\n" + "concore_default_maxtime(100);\n\n" + "init_val = '[0.0, 0.0]';\n" + "val = concore_initval(init_val);\n\n" + "while concore.simtime < concore.maxtime\n" + " while concore_unchanged()\n" + " val = concore_read(1, 'data', init_val);\n" + " end\n" + " result = val * 2;\n" + " concore_write(1, 'result', result, 0);\n" + "end\n" + ), + }, + "verilog": { + "label": "Verilog", + "filename": "script.v", + "color": "#f28c8c", + "stub": ( + '`include "concore.v"\n\n' + "module script;\n" + " // concore module provides: simtime, maxtime, readdata, writedata, unchanged\n" + " // data[] and datasize are global arrays filled by readdata\n\n" + " real init_val[1:0]; // [simtime, value]\n" + " integer i;\n\n" + " initial begin\n" + " concore.simtime = 0;\n" + " // set your maxtime (or let concore.maxtime file override)\n\n" + " while (concore.simtime < 100) begin\n" + ' while (concore.unchanged(0)) begin\n' + " // readdata fills concore.data[] and updates concore.simtime\n" + ' concore.readdata(1, "data", "[0.0,0.0]");\n' + " end\n" + " // TODO: process concore.data[0..datasize-1]\n" + " concore.data[0] = concore.data[0] * 2;\n" + " concore.datasize = 1;\n" + ' concore.writedata(1, "result", 0); // delta=0\n' + " end\n" + " $finish;\n" + " end\n" + "endmodule\n" + ), + }, + "java": { + "label": "Java", + "filename": "Script.java", + "color": "#a8d8a8", + "stub": ( + "public class Script {\n" + " public static void main(String[] args) throws Exception {\n" + " concoredocker cd = new concoredocker();\n" + " double maxtime = 100;\n" + " double delay = 0.02;\n" + ' String init_val = "[0.0, 0.0]";\n\n' + " String val = cd.initval(init_val);\n" + " while (cd.simtime() < maxtime) {\n" + " while (cd.unchanged()) {\n" + ' val = cd.read(1, "data", init_val);\n' + " }\n" + " // TODO: process val\n" + ' cd.write(1, "result", val, 0);\n' + " }\n" + " }\n" + "}\n" + ), + }, +} README_TEMPLATE = """# {project_name} @@ -59,14 +193,132 @@ ## Next Steps -- Modify `workflow.graphml` to define your processing pipeline -- Add Python/C++/MATLAB scripts to `src/` +- Open `workflow.graphml` in yEd and connect the nodes with edges - Use `concore validate workflow.graphml` to check your workflow - Use `concore status` to monitor running processes """ +# --------------------------------------------------------------------------- +# Interactive wizard +# --------------------------------------------------------------------------- + +def run_wizard(console): + """Ask y/n for each supported language. Returns list of selected lang keys.""" + console.print() + console.print( + "[bold cyan]Select the node types to include[/bold cyan] " + "[dim](Enter = yes)[/dim]" + ) + console.print() + + selected = [] + for key, info in LANGUAGE_NODES.items(): + raw = console.input( + f" Include [bold]{info['label']}[/bold] node? [Y/n] " + ).strip().lower() + if raw in ("", "y", "yes"): + selected.append(key) + + return selected + + +# --------------------------------------------------------------------------- +# GraphML builder +# --------------------------------------------------------------------------- + +def _build_graphml(project_name, selected_langs): + """Return a GraphML string with one unconnected node per selected language.""" + node_blocks = [] + for idx, lang_key in enumerate(selected_langs, start=1): + info = LANGUAGE_NODES[lang_key] + node_blocks.append( + GRAPHML_NODE.format( + idx=idx, + y=100 + (idx - 1) * 100, # stack vertically, 100 px apart + color=info["color"], + filename=info["filename"], + ) + ) + return GRAPHML_HEADER.format( + project_name=project_name, + nodes="\n".join(node_blocks), + ) + + +# --------------------------------------------------------------------------- +# Public entry points +# --------------------------------------------------------------------------- + +def init_project_interactive(name, selected_langs, console): + """Create a project with one node per selected language (no edges).""" + project_path = Path(name) + + if project_path.exists(): + raise FileExistsError(f"Directory '{name}' already exists") + + if not selected_langs: + console.print("[yellow]No languages selected — nothing to create.[/yellow]") + return + + console.print() + console.print(f"[cyan]Creating project:[/cyan] {name}") + + project_path.mkdir() + src_path = project_path / "src" + src_path.mkdir() + + # workflow.graphml + workflow_file = project_path / "workflow.graphml" + workflow_file.write_text(_build_graphml(name, selected_langs)) + + # one source stub per selected language + for lang_key in selected_langs: + info = LANGUAGE_NODES[lang_key] + (src_path / info["filename"]).write_text(info["stub"]) + + # README + (project_path / "README.md").write_text( + README_TEMPLATE.format(project_name=name) + ) + + # Metadata + metadata_info = "" + try: + metadata_path = write_study_metadata( + project_path, + generated_by="concore init --interactive", + workflow_file=workflow_file, + ) + metadata_info = f"Metadata:\n {metadata_path.name}\n\n" + except Exception as exc: + console.print( + f"[yellow]Warning:[/yellow] Failed to write study metadata: {exc}" + ) + + node_lines = "\n".join( + f" N{i}: {LANGUAGE_NODES[k]['filename']}" + for i, k in enumerate(selected_langs, 1) + ) + + console.print() + console.print( + Panel.fit( + f"[green]✓[/green] Project created with {len(selected_langs)} node(s)!\n\n" + f"{metadata_info}" + f"Nodes (unconnected — connect them in yEd):\n{node_lines}\n\n" + f"Next steps:\n" + f" cd {name}\n" + f" concore validate workflow.graphml\n" + f" concore run workflow.graphml", + title="Success", + border_style="green", + ) + ) + + def init_project(name, template, console): + """Non-interactive init — single Python node skeleton.""" project_path = Path(name) if project_path.exists(): @@ -81,13 +333,13 @@ def init_project(name, template, console): with open(workflow_file, "w") as f: f.write(SAMPLE_GRAPHML) - sample_script = project_path / "src" / "script.py" - with open(sample_script, "w") as f: - f.write(SAMPLE_PYTHON) + (project_path / "src" / "script.py").write_text( + LANGUAGE_NODES["python"]["stub"] + ) - readme_file = project_path / "README.md" - with open(readme_file, "w") as f: - f.write(README_TEMPLATE.format(project_name=name)) + (project_path / "README.md").write_text( + README_TEMPLATE.format(project_name=name) + ) metadata_info = "" try: From b58ce09de1411d54cf7d8b38ac8716c0739aee14 Mon Sep 17 00:00:00 2001 From: GREENRAT-K405 Date: Wed, 25 Mar 2026 23:28:51 +0530 Subject: [PATCH 252/275] Add interactive mode to concore init CLI --- concore_cli/cli.py | 7 +++++-- concore_cli/commands/init.py | 27 +++++++++++++-------------- 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/concore_cli/cli.py b/concore_cli/cli.py index 3f4f9634..b5336aab 100644 --- a/concore_cli/cli.py +++ b/concore_cli/cli.py @@ -27,7 +27,8 @@ def cli(): @click.argument("name", required=False, default=None) @click.option("--template", default="basic", help="Template type to use") @click.option( - "--interactive", "-i", + "--interactive", + "-i", is_flag=True, help="Launch guided wizard to select node types", ) @@ -44,7 +45,9 @@ def init(name, template, interactive): init_project_interactive(name, selected, console) else: if not name: - console.print("[red]Error:[/red] Provide a project name or use --interactive.") + console.print( + "[red]Error:[/red] Provide a project name or use --interactive." + ) sys.exit(1) init_project(name, template, console) except Exception as e: diff --git a/concore_cli/commands/init.py b/concore_cli/commands/init.py index 388871d3..4137a11a 100644 --- a/concore_cli/commands/init.py +++ b/concore_cli/commands/init.py @@ -133,7 +133,7 @@ " concore.simtime = 0;\n" " // set your maxtime (or let concore.maxtime file override)\n\n" " while (concore.simtime < 100) begin\n" - ' while (concore.unchanged(0)) begin\n' + " while (concore.unchanged(0)) begin\n" " // readdata fills concore.data[] and updates concore.simtime\n" ' concore.readdata(1, "data", "[0.0,0.0]");\n' " end\n" @@ -203,6 +203,7 @@ # Interactive wizard # --------------------------------------------------------------------------- + def run_wizard(console): """Ask y/n for each supported language. Returns list of selected lang keys.""" console.print() @@ -214,9 +215,11 @@ def run_wizard(console): selected = [] for key, info in LANGUAGE_NODES.items(): - raw = console.input( - f" Include [bold]{info['label']}[/bold] node? [Y/n] " - ).strip().lower() + raw = ( + console.input(f" Include [bold]{info['label']}[/bold] node? [Y/n] ") + .strip() + .lower() + ) if raw in ("", "y", "yes"): selected.append(key) @@ -227,6 +230,7 @@ def run_wizard(console): # GraphML builder # --------------------------------------------------------------------------- + def _build_graphml(project_name, selected_langs): """Return a GraphML string with one unconnected node per selected language.""" node_blocks = [] @@ -235,7 +239,7 @@ def _build_graphml(project_name, selected_langs): node_blocks.append( GRAPHML_NODE.format( idx=idx, - y=100 + (idx - 1) * 100, # stack vertically, 100 px apart + y=100 + (idx - 1) * 100, # stack vertically, 100 px apart color=info["color"], filename=info["filename"], ) @@ -250,6 +254,7 @@ def _build_graphml(project_name, selected_langs): # Public entry points # --------------------------------------------------------------------------- + def init_project_interactive(name, selected_langs, console): """Create a project with one node per selected language (no edges).""" project_path = Path(name) @@ -278,9 +283,7 @@ def init_project_interactive(name, selected_langs, console): (src_path / info["filename"]).write_text(info["stub"]) # README - (project_path / "README.md").write_text( - README_TEMPLATE.format(project_name=name) - ) + (project_path / "README.md").write_text(README_TEMPLATE.format(project_name=name)) # Metadata metadata_info = "" @@ -333,13 +336,9 @@ def init_project(name, template, console): with open(workflow_file, "w") as f: f.write(SAMPLE_GRAPHML) - (project_path / "src" / "script.py").write_text( - LANGUAGE_NODES["python"]["stub"] - ) + (project_path / "src" / "script.py").write_text(LANGUAGE_NODES["python"]["stub"]) - (project_path / "README.md").write_text( - README_TEMPLATE.format(project_name=name) - ) + (project_path / "README.md").write_text(README_TEMPLATE.format(project_name=name)) metadata_info = "" try: From e17ad451e81a940265ebd0f4ea2a380e247e6514 Mon Sep 17 00:00:00 2001 From: PARAM KANADA Date: Thu, 26 Mar 2026 02:14:30 +0530 Subject: [PATCH 253/275] Update concore_cli/commands/init.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- concore_cli/commands/init.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/concore_cli/commands/init.py b/concore_cli/commands/init.py index 4137a11a..77a0c803 100644 --- a/concore_cli/commands/init.py +++ b/concore_cli/commands/init.py @@ -14,7 +14,7 @@ xmlns:y="http://www.yworks.com/xml/graphml"> - + {nodes} From a3a2b0c41b4f62bf7109857c955a8f50bd77b47a Mon Sep 17 00:00:00 2001 From: PARAM KANADA Date: Thu, 26 Mar 2026 02:15:41 +0530 Subject: [PATCH 254/275] Update concore_cli/commands/init.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- concore_cli/commands/init.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/concore_cli/commands/init.py b/concore_cli/commands/init.py index 77a0c803..ed72c907 100644 --- a/concore_cli/commands/init.py +++ b/concore_cli/commands/init.py @@ -270,16 +270,17 @@ def init_project_interactive(name, selected_langs, console): console.print(f"[cyan]Creating project:[/cyan] {name}") project_path.mkdir() - src_path = project_path / "src" - src_path.mkdir() - - # workflow.graphml - workflow_file = project_path / "workflow.graphml" - workflow_file.write_text(_build_graphml(name, selected_langs)) + workflow_file.write_text(_build_graphml(name, selected_langs), encoding="utf-8") # one source stub per selected language for lang_key in selected_langs: info = LANGUAGE_NODES[lang_key] + (src_path / info["filename"]).write_text(info["stub"], encoding="utf-8") + + # README + (project_path / "README.md").write_text( + README_TEMPLATE.format(project_name=name), + encoding="utf-8", (src_path / info["filename"]).write_text(info["stub"]) # README From a277b6f4552a22c1e766f170e8181e457ab94ac0 Mon Sep 17 00:00:00 2001 From: GREENRAT-K405 Date: Thu, 26 Mar 2026 02:53:47 +0530 Subject: [PATCH 255/275] style: apply ruff formatting to init.py and cli.py --- concore_cli/commands/init.py | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/concore_cli/commands/init.py b/concore_cli/commands/init.py index ed72c907..7d136a04 100644 --- a/concore_cli/commands/init.py +++ b/concore_cli/commands/init.py @@ -1,4 +1,5 @@ from pathlib import Path +from xml.sax.saxutils import quoteattr from rich.panel import Panel from .metadata import write_study_metadata @@ -14,7 +15,7 @@ xmlns:y="http://www.yworks.com/xml/graphml"> - + {nodes} @@ -245,7 +246,7 @@ def _build_graphml(project_name, selected_langs): ) ) return GRAPHML_HEADER.format( - project_name=project_name, + project_name=quoteattr(project_name), nodes="\n".join(node_blocks), ) @@ -270,6 +271,11 @@ def init_project_interactive(name, selected_langs, console): console.print(f"[cyan]Creating project:[/cyan] {name}") project_path.mkdir() + src_path = project_path / "src" + src_path.mkdir() + + # workflow.graphml + workflow_file = project_path / "workflow.graphml" workflow_file.write_text(_build_graphml(name, selected_langs), encoding="utf-8") # one source stub per selected language @@ -279,12 +285,8 @@ def init_project_interactive(name, selected_langs, console): # README (project_path / "README.md").write_text( - README_TEMPLATE.format(project_name=name), - encoding="utf-8", - (src_path / info["filename"]).write_text(info["stub"]) - - # README - (project_path / "README.md").write_text(README_TEMPLATE.format(project_name=name)) + README_TEMPLATE.format(project_name=name), encoding="utf-8" + ) # Metadata metadata_info = "" @@ -334,12 +336,16 @@ def init_project(name, template, console): (project_path / "src").mkdir() workflow_file = project_path / "workflow.graphml" - with open(workflow_file, "w") as f: + with open(workflow_file, "w", encoding="utf-8") as f: f.write(SAMPLE_GRAPHML) - (project_path / "src" / "script.py").write_text(LANGUAGE_NODES["python"]["stub"]) + (project_path / "src" / "script.py").write_text( + LANGUAGE_NODES["python"]["stub"], encoding="utf-8" + ) - (project_path / "README.md").write_text(README_TEMPLATE.format(project_name=name)) + (project_path / "README.md").write_text( + README_TEMPLATE.format(project_name=name), encoding="utf-8" + ) metadata_info = "" try: From 6c4f0f46fe1de3057b37c5a499fff7cb7f0c0217 Mon Sep 17 00:00:00 2001 From: GREENRAT-K405 Date: Thu, 26 Mar 2026 03:16:49 +0530 Subject: [PATCH 256/275] style: add TODO comment to C++ stub for consistency --- concore_cli/commands/init.py | 1 + 1 file changed, 1 insertion(+) diff --git a/concore_cli/commands/init.py b/concore_cli/commands/init.py index 7d136a04..8e37eb7a 100644 --- a/concore_cli/commands/init.py +++ b/concore_cli/commands/init.py @@ -93,6 +93,7 @@ " while (concore.unchanged()) {\n" ' val = concore.read(1, "data", init_val);\n' " }\n" + " // TODO: process val (e.g. multiply by 2)\n" ' concore.write(1, "result", val, 0);\n' " }\n" " return 0;\n" From ec7f18a46b1328a757b2e8713aaf4eb45c567011 Mon Sep 17 00:00:00 2001 From: GREENRAT-K405 Date: Thu, 26 Mar 2026 05:35:01 +0530 Subject: [PATCH 257/275] fix: update java stub in init interactive, make DEV-GUIDE.md containing guide for future julia integration for init --interactive --- DEV-GUIDE.md | 120 +++++++++++++++++++++++++++++++++++ concore_cli/commands/init.py | 17 ++--- 2 files changed, 129 insertions(+), 8 deletions(-) create mode 100644 DEV-GUIDE.md diff --git a/DEV-GUIDE.md b/DEV-GUIDE.md new file mode 100644 index 00000000..6c0eb772 --- /dev/null +++ b/DEV-GUIDE.md @@ -0,0 +1,120 @@ +# Concore Developer Guide + +## `concore init --interactive` + +The interactive init wizard lets users scaffold a new multi-language concore project without writing any boilerplate by hand. + +### Usage + +```bash +concore init --interactive +# or shorthand +concore init -i +``` + +The wizard prompts for each supported language with a `y/n` question (default: yes): + +``` +Select the node types to include (Enter = yes) + + Include Python node? [Y/n] + Include C++ node? [Y/n] + Include Octave/MATLAB node? [Y/n] + Include Verilog node? [Y/n] + Include Java node? [Y/n] +``` + +### What gets generated + +For each selected language, the wizard: + +1. Creates a **source stub** in `src/` with the correct concore API calls for that language. +2. Adds an **unconnected node** to `workflow.graphml`, colour-coded and vertically positioned for easy viewing in yEd. + +Example output for Python + Java selected: + +``` +my_project/ +├── workflow.graphml ← 2 unconnected nodes +├── src/ +│ ├── script.py ← Python stub +│ └── Script.java ← Java stub +├── README.md +└── STUDY.json +``` + +### Architecture + +The feature lives entirely in `concore_cli/commands/init.py`: + +| Symbol | Role | +|---|---| +| `LANGUAGE_NODES` | Dict mapping language key → `label`, `filename`, node `color`, source `stub` | +| `GRAPHML_HEADER` | XML template for the GraphML wrapper; `project_name` is escaped via `xml.sax.saxutils.quoteattr` | +| `GRAPHML_NODE` | XML template for a single node block | +| `run_wizard()` | Prompts y/n for each language; returns list of selected keys | +| `_build_graphml()` | Assembles the full GraphML string from selected languages | +| `init_project_interactive()` | Orchestrates directory creation, file writes, and success output | + +--- + +## Adding a New Language + +Adding Julia (or any other language) to the interactive wizard is a **one-file change** — just add an entry to the `LANGUAGE_NODES` dictionary in `concore_cli/commands/init.py`. + +### Step-by-step: adding Julia + +**1. Add the entry to `LANGUAGE_NODES`** in `concore_cli/commands/init.py`: + +```python +"julia": { + "label": "Julia", + "filename": "script.jl", + "color": "#9558b2", # Julia purple + "stub": ( + "using Concore\n\n" + "Concore.state.delay = 0.02\n" + "Concore.state.inpath = \"./in\"\n" + "Concore.state.outpath = \"./out\"\n\n" + "maxtime = default_maxtime(100.0)\n" + 'init_val = "[0.0, 0.0]"\n\n' + "while Concore.state.simtime < maxtime\n" + " while unchanged()\n" + ' val = Float64.(concore_read(1, "data", init_val))\n' + " end\n" + " # TODO: process val\n" + " result = val .* 2\n" + ' concore_write(1, "result", result, 0.0)\n' + "end\n" + "println(\"retry=$(Concore.state.retrycount)\")\n" + ), +}, +``` + +Key Julia API points (based on real concore Julia scripts): + +| Element | Julia equivalent | +|---|---| +| Import | `using Concore` | +| Setup | `Concore.state.delay`, `.inpath`, `.outpath` | +| Max time | `default_maxtime(100.0)` — returns the value | +| Sim time | `Concore.state.simtime` | +| Unchanged check | `unchanged()` — no module prefix | +| Read | `concore_read(port, name, initstr)` — snake\_case, no prefix | +| Write | `concore_write(port, name, val, delta)` — snake\_case, no prefix | +| Type cast | `Float64.(concore_read(...))` to ensure numeric type | + +**2. That's it.** No other files need to change — the wizard, GraphML builder, and file writer all iterate over `LANGUAGE_NODES` dynamically. + +### Node colours used + +| Language | Hex colour | +|---|---| +| Python | `#ffcc00` (yellow) | +| C++ | `#ae85ca` (purple) | +| Octave/MATLAB | `#6db3f2` (blue) | +| Verilog | `#f28c8c` (red) | +| Java | `#a8d8a8` (green) | +| Julia *(proposed)* | `#9558b2` (Julia purple) | + +--- diff --git a/concore_cli/commands/init.py b/concore_cli/commands/init.py index 8e37eb7a..64739b52 100644 --- a/concore_cli/commands/init.py +++ b/concore_cli/commands/init.py @@ -154,19 +154,20 @@ "filename": "Script.java", "color": "#a8d8a8", "stub": ( + "import java.util.List;\n\n" "public class Script {\n" " public static void main(String[] args) throws Exception {\n" - " concoredocker cd = new concoredocker();\n" " double maxtime = 100;\n" - " double delay = 0.02;\n" ' String init_val = "[0.0, 0.0]";\n\n' - " String val = cd.initval(init_val);\n" - " while (cd.simtime() < maxtime) {\n" - " while (cd.unchanged()) {\n" - ' val = cd.read(1, "data", init_val);\n' + " // All concore methods are static\n" + " List val = concore.initVal(init_val);\n" + " while (concore.getSimtime() < maxtime) {\n" + " while (concore.unchanged()) {\n" + ' concore.ReadResult r = concore.read(1, "data", init_val);\n' + " val = r.data;\n" " }\n" - " // TODO: process val\n" - ' cd.write(1, "result", val, 0);\n' + " // TODO: process val (List)\n" + ' concore.write(1, "result", val, 0);\n' " }\n" " }\n" "}\n" From d86408e3b5d996e8e067e3f04d465c9a889a9607 Mon Sep 17 00:00:00 2001 From: Avinash Kumar Deepak Date: Thu, 26 Mar 2026 12:20:13 +0530 Subject: [PATCH 258/275] feat(cli): add concore setup autodetect command --- concore_cli/cli.py | 17 +++++ concore_cli/commands/setup.py | 107 ++++++++++++++++++++++++++ tests/test_setup.py | 139 ++++++++++++++++++++++++++++++++++ 3 files changed, 263 insertions(+) create mode 100644 concore_cli/commands/setup.py create mode 100644 tests/test_setup.py diff --git a/concore_cli/cli.py b/concore_cli/cli.py index b5336aab..67d945f5 100644 --- a/concore_cli/cli.py +++ b/concore_cli/cli.py @@ -11,6 +11,7 @@ from .commands.inspect import inspect_workflow from .commands.watch import watch_study from .commands.doctor import doctor_check +from .commands.setup import setup_concore from . import __version__ console = Console() @@ -151,5 +152,21 @@ def doctor(): sys.exit(1) +@cli.command() +@click.option( + "--dry-run", is_flag=True, help="Preview detected config without writing" +) +@click.option("--force", is_flag=True, help="Overwrite existing config files") +def setup(dry_run, force): + """Auto-detect tools and write concore config files""" + try: + ok = setup_concore(console, dry_run=dry_run, force=force) + if not ok: + sys.exit(1) + except Exception as e: + console.print(f"[red]Error:[/red] {str(e)}") + sys.exit(1) + + if __name__ == "__main__": cli() diff --git a/concore_cli/commands/setup.py b/concore_cli/commands/setup.py new file mode 100644 index 00000000..6088c483 --- /dev/null +++ b/concore_cli/commands/setup.py @@ -0,0 +1,107 @@ +from pathlib import Path + +from .doctor import ( + TOOL_DEFINITIONS, + _detect_tool, + _get_platform_key, + _resolve_concore_path, +) + + +def _pick_config_key(config_keys, plat_key): + if plat_key == "windows": + for key in config_keys: + if key.endswith("WIN"): + return key + else: + for key in config_keys: + if key.endswith("EXE"): + return key + return config_keys[0] if config_keys else None + + +def _detect_tool_overrides(plat_key): + found = [] + for tool_def in TOOL_DEFINITIONS.values(): + config_keys = tool_def.get("config_keys", []) + if not config_keys: + continue + candidates = tool_def["names"].get(plat_key, []) + path, _ = _detect_tool(candidates) + if not path: + continue + config_key = _pick_config_key(config_keys, plat_key) + if config_key: + found.append((config_key, path)) + return found + + +def _write_text(path, content, dry_run, force, console): + if path.exists() and not force: + console.print( + f"[yellow]![/yellow] Skipping {path.name} " + "(already exists; use --force)" + ) + return True + if dry_run: + preview = content if content else "" + console.print(f"[dim]-[/dim] Would write {path.name}:\n{preview}") + return True + path.write_text(content) + console.print(f"[green]+[/green] Wrote {path.name}") + return True + + +def setup_concore(console, dry_run=False, force=False): + plat_key = _get_platform_key() + concore_path = _resolve_concore_path() + + console.print(f"[cyan]CONCOREPATH:[/cyan] {concore_path}") + + tool_overrides = _detect_tool_overrides(plat_key) + docker_candidates = TOOL_DEFINITIONS["Docker"]["names"].get(plat_key, []) + _, docker_command = _detect_tool(docker_candidates) + octave_candidates = TOOL_DEFINITIONS["Octave"]["names"].get(plat_key, []) + octave_path, _ = _detect_tool(octave_candidates) + octave_found = bool(octave_path) + + wrote_any = False + + tools_file = Path(concore_path) / "concore.tools" + if tool_overrides: + tools_content = "\n".join(f"{k}={v}" for k, v in tool_overrides) + "\n" + wrote_any = ( + _write_text(tools_file, tools_content, dry_run, force, console) + or wrote_any + ) + else: + console.print("[yellow]![/yellow] No tool paths detected for concore.tools") + + sudo_file = Path(concore_path) / "concore.sudo" + if docker_command: + sudo_content = f"{docker_command}\n" + wrote_any = ( + _write_text(sudo_file, sudo_content, dry_run, force, console) + or wrote_any + ) + else: + console.print("[yellow]![/yellow] Docker/Podman not detected; not writing concore.sudo") + + octave_file = Path(concore_path) / "concore.octave" + if octave_found: + wrote_any = ( + _write_text(octave_file, "", dry_run, force, console) + or wrote_any + ) + else: + console.print("[dim]-[/dim] Octave not detected; not writing concore.octave") + + if not wrote_any: + console.print("[yellow]No files written.[/yellow]") + return False + + if dry_run: + console.print("[green]Dry run complete.[/green]") + else: + console.print("[green]Setup complete.[/green]") + return True \ No newline at end of file diff --git a/tests/test_setup.py b/tests/test_setup.py new file mode 100644 index 00000000..42a05fe2 --- /dev/null +++ b/tests/test_setup.py @@ -0,0 +1,139 @@ +import tempfile +import shutil +import unittest +from pathlib import Path +from unittest.mock import patch + +from click.testing import CliRunner +from concore_cli.cli import cli + + +class TestSetupCommand(unittest.TestCase): + def setUp(self): + self.runner = CliRunner() + self.temp_dir = tempfile.mkdtemp() + + def tearDown(self): + shutil.rmtree(self.temp_dir) + + @patch("concore_cli.commands.setup._resolve_concore_path") + @patch("concore_cli.commands.setup._detect_tool") + @patch("concore_cli.commands.setup._get_platform_key") + def test_setup_dry_run_does_not_write(self, mock_plat, mock_detect, mock_path): + mock_plat.return_value = "posix" + mock_path.return_value = Path(self.temp_dir) + + def detect_side_effect(names): + if "g++" in names: + return "/usr/bin/g++", "g++" + if "python3" in names: + return "/usr/bin/python3", "python3" + if "iverilog" in names: + return "/usr/bin/iverilog", "iverilog" + if "octave" in names: + return "/usr/bin/octave", "octave" + if "docker" in names: + return "/usr/bin/docker", "docker" + return None, None + + mock_detect.side_effect = detect_side_effect + + result = self.runner.invoke(cli, ["setup", "--dry-run"]) + self.assertEqual(result.exit_code, 0) + + self.assertFalse((Path(self.temp_dir) / "concore.tools").exists()) + self.assertFalse((Path(self.temp_dir) / "concore.sudo").exists()) + self.assertFalse((Path(self.temp_dir) / "concore.octave").exists()) + + @patch("concore_cli.commands.setup._resolve_concore_path") + @patch("concore_cli.commands.setup._detect_tool") + @patch("concore_cli.commands.setup._get_platform_key") + def test_setup_writes_files(self, mock_plat, mock_detect, mock_path): + mock_plat.return_value = "posix" + mock_path.return_value = Path(self.temp_dir) + + def detect_side_effect(names): + if "g++" in names: + return "/usr/bin/g++", "g++" + if "python3" in names: + return "/usr/bin/python3", "python3" + if "iverilog" in names: + return "/usr/bin/iverilog", "iverilog" + if "octave" in names: + return "/usr/bin/octave", "octave" + if "docker" in names: + return "/usr/bin/docker", "docker" + return None, None + + mock_detect.side_effect = detect_side_effect + + result = self.runner.invoke(cli, ["setup"]) + self.assertEqual(result.exit_code, 0) + + tools_file = Path(self.temp_dir) / "concore.tools" + sudo_file = Path(self.temp_dir) / "concore.sudo" + octave_file = Path(self.temp_dir) / "concore.octave" + + self.assertTrue(tools_file.exists()) + self.assertTrue(sudo_file.exists()) + self.assertTrue(octave_file.exists()) + + tools_content = tools_file.read_text() + self.assertIn("CPPEXE=/usr/bin/g++", tools_content) + self.assertIn("PYTHONEXE=/usr/bin/python3", tools_content) + self.assertIn("VEXE=/usr/bin/iverilog", tools_content) + self.assertIn("OCTAVEEXE=/usr/bin/octave", tools_content) + self.assertEqual(sudo_file.read_text().strip(), "docker") + + @patch("concore_cli.commands.setup._resolve_concore_path") + @patch("concore_cli.commands.setup._detect_tool") + @patch("concore_cli.commands.setup._get_platform_key") + def test_setup_no_force_keeps_existing(self, mock_plat, mock_detect, mock_path): + mock_plat.return_value = "posix" + mock_path.return_value = Path(self.temp_dir) + + tools_file = Path(self.temp_dir) / "concore.tools" + tools_file.write_text("CPPEXE=/old/path\n") + + def detect_side_effect(names): + if "g++" in names: + return "/usr/bin/g++", "g++" + if "python3" in names: + return "/usr/bin/python3", "python3" + return None, None + + mock_detect.side_effect = detect_side_effect + + result = self.runner.invoke(cli, ["setup"]) + self.assertEqual(result.exit_code, 0) + self.assertEqual(tools_file.read_text(), "CPPEXE=/old/path\n") + + @patch("concore_cli.commands.setup._resolve_concore_path") + @patch("concore_cli.commands.setup._detect_tool") + @patch("concore_cli.commands.setup._get_platform_key") + def test_setup_force_overwrites_existing(self, mock_plat, mock_detect, mock_path): + mock_plat.return_value = "posix" + mock_path.return_value = Path(self.temp_dir) + + tools_file = Path(self.temp_dir) / "concore.tools" + tools_file.write_text("CPPEXE=/old/path\n") + + def detect_side_effect(names): + if "g++" in names: + return "/usr/bin/g++", "g++" + if "python3" in names: + return "/usr/bin/python3", "python3" + return None, None + + mock_detect.side_effect = detect_side_effect + + result = self.runner.invoke(cli, ["setup", "--force"]) + self.assertEqual(result.exit_code, 0) + + content = tools_file.read_text() + self.assertIn("CPPEXE=/usr/bin/g++", content) + self.assertIn("PYTHONEXE=/usr/bin/python3", content) + + +if __name__ == "__main__": + unittest.main() From 336f4f8cef7a950d13431af42b89b378b0cfeade Mon Sep 17 00:00:00 2001 From: Avinash Kumar Deepak Date: Thu, 26 Mar 2026 12:30:45 +0530 Subject: [PATCH 259/275] style: apply ruff formatting --- concore_cli/cli.py | 4 +--- concore_cli/commands/setup.py | 20 ++++++++------------ 2 files changed, 9 insertions(+), 15 deletions(-) diff --git a/concore_cli/cli.py b/concore_cli/cli.py index 67d945f5..90cf112b 100644 --- a/concore_cli/cli.py +++ b/concore_cli/cli.py @@ -153,9 +153,7 @@ def doctor(): @cli.command() -@click.option( - "--dry-run", is_flag=True, help="Preview detected config without writing" -) +@click.option("--dry-run", is_flag=True, help="Preview detected config without writing") @click.option("--force", is_flag=True, help="Overwrite existing config files") def setup(dry_run, force): """Auto-detect tools and write concore config files""" diff --git a/concore_cli/commands/setup.py b/concore_cli/commands/setup.py index 6088c483..829f24d4 100644 --- a/concore_cli/commands/setup.py +++ b/concore_cli/commands/setup.py @@ -39,8 +39,7 @@ def _detect_tool_overrides(plat_key): def _write_text(path, content, dry_run, force, console): if path.exists() and not force: console.print( - f"[yellow]![/yellow] Skipping {path.name} " - "(already exists; use --force)" + f"[yellow]![/yellow] Skipping {path.name} (already exists; use --force)" ) return True if dry_run: @@ -71,8 +70,7 @@ def setup_concore(console, dry_run=False, force=False): if tool_overrides: tools_content = "\n".join(f"{k}={v}" for k, v in tool_overrides) + "\n" wrote_any = ( - _write_text(tools_file, tools_content, dry_run, force, console) - or wrote_any + _write_text(tools_file, tools_content, dry_run, force, console) or wrote_any ) else: console.print("[yellow]![/yellow] No tool paths detected for concore.tools") @@ -81,18 +79,16 @@ def setup_concore(console, dry_run=False, force=False): if docker_command: sudo_content = f"{docker_command}\n" wrote_any = ( - _write_text(sudo_file, sudo_content, dry_run, force, console) - or wrote_any + _write_text(sudo_file, sudo_content, dry_run, force, console) or wrote_any ) else: - console.print("[yellow]![/yellow] Docker/Podman not detected; not writing concore.sudo") + console.print( + "[yellow]![/yellow] Docker/Podman not detected; not writing concore.sudo" + ) octave_file = Path(concore_path) / "concore.octave" if octave_found: - wrote_any = ( - _write_text(octave_file, "", dry_run, force, console) - or wrote_any - ) + wrote_any = _write_text(octave_file, "", dry_run, force, console) or wrote_any else: console.print("[dim]-[/dim] Octave not detected; not writing concore.octave") @@ -104,4 +100,4 @@ def setup_concore(console, dry_run=False, force=False): console.print("[green]Dry run complete.[/green]") else: console.print("[green]Setup complete.[/green]") - return True \ No newline at end of file + return True From 785b58c117b13d0793d13420ce7408f25055aa91 Mon Sep 17 00:00:00 2001 From: GREENRAT-K405 Date: Fri, 27 Mar 2026 06:27:08 +0530 Subject: [PATCH 260/275] Add concore editor to CLI --- concore_cli/cli.py | 11 +++++++++++ concore_cli/commands/editor.py | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+) create mode 100644 concore_cli/commands/editor.py diff --git a/concore_cli/cli.py b/concore_cli/cli.py index 90cf112b..9227838d 100644 --- a/concore_cli/cli.py +++ b/concore_cli/cli.py @@ -166,5 +166,16 @@ def setup(dry_run, force): sys.exit(1) +@cli.command() +def editor(): + """Launch concore-editor""" + try: + from .commands.editor import launch_editor + launch_editor() + except Exception as e: + console.print(f"[red]Error:[/red] {str(e)}") + sys.exit(1) + + if __name__ == "__main__": cli() diff --git a/concore_cli/commands/editor.py b/concore_cli/commands/editor.py new file mode 100644 index 00000000..5ad85623 --- /dev/null +++ b/concore_cli/commands/editor.py @@ -0,0 +1,33 @@ +import click +import os +import sys +import shutil +import subprocess +import urllib.parse +from pathlib import Path +from rich.console import Console + +console = Console() +EDITOR_URL = "https://controlcore-project.github.io/concore-editor/" + +def open_editor_url(url): + try: + if sys.platform == "win32": + os.system(f'start "" chrome "{url}"') + else: + if shutil.which("open"): + if sys.platform == "darwin": + subprocess.run(["open", "-a", "Google Chrome", url]) + elif sys.platform.startswith("linux"): + subprocess.run(["xdg-open", url]) + else: + if shutil.which("xdg-open"): + subprocess.run(["xdg-open", url]) + else: + console.print("unable to open browser for the concore editor.") + except Exception as e: + console.print(f"unable to open browser for the concore editor. ({e})") + +def launch_editor(): + console.print("[cyan]Opening concore-editor...[/cyan]") + open_editor_url(EDITOR_URL) From 98e1a6feeb4d106e9bc0312d970c1c0b82590c0b Mon Sep 17 00:00:00 2001 From: GREENRAT-K405 Date: Fri, 27 Mar 2026 06:31:06 +0530 Subject: [PATCH 261/275] Add concore editor to CLI --- concore_cli/cli.py | 1 + concore_cli/commands/editor.py | 5 ++--- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/concore_cli/cli.py b/concore_cli/cli.py index 9227838d..32061c0b 100644 --- a/concore_cli/cli.py +++ b/concore_cli/cli.py @@ -171,6 +171,7 @@ def editor(): """Launch concore-editor""" try: from .commands.editor import launch_editor + launch_editor() except Exception as e: console.print(f"[red]Error:[/red] {str(e)}") diff --git a/concore_cli/commands/editor.py b/concore_cli/commands/editor.py index 5ad85623..b40962da 100644 --- a/concore_cli/commands/editor.py +++ b/concore_cli/commands/editor.py @@ -1,15 +1,13 @@ -import click import os import sys import shutil import subprocess -import urllib.parse -from pathlib import Path from rich.console import Console console = Console() EDITOR_URL = "https://controlcore-project.github.io/concore-editor/" + def open_editor_url(url): try: if sys.platform == "win32": @@ -28,6 +26,7 @@ def open_editor_url(url): except Exception as e: console.print(f"unable to open browser for the concore editor. ({e})") + def launch_editor(): console.print("[cyan]Opening concore-editor...[/cyan]") open_editor_url(EDITOR_URL) From 8d6d98992a164ee82dbe423564aee792b27940a3 Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Fri, 27 Mar 2026 11:01:39 +0530 Subject: [PATCH 262/275] Fix: Replace hardcoded 256-byte shared memory buffer with 4096-byte SHM_SIZE constant (#514) The C++ shared memory implementation in concore.hpp and concoredocker.hpp allocated a fixed 256-byte buffer for inter-node data exchange and silently truncated any payload exceeding 255 characters. Changes: - Add static constexpr SHM_SIZE = 4096 in both concore.hpp and concoredocker.hpp - Replace all hardcoded 256 in shmget(), strnlen(), and strncpy() with SHM_SIZE - Add overflow detection: stderr error message when payload exceeds SHM_SIZE - Add explicit null termination after strncpy() to prevent read overruns - Buffer now supports ~200+ double values (up from ~25) Fixes #514 --- concore.hpp | 26 ++++++++++++++++++++------ concoredocker.hpp | 18 +++++++++++++----- 2 files changed, 33 insertions(+), 11 deletions(-) diff --git a/concore.hpp b/concore.hpp index eb88108e..9e981151 100644 --- a/concore.hpp +++ b/concore.hpp @@ -40,6 +40,8 @@ class Concore{ string inpath = "./in"; string outpath = "./out"; + static constexpr size_t SHM_SIZE = 4096; + int shmId_create = -1; int shmId_get = -1; @@ -259,7 +261,7 @@ class Concore{ */ void createSharedMemory(key_t key) { - shmId_create = shmget(key, 256, IPC_CREAT | 0666); + shmId_create = shmget(key, SHM_SIZE, IPC_CREAT | 0666); if (shmId_create == -1) { std::cerr << "Failed to create shared memory segment." << std::endl; @@ -284,7 +286,7 @@ class Concore{ const int MAX_RETRY = 100; while (retry < MAX_RETRY) { // Get the shared memory segment created by Writer - shmId_get = shmget(key, 256, 0666); + shmId_get = shmget(key, SHM_SIZE, 0666); // Check if shared memory exists if (shmId_get != -1) { break; // Break the loop if shared memory exists @@ -490,7 +492,7 @@ class Concore{ try { if (shmId_get != -1) { if (sharedData_get && sharedData_get[0] != '\0') { - std::string message(sharedData_get, strnlen(sharedData_get, 256)); + std::string message(sharedData_get, strnlen(sharedData_get, SHM_SIZE)); ins = message; } else @@ -515,7 +517,7 @@ class Concore{ this_thread::sleep_for(timespan); try{ if(shmId_get != -1) { - std::string message(sharedData_get, strnlen(sharedData_get, 256)); + std::string message(sharedData_get, strnlen(sharedData_get, SHM_SIZE)); ins = message; retrycount++; } @@ -664,7 +666,13 @@ class Concore{ outfile<= SHM_SIZE) { + std::cerr << "ERROR: write_SM payload (" << result.size() + << " bytes) exceeds " << SHM_SIZE - 1 + << "-byte shared memory limit. Data truncated!" << std::endl; + } + std::strncpy(sharedData_create, result.c_str(), SHM_SIZE - 1); + sharedData_create[SHM_SIZE - 1] = '\0'; // simtime must not be mutated here (issue #385). } else{ @@ -689,7 +697,13 @@ class Concore{ this_thread::sleep_for(timespan); try { if(shmId_create != -1){ - std::strncpy(sharedData_create, val.c_str(), 256 - 1); + if (val.size() >= SHM_SIZE) { + std::cerr << "ERROR: write_SM payload (" << val.size() + << " bytes) exceeds " << SHM_SIZE - 1 + << "-byte shared memory limit. Data truncated!" << std::endl; + } + std::strncpy(sharedData_create, val.c_str(), SHM_SIZE - 1); + sharedData_create[SHM_SIZE - 1] = '\0'; } else throw 505; } diff --git a/concoredocker.hpp b/concoredocker.hpp index 51d6ca5f..1218aaee 100644 --- a/concoredocker.hpp +++ b/concoredocker.hpp @@ -26,6 +26,8 @@ class Concore { private: + static constexpr size_t SHM_SIZE = 4096; + int shmId_create = -1; int shmId_get = -1; char* sharedData_create = nullptr; @@ -233,7 +235,7 @@ class Concore { #ifdef __linux__ void createSharedMemory(key_t key) { - shmId_create = shmget(key, 256, IPC_CREAT | 0666); + shmId_create = shmget(key, SHM_SIZE, IPC_CREAT | 0666); if (shmId_create == -1) { std::cerr << "Failed to create shared memory segment.\n"; return; @@ -249,7 +251,7 @@ class Concore { int retry = 0; const int MAX_RETRY = 100; while (retry < MAX_RETRY) { - shmId_get = shmget(key, 256, 0666); + shmId_get = shmget(key, SHM_SIZE, 0666); if (shmId_get != -1) break; std::cout << "Shared memory does not exist. Make sure the writer process is running.\n"; @@ -345,7 +347,7 @@ class Concore { std::string ins; try { if (shmId_get != -1 && sharedData_get && sharedData_get[0] != '\0') - ins = std::string(sharedData_get, strnlen(sharedData_get, 256)); + ins = std::string(sharedData_get, strnlen(sharedData_get, SHM_SIZE)); else throw 505; } catch (...) { @@ -359,7 +361,7 @@ class Concore { std::this_thread::sleep_for(std::chrono::seconds(delay)); try { if (shmId_get != -1 && sharedData_get) { - ins = std::string(sharedData_get, strnlen(sharedData_get, 256)); + ins = std::string(sharedData_get, strnlen(sharedData_get, SHM_SIZE)); retrycount++; } else { retrycount++; @@ -426,7 +428,13 @@ class Concore { outfile << val[i] << ','; outfile << val[val.size() - 1] << ']'; std::string result = outfile.str(); - std::strncpy(sharedData_create, result.c_str(), 256 - 1); + if (result.size() >= SHM_SIZE) { + std::cerr << "ERROR: write_SM payload (" << result.size() + << " bytes) exceeds " << SHM_SIZE - 1 + << "-byte shared memory limit. Data truncated!" << std::endl; + } + std::strncpy(sharedData_create, result.c_str(), SHM_SIZE - 1); + sharedData_create[SHM_SIZE - 1] = '\0'; } catch (...) { std::cerr << "skipping +" << outpath << port << "/" << name << "\n"; } From 0af338c0cc2c3b7fab1cd2d09db69fef1bf3ab9d Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Fri, 27 Mar 2026 11:57:09 +0530 Subject: [PATCH 263/275] Fix: Add nullptr guard for sharedData_create and shmctl segment size verification Address Copilot review feedback: - Add sharedData_create != nullptr check before strncpy/null-termination in all write_SM() overloads to prevent null pointer dereference when shmat() fails in createSharedMemory() - Add shmctl(IPC_STAT) verification in createSharedMemory() to detect stale segments smaller than SHM_SIZE (shmget won't resize existing segments); removes and recreates the segment when too small - Add early return in concore.hpp createSharedMemory() on shmget failure (was missing, unlike concoredocker.hpp which already had it) --- concore.hpp | 18 ++++++++++++++++++ concoredocker.hpp | 16 ++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/concore.hpp b/concore.hpp index 9e981151..e3fb5e1a 100644 --- a/concore.hpp +++ b/concore.hpp @@ -265,6 +265,20 @@ class Concore{ if (shmId_create == -1) { std::cerr << "Failed to create shared memory segment." << std::endl; + return; + } + + // Verify the segment is large enough (shmget won't resize an existing segment) + struct shmid_ds shm_info; + if (shmctl(shmId_create, IPC_STAT, &shm_info) == 0 && shm_info.shm_segsz < SHM_SIZE) { + std::cerr << "Shared memory segment too small (" << shm_info.shm_segsz + << " bytes, need " << SHM_SIZE << "). Removing and recreating." << std::endl; + shmctl(shmId_create, IPC_RMID, nullptr); + shmId_create = shmget(key, SHM_SIZE, IPC_CREAT | 0666); + if (shmId_create == -1) { + std::cerr << "Failed to recreate shared memory segment." << std::endl; + return; + } } // Attach the shared memory segment to the process's address space @@ -660,6 +674,8 @@ class Concore{ try { std::ostringstream outfile; if(shmId_create != -1){ + if (sharedData_create == nullptr) + throw 506; val.insert(val.begin(),simtime+delta); outfile<<'['; for(int i=0;i= SHM_SIZE) { std::cerr << "ERROR: write_SM payload (" << val.size() << " bytes) exceeds " << SHM_SIZE - 1 diff --git a/concoredocker.hpp b/concoredocker.hpp index 1218aaee..d9c9a1e0 100644 --- a/concoredocker.hpp +++ b/concoredocker.hpp @@ -240,6 +240,20 @@ class Concore { std::cerr << "Failed to create shared memory segment.\n"; return; } + + // Verify the segment is large enough (shmget won't resize an existing segment) + struct shmid_ds shm_info; + if (shmctl(shmId_create, IPC_STAT, &shm_info) == 0 && shm_info.shm_segsz < SHM_SIZE) { + std::cerr << "Shared memory segment too small (" << shm_info.shm_segsz + << " bytes, need " << SHM_SIZE << "). Removing and recreating.\n"; + shmctl(shmId_create, IPC_RMID, nullptr); + shmId_create = shmget(key, SHM_SIZE, IPC_CREAT | 0666); + if (shmId_create == -1) { + std::cerr << "Failed to recreate shared memory segment.\n"; + return; + } + } + sharedData_create = static_cast(shmat(shmId_create, NULL, 0)); if (sharedData_create == reinterpret_cast(-1)) { std::cerr << "Failed to attach shared memory segment.\n"; @@ -421,6 +435,8 @@ class Concore { try { if (shmId_create == -1) throw 505; + if (sharedData_create == nullptr) + throw 506; val.insert(val.begin(), simtime + delta); std::ostringstream outfile; outfile << '['; From 4f1454c445a903562be0f3613be548131dccb086 Mon Sep 17 00:00:00 2001 From: Avinash Kumar Deepak Date: Fri, 27 Mar 2026 22:11:46 +0530 Subject: [PATCH 264/275] feat(cli): add optional docker-compose generation for docker runs --- concore_cli/README.md | 9 +++ concore_cli/cli.py | 17 ++++- concore_cli/commands/run.py | 135 +++++++++++++++++++++++++++++++++++- tests/test_cli.py | 99 ++++++++++++++++++++++++++ 4 files changed, 255 insertions(+), 5 deletions(-) diff --git a/concore_cli/README.md b/concore_cli/README.md index 55546c00..7264d144 100644 --- a/concore_cli/README.md +++ b/concore_cli/README.md @@ -62,12 +62,21 @@ Generates and optionally builds a workflow from a GraphML file. - `-o, --output ` - Output directory (default: out) - `-t, --type ` - Execution type: windows, posix, or docker (default: windows) - `--auto-build` - Automatically run build script after generation +- `--compose` - Generate `docker-compose.yml` (only valid with `--type docker`) **Example:** ```bash concore run workflow.graphml --source ./src --output ./build --auto-build ``` +Docker compose example: + +```bash +concore run workflow.graphml --source ./src --output ./out --type docker --compose +cd out +docker compose up +``` + ### `concore validate ` Validates a GraphML workflow file before running. diff --git a/concore_cli/cli.py b/concore_cli/cli.py index 32061c0b..4d7dd64a 100644 --- a/concore_cli/cli.py +++ b/concore_cli/cli.py @@ -70,10 +70,23 @@ def init(name, template, interactive): @click.option( "--auto-build", is_flag=True, help="Automatically run build after generation" ) -def run(workflow_file, source, output, type, auto_build): +@click.option( + "--compose", + is_flag=True, + help="Generate docker-compose.yml in output directory (docker type only)", +) +def run(workflow_file, source, output, type, auto_build, compose): """Run a concore workflow""" try: - run_workflow(workflow_file, source, output, type, auto_build, console) + run_workflow( + workflow_file, + source, + output, + type, + auto_build, + console, + compose=compose, + ) except Exception as e: console.print(f"[red]Error:[/red] {str(e)}") sys.exit(1) diff --git a/concore_cli/commands/run.py b/concore_cli/commands/run.py index ad1c23c6..de7bed27 100644 --- a/concore_cli/commands/run.py +++ b/concore_cli/commands/run.py @@ -1,5 +1,7 @@ -import sys +import re +import shlex import subprocess +import sys from pathlib import Path from rich.panel import Panel from rich.progress import Progress, SpinnerColumn, TextColumn @@ -15,7 +17,113 @@ def _find_mkconcore_path(): return None -def run_workflow(workflow_file, source, output, exec_type, auto_build, console): +def _yaml_quote(value): + return "'" + value.replace("'", "''") + "'" + + +def _parse_docker_run_line(line): + text = line.strip() + if not text or text.startswith("#"): + return None + + if text.endswith("&"): + text = text[:-1].strip() + + try: + tokens = shlex.split(text) + except ValueError: + return None + + if "run" not in tokens: + return None + + run_index = tokens.index("run") + args = tokens[run_index + 1 :] + + container_name = None + volumes = [] + image = None + + i = 0 + while i < len(args): + token = args[i] + if token.startswith("--name="): + container_name = token.split("=", 1)[1] + elif token == "--name" and i + 1 < len(args): + container_name = args[i + 1] + i += 1 + elif token in ("-v", "--volume") and i + 1 < len(args): + volumes.append(args[i + 1]) + i += 1 + elif token.startswith("--volume="): + volumes.append(token.split("=", 1)[1]) + elif token.startswith("-"): + pass + else: + image = token + break + i += 1 + + if not container_name or not image: + return None + + return { + "container_name": container_name, + "volumes": volumes, + "image": image, + } + + +def _write_docker_compose(output_path): + run_script = output_path / "run" + if not run_script.exists(): + return None + + services = [] + for line in run_script.read_text(encoding="utf-8").splitlines(): + parsed = _parse_docker_run_line(line) + if parsed is not None: + services.append(parsed) + + if not services: + return None + + compose_lines = ["services:"] + + for index, service in enumerate(services, start=1): + service_name = re.sub(r"[^A-Za-z0-9_.-]", "-", service["container_name"]).strip( + "-." + ) + if not service_name: + service_name = f"service-{index}" + elif not service_name[0].isalnum(): + service_name = f"service-{service_name}" + + compose_lines.append(f" {service_name}:") + compose_lines.append(f" image: {_yaml_quote(service['image'])}") + compose_lines.append( + f" container_name: {_yaml_quote(service['container_name'])}" + ) + if service["volumes"]: + compose_lines.append(" volumes:") + for volume_spec in service["volumes"]: + compose_lines.append(f" - {_yaml_quote(volume_spec)}") + + compose_lines.append("") + compose_path = output_path / "docker-compose.yml" + compose_path.write_text("\n".join(compose_lines), encoding="utf-8") + return compose_path + + +def run_workflow( + workflow_file, + source, + output, + exec_type, + auto_build, + console, + compose=False, +): workflow_path = Path(workflow_file).resolve() source_path = Path(source).resolve() output_path = Path(output).resolve() @@ -34,8 +142,13 @@ def run_workflow(workflow_file, source, output, exec_type, auto_build, console): console.print(f"[cyan]Source:[/cyan] {source_path}") console.print(f"[cyan]Output:[/cyan] {output_path}") console.print(f"[cyan]Type:[/cyan] {exec_type}") + if compose: + console.print("[cyan]Compose:[/cyan] enabled") console.print() + if compose and exec_type != "docker": + raise ValueError("--compose can only be used with --type docker") + mkconcore_path = _find_mkconcore_path() if mkconcore_path is None: raise FileNotFoundError( @@ -73,6 +186,18 @@ def run_workflow(workflow_file, source, output, exec_type, auto_build, console): console.print( f"[green]✓[/green] Workflow generated in [cyan]{output_path}[/cyan]" ) + + if compose: + compose_path = _write_docker_compose(output_path) + if compose_path is not None: + console.print( + f"[green]✓[/green] Compose file written to [cyan]{compose_path}[/cyan]" + ) + else: + console.print( + "[yellow]Warning:[/yellow] Could not generate docker-compose.yml from run script" + ) + try: metadata_path = write_study_metadata( output_path, @@ -128,6 +253,10 @@ def run_workflow(workflow_file, source, output, exec_type, auto_build, console): if e.stderr: console.print(e.stderr) + run_command = "docker compose up" if compose else "./run" + if exec_type == "windows": + run_command = "run.bat" + console.print() console.print( Panel.fit( @@ -135,7 +264,7 @@ def run_workflow(workflow_file, source, output, exec_type, auto_build, console): f"To run your workflow:\n" f" cd {output_path}\n" f" {'build.bat' if exec_type == 'windows' else './build'}\n" - f" {'run.bat' if exec_type == 'windows' else './run'}", + f" {run_command}", title="Next Steps", border_style="green", ) diff --git a/tests/test_cli.py b/tests/test_cli.py index 63cd0f23..4f8f57b3 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -233,6 +233,105 @@ def test_run_command_docker_subdir_source_build_paths(self): self.assertIn("cp ../src/subdir/script.iport concore.iport", build_script) self.assertIn("cd ..", build_script) + def test_run_command_compose_requires_docker_type(self): + with self.runner.isolated_filesystem(temp_dir=self.temp_dir): + result = self.runner.invoke(cli, ["init", "test-project"]) + self.assertEqual(result.exit_code, 0) + + result = self.runner.invoke( + cli, + [ + "run", + "test-project/workflow.graphml", + "--source", + "test-project/src", + "--output", + "out", + "--type", + "posix", + "--compose", + ], + ) + self.assertNotEqual(result.exit_code, 0) + self.assertIn( + "--compose can only be used with --type docker", result.output + ) + + def test_run_command_docker_compose_single_node(self): + with self.runner.isolated_filesystem(temp_dir=self.temp_dir): + result = self.runner.invoke(cli, ["init", "test-project"]) + self.assertEqual(result.exit_code, 0) + + result = self.runner.invoke( + cli, + [ + "run", + "test-project/workflow.graphml", + "--source", + "test-project/src", + "--output", + "out", + "--type", + "docker", + "--compose", + ], + ) + self.assertEqual(result.exit_code, 0) + + compose_path = Path("out/docker-compose.yml") + self.assertTrue(compose_path.exists()) + compose_content = compose_path.read_text() + self.assertIn("services:", compose_content) + self.assertIn("container_name: 'N1'", compose_content) + self.assertIn("image: 'docker-script'", compose_content) + + metadata = json.loads(Path("out/STUDY.json").read_text()) + self.assertIn("docker-compose.yml", metadata["checksums"]) + + def test_run_command_docker_compose_multi_node(self): + with self.runner.isolated_filesystem(temp_dir=self.temp_dir): + Path("src").mkdir() + Path("src/common.py").write_text( + "import concore\n\ndef step():\n return None\n" + ) + + workflow = """ + + + + + A:common.py + B:common.py + C:common.py + 0x1000_AB + 0x1001_BC + + +""" + Path("workflow.graphml").write_text(workflow) + + result = self.runner.invoke( + cli, + [ + "run", + "workflow.graphml", + "--source", + "src", + "--output", + "out", + "--type", + "docker", + "--compose", + ], + ) + self.assertEqual(result.exit_code, 0) + + compose_content = Path("out/docker-compose.yml").read_text() + self.assertIn("container_name: 'A'", compose_content) + self.assertIn("container_name: 'B'", compose_content) + self.assertIn("container_name: 'C'", compose_content) + self.assertIn("image: 'docker-common'", compose_content) + def test_run_command_shared_source_specialization_merges_edge_params(self): with self.runner.isolated_filesystem(temp_dir=self.temp_dir): Path("src").mkdir() From 01178383f18555db8f19591676fc55b3b2c24569 Mon Sep 17 00:00:00 2001 From: GREENRAT-K405 Date: Sat, 28 Mar 2026 12:21:16 +0530 Subject: [PATCH 265/275] replace concore run with concore build --- README.md | 4 +- concore_cli/README.md | 14 +++---- concore_cli/cli.py | 10 ++--- concore_cli/commands/__init__.py | 4 +- concore_cli/commands/{run.py => build.py} | 4 +- concore_cli/commands/init.py | 8 ++-- tests/test_cli.py | 47 ++++++++++++----------- 7 files changed, 46 insertions(+), 45 deletions(-) rename concore_cli/commands/{run.py => build.py} (99%) diff --git a/README.md b/README.md index 0f0ba7a5..c5c570d4 100644 --- a/README.md +++ b/README.md @@ -55,8 +55,8 @@ concore init my-project # Validate your workflow concore validate workflow.graphml -# Run your workflow -concore run workflow.graphml --auto-build +# Compile your workflow +concore build workflow.graphml --auto-build # Monitor running processes concore status diff --git a/concore_cli/README.md b/concore_cli/README.md index 7264d144..78f98b1a 100644 --- a/concore_cli/README.md +++ b/concore_cli/README.md @@ -20,8 +20,8 @@ cd my-project # Validate your workflow concore validate workflow.graphml -# Run your workflow -concore run workflow.graphml +# Build your workflow +concore build workflow.graphml # Check running processes concore status @@ -53,9 +53,9 @@ my-workflow/ └── README.md # Project documentation ``` -### `concore run ` +### `concore build ` -Generates and optionally builds a workflow from a GraphML file. +Compiles a concore workflow GraphML file into executable scripts (POSIX, Windows, or Docker). **Options:** - `-s, --source ` - Source directory (default: src) @@ -66,13 +66,13 @@ Generates and optionally builds a workflow from a GraphML file. **Example:** ```bash -concore run workflow.graphml --source ./src --output ./build --auto-build +concore build workflow.graphml --source ./src --output ./build --auto-build ``` Docker compose example: ```bash -concore run workflow.graphml --source ./src --output ./out --type docker --compose +concore build workflow.graphml --source ./src --output ./out --type docker --compose cd out docker compose up ``` @@ -150,7 +150,7 @@ concore stop 5. **Generate and run** ```bash - concore run workflow.graphml --auto-build + concore build workflow.graphml --auto-build cd out ./run.bat # or ./run on Linux/Mac ``` diff --git a/concore_cli/cli.py b/concore_cli/cli.py index 4d7dd64a..aeceb995 100644 --- a/concore_cli/cli.py +++ b/concore_cli/cli.py @@ -4,7 +4,7 @@ import sys from .commands.init import init_project, init_project_interactive, run_wizard -from .commands.run import run_workflow +from .commands.build import build_workflow from .commands.validate import validate_workflow from .commands.status import show_status from .commands.stop import stop_all @@ -68,17 +68,17 @@ def init(name, template, interactive): help="Execution type", ) @click.option( - "--auto-build", is_flag=True, help="Automatically run build after generation" + "--auto-build", is_flag=True, help="Automatically run build script after generation" ) @click.option( "--compose", is_flag=True, help="Generate docker-compose.yml in output directory (docker type only)", ) -def run(workflow_file, source, output, type, auto_build, compose): - """Run a concore workflow""" +def build(workflow_file, source, output, type, auto_build, compose): + """Compile a concore workflow into executable scripts""" try: - run_workflow( + build_workflow( workflow_file, source, output, diff --git a/concore_cli/commands/__init__.py b/concore_cli/commands/__init__.py index b9771e4c..fafdf6ec 100644 --- a/concore_cli/commands/__init__.py +++ b/concore_cli/commands/__init__.py @@ -1,5 +1,5 @@ from .init import init_project -from .run import run_workflow +from .build import build_workflow from .validate import validate_workflow from .status import show_status from .stop import stop_all @@ -8,7 +8,7 @@ __all__ = [ "init_project", - "run_workflow", + "build_workflow", "validate_workflow", "show_status", "stop_all", diff --git a/concore_cli/commands/run.py b/concore_cli/commands/build.py similarity index 99% rename from concore_cli/commands/run.py rename to concore_cli/commands/build.py index de7bed27..5865500b 100644 --- a/concore_cli/commands/run.py +++ b/concore_cli/commands/build.py @@ -115,7 +115,7 @@ def _write_docker_compose(output_path): return compose_path -def run_workflow( +def build_workflow( workflow_file, source, output, @@ -201,7 +201,7 @@ def run_workflow( try: metadata_path = write_study_metadata( output_path, - generated_by="concore run", + generated_by="concore build", workflow_file=workflow_path, ) console.print( diff --git a/concore_cli/commands/init.py b/concore_cli/commands/init.py index 64739b52..53fd53f7 100644 --- a/concore_cli/commands/init.py +++ b/concore_cli/commands/init.py @@ -183,9 +183,9 @@ 1. Edit your workflow in `workflow.graphml` using yEd or similar GraphML editor 2. Add your processing scripts to the `src/` directory -3. Run your workflow: +3. Build your workflow: ``` - concore run workflow.graphml + concore build workflow.graphml ``` ## Project Structure @@ -318,7 +318,7 @@ def init_project_interactive(name, selected_langs, console): f"Next steps:\n" f" cd {name}\n" f" concore validate workflow.graphml\n" - f" concore run workflow.graphml", + f" concore build workflow.graphml", title="Success", border_style="green", ) @@ -371,7 +371,7 @@ def init_project(name, template, console): f"Next steps:\n" f" cd {name}\n" f" concore validate workflow.graphml\n" - f" concore run workflow.graphml", + f" concore build workflow.graphml", title="Success", border_style="green", ) diff --git a/tests/test_cli.py b/tests/test_cli.py index 4f8f57b3..d5025523 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -88,15 +88,16 @@ def test_status_command(self): result = self.runner.invoke(cli, ["status"]) self.assertEqual(result.exit_code, 0) - def test_run_command_missing_source(self): + def test_build_command_missing_source(self): with self.runner.isolated_filesystem(temp_dir=self.temp_dir): result = self.runner.invoke(cli, ["init", "test-project"]) result = self.runner.invoke( - cli, ["run", "test-project/workflow.graphml", "--source", "nonexistent"] + cli, + ["build", "test-project/workflow.graphml", "--source", "nonexistent"], ) self.assertNotEqual(result.exit_code, 0) - def test_run_command_from_project_dir(self): + def test_build_command_from_project_dir(self): with self.runner.isolated_filesystem(temp_dir=self.temp_dir): result = self.runner.invoke(cli, ["init", "test-project"]) self.assertEqual(result.exit_code, 0) @@ -104,7 +105,7 @@ def test_run_command_from_project_dir(self): result = self.runner.invoke( cli, [ - "run", + "build", "test-project/workflow.graphml", "--source", "test-project/src", @@ -119,12 +120,12 @@ def test_run_command_from_project_dir(self): self.assertTrue(Path("out/STUDY.json").exists()) metadata = json.loads(Path("out/STUDY.json").read_text()) - self.assertEqual(metadata["generated_by"], "concore run") + self.assertEqual(metadata["generated_by"], "concore build") self.assertEqual(metadata["study_name"], "out") self.assertEqual(metadata["schema_version"], 1) self.assertIn("workflow.graphml", metadata["checksums"]) - def test_run_command_default_type(self): + def test_build_command_default_type(self): with self.runner.isolated_filesystem(temp_dir=self.temp_dir): result = self.runner.invoke(cli, ["init", "test-project"]) self.assertEqual(result.exit_code, 0) @@ -132,7 +133,7 @@ def test_run_command_default_type(self): result = self.runner.invoke( cli, [ - "run", + "build", "test-project/workflow.graphml", "--source", "test-project/src", @@ -146,7 +147,7 @@ def test_run_command_default_type(self): else: self.assertTrue(Path("out/build").exists()) - def test_run_command_nested_output_path(self): + def test_build_command_nested_output_path(self): with self.runner.isolated_filesystem(temp_dir=self.temp_dir): result = self.runner.invoke(cli, ["init", "test-project"]) self.assertEqual(result.exit_code, 0) @@ -154,7 +155,7 @@ def test_run_command_nested_output_path(self): result = self.runner.invoke( cli, [ - "run", + "build", "test-project/workflow.graphml", "--source", "test-project/src", @@ -167,7 +168,7 @@ def test_run_command_nested_output_path(self): self.assertEqual(result.exit_code, 0) self.assertTrue(Path("build/out/src/concore.py").exists()) - def test_run_command_subdir_source(self): + def test_build_command_subdir_source(self): with self.runner.isolated_filesystem(temp_dir=self.temp_dir): result = self.runner.invoke(cli, ["init", "test-project"]) self.assertEqual(result.exit_code, 0) @@ -184,7 +185,7 @@ def test_run_command_subdir_source(self): result = self.runner.invoke( cli, [ - "run", + "build", "test-project/workflow.graphml", "--source", "test-project/src", @@ -197,7 +198,7 @@ def test_run_command_subdir_source(self): self.assertEqual(result.exit_code, 0) self.assertTrue(Path("out/src/subdir/script.py").exists()) - def test_run_command_docker_subdir_source_build_paths(self): + def test_build_command_docker_subdir_source_build_paths(self): with self.runner.isolated_filesystem(temp_dir=self.temp_dir): result = self.runner.invoke(cli, ["init", "test-project"]) self.assertEqual(result.exit_code, 0) @@ -214,7 +215,7 @@ def test_run_command_docker_subdir_source_build_paths(self): result = self.runner.invoke( cli, [ - "run", + "build", "test-project/workflow.graphml", "--source", "test-project/src", @@ -233,7 +234,7 @@ def test_run_command_docker_subdir_source_build_paths(self): self.assertIn("cp ../src/subdir/script.iport concore.iport", build_script) self.assertIn("cd ..", build_script) - def test_run_command_compose_requires_docker_type(self): + def test_build_command_compose_requires_docker_type(self): with self.runner.isolated_filesystem(temp_dir=self.temp_dir): result = self.runner.invoke(cli, ["init", "test-project"]) self.assertEqual(result.exit_code, 0) @@ -241,7 +242,7 @@ def test_run_command_compose_requires_docker_type(self): result = self.runner.invoke( cli, [ - "run", + "build", "test-project/workflow.graphml", "--source", "test-project/src", @@ -257,7 +258,7 @@ def test_run_command_compose_requires_docker_type(self): "--compose can only be used with --type docker", result.output ) - def test_run_command_docker_compose_single_node(self): + def test_build_command_docker_compose_single_node(self): with self.runner.isolated_filesystem(temp_dir=self.temp_dir): result = self.runner.invoke(cli, ["init", "test-project"]) self.assertEqual(result.exit_code, 0) @@ -265,7 +266,7 @@ def test_run_command_docker_compose_single_node(self): result = self.runner.invoke( cli, [ - "run", + "build", "test-project/workflow.graphml", "--source", "test-project/src", @@ -288,7 +289,7 @@ def test_run_command_docker_compose_single_node(self): metadata = json.loads(Path("out/STUDY.json").read_text()) self.assertIn("docker-compose.yml", metadata["checksums"]) - def test_run_command_docker_compose_multi_node(self): + def test_build_command_docker_compose_multi_node(self): with self.runner.isolated_filesystem(temp_dir=self.temp_dir): Path("src").mkdir() Path("src/common.py").write_text( @@ -313,7 +314,7 @@ def test_run_command_docker_compose_multi_node(self): result = self.runner.invoke( cli, [ - "run", + "build", "workflow.graphml", "--source", "src", @@ -332,7 +333,7 @@ def test_run_command_docker_compose_multi_node(self): self.assertIn("container_name: 'C'", compose_content) self.assertIn("image: 'docker-common'", compose_content) - def test_run_command_shared_source_specialization_merges_edge_params(self): + def test_build_command_shared_source_specialization_merges_edge_params(self): with self.runner.isolated_filesystem(temp_dir=self.temp_dir): Path("src").mkdir() Path("src/common.py").write_text( @@ -357,7 +358,7 @@ def test_run_command_shared_source_specialization_merges_edge_params(self): result = self.runner.invoke( cli, [ - "run", + "build", "workflow.graphml", "--source", "src", @@ -377,7 +378,7 @@ def test_run_command_shared_source_specialization_merges_edge_params(self): self.assertIn("PORT_NAME_B_C", content) self.assertIn("PORT_B_C", content) - def test_run_command_existing_output(self): + def test_build_command_existing_output(self): with self.runner.isolated_filesystem(temp_dir=self.temp_dir): result = self.runner.invoke(cli, ["init", "test-project"]) Path("output").mkdir() @@ -385,7 +386,7 @@ def test_run_command_existing_output(self): result = self.runner.invoke( cli, [ - "run", + "build", "test-project/workflow.graphml", "--source", "test-project/src", From 6e59e9d6557b29f003ad9c627e4abe5fe8259c7f Mon Sep 17 00:00:00 2001 From: Avinash Kumar Deepak Date: Sat, 28 Mar 2026 19:08:14 +0530 Subject: [PATCH 266/275] ci: add java phase-2 conformance coverage --- .github/workflows/ci.yml | 23 ++++++++++++++ tests/protocol_fixtures/PROTOCOL_FIXTURES.md | 10 +++++- .../cross_runtime_matrix.phase2.json | 31 +++++++++++++++++++ tests/test_protocol_conformance_phase2.py | 13 +++++++- 4 files changed, 75 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5cfdc7e4..c61a1679 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -55,6 +55,29 @@ jobs: --ignore=ratc/ \ --ignore=linktest/ + java-test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: '17' + + - name: Download jeromq + run: | + curl -fsSL -o jeromq.jar https://repo1.maven.org/maven2/org/zeromq/jeromq/0.6.0/jeromq-0.6.0.jar + + - name: Compile Java tests + run: | + javac -cp jeromq.jar concoredocker.java TestLiteralEval.java TestConcoredockerApi.java + + - name: Run Java tests + run: | + java -cp .:jeromq.jar TestLiteralEval + java -cp .:jeromq.jar TestConcoredockerApi + docker-build: runs-on: ubuntu-latest steps: diff --git a/tests/protocol_fixtures/PROTOCOL_FIXTURES.md b/tests/protocol_fixtures/PROTOCOL_FIXTURES.md index 1bf953e7..79b06ae3 100644 --- a/tests/protocol_fixtures/PROTOCOL_FIXTURES.md +++ b/tests/protocol_fixtures/PROTOCOL_FIXTURES.md @@ -16,4 +16,12 @@ Phase-2 scope (mapping only): - No runtime behavior changes. - Adds a cross-runtime matrix to track per-case audit status and classification. -- Keeps CI non-blocking for non-Python runtimes by marking them as `not_audited` until adapters are added. +- Java runtime entries are tracked with observed status from the Java regression suite (`TestLiteralEval.java`, `TestConcoredockerApi.java`). +- Current baseline records Java as `observed_pass` for the listed phase-2 cases. +- Keeps CI non-blocking for non-Python runtimes that are not yet audited by marking them as `not_audited`. + +Java conformance execution in CI: + +- The `java-test` job in `.github/workflows/ci.yml` downloads `jeromq` for classpath compatibility. +- It compiles `concoredocker.java`, `TestLiteralEval.java`, and `TestConcoredockerApi.java`. +- It runs both Java test classes and records initial phase-2 matrix status as observed in CI. diff --git a/tests/protocol_fixtures/cross_runtime_matrix.phase2.json b/tests/protocol_fixtures/cross_runtime_matrix.phase2.json index 746f2f1a..ed642cdd 100644 --- a/tests/protocol_fixtures/cross_runtime_matrix.phase2.json +++ b/tests/protocol_fixtures/cross_runtime_matrix.phase2.json @@ -6,6 +6,7 @@ "runtimes": [ "python", "cpp", + "java", "matlab", "octave", "verilog" @@ -35,6 +36,11 @@ "classification": "required", "note": "Audit planned in phase 2." }, + "java": { + "status": "observed_pass", + "classification": "required", + "note": "Validated by TestLiteralEval.java and TestConcoredockerApi.java." + }, "matlab": { "status": "not_audited", "classification": "required", @@ -66,6 +72,11 @@ "classification": "required", "note": "Audit planned in phase 2." }, + "java": { + "status": "observed_pass", + "classification": "required", + "note": "Validated by TestLiteralEval.java and TestConcoredockerApi.java." + }, "matlab": { "status": "not_audited", "classification": "required", @@ -97,6 +108,11 @@ "classification": "required", "note": "Audit planned in phase 2." }, + "java": { + "status": "observed_pass", + "classification": "required", + "note": "Validated by TestLiteralEval.java and TestConcoredockerApi.java." + }, "matlab": { "status": "not_audited", "classification": "required", @@ -128,6 +144,11 @@ "classification": "required", "note": "Audit planned in phase 2." }, + "java": { + "status": "observed_pass", + "classification": "required", + "note": "Validated by TestLiteralEval.java and TestConcoredockerApi.java." + }, "matlab": { "status": "not_audited", "classification": "required", @@ -159,6 +180,11 @@ "classification": "required", "note": "Audit planned in phase 2." }, + "java": { + "status": "observed_pass", + "classification": "required", + "note": "Validated by TestLiteralEval.java and TestConcoredockerApi.java." + }, "matlab": { "status": "not_audited", "classification": "required", @@ -190,6 +216,11 @@ "classification": "required", "note": "Audit planned in phase 2." }, + "java": { + "status": "observed_pass", + "classification": "required", + "note": "Validated by TestLiteralEval.java and TestConcoredockerApi.java." + }, "matlab": { "status": "not_audited", "classification": "required", diff --git a/tests/test_protocol_conformance_phase2.py b/tests/test_protocol_conformance_phase2.py index 01d8c3f8..787e918c 100644 --- a/tests/test_protocol_conformance_phase2.py +++ b/tests/test_protocol_conformance_phase2.py @@ -6,7 +6,7 @@ PHASE1_CASES_PATH = FIXTURE_DIR / "python_phase1_cases.json" PHASE2_MATRIX_PATH = FIXTURE_DIR / "cross_runtime_matrix.phase2.json" -EXPECTED_RUNTIMES = {"python", "cpp", "matlab", "octave", "verilog"} +EXPECTED_RUNTIMES = {"python", "cpp", "java", "matlab", "octave", "verilog"} EXPECTED_CLASSIFICATIONS = {"required", "implementation_defined", "known_deviation"} EXPECTED_STATUSES = {"observed_pass", "observed_fail", "not_audited"} @@ -30,6 +30,8 @@ def test_phase2_matrix_metadata_and_enums(): assert doc["phase"] == "2" assert doc["mode"] == "report_only" assert doc["source_fixture"] == "python_phase1_cases.json" + assert "java" in doc["runtimes"] + assert len(doc["runtimes"]) == len(set(doc["runtimes"])) assert set(doc["runtimes"]) == EXPECTED_RUNTIMES assert set(doc["classifications"]) == EXPECTED_CLASSIFICATIONS assert set(doc["statuses"]) == EXPECTED_STATUSES @@ -55,3 +57,12 @@ def test_phase2_matrix_rows_have_consistent_shape(): assert isinstance(result["note"], str) and result["note"].strip() if runtime == "python": assert result["status"] == "observed_pass" + + +def test_phase2_matrix_java_status_is_recorded_for_each_case(): + for row in _phase2_matrix()["cases"]: + java_result = row["runtime_results"]["java"] + assert java_result["status"] in EXPECTED_STATUSES + assert java_result["classification"] in EXPECTED_CLASSIFICATIONS + assert isinstance(java_result["note"], str) and java_result["note"].strip() + assert java_result["status"] == "observed_pass" From 8e1c043313cbfccd4bccf853e9326672d1d9853d Mon Sep 17 00:00:00 2001 From: Avinash Kumar Deepak Date: Sun, 29 Mar 2026 22:40:13 +0530 Subject: [PATCH 267/275] add java end-to-end study example --- example/java_e2e/README.md | 36 ++++++++++ example/java_e2e/controller.py | 45 ++++++++++++ example/java_e2e/java_e2e.graphml | 47 ++++++++++++ example/java_e2e/pm_java.java | 43 +++++++++++ example/java_e2e/smoke_check.py | 114 ++++++++++++++++++++++++++++++ 5 files changed, 285 insertions(+) create mode 100644 example/java_e2e/README.md create mode 100644 example/java_e2e/controller.py create mode 100644 example/java_e2e/java_e2e.graphml create mode 100644 example/java_e2e/pm_java.java create mode 100644 example/java_e2e/smoke_check.py diff --git a/example/java_e2e/README.md b/example/java_e2e/README.md new file mode 100644 index 00000000..75c0e812 --- /dev/null +++ b/example/java_e2e/README.md @@ -0,0 +1,36 @@ +# Java end-to-end example + +This example runs a Python controller (`controller.py`) and a Java PM node (`pm_java.java`) over the standard concore file-based exchange. + +## Files + +- `controller.py` - Python controller node +- `pm_java.java` - Java PM node using `concoredocker.java` +- `java_e2e.graphml` - workflow graph for the example +- `smoke_check.py` - lightweight verification script + +## Prerequisites + +- Python environment with project dependencies installed +- JDK (for `javac` and `java`) +- `jeromq-0.6.0.jar` + +Download jar (from repo root): + +```bash +mkdir -p .ci-cache/java +curl -fsSL -o .ci-cache/java/jeromq-0.6.0.jar https://repo1.maven.org/maven2/org/zeromq/jeromq/0.6.0/jeromq-0.6.0.jar +``` + +## Run smoke check + +From repo root: + +```bash +python example/java_e2e/smoke_check.py --jar .ci-cache/java/jeromq-0.6.0.jar +``` + +Expected result: + +- script prints `smoke_check passed` +- final `u` and `ym` payloads are printed in concore wire format diff --git a/example/java_e2e/controller.py b/example/java_e2e/controller.py new file mode 100644 index 00000000..e5faa0b7 --- /dev/null +++ b/example/java_e2e/controller.py @@ -0,0 +1,45 @@ +import ast +import os +from pathlib import Path + +import concore +import numpy as np + + +ysp = 3.0 + + +def controller(ym): + if ym[0] < ysp: + return 1.01 * ym + else: + return 0.9 * ym + + +study_dir = os.environ.get("CONCORE_STUDY_DIR", str(Path(__file__).resolve().parent / "study")) +os.makedirs(os.path.join(study_dir, "1"), exist_ok=True) + +concore.inpath = os.path.join(study_dir, "") +concore.outpath = os.path.join(study_dir, "") +concore.default_maxtime(20) +concore.delay = 0.02 + +init_simtime_u = "[0.0, 0.0]" +init_simtime_ym = "[0.0, 0.0]" + +u = np.array([concore.initval(init_simtime_u)]).T +while(concore.simtime + + + + + + + + + + + CJ:example/java_e2e/controller.py + + + + + + + + + + + PJ:example/java_e2e/pm_java.java + + + + + + + + + + CU + + + + + + + + + PYM + + + + + diff --git a/example/java_e2e/pm_java.java b/example/java_e2e/pm_java.java new file mode 100644 index 00000000..1b11e075 --- /dev/null +++ b/example/java_e2e/pm_java.java @@ -0,0 +1,43 @@ +import java.util.ArrayList; +import java.util.List; + +public class pm_java { + private static final String INIT_SIMTIME_U = "[0.0, 0.0]"; + + private static double toDouble(Object value) { + if (value instanceof Number) { + return ((Number) value).doubleValue(); + } + return 0.0; + } + + public static void main(String[] args) { + String studyDir = System.getenv("CONCORE_STUDY_DIR"); + if (studyDir == null || studyDir.isEmpty()) { + studyDir = "example/java_e2e/study"; + } + double maxTime = 20.0; + + concoredocker.setInPath(studyDir); + concoredocker.setOutPath(studyDir); + concoredocker.setDelay(20); + concoredocker.defaultMaxTime(maxTime); + + while (concoredocker.getSimtime() < maxTime) { + concoredocker.ReadResult readResult = concoredocker.read(1, "u", INIT_SIMTIME_U); + List u = readResult.data; + + double u0 = 0.0; + if (!u.isEmpty()) { + u0 = toDouble(u.get(0)); + } + + double ym0 = u0 + 0.01; + List ym = new ArrayList<>(); + ym.add(ym0); + + System.out.println(concoredocker.getSimtime() + ". u=" + u + " ym=" + ym); + concoredocker.write(1, "ym", ym, 1); + } + } +} diff --git a/example/java_e2e/smoke_check.py b/example/java_e2e/smoke_check.py new file mode 100644 index 00000000..5fc04e92 --- /dev/null +++ b/example/java_e2e/smoke_check.py @@ -0,0 +1,114 @@ +import argparse +import ast +import os +from pathlib import Path +import shutil +import subprocess +import sys + + +JEROMQ_URL = "https://repo1.maven.org/maven2/org/zeromq/jeromq/0.6.0/jeromq-0.6.0.jar" + + +def parse_args(): + parser = argparse.ArgumentParser(description="Run Java e2e example smoke check") + parser.add_argument("--jar", type=Path, help="Path to jeromq jar") + parser.add_argument("--keep-study", action="store_true", help="Keep generated study files") + return parser.parse_args() + +def main(): + args = parse_args() + here = Path(__file__).resolve().parent + repo_root = here.parents[1] + study_dir = here / "study" + jar_path = args.jar or (repo_root / ".ci-cache" / "java" / "jeromq-0.6.0.jar") + + if not jar_path.exists(): + raise FileNotFoundError( + f"Missing jeromq jar at {jar_path}. Download from {JEROMQ_URL} or pass --jar." + ) + + if study_dir.exists(): + shutil.rmtree(study_dir) + (study_dir / "1").mkdir(parents=True, exist_ok=True) + (study_dir / "1" / "concore.maxtime").write_text("20", encoding="utf-8") + + subprocess.run( + [ + "javac", + "-cp", + str(jar_path), + str(repo_root / "concoredocker.java"), + str(here / "pm_java.java"), + ], + check=True, + cwd=repo_root, + ) + + env = os.environ.copy() + env["CONCORE_STUDY_DIR"] = str(study_dir) + classpath = os.pathsep.join([str(repo_root), str(here), str(jar_path)]) + + py_log = open(study_dir / "controller.log", "w", encoding="utf-8") + java_log = open(study_dir / "pm_java.log", "w", encoding="utf-8") + + py_proc = subprocess.Popen( + [sys.executable, str(here / "controller.py")], + cwd=repo_root, + env=env, + stdout=py_log, + stderr=subprocess.STDOUT, + ) + java_proc = subprocess.Popen( + ["java", "-cp", classpath, "pm_java"], + cwd=repo_root, + env=env, + stdout=java_log, + stderr=subprocess.STDOUT, + ) + + try: + py_rc = py_proc.wait(timeout=45) + java_rc = java_proc.wait(timeout=45) + except subprocess.TimeoutExpired: + py_proc.kill() + java_proc.kill() + py_log.close() + java_log.close() + raise RuntimeError("Timed out waiting for node processes") + + py_log.close() + java_log.close() + + if py_rc != 0 or java_rc != 0: + raise RuntimeError( + f"Node process failed (controller={py_rc}, pm_java={java_rc}). " + f"See {study_dir / 'controller.log'} and {study_dir / 'pm_java.log'}." + ) + + u_path = study_dir / "1" / "u" + ym_path = study_dir / "1" / "ym" + if not u_path.exists() or not ym_path.exists(): + raise RuntimeError("Expected output files were not produced") + + u_val = ast.literal_eval(u_path.read_text(encoding="utf-8")) + ym_val = ast.literal_eval(ym_path.read_text(encoding="utf-8")) + if not isinstance(u_val, list) or len(u_val) < 2: + raise RuntimeError("u output did not match expected wire format") + if not isinstance(ym_val, list) or len(ym_val) < 2: + raise RuntimeError("ym output did not match expected wire format") + + print("smoke_check passed") + print(f"u: {u_val}") + print(f"ym: {ym_val}") + + if not args.keep_study and study_dir.exists(): + shutil.rmtree(study_dir) + + class_file = here / "pm_java.class" + if class_file.exists(): + class_file.unlink() + + +if __name__ == "__main__": + main() From 9c50325a0164990dc14673b256922ccf57194766 Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Wed, 1 Apr 2026 09:21:50 +0530 Subject: [PATCH 268/275] fix: resolve broken imports in cardiac_pm.py (#290) cardiac_pm.py in humanc/ and tools/ (symlinked by linktest/) imported pulsatile_model_functions and healthy_params directly, but these modules only exist inside the cardiac_pm.dir/ subdirectories. Add sys.path manipulation to insert the cardiac_pm.dir/ directory into the import path so the modules are found both during development and after deployment via mkconcore.py. --- humanc/cardiac_pm.py | 4 ++++ tools/cardiac_pm.py | 6 +++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/humanc/cardiac_pm.py b/humanc/cardiac_pm.py index d3edb300..398e2fc7 100644 --- a/humanc/cardiac_pm.py +++ b/humanc/cardiac_pm.py @@ -1,4 +1,8 @@ import numpy as np +import sys +import os + +sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), 'cardiac_pm.dir')) import pulsatile_model_functions as pmf import healthy_params as K import concore diff --git a/tools/cardiac_pm.py b/tools/cardiac_pm.py index 0fd5924e..0cfe34c9 100644 --- a/tools/cardiac_pm.py +++ b/tools/cardiac_pm.py @@ -1,8 +1,12 @@ import numpy as np +import sys +import os +import logging + +sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), 'cardiac_pm.dir')) import pulsatile_model_functions as pmf import healthy_params as K import concore -import logging #x0 = np.loadtxt('pulsatile_steady.txt') From 011daaa5e7a92097648fe2d197dd84111abb6ac0 Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Wed, 1 Apr 2026 09:35:45 +0530 Subject: [PATCH 269/275] refactor: guard sys.path.insert and fix bare except (E722) - Guard sys.path.insert with a duplicate check to avoid accumulating entries on repeated imports/reloads (Copilot review suggestion). - Replace bare except with except (ValueError, SyntaxError) in sample/PZ/pm.py and sample/src/pm.py to fix ruff E722. --- humanc/cardiac_pm.py | 4 +++- sample/PZ/pm.py | 34 ++++++++++++++++++++++++++++++++++ sample/src/pm.py | 34 ++++++++++++++++++++++++++++++++++ tools/cardiac_pm.py | 4 +++- 4 files changed, 74 insertions(+), 2 deletions(-) create mode 100644 sample/PZ/pm.py create mode 100644 sample/src/pm.py diff --git a/humanc/cardiac_pm.py b/humanc/cardiac_pm.py index 398e2fc7..67a55cf7 100644 --- a/humanc/cardiac_pm.py +++ b/humanc/cardiac_pm.py @@ -2,7 +2,9 @@ import sys import os -sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), 'cardiac_pm.dir')) +cardiac_pm_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'cardiac_pm.dir') +if cardiac_pm_dir not in sys.path: + sys.path.insert(0, cardiac_pm_dir) import pulsatile_model_functions as pmf import healthy_params as K import concore diff --git a/sample/PZ/pm.py b/sample/PZ/pm.py new file mode 100644 index 00000000..8eff2ca3 --- /dev/null +++ b/sample/PZ/pm.py @@ -0,0 +1,34 @@ +import concore +import numpy as np +import ast + + +def pm(u): + return u + 0.01 + + +concore.default_maxtime(150) +concore.delay = 0.02 + +init_simtime_u = "[0.0, 0.0]" +init_simtime_ym = "[0.0, 0.0]" + +ym = np.array([concore.initval(init_simtime_ym)], dtype=np.float64).T + +while concore.simtime < concore.maxtime: + while concore.unchanged(): + u_raw = concore.read(1, "u", init_simtime_u) + if isinstance(u_raw, str): + try: + u_raw = ast.literal_eval(u_raw) + except (ValueError, SyntaxError): + print("Failed to parse fallback u string:", u_raw) + u_raw = [0.0] + u = np.array([u_raw], dtype=np.float64).T + + ym = pm(u) + + print(f"{concore.simtime}. u={u} ym={ym}") + concore.write(1, "ym", [float(x) for x in ym.T[0]], delta=1) + +print("retry=" + str(concore.retrycount)) diff --git a/sample/src/pm.py b/sample/src/pm.py new file mode 100644 index 00000000..8eff2ca3 --- /dev/null +++ b/sample/src/pm.py @@ -0,0 +1,34 @@ +import concore +import numpy as np +import ast + + +def pm(u): + return u + 0.01 + + +concore.default_maxtime(150) +concore.delay = 0.02 + +init_simtime_u = "[0.0, 0.0]" +init_simtime_ym = "[0.0, 0.0]" + +ym = np.array([concore.initval(init_simtime_ym)], dtype=np.float64).T + +while concore.simtime < concore.maxtime: + while concore.unchanged(): + u_raw = concore.read(1, "u", init_simtime_u) + if isinstance(u_raw, str): + try: + u_raw = ast.literal_eval(u_raw) + except (ValueError, SyntaxError): + print("Failed to parse fallback u string:", u_raw) + u_raw = [0.0] + u = np.array([u_raw], dtype=np.float64).T + + ym = pm(u) + + print(f"{concore.simtime}. u={u} ym={ym}") + concore.write(1, "ym", [float(x) for x in ym.T[0]], delta=1) + +print("retry=" + str(concore.retrycount)) diff --git a/tools/cardiac_pm.py b/tools/cardiac_pm.py index 0cfe34c9..7741e628 100644 --- a/tools/cardiac_pm.py +++ b/tools/cardiac_pm.py @@ -3,7 +3,9 @@ import os import logging -sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), 'cardiac_pm.dir')) +_cardiac_pm_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'cardiac_pm.dir') +if _cardiac_pm_dir not in sys.path: + sys.path.insert(0, _cardiac_pm_dir) import pulsatile_model_functions as pmf import healthy_params as K import concore From 602d7f2d5fb8bd6efb67a20d1fbcf486e5f5cbcc Mon Sep 17 00:00:00 2001 From: Avinash Kumar Deepak Date: Wed, 1 Apr 2026 16:03:40 +0530 Subject: [PATCH 270/275] extend conformance matrix with read_file cases --- tests/protocol_fixtures/PROTOCOL_FIXTURES.md | 2 + .../cross_runtime_matrix.phase2.json | 72 +++++++++++++++++++ .../python_phase1_cases.json | 37 ++++++++++ tests/protocol_fixtures/schema.phase1.json | 3 +- tests/test_protocol_conformance.py | 42 ++++++++++- 5 files changed, 154 insertions(+), 2 deletions(-) diff --git a/tests/protocol_fixtures/PROTOCOL_FIXTURES.md b/tests/protocol_fixtures/PROTOCOL_FIXTURES.md index 79b06ae3..dd42bc6c 100644 --- a/tests/protocol_fixtures/PROTOCOL_FIXTURES.md +++ b/tests/protocol_fixtures/PROTOCOL_FIXTURES.md @@ -11,6 +11,7 @@ Phase-1 scope: - No runtime behavior changes. - Python-only execution through `tests/test_protocol_conformance.py`. - Fixture format is language-neutral to enable future cross-binding runners. +- Baseline now includes `read_file` runtime-behavior checks in addition to parser/API targets. Phase-2 scope (mapping only): @@ -18,6 +19,7 @@ Phase-2 scope (mapping only): - Adds a cross-runtime matrix to track per-case audit status and classification. - Java runtime entries are tracked with observed status from the Java regression suite (`TestLiteralEval.java`, `TestConcoredockerApi.java`). - Current baseline records Java as `observed_pass` for the listed phase-2 cases. +- Phase-2 matrix includes `read_file` status rows for cross-runtime tracking. - Keeps CI non-blocking for non-Python runtimes that are not yet audited by marking them as `not_audited`. Java conformance execution in CI: diff --git a/tests/protocol_fixtures/cross_runtime_matrix.phase2.json b/tests/protocol_fixtures/cross_runtime_matrix.phase2.json index ed642cdd..712ba967 100644 --- a/tests/protocol_fixtures/cross_runtime_matrix.phase2.json +++ b/tests/protocol_fixtures/cross_runtime_matrix.phase2.json @@ -237,6 +237,78 @@ "note": "May require binding-specific interpretation." } } + }, + { + "id": "read_file/missing_file_returns_default_and_false", + "target": "read_file", + "runtime_results": { + "python": { + "status": "observed_pass", + "classification": "required", + "note": "Phase-1 baseline execution." + }, + "cpp": { + "status": "not_audited", + "classification": "required", + "note": "Audit planned in phase 2." + }, + "java": { + "status": "observed_pass", + "classification": "required", + "note": "Validated by TestConcoredockerApi.java." + }, + "matlab": { + "status": "not_audited", + "classification": "required", + "note": "Audit planned in phase 2." + }, + "octave": { + "status": "not_audited", + "classification": "required", + "note": "Audit planned in phase 2." + }, + "verilog": { + "status": "not_audited", + "classification": "implementation_defined", + "note": "May require binding-specific interpretation." + } + } + }, + { + "id": "read_file/older_timestamp_does_not_decrease_simtime", + "target": "read_file", + "runtime_results": { + "python": { + "status": "observed_pass", + "classification": "required", + "note": "Phase-1 baseline execution." + }, + "cpp": { + "status": "not_audited", + "classification": "required", + "note": "Audit planned in phase 2." + }, + "java": { + "status": "observed_pass", + "classification": "required", + "note": "Validated by TestConcoredockerApi.java simtime progression checks." + }, + "matlab": { + "status": "not_audited", + "classification": "required", + "note": "Audit planned in phase 2." + }, + "octave": { + "status": "not_audited", + "classification": "required", + "note": "Audit planned in phase 2." + }, + "verilog": { + "status": "not_audited", + "classification": "implementation_defined", + "note": "May require binding-specific interpretation." + } + } } ] } diff --git a/tests/protocol_fixtures/python_phase1_cases.json b/tests/protocol_fixtures/python_phase1_cases.json index f8a72185..ccdeb02d 100644 --- a/tests/protocol_fixtures/python_phase1_cases.json +++ b/tests/protocol_fixtures/python_phase1_cases.json @@ -100,6 +100,43 @@ "sent_payload": "ok", "simtime_after": 10 } + }, + { + "id": "read_file/missing_file_returns_default_and_false", + "target": "read_file", + "description": "read() returns init default with ok=False when file is missing.", + "input": { + "initial_simtime": 4, + "port": 1, + "name": "missing", + "initstr_val": "[0.0, 5.0]" + }, + "expected": { + "result": [ + 5.0 + ], + "ok": false, + "simtime_after": 4 + } + }, + { + "id": "read_file/older_timestamp_does_not_decrease_simtime", + "target": "read_file", + "description": "read() keeps simtime monotonic when incoming file timestamp is older.", + "input": { + "initial_simtime": 10, + "port": 1, + "name": "ym", + "file_content": "[7.0, 3.14]", + "initstr_val": "[0.0, 0.0]" + }, + "expected": { + "result": [ + 3.14 + ], + "ok": true, + "simtime_after": 10 + } } ] } diff --git a/tests/protocol_fixtures/schema.phase1.json b/tests/protocol_fixtures/schema.phase1.json index 3b54e135..33495b82 100644 --- a/tests/protocol_fixtures/schema.phase1.json +++ b/tests/protocol_fixtures/schema.phase1.json @@ -41,7 +41,8 @@ "enum": [ "parse_params", "initval", - "write_zmq" + "write_zmq", + "read_file" ] }, "description": { diff --git a/tests/test_protocol_conformance.py b/tests/test_protocol_conformance.py index 9bb5ab3c..e12ad2b7 100644 --- a/tests/test_protocol_conformance.py +++ b/tests/test_protocol_conformance.py @@ -1,5 +1,7 @@ import json +import os from pathlib import Path +import tempfile import pytest @@ -9,7 +11,7 @@ FIXTURE_DIR = Path(__file__).parent / "protocol_fixtures" SCHEMA_PATH = FIXTURE_DIR / "schema.phase1.json" CASES_PATH = FIXTURE_DIR / "python_phase1_cases.json" -SUPPORTED_TARGETS = {"parse_params", "initval", "write_zmq"} +SUPPORTED_TARGETS = {"parse_params", "initval", "write_zmq", "read_file"} def _load_json(path): @@ -93,6 +95,42 @@ def send_json_with_retry(self, message): concore.zmq_ports[port_name] = existing_port +def _run_read_file_case(case): + old_simtime = concore.simtime + old_inpath = concore.inpath + old_delay = concore.delay + try: + with tempfile.TemporaryDirectory() as temp_dir: + concore.simtime = case["input"]["initial_simtime"] + concore.inpath = os.path.join(temp_dir, "in") + concore.delay = 0 + + port_dir = os.path.join(temp_dir, f"in{case['input']['port']}") + os.makedirs(port_dir, exist_ok=True) + + if "file_content" in case["input"]: + with open( + os.path.join(port_dir, case["input"]["name"]), + "w", + encoding="utf-8", + ) as f: + f.write(case["input"]["file_content"]) + + result, ok = concore.read( + case["input"]["port"], + case["input"]["name"], + case["input"]["initstr_val"], + ) + + assert result == case["expected"]["result"] + assert ok == case["expected"]["ok"] + assert concore.simtime == case["expected"]["simtime_after"] + finally: + concore.simtime = old_simtime + concore.inpath = old_inpath + concore.delay = old_delay + + def _run_case(case): if case["target"] == "parse_params": _run_parse_params_case(case) @@ -100,6 +138,8 @@ def _run_case(case): _run_initval_case(case) elif case["target"] == "write_zmq": _run_write_zmq_case(case) + elif case["target"] == "read_file": + _run_read_file_case(case) else: raise AssertionError(f"Unsupported target: {case['target']}") From 3a44dbd0519e1a0dda9c4226d1e98bf9d92f2cd9 Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Thu, 2 Apr 2026 12:30:52 +0530 Subject: [PATCH 271/275] Docs: Add CONFIG.md documenting config files, env vars, and precedence (#503) --- docs/CONFIG.md | 226 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 226 insertions(+) create mode 100644 docs/CONFIG.md diff --git a/docs/CONFIG.md b/docs/CONFIG.md new file mode 100644 index 00000000..1bc36433 --- /dev/null +++ b/docs/CONFIG.md @@ -0,0 +1,226 @@ +# concore Configuration Reference + +This document describes the configuration files used by `mkconcore.py` to locate compilers, runtimes, and Docker settings. All config files are read from `CONCOREPATH` — the directory containing `concore.py`. + +## How CONCOREPATH Is Resolved + +`mkconcore.py` resolves `CONCOREPATH` in the following order: + +1. The directory containing `mkconcore.py` itself (if `concore.py` exists there) +2. The current working directory (if `concore.py` exists there) +3. Falls back to the `mkconcore.py` directory + +--- + +## Configuration Files + +### `concore.tools` + +**Purpose:** Override compiler and runtime paths for all supported languages. + +**Format:** Key-value pairs, one per line. Lines starting with `#` and blank lines are ignored. + +``` +KEY=value +``` + +**Supported Keys:** + +| Key | Platform | Default | Description | +|-----|----------|---------|-------------| +| `CPPWIN` | Windows | `g++` | C++ compiler | +| `CPPEXE` | POSIX | `g++` | C++ compiler | +| `VWIN` | Windows | `iverilog` | Verilog compiler | +| `VEXE` | POSIX | `iverilog` | Verilog compiler | +| `PYTHONWIN` | Windows | `python` | Python interpreter | +| `PYTHONEXE` | POSIX | `python3` | Python interpreter | +| `MATLABWIN` | Windows | `matlab` | MATLAB executable | +| `MATLABEXE` | POSIX | `matlab` | MATLAB executable | +| `OCTAVEWIN` | Windows | `octave` | GNU Octave executable | +| `OCTAVEEXE` | POSIX | `octave` | GNU Octave executable | +| `JAVACWIN` | Windows | `javac` | Java compiler | +| `JAVACEXE` | POSIX | `javac` | Java compiler | +| `JAVAWIN` | Windows | `java` | Java runtime | +| `JAVAEXE` | POSIX | `java` | Java runtime | + +**Example (`concore.tools`):** + +``` +CPPEXE=/usr/bin/g++ +PYTHONEXE=/opt/python3.11/bin/python3 +VEXE=/usr/local/bin/iverilog +OCTAVEEXE=/usr/bin/octave +JAVACEXE=/usr/lib/jvm/java-17/bin/javac +JAVAEXE=/usr/lib/jvm/java-17/bin/java +``` + +**Windows Example:** + +``` +CPPWIN=C:\MinGW\bin\g++.exe +PYTHONWIN=C:\Python313\python.exe +JAVACWIN=C:\Program Files\Java\jdk-17\bin\javac.exe +JAVAWIN=C:\Program Files\Java\jdk-17\bin\java.exe +``` + +--- + +### `concore.octave` + +**Purpose:** When this file is present, `.m` files are treated as GNU Octave instead of MATLAB. The file content is ignored — only its existence matters. + +**Format:** Empty file (presence-based flag). + +**How to enable:** + +```bash +touch $CONCOREPATH/concore.octave +``` + +**How to disable:** Delete the file. + +--- + +### `concore.mcr` + +**Purpose:** Path to the local MATLAB Compiler Runtime (MCR) installation. Used when running compiled MATLAB executables instead of MATLAB directly. + +**Format:** Single line containing the absolute path to the MCR installation. + +**Default:** `~/MATLAB/R2021a` + +**Example (`concore.mcr`):** + +``` +/opt/matlab/mcr/v913 +``` + +**Windows Example:** + +``` +C:\Program Files\MATLAB\R2021a\runtime\win64 +``` + +--- + +### `concore.sudo` + +**Purpose:** Override the Docker executable command. Originally intended to control whether Docker runs with `sudo`, but can specify any container runtime (Docker, Podman, etc.). + +**Format:** Single line containing the Docker executable command. + +**Default:** `docker` (from `DOCKEREXE` environment variable, or the literal string `docker`) + +**Example (`concore.sudo`):** + +``` +sudo docker +``` + +**To use Podman instead of Docker:** + +``` +podman +``` + +--- + +### `concore.repo` + +**Purpose:** Override the Docker Hub repository prefix used when pulling pre-built images. + +**Format:** Single line containing the Docker Hub username or organization. + +**Default:** `markgarnold` + +**Example (`concore.repo`):** + +``` +myorganization +``` + +This means Docker images will be pulled from `myorganization/` instead of `markgarnold/`. + +--- + +## Environment Variables + +Environment variables are read **before** config files and provide initial defaults. Config files override environment variables. + +| Variable | Default | Description | +|----------|---------|-------------| +| `CONCORE_CPPWIN` | `g++` | Windows C++ compiler | +| `CONCORE_CPPEXE` | `g++` | POSIX C++ compiler | +| `CONCORE_VWIN` | `iverilog` | Windows Verilog compiler | +| `CONCORE_VEXE` | `iverilog` | POSIX Verilog compiler | +| `CONCORE_PYTHONEXE` | `python3` | POSIX Python interpreter | +| `CONCORE_PYTHONWIN` | `python` | Windows Python interpreter | +| `CONCORE_MATLABEXE` | `matlab` | POSIX MATLAB executable | +| `CONCORE_MATLABWIN` | `matlab` | Windows MATLAB executable | +| `CONCORE_OCTAVEEXE` | `octave` | POSIX GNU Octave executable | +| `CONCORE_OCTAVEWIN` | `octave` | Windows GNU Octave executable | +| `CONCORE_JAVACEXE` | `javac` | POSIX Java compiler | +| `CONCORE_JAVACWIN` | `javac` | Windows Java compiler | +| `CONCORE_JAVAEXE` | `java` | POSIX Java runtime | +| `CONCORE_JAVAWIN` | `java` | Windows Java runtime | +| `DOCKEREXE` | `docker` | Docker executable | + +--- + +## Precedence (Priority Order) + +When `mkconcore.py` resolves a tool path, the following precedence applies (highest priority first): + +1. **`concore.tools` file** — overrides everything for compiler/runtime paths +2. **`concore.sudo` file** — overrides `DOCKEREXE` for Docker executable +3. **`concore.repo` file** — overrides `DOCKEREPO` for Docker repository +4. **Environment variables** (`CONCORE_*`, `DOCKEREXE`) — initial defaults +5. **Hardcoded defaults** — `g++`, `python3`, `iverilog`, `octave`, `docker`, etc. + +For `concore.octave` and `concore.mcr`, there is no environment variable equivalent — they are controlled only by file presence/content. + +--- + +## Quick Start + +**Minimal setup (Python-only study on Linux):** + +No config files needed — defaults work out of the box. + +**Typical setup (Python + C++ on Linux):** + +```bash +# Only needed if g++ is not on PATH +echo "CPPEXE=/usr/local/bin/g++-13" > $CONCOREPATH/concore.tools +``` + +**Windows setup (Python + C++):** + +``` +CPPWIN=C:\MinGW\bin\g++.exe +PYTHONWIN=C:\Python313\python.exe +``` + +**Docker without sudo:** + +```bash +echo "docker" > $CONCOREPATH/concore.sudo +``` + +**Using Octave instead of MATLAB:** + +```bash +touch $CONCOREPATH/concore.octave +``` + +--- + +## Diagnostic + +Run `concore doctor` to see all detected tools, config file status, and environment variables in a single readiness report: + +``` +$ concore doctor +``` + +See the [CLI README](../concore_cli/README.md) for details. From 1d833d4895eb65c43254466fa98daa455726247e Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Thu, 2 Apr 2026 12:42:19 +0530 Subject: [PATCH 272/275] Docs: Apply review suggestions for CONCOREPATH and concore.repo descriptions --- docs/CONFIG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/CONFIG.md b/docs/CONFIG.md index 1bc36433..f72e6289 100644 --- a/docs/CONFIG.md +++ b/docs/CONFIG.md @@ -1,6 +1,6 @@ # concore Configuration Reference -This document describes the configuration files used by `mkconcore.py` to locate compilers, runtimes, and Docker settings. All config files are read from `CONCOREPATH` — the directory containing `concore.py`. +This document describes the configuration files used by `mkconcore.py` to locate compilers, runtimes, and Docker settings. All config files are read from `CONCOREPATH` — the resolved concore install directory (usually the directory containing `concore.py`). ## How CONCOREPATH Is Resolved @@ -173,7 +173,7 @@ When `mkconcore.py` resolves a tool path, the following precedence applies (high 1. **`concore.tools` file** — overrides everything for compiler/runtime paths 2. **`concore.sudo` file** — overrides `DOCKEREXE` for Docker executable -3. **`concore.repo` file** — overrides `DOCKEREPO` for Docker repository +3. **`concore.repo` file** — overrides the hardcoded default Docker repository prefix (for example, `markgarnold`) 4. **Environment variables** (`CONCORE_*`, `DOCKEREXE`) — initial defaults 5. **Hardcoded defaults** — `g++`, `python3`, `iverilog`, `octave`, `docker`, etc. From 9d9e038dde0f4f5f132679cc616e8d287655aa5a Mon Sep 17 00:00:00 2001 From: Titas-Ghosh Date: Fri, 3 Apr 2026 02:41:20 +0530 Subject: [PATCH 273/275] cli: add JSON output for validate --- concore_cli/cli.py | 16 ++- concore_cli/commands/validate.py | 183 +++++++++++++++++++++++++++++-- tests/test_cli.py | 38 +++++++ 3 files changed, 228 insertions(+), 9 deletions(-) diff --git a/concore_cli/cli.py b/concore_cli/cli.py index aeceb995..a2736755 100644 --- a/concore_cli/cli.py +++ b/concore_cli/cli.py @@ -95,10 +95,22 @@ def build(workflow_file, source, output, type, auto_build, compose): @cli.command() @click.argument("workflow_file", type=click.Path(exists=True)) @click.option("--source", "-s", default="src", help="Source directory") -def validate(workflow_file, source): +@click.option( + "--format", + "output_format", + default="text", + type=click.Choice(["text", "json"]), + help="Validation output format", +) +def validate(workflow_file, source, output_format): """Validate a workflow file""" try: - ok = validate_workflow(workflow_file, source, console) + ok = validate_workflow( + workflow_file, + source, + console, + output_format=output_format, + ) if not ok: sys.exit(1) except Exception as e: diff --git a/concore_cli/commands/validate.py b/concore_cli/commands/validate.py index e987c8ad..48a69b05 100644 --- a/concore_cli/commands/validate.py +++ b/concore_cli/commands/validate.py @@ -1,3 +1,4 @@ +import json from pathlib import Path from bs4 import BeautifulSoup from rich.panel import Panel @@ -5,23 +6,160 @@ import xml.etree.ElementTree as ET -def validate_workflow(workflow_file, source_dir, console): +def _classify_message(message, bucket_name): + if bucket_name == "info": + if message.startswith("Found ") and "node(s)" in message: + return {"info_type": "node_count"} + if message.startswith("Found ") and "edge(s)" in message: + return {"info_type": "edge_count"} + if message.startswith("ZMQ-based edges:"): + return {"info_type": "zmq_edges"} + if message.startswith("File-based edges:"): + return {"info_type": "file_edges"} + return {"info_type": "info"} + + if message == "File is empty": + return {"error_type": "empty_file"} + if message.startswith("Invalid XML:"): + return {"error_type": "invalid_xml"} + if message == "Not a valid GraphML file - missing root element": + return {"error_type": "invalid_graphml"} + if message == "Missing element": + return {"error_type": "missing_graph_element"} + if message == "Graph missing required 'edgedefault' attribute": + return {"error_type": "missing_edgedefault"} + if message.startswith("Invalid edgedefault value"): + return {"error_type": "invalid_edgedefault"} + if message == "No nodes found in workflow": + return {"error_type": "no_nodes"} + if message == "No edges found in workflow": + return {"error_type": "no_edges"} + if message.startswith("Source directory not found:"): + return {"error_type": "missing_source_dir"} + if message == "Node missing required 'id' attribute": + return {"error_type": "missing_node_id"} + if message.startswith("Node '") and message.endswith("contains unsafe shell characters"): + return {"error_type": "unsafe_node_label"} + if message.startswith("Node '") and "missing format 'ID:filename'" in message: + return {"error_type": "invalid_node_label_format"} + if message.startswith("Node '") and message.endswith("has invalid format"): + return {"error_type": "invalid_node_label_format"} + if message.startswith("Node '") and message.endswith("has no filename"): + return {"error_type": "missing_node_filename"} + if message.startswith("Node '") and message.endswith("has unusual file extension"): + return {"error_type": "unusual_file_extension"} + if message.startswith("Missing source file:"): + return {"error_type": "missing_source_file"} + if message.startswith("Node ") and message.endswith(" has no label"): + return {"error_type": "missing_node_label"} + if message.startswith("Error parsing node:"): + return {"error_type": "node_parse_error"} + if message.startswith("Duplicate node label:"): + return {"error_type": "duplicate_node_label"} + if message == "Edge missing source or target": + return {"error_type": "missing_edge_endpoint"} + if message.startswith("Edge references non-existent source node:"): + return {"error_type": "missing_edge_source"} + if message.startswith("Edge references non-existent target node:"): + return {"error_type": "missing_edge_target"} + if message == "Workflow contains cycles (expected for control loops)": + return {"error_type": "cycle_detected"} + if message.startswith("Invalid port number:"): + return {"error_type": "invalid_port_number"} + if message.startswith("Port conflict:"): + return {"error_type": "port_conflict"} + if message.startswith("Port ") and "is in reserved range" in message: + return {"error_type": "reserved_port"} + if message.startswith("File not found:"): + return {"error_type": "file_not_found"} + if message.startswith("Validation failed:"): + return {"error_type": "validation_exception"} + return {"error_type": "validation_message"} + + +def _build_entries(bucket_name, messages, source_nodes): + entries = [] + for message in messages: + entry = {"message": message} + entry.update(_classify_message(message, bucket_name)) + + if message.startswith("Missing source file:"): + filename = message.split(":", 1)[1].strip() + node_id = source_nodes.get(filename) + if node_id: + entry["node_id"] = node_id + elif message.startswith("Node ") and message.endswith(" has no label"): + entry["node_id"] = message[5:-9] + elif message.startswith("Edge references non-existent source node:"): + entry["node_id"] = message.split(":", 1)[1].strip() + elif message.startswith("Edge references non-existent target node:"): + entry["node_id"] = message.split(":", 1)[1].strip() + + entries.append(entry) + return entries + + +def _build_payload(workflow_path, source_root, errors, warnings, info, source_nodes): + error_entries = _build_entries("errors", errors, source_nodes) + warning_entries = _build_entries("warnings", warnings, source_nodes) + info_entries = _build_entries("info", info, source_nodes) + + nodes_affected = [] + for entry in error_entries + warning_entries: + node_id = entry.get("node_id") + if node_id and node_id not in nodes_affected: + nodes_affected.append(node_id) + + return { + "workflow": workflow_path.name, + "source_dir": str(source_root), + "valid": len(errors) == 0, + "errors": error_entries, + "warnings": warning_entries, + "info": info_entries, + "summary": { + "error_count": len(error_entries), + "warning_count": len(warning_entries), + "info_count": len(info_entries), + "nodes_affected": nodes_affected, + }, + } + + +def validate_workflow(workflow_file, source_dir, console, output_format="text"): workflow_path = Path(workflow_file) source_root = workflow_path.parent / source_dir - console.print(f"[cyan]Validating:[/cyan] {workflow_path.name}") - console.print() + if output_format == "text": + console.print(f"[cyan]Validating:[/cyan] {workflow_path.name}") + console.print() errors = [] warnings = [] info = [] + source_nodes = {} def finalize(): - show_results(console, errors, warnings, info) + if output_format == "json": + print( + json.dumps( + _build_payload( + workflow_path, + source_root, + errors, + warnings, + info, + source_nodes, + ), + indent=2, + ) + ) + else: + show_results(console, errors, warnings, info) return len(errors) == 0 try: - with open(workflow_path, "r") as f: + with open(workflow_path, "r", encoding="utf-8") as f: content = f.read() if not content.strip(): @@ -109,6 +247,7 @@ def finalize(): warnings.append(f"Node '{label}' has invalid format") else: nodeId_part, filename = parts + source_nodes[filename] = node_id if not filename: errors.append(f"Node '{label}' has no filename") elif not any( @@ -177,10 +316,40 @@ def finalize(): return finalize() except FileNotFoundError: - console.print(f"[red]Error:[/red] File not found: {workflow_path}") + if output_format == "json": + print( + json.dumps( + _build_payload( + workflow_path, + source_root, + [f"File not found: {workflow_path}"], + [], + [], + source_nodes, + ), + indent=2, + ) + ) + else: + console.print(f"[red]Error:[/red] File not found: {workflow_path}") return False except Exception as e: - console.print(f"[red]Validation failed:[/red] {str(e)}") + if output_format == "json": + print( + json.dumps( + _build_payload( + workflow_path, + source_root, + [f"Validation failed: {str(e)}"], + [], + [], + source_nodes, + ), + indent=2, + ) + ) + else: + console.print(f"[red]Validation failed:[/red] {str(e)}") return False diff --git a/tests/test_cli.py b/tests/test_cli.py index d5025523..63d3cb4d 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -84,6 +84,44 @@ def test_validate_missing_node_file(self): self.assertNotEqual(result.exit_code, 0) self.assertIn("Missing source file", result.output) + def test_validate_json_output_for_valid_file(self): + with self.runner.isolated_filesystem(temp_dir=self.temp_dir): + result = self.runner.invoke(cli, ["init", "test-project"]) + self.assertEqual(result.exit_code, 0) + + result = self.runner.invoke( + cli, + ["validate", "test-project/workflow.graphml", "--format", "json"], + ) + self.assertEqual(result.exit_code, 0) + + payload = json.loads(result.output) + self.assertTrue(payload["valid"]) + self.assertEqual(payload["summary"]["error_count"], 0) + self.assertEqual(payload["workflow"], "workflow.graphml") + self.assertIn("src", payload["source_dir"]) + + def test_validate_json_output_for_missing_source_file(self): + with self.runner.isolated_filesystem(temp_dir=self.temp_dir): + result = self.runner.invoke(cli, ["init", "test-project"]) + self.assertEqual(result.exit_code, 0) + + missing_file = Path("test-project/src/script.py") + if missing_file.exists(): + missing_file.unlink() + + result = self.runner.invoke( + cli, + ["validate", "test-project/workflow.graphml", "--format", "json"], + ) + self.assertNotEqual(result.exit_code, 0) + + payload = json.loads(result.output) + self.assertFalse(payload["valid"]) + self.assertEqual(payload["summary"]["error_count"], 1) + self.assertEqual(payload["errors"][0]["error_type"], "missing_source_file") + self.assertEqual(payload["errors"][0]["node_id"], "n1") + def test_status_command(self): result = self.runner.invoke(cli, ["status"]) self.assertEqual(result.exit_code, 0) From 96b13301a761d32c49e96caa87121e9b70a73f32 Mon Sep 17 00:00:00 2001 From: Titas-Ghosh Date: Fri, 3 Apr 2026 03:07:12 +0530 Subject: [PATCH 274/275] style: format validate JSON output changes --- concore_cli/commands/validate.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/concore_cli/commands/validate.py b/concore_cli/commands/validate.py index 48a69b05..a7168b3b 100644 --- a/concore_cli/commands/validate.py +++ b/concore_cli/commands/validate.py @@ -38,7 +38,9 @@ def _classify_message(message, bucket_name): return {"error_type": "missing_source_dir"} if message == "Node missing required 'id' attribute": return {"error_type": "missing_node_id"} - if message.startswith("Node '") and message.endswith("contains unsafe shell characters"): + if message.startswith("Node '") and message.endswith( + "contains unsafe shell characters" + ): return {"error_type": "unsafe_node_label"} if message.startswith("Node '") and "missing format 'ID:filename'" in message: return {"error_type": "invalid_node_label_format"} From 940a65a5a821d046a352af47207a4ec8bb0aa70c Mon Sep 17 00:00:00 2001 From: Pradeeban Kathiravelu Date: Thu, 2 Apr 2026 20:09:39 -0800 Subject: [PATCH 275/275] Attempt to fix the workload integration failure. --- .gitignore | 5 +++++ Dockerfile.py | 2 +- testsou/renameDockerfile.cpymat | 2 +- testsou/renameDockerfile.pmpymat | 2 +- 4 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 81647a6f..b323b3eb 100644 --- a/.gitignore +++ b/.gitignore @@ -31,3 +31,8 @@ htmlcov/ # Concore specific concorekill.bat + +.claude +.codex +.cursor +_bmad diff --git a/Dockerfile.py b/Dockerfile.py index 1a2a5169..f769c959 100644 --- a/Dockerfile.py +++ b/Dockerfile.py @@ -1,7 +1,7 @@ FROM python:3.10-slim WORKDIR /src -RUN apt-get update && apt-get install -y build-essential g++ libgl1-mesa-glx libx11-6 && rm -rf /var/lib/apt/lists/* +RUN apt-get update && apt-get install -y build-essential g++ libgl1 libx11-6 && rm -rf /var/lib/apt/lists/* COPY requirements.txt . RUN pip install --no-cache-dir -r requirements.txt COPY . . diff --git a/testsou/renameDockerfile.cpymat b/testsou/renameDockerfile.cpymat index 31dbade5..88326efc 100644 --- a/testsou/renameDockerfile.cpymat +++ b/testsou/renameDockerfile.cpymat @@ -2,7 +2,7 @@ FROM jupyter/base-notebook USER root RUN apt-get update -RUN apt-get install -y build-essential g++ libgl1-mesa-glx libx11-6 +RUN apt-get install -y build-essential g++ libgl1 libx11-6 RUN conda install matplotlib scipy RUN pip install cvxopt COPY . /src diff --git a/testsou/renameDockerfile.pmpymat b/testsou/renameDockerfile.pmpymat index df4a0809..8045b470 100644 --- a/testsou/renameDockerfile.pmpymat +++ b/testsou/renameDockerfile.pmpymat @@ -2,7 +2,7 @@ FROM jupyter/base-notebook USER root RUN apt-get update -RUN apt-get install -y build-essential g++ libgl1-mesa-glx libx11-6 +RUN apt-get install -y build-essential g++ libgl1 libx11-6 RUN conda install matplotlib scipy RUN pip install cvxopt COPY . /src