diff --git a/README.md b/README.md index 1d59a86..e76f649 100644 --- a/README.md +++ b/README.md @@ -78,7 +78,7 @@ _concore_ supports customization through configuration files in the `CONCOREPATH VEXE=/opt/iverilog/bin/iverilog OCTAVEEXE=/snap/bin/octave ``` - Supported keys: `CPPWIN`, `CPPEXE`, `VWIN`, `VEXE`, `PYTHONEXE`, `PYTHONWIN`, `MATLABEXE`, `MATLABWIN`, `OCTAVEEXE`, `OCTAVEWIN` + Supported keys: `CPPWIN`, `CPPEXE`, `VWIN`, `VEXE`, `PYTHONEXE`, `PYTHONWIN`, `MATLABEXE`, `MATLABWIN`, `OCTAVEEXE`, `OCTAVEWIN`, `JULIAEXE`, `JULIAWIN` - **concore.octave** - Treat `.m` files as Octave instead of MATLAB (presence = enabled) - **concore.mcr** - MATLAB Compiler Runtime path (single line) diff --git a/concore_cli/commands/_process_match.py b/concore_cli/commands/_process_match.py index f1b143e..726d694 100644 --- a/concore_cli/commands/_process_match.py +++ b/concore_cli/commands/_process_match.py @@ -12,6 +12,8 @@ "concoredocker.java", "concore.v", "concoredocker.v", + "concore.jl", + "concoredocker.jl", "concore_read.m", ) diff --git a/concore_cli/commands/validate.py b/concore_cli/commands/validate.py index c8972d7..288694b 100644 --- a/concore_cli/commands/validate.py +++ b/concore_cli/commands/validate.py @@ -258,7 +258,7 @@ def finalize(): errors.append(f"Node '{label}' has no filename") elif not any( filename.endswith(ext) - for ext in [".py", ".cpp", ".m", ".v", ".java"] + for ext in [".py", ".cpp", ".m", ".v", ".java", ".jl"] ): warnings.append( f"Node '{label}' has unusual file extension" diff --git a/mkconcore.py b/mkconcore.py index a1f44ef..f5db91b 100644 --- a/mkconcore.py +++ b/mkconcore.py @@ -204,6 +204,8 @@ def _resolve_concore_path(): JAVACWIN = os.environ.get("CONCORE_JAVACWIN", "javac") #Windows javac JAVAEXE = os.environ.get("CONCORE_JAVAEXE", "java") #Ubuntu/macOS java JAVAWIN = os.environ.get("CONCORE_JAVAWIN", "java") #Windows java +JULIAEXE = os.environ.get("CONCORE_JULIAEXE", "julia") #Ubuntu/macOS julia +JULIAWIN = os.environ.get("CONCORE_JULIAWIN", "julia") #Windows julia JEROMQ_URL = "https://repo1.maven.org/maven2/org/zeromq/jeromq/0.6.0/jeromq-0.6.0.jar" M_IS_OCTAVE = False #treat .m as octave MCRPATH = "~/MATLAB/R2021a" #path to local Ubunta Matlab Compiler Runtime @@ -249,6 +251,8 @@ def _resolve_concore_path(): JAVACWIN = _tools.get("JAVACWIN", JAVACWIN) JAVAEXE = _tools.get("JAVAEXE", JAVAEXE) JAVAWIN = _tools.get("JAVAWIN", JAVAWIN) + JULIAEXE = _tools.get("JULIAEXE", JULIAEXE) + JULIAWIN = _tools.get("JULIAWIN", JULIAWIN) prefixedgenode = "" sourcedir = os.path.abspath(sys.argv[2]) @@ -644,11 +648,12 @@ def cleanup_script_files(): print(CONCOREPATH+" is not correct path to concore (missing concore_base.py)") sys.exit(1) -if 'jl' in required_langs and concoretype=="docker": +if 'jl' in required_langs: + julia_runtime = "concoredocker.jl" if concoretype=="docker" else "concore.jl" try: - fsource = open(CONCOREPATH+"/concoredocker.jl") + fsource = open(CONCOREPATH+"/"+julia_runtime) except (FileNotFoundError, IOError): - print(CONCOREPATH+" is not correct path to concore (missing concoredocker.jl)") + print(CONCOREPATH+" is not correct path to concore (missing "+julia_runtime+")") sys.exit(1) with open(outdir+"/src/concore.jl","w") as fcopy: fcopy.write(fsource.read()) @@ -1140,6 +1145,8 @@ def cleanup_script_files(): fbuild.write("copy .\\src\\concore.v .\\" + containername + "\\concore.v\n") elif langext == "java": fbuild.write("copy .\\src\\concore.java .\\" + containername + "\\concore.java\n") + elif langext == "jl": + fbuild.write("copy .\\src\\concore.jl .\\" + containername + "\\concore.jl\n") elif langext == "m": # 4/2/21 fbuild.write("copy .\\src\\concore_*.m .\\" + containername + "\\\n") fbuild.write("copy .\\src\\import_concore.m .\\" + containername + "\\\n") @@ -1160,6 +1167,8 @@ def cleanup_script_files(): fbuild.write("cp ./src/concore.v ./"+containername+"/concore.v\n") elif langext == "java": fbuild.write("cp ./src/concore.java ./"+containername+"/concore.java\n") + elif langext == "jl": + fbuild.write("cp ./src/concore.jl ./"+containername+"/concore.jl\n") elif langext == "m": # 4/2/21 fbuild.write("cp ./src/concore_*.m ./"+containername+"/\n") fbuild.write("cp ./src/import_concore.m ./"+containername+"/\n") @@ -1219,6 +1228,7 @@ def cleanup_script_files(): VWIN = _quote_executable_windows(VWIN) JAVACWIN = _quote_executable_windows(JAVACWIN) JAVAWIN = _quote_executable_windows(JAVAWIN) + JULIAWIN = _quote_executable_windows(JULIAWIN) OCTAVEWIN = _quote_executable_windows(OCTAVEWIN) MATLABWIN = _quote_executable_windows(MATLABWIN) else: @@ -1227,6 +1237,7 @@ def cleanup_script_files(): VEXE = _quote_executable_posix(VEXE) JAVACEXE = _quote_executable_posix(JAVACEXE) JAVAEXE = _quote_executable_posix(JAVAEXE) + JULIAEXE = _quote_executable_posix(JULIAEXE) OCTAVEEXE = _quote_executable_posix(OCTAVEEXE) MATLABEXE = _quote_executable_posix(MATLABEXE) @@ -1236,7 +1247,7 @@ def cleanup_script_files(): containername,sourcecode = nodes_dict[node].split(':') if len(sourcecode)!=0: dockername,langext = sourcecode.rsplit(".", 1) - if not (langext in ["py","m","sh","cpp","v","java"]): # 6/22/21 + if not (langext in ["py","m","sh","cpp","v","java","jl"]): # 6/22/21 logging.error(f"Extension .{langext} is unsupported") sys.exit(1) if concoretype=="windows": @@ -1247,6 +1258,9 @@ def cleanup_script_files(): if langext=="py": frun.write('start /B /D '+q_container+" "+PYTHONWIN+" "+q_source+" >"+q_container+"\\concoreout.txt\n") fdebug.write('start /D '+q_container+" cmd /K "+PYTHONWIN+" "+q_source+"\n") + elif langext=="jl": + frun.write('start /B /D '+q_container+" "+JULIAWIN+" "+q_source+" >"+q_container+"\\concoreout.txt\n") + fdebug.write('start /D '+q_container+" cmd /K "+JULIAWIN+" "+q_source+"\n") elif langext=="cpp": #6/25/21 frun.write('cd '+q_container+'\n') frun.write(CPPWIN+' '+q_source+'\n') @@ -1301,6 +1315,15 @@ def cleanup_script_files(): fdebug.write('concorewd="$(pwd)"\n') fdebug.write('osascript -e "tell application \\"Terminal\\" to do script \\"cd \\\\\\"$concorewd/' + safe_container + '\\\\\\"; ' + PYTHONEXE + ' ' + safe_source + '\\"" \n') + elif langext == "jl": + frun.write('(cd ' + safe_container + '; ' + JULIAEXE + ' ' + safe_source + ' >concoreout.txt & echo $! >concorepid) &\n') + if ubuntu: + fdebug.write('concorewd="$(pwd)"\n') + fdebug.write('xterm -e bash -c "cd \\"$concorewd/' + safe_container + '\\"; ' + JULIAEXE + ' ' + safe_source + '; bash" &\n') + else: + fdebug.write('concorewd="$(pwd)"\n') + fdebug.write('osascript -e "tell application \\"Terminal\\" to do script \\"cd \\\\\\"$concorewd/' + safe_container + '\\\\\\"; ' + JULIAEXE + ' ' + safe_source + '\\"" \n') + elif langext == "cpp": # 6/22/21 frun.write('(cd ' + safe_container + '; ' + CPPEXE + ' ' + safe_source + '; ./a.out >concoreout.txt & echo $! >concorepid) &\n') if ubuntu: diff --git a/tests/test_julia_local_build.py b/tests/test_julia_local_build.py new file mode 100644 index 0000000..4d0f658 --- /dev/null +++ b/tests/test_julia_local_build.py @@ -0,0 +1,143 @@ +"""Tests for Julia (.jl) nodes in local (non-Docker) builds. + +Julia nodes used to work only with ``--type docker``. For every other build +type ``mkconcore.py`` stopped with "Extension .jl is unsupported", so a mixed +Julia/Python study could not be run on a plain machine. + +These tests run ``mkconcore.py`` on the repository's own mixed demo study +(``demo/sampleJ.graphml``: a Julia controller ``CZ`` and a Python plant ``PZ``) +and check the generated build, run and debug scripts. They need no Julia +installation because they only look at the generated files. +""" + +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[1] +MKCONCORE = REPO_ROOT / "mkconcore.py" +DEMO_DIR = REPO_ROOT / "demo" +GRAPH = DEMO_DIR / "sampleJ.graphml" + +POSIX_TYPES = ["posix", "macos", "ubuntu"] + + +def build_study(tmp_path, build_type, extra_env=None): + """Run mkconcore.py and return (result, output_dir).""" + out_dir = tmp_path / "study" + env = {k: v for k, v in os.environ.items() if not k.startswith("CONCORE_JULIA")} + env.update(extra_env or {}) + result = subprocess.run( + [ + sys.executable, + str(MKCONCORE), + str(GRAPH), + str(DEMO_DIR), + str(out_dir), + build_type, + ], + cwd=str(REPO_ROOT), + env=env, + capture_output=True, + text=True, + timeout=120, + ) + return result, out_dir + + +def read_script(out_dir, name, windows=False): + return (out_dir / (name + (".bat" if windows else ""))).read_text() + + +@pytest.mark.parametrize("build_type", POSIX_TYPES + ["windows"]) +def test_local_build_accepts_julia_node(tmp_path, build_type): + result, out_dir = build_study(tmp_path, build_type) + assert result.returncode == 0, result.stdout + result.stderr + assert "unsupported" not in (result.stdout + result.stderr) + assert (out_dir / "src" / "controller_jl.jl").exists() + + +@pytest.mark.parametrize("build_type", POSIX_TYPES) +def test_posix_build_copies_local_julia_runtime(tmp_path, build_type): + result, out_dir = build_study(tmp_path, build_type) + assert result.returncode == 0, result.stdout + result.stderr + + # The local runtime (relative ./in and ./out paths), not the Docker one. + assert (out_dir / "src" / "concore.jl").read_text() == ( + REPO_ROOT / "concore.jl" + ).read_text() + + build = read_script(out_dir, "build") + assert "cp ./src/controller_jl.jl ./CZ/controller_jl.jl" in build + assert "cp ./src/concore.jl ./CZ/concore.jl" in build + # The Python node keeps getting its own runtime, and no Julia files. + assert "cp ./src/concore.py ./PZ/concore.py" in build + assert "./PZ/concore.jl" not in build + + +def test_posix_run_script_launches_julia_node(tmp_path): + result, out_dir = build_study(tmp_path, "posix") + assert result.returncode == 0, result.stdout + result.stderr + + run = read_script(out_dir, "run") + assert ( + "(cd CZ; julia controller_jl.jl >concoreout.txt & echo $! >concorepid) &" in run + ) + # concore stop relies on this pid file + assert "pm.py" in run + + +def test_ubuntu_debug_script_opens_julia_in_xterm(tmp_path): + result, out_dir = build_study(tmp_path, "ubuntu") + assert result.returncode == 0, result.stdout + result.stderr + + debug = read_script(out_dir, "debug") + assert ( + 'xterm -e bash -c "cd \\"$concorewd/CZ\\"; julia controller_jl.jl; bash" &' + in debug + ) + + +def test_macos_debug_script_opens_julia_in_terminal(tmp_path): + result, out_dir = build_study(tmp_path, "macos") + assert result.returncode == 0, result.stdout + result.stderr + + debug = read_script(out_dir, "debug") + assert "osascript" in debug + assert "julia controller_jl.jl" in debug + + +def test_windows_scripts_copy_and_launch_julia_node(tmp_path): + result, out_dir = build_study(tmp_path, "windows") + assert result.returncode == 0, result.stdout + result.stderr + + build = read_script(out_dir, "build", windows=True) + assert "copy .\\src\\concore.jl .\\CZ\\concore.jl" in build + + run = read_script(out_dir, "run", windows=True) + assert 'start /B /D "CZ" "julia" "controller_jl.jl" >"CZ"\\concoreout.txt' in run + + debug = read_script(out_dir, "debug", windows=True) + assert 'start /D "CZ" cmd /K "julia" "controller_jl.jl"' in debug + + +def test_julia_executable_can_be_overridden_with_env(tmp_path): + result, out_dir = build_study( + tmp_path, "posix", {"CONCORE_JULIAEXE": "/opt/julia/bin/julia"} + ) + assert result.returncode == 0, result.stdout + result.stderr + + run = read_script(out_dir, "run") + assert "(cd CZ; /opt/julia/bin/julia controller_jl.jl >" in run + + +def test_docker_build_still_uses_docker_runtime(tmp_path): + result, out_dir = build_study(tmp_path, "docker") + assert result.returncode == 0, result.stdout + result.stderr + + assert (out_dir / "src" / "concore.jl").read_text() == ( + REPO_ROOT / "concoredocker.jl" + ).read_text() diff --git a/tests/test_process_match.py b/tests/test_process_match.py index b218d19..f86e9cd 100644 --- a/tests/test_process_match.py +++ b/tests/test_process_match.py @@ -27,10 +27,17 @@ def test_true_with_docker_runtime_file(self, tmp_path): @pytest.mark.parametrize( "runtime_file", - ["concore.hpp", "concore.java", "concore.v", "concore_read.m"], + [ + "concore.hpp", + "concore.java", + "concore.v", + "concore_read.m", + "concore.jl", + "concoredocker.jl", + ], ) def test_true_for_non_python_nodes(self, tmp_path, runtime_file): - # C++, Java, Verilog and MATLAB nodes never get concore.py + # C++, Java, Verilog, MATLAB and Julia nodes never get concore.py (tmp_path / "concore.iport").write_text("{}") (tmp_path / runtime_file).write_text("") assert has_concore_markers(str(tmp_path)) is True