feat(build): support Julia (.jl) nodes in posix and windows builds

README.md

@@ -78,7 +78,7 @@ _concore_ supports customization through configuration files in the `CONCOREPATH
   VEXE=/opt/iverilog/bin/iverilog
   OCTAVEEXE=/snap/bin/octave
   ```
-  Supported keys: `CPPWIN`, `CPPEXE`, `VWIN`, `VEXE`, `PYTHONEXE`, `PYTHONWIN`, `MATLABEXE`, `MATLABWIN`, `OCTAVEEXE`, `OCTAVEWIN`
+  Supported keys: `CPPWIN`, `CPPEXE`, `VWIN`, `VEXE`, `PYTHONEXE`, `PYTHONWIN`, `MATLABEXE`, `MATLABWIN`, `OCTAVEEXE`, `OCTAVEWIN`, `JULIAEXE`, `JULIAWIN`
 
 - **concore.octave** - Treat `.m` files as Octave instead of MATLAB (presence = enabled)
 - **concore.mcr** - MATLAB Compiler Runtime path (single line)

concore_cli/commands/_process_match.py

@@ -12,6 +12,8 @@
     "concoredocker.java",
     "concore.v",
     "concoredocker.v",
+    "concore.jl",
+    "concoredocker.jl",
     "concore_read.m",
 )
 

concore_cli/commands/validate.py

@@ -258,7 +258,7 @@ def finalize():
                                 errors.append(f"Node '{label}' has no filename")
                             elif not any(
                                 filename.endswith(ext)
-                                for ext in [".py", ".cpp", ".m", ".v", ".java"]
+                                for ext in [".py", ".cpp", ".m", ".v", ".java", ".jl"]
                             ):
                                 warnings.append(
                                     f"Node '{label}' has unusual file extension"

mkconcore.py

@@ -204,6 +204,8 @@ def _resolve_concore_path():
 JAVACWIN  = os.environ.get("CONCORE_JAVACWIN", "javac")      #Windows javac
 JAVAEXE   = os.environ.get("CONCORE_JAVAEXE", "java")        #Ubuntu/macOS java
 JAVAWIN   = os.environ.get("CONCORE_JAVAWIN", "java")        #Windows java
+JULIAEXE  = os.environ.get("CONCORE_JULIAEXE", "julia")      #Ubuntu/macOS julia
+JULIAWIN  = os.environ.get("CONCORE_JULIAWIN", "julia")      #Windows julia
 JEROMQ_URL = "https://repo1.maven.org/maven2/org/zeromq/jeromq/0.6.0/jeromq-0.6.0.jar"
 M_IS_OCTAVE = False      #treat .m as octave
 MCRPATH  = "~/MATLAB/R2021a" #path to local Ubunta Matlab Compiler Runtime
@@ -249,6 +251,8 @@ def _resolve_concore_path():
     JAVACWIN  = _tools.get("JAVACWIN", JAVACWIN)
     JAVAEXE   = _tools.get("JAVAEXE", JAVAEXE)
     JAVAWIN   = _tools.get("JAVAWIN", JAVAWIN)
+    JULIAEXE  = _tools.get("JULIAEXE", JULIAEXE)
+    JULIAWIN  = _tools.get("JULIAWIN", JULIAWIN)
 
 prefixedgenode = ""
 sourcedir = os.path.abspath(sys.argv[2])
@@ -644,11 +648,12 @@ def cleanup_script_files():
         print(CONCOREPATH+" is not correct path to concore (missing concore_base.py)")
         sys.exit(1)
 
-if 'jl' in required_langs and concoretype=="docker":
+if 'jl' in required_langs:
+    julia_runtime = "concoredocker.jl" if concoretype=="docker" else "concore.jl"
     try:
-        fsource = open(CONCOREPATH+"/concoredocker.jl")
+        fsource = open(CONCOREPATH+"/"+julia_runtime)
     except (FileNotFoundError, IOError):
-        print(CONCOREPATH+" is not correct path to concore (missing concoredocker.jl)")
+        print(CONCOREPATH+" is not correct path to concore (missing "+julia_runtime+")")
         sys.exit(1)
     with open(outdir+"/src/concore.jl","w") as fcopy:
         fcopy.write(fsource.read())
@@ -1140,6 +1145,8 @@ def cleanup_script_files():
                 fbuild.write("copy .\\src\\concore.v .\\" + containername + "\\concore.v\n")
             elif langext == "java":
                 fbuild.write("copy .\\src\\concore.java .\\" + containername + "\\concore.java\n")
+            elif langext == "jl":
+                fbuild.write("copy .\\src\\concore.jl .\\" + containername + "\\concore.jl\n")
             elif langext == "m":   #  4/2/21
                 fbuild.write("copy .\\src\\concore_*.m .\\" + containername + "\\\n")
                 fbuild.write("copy .\\src\\import_concore.m .\\" + containername + "\\\n")
@@ -1160,6 +1167,8 @@ def cleanup_script_files():
                 fbuild.write("cp ./src/concore.v ./"+containername+"/concore.v\n")
             elif langext == "java":
                 fbuild.write("cp ./src/concore.java ./"+containername+"/concore.java\n")
+            elif langext == "jl":
+                fbuild.write("cp ./src/concore.jl ./"+containername+"/concore.jl\n")
             elif langext == "m":  # 4/2/21
                 fbuild.write("cp ./src/concore_*.m ./"+containername+"/\n")
                 fbuild.write("cp ./src/import_concore.m ./"+containername+"/\n")
@@ -1219,6 +1228,7 @@ def cleanup_script_files():
     VWIN = _quote_executable_windows(VWIN)
     JAVACWIN = _quote_executable_windows(JAVACWIN)
     JAVAWIN = _quote_executable_windows(JAVAWIN)
+    JULIAWIN = _quote_executable_windows(JULIAWIN)
     OCTAVEWIN = _quote_executable_windows(OCTAVEWIN)
     MATLABWIN = _quote_executable_windows(MATLABWIN)
 else:
@@ -1227,6 +1237,7 @@ def cleanup_script_files():
     VEXE = _quote_executable_posix(VEXE)
     JAVACEXE = _quote_executable_posix(JAVACEXE)
     JAVAEXE = _quote_executable_posix(JAVAEXE)
+    JULIAEXE = _quote_executable_posix(JULIAEXE)
     OCTAVEEXE = _quote_executable_posix(OCTAVEEXE)
     MATLABEXE = _quote_executable_posix(MATLABEXE)
 
@@ -1236,7 +1247,7 @@ def cleanup_script_files():
   containername,sourcecode = nodes_dict[node].split(':')
   if len(sourcecode)!=0:
       dockername,langext = sourcecode.rsplit(".", 1)
-      if not (langext in ["py","m","sh","cpp","v","java"]): # 6/22/21
+      if not (langext in ["py","m","sh","cpp","v","java","jl"]): # 6/22/21
           logging.error(f"Extension .{langext} is unsupported")
           sys.exit(1)
       if concoretype=="windows":
@@ -1247,6 +1258,9 @@ def cleanup_script_files():
           if langext=="py":
               frun.write('start /B /D '+q_container+" "+PYTHONWIN+" "+q_source+" >"+q_container+"\\concoreout.txt\n")
               fdebug.write('start /D '+q_container+" cmd /K "+PYTHONWIN+" "+q_source+"\n")
+          elif langext=="jl":
+              frun.write('start /B /D '+q_container+" "+JULIAWIN+" "+q_source+" >"+q_container+"\\concoreout.txt\n")
+              fdebug.write('start /D '+q_container+" cmd /K "+JULIAWIN+" "+q_source+"\n")
           elif langext=="cpp":  #6/25/21
               frun.write('cd '+q_container+'\n')
               frun.write(CPPWIN+' '+q_source+'\n')
@@ -1301,6 +1315,15 @@ def cleanup_script_files():
                     fdebug.write('concorewd="$(pwd)"\n')
                     fdebug.write('osascript -e "tell application \\"Terminal\\" to do script \\"cd \\\\\\"$concorewd/' + safe_container + '\\\\\\"; ' + PYTHONEXE + ' ' + safe_source + '\\"" \n')
 
+            elif langext == "jl":
+                frun.write('(cd ' + safe_container + '; ' + JULIAEXE + ' ' + safe_source + ' >concoreout.txt & echo $! >concorepid) &\n')
+                if ubuntu:
+                    fdebug.write('concorewd="$(pwd)"\n')
+                    fdebug.write('xterm -e bash -c "cd \\"$concorewd/' + safe_container + '\\"; ' + JULIAEXE + ' ' + safe_source + '; bash" &\n')
+                else:
+                    fdebug.write('concorewd="$(pwd)"\n')
+                    fdebug.write('osascript -e "tell application \\"Terminal\\" to do script \\"cd \\\\\\"$concorewd/' + safe_container + '\\\\\\"; ' + JULIAEXE + ' ' + safe_source + '\\"" \n')
+
             elif langext == "cpp":  # 6/22/21
                 frun.write('(cd ' + safe_container + '; ' + CPPEXE + ' ' + safe_source + '; ./a.out >concoreout.txt & echo $! >concorepid) &\n')
                 if ubuntu:

tests/test_julia_local_build.py

@@ -0,0 +1,143 @@
+"""Tests for Julia (.jl) nodes in local (non-Docker) builds.
+
+Julia nodes used to work only with ``--type docker``. For every other build
+type ``mkconcore.py`` stopped with "Extension .jl is unsupported", so a mixed
+Julia/Python study could not be run on a plain machine.
+
+These tests run ``mkconcore.py`` on the repository's own mixed demo study
+(``demo/sampleJ.graphml``: a Julia controller ``CZ`` and a Python plant ``PZ``)
+and check the generated build, run and debug scripts. They need no Julia
+installation because they only look at the generated files.
+"""
+
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+import pytest
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+MKCONCORE = REPO_ROOT / "mkconcore.py"
+DEMO_DIR = REPO_ROOT / "demo"
+GRAPH = DEMO_DIR / "sampleJ.graphml"
+
+POSIX_TYPES = ["posix", "macos", "ubuntu"]
+
+
+def build_study(tmp_path, build_type, extra_env=None):
+    """Run mkconcore.py and return (result, output_dir)."""
+    out_dir = tmp_path / "study"
+    env = {k: v for k, v in os.environ.items() if not k.startswith("CONCORE_JULIA")}
+    env.update(extra_env or {})
+    result = subprocess.run(
+        [
+            sys.executable,
+            str(MKCONCORE),
+            str(GRAPH),
+            str(DEMO_DIR),
+            str(out_dir),
+            build_type,
+        ],
+        cwd=str(REPO_ROOT),
+        env=env,
+        capture_output=True,
+        text=True,
+        timeout=120,
+    )
+    return result, out_dir
+
+
+def read_script(out_dir, name, windows=False):
+    return (out_dir / (name + (".bat" if windows else ""))).read_text()
+
+
+@pytest.mark.parametrize("build_type", POSIX_TYPES + ["windows"])
+def test_local_build_accepts_julia_node(tmp_path, build_type):
+    result, out_dir = build_study(tmp_path, build_type)
+    assert result.returncode == 0, result.stdout + result.stderr
+    assert "unsupported" not in (result.stdout + result.stderr)
+    assert (out_dir / "src" / "controller_jl.jl").exists()
+
+
+@pytest.mark.parametrize("build_type", POSIX_TYPES)
+def test_posix_build_copies_local_julia_runtime(tmp_path, build_type):
+    result, out_dir = build_study(tmp_path, build_type)
+    assert result.returncode == 0, result.stdout + result.stderr
+
+    # The local runtime (relative ./in and ./out paths), not the Docker one.
+    assert (out_dir / "src" / "concore.jl").read_text() == (
+        REPO_ROOT / "concore.jl"
+    ).read_text()
+
+    build = read_script(out_dir, "build")
+    assert "cp ./src/controller_jl.jl ./CZ/controller_jl.jl" in build
+    assert "cp ./src/concore.jl ./CZ/concore.jl" in build
+    # The Python node keeps getting its own runtime, and no Julia files.
+    assert "cp ./src/concore.py ./PZ/concore.py" in build
+    assert "./PZ/concore.jl" not in build
+
+
+def test_posix_run_script_launches_julia_node(tmp_path):
+    result, out_dir = build_study(tmp_path, "posix")
+    assert result.returncode == 0, result.stdout + result.stderr
+
+    run = read_script(out_dir, "run")
+    assert (
+        "(cd CZ; julia controller_jl.jl >concoreout.txt & echo $! >concorepid) &" in run
+    )
+    # concore stop relies on this pid file
+    assert "pm.py" in run
+
+
+def test_ubuntu_debug_script_opens_julia_in_xterm(tmp_path):
+    result, out_dir = build_study(tmp_path, "ubuntu")
+    assert result.returncode == 0, result.stdout + result.stderr
+
+    debug = read_script(out_dir, "debug")
+    assert (
+        'xterm -e bash -c "cd \\"$concorewd/CZ\\"; julia controller_jl.jl; bash" &'
+        in debug
+    )
+
+
+def test_macos_debug_script_opens_julia_in_terminal(tmp_path):
+    result, out_dir = build_study(tmp_path, "macos")
+    assert result.returncode == 0, result.stdout + result.stderr
+
+    debug = read_script(out_dir, "debug")
+    assert "osascript" in debug
+    assert "julia controller_jl.jl" in debug
+
+
+def test_windows_scripts_copy_and_launch_julia_node(tmp_path):
+    result, out_dir = build_study(tmp_path, "windows")
+    assert result.returncode == 0, result.stdout + result.stderr
+
+    build = read_script(out_dir, "build", windows=True)
+    assert "copy .\\src\\concore.jl .\\CZ\\concore.jl" in build
+
+    run = read_script(out_dir, "run", windows=True)
+    assert 'start /B /D "CZ" "julia" "controller_jl.jl" >"CZ"\\concoreout.txt' in run
+
+    debug = read_script(out_dir, "debug", windows=True)
+    assert 'start /D "CZ" cmd /K "julia" "controller_jl.jl"' in debug
+
+
+def test_julia_executable_can_be_overridden_with_env(tmp_path):
+    result, out_dir = build_study(
+        tmp_path, "posix", {"CONCORE_JULIAEXE": "/opt/julia/bin/julia"}
+    )
+    assert result.returncode == 0, result.stdout + result.stderr
+
+    run = read_script(out_dir, "run")
+    assert "(cd CZ; /opt/julia/bin/julia controller_jl.jl >" in run
+
+
+def test_docker_build_still_uses_docker_runtime(tmp_path):
+    result, out_dir = build_study(tmp_path, "docker")
+    assert result.returncode == 0, result.stdout + result.stderr
+
+    assert (out_dir / "src" / "concore.jl").read_text() == (
+        REPO_ROOT / "concoredocker.jl"
+    ).read_text()

tests/test_process_match.py

@@ -27,10 +27,17 @@ def test_true_with_docker_runtime_file(self, tmp_path):
 
     @pytest.mark.parametrize(
         "runtime_file",
-        ["concore.hpp", "concore.java", "concore.v", "concore_read.m"],
+        [
+            "concore.hpp",
+            "concore.java",
+            "concore.v",
+            "concore_read.m",
+            "concore.jl",
+            "concoredocker.jl",
+        ],
     )
     def test_true_for_non_python_nodes(self, tmp_path, runtime_file):
-        # C++, Java, Verilog and MATLAB nodes never get concore.py
+        # C++, Java, Verilog, MATLAB and Julia nodes never get concore.py
         (tmp_path / "concore.iport").write_text("{}")
         (tmp_path / runtime_file).write_text("")
         assert has_concore_markers(str(tmp_path)) is True